From 23a9e249930fbd604df5433f510c2289d533a7ef Mon Sep 17 00:00:00 2001
From: disksing <i@disksing.com>
Date: Tue, 16 Sep 2025 16:16:50 +0800
Subject: [PATCH 01/50] resource_group: implement async loading for resource
 groups to improve startup performance (#411)

* feat: implement async loading for resource groups

- Add async loading mechanism to reduce startup time
- Use atomic operations for loading state management
- Implement lazy loading for individual resource groups
- Add retry mechanism with infinite retries for reliability
- Return error for list requests during loading
- Extend storage interface for single group loading
- Optimize loading logic to avoid partial loading issues

This change significantly improves startup performance by loading
resource groups asynchronously while maintaining data integrity.

Signed-off-by: disksing <i@disksing.com>

* tiny fix

Signed-off-by: disksing <i@disksing.com>

* test: add comprehensive async loading test with simplified blocking control

- Simplify control mechanism from 4 to 2 control points:
  * blockBeforeLoad: blocks before starting load operation
  * blockAfterLoad: blocks after loading is completed
- Add more test data (test-group-3, test-group-4) for better coverage
- Test operations during async loading (read, update, delete, list)
- Test operations after async loading completes
- Verify syncLoadedGroups mechanism prevents group resurrection
- Ensure proper error handling during loading state

This test validates the complete async loading workflow with
simplified control and comprehensive scenario coverage.

Signed-off-by: disksing <i@disksing.com>

* fix: resolve testifylint issues in test files

- Remove unnecessary fmt.Sprintf calls in assert messages
- Use require instead of assert for error assertions
- Remove unused fmt imports

This fixes all testifylint warnings in the test files.

Signed-off-by: disksing <i@disksing.com>

* fix: replace assert.NoError with require.NoError in manager_async_test.go

- Fix testifylint require-error violations on lines 342, 347, and 353
- Use require.NoError for error assertions to ensure test stops on failure

Signed-off-by: disksing <i@disksing.com>

* feat: add metrics for resource group loading operations

- Add asyncLoadGroupDuration histogram to track async loading performance
- Add syncLoadGroupCounter to count synchronous loading operations
- Include duration in async loading completion logs for better observability

This helps monitor the performance of resource group loading and understand
the loading patterns in the system.

Signed-off-by: disksing <i@disksing.com>

* update error code

Signed-off-by: disksing <i@disksing.com>

* minor fix

Signed-off-by: disksing <i@disksing.com>

* fix default group

Signed-off-by: disksing <i@disksing.com>

* extract addDefaultGroup

Signed-off-by: disksing <i@disksing.com>

* minor fix

Signed-off-by: disksing <i@disksing.com>

* fix static check

Signed-off-by: disksing <i@disksing.com>

* fix lint

Signed-off-by: disksing <i@disksing.com>

* fix manager reload

Signed-off-by: disksing <i@disksing.com>

* fix update default group

Signed-off-by: disksing <i@disksing.com>

* fix test

Signed-off-by: disksing <i@disksing.com>

* fix when load failed

Signed-off-by: disksing <i@disksing.com>

---------

Signed-off-by: disksing <i@disksing.com>
(cherry picked from commit da1b8ba1e3873401aef0fcbd99c0898a654952e5)
---
 errors.toml                                   |   5 +
 pkg/errs/errno.go                             |   1 +
 pkg/mcs/resourcemanager/server/manager.go     | 282 ++++++++++++++++--
 .../server/manager_async_test.go              | 150 ++++++++++
 .../resourcemanager/server/manager_test.go    |   4 +-
 pkg/mcs/resourcemanager/server/metrics.go     |  18 ++
 pkg/storage/endpoint/resource_group.go        |  12 +
 .../resourcemanager/resource_manager_test.go  |   9 +
 8 files changed, 454 insertions(+), 27 deletions(-)
 create mode 100644 pkg/mcs/resourcemanager/server/manager_async_test.go

diff --git a/errors.toml b/errors.toml
index 33acad6fbf..791560c9de 100644
--- a/errors.toml
+++ b/errors.toml
@@ -926,6 +926,11 @@ error = '''
 empty region
 '''
 
+["PD:resourcemanager:ErrResourceGroupsLoading"]
+error = '''
+resource groups are still being loaded, please try again later
+'''
+
 ["PD:schedule:ErrCreateOperator"]
 error = '''
 unable to create operator, %s
diff --git a/pkg/errs/errno.go b/pkg/errs/errno.go
index a887d539f6..415f51061a 100644
--- a/pkg/errs/errno.go
+++ b/pkg/errs/errno.go
@@ -541,6 +541,7 @@ var (
 	ErrResourceGroupNotExists  = errors.Normalize("the %s resource group does not exist", errors.RFCCodeText("PD:resourcemanager:ErrGroupNotExists"))
 	ErrDeleteReservedGroup     = errors.Normalize("cannot delete reserved group", errors.RFCCodeText("PD:resourcemanager:ErrDeleteReservedGroup"))
 	ErrInvalidGroup            = errors.Normalize("invalid group settings, please check %s", errors.RFCCodeText("PD:resourcemanager:ErrInvalidGroup"))
+	ErrResourceGroupsLoading  = errors.Normalize("resource groups are still being loaded, please try again later", errors.RFCCodeText("PD:resourcemanager:ErrResourceGroupsLoading"))
 )
 
 // Microservice errors
diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go
index 4f811a0392..606223a5b9 100644
--- a/pkg/mcs/resourcemanager/server/manager.go
+++ b/pkg/mcs/resourcemanager/server/manager.go
@@ -21,6 +21,7 @@ import (
 	"os"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus/push"
@@ -114,8 +115,22 @@ type Manager struct {
 	metrics *metrics
 	// ruCollector is used to collect the RU metering data.
 	ruCollector *ruCollector
+	// async loading state management
+	loadingState int32 // atomic access
+	// syncLoadedGroups records groups that were loaded synchronously (e.g., by lazy loading)
+	syncLoadedGroups map[trackerKey]bool
 }
 
+// LoadingState represents the current loading state of resource groups
+const (
+	// LoadingStateNotStarted means resource groups haven't started loading
+	LoadingStateNotStarted int32 = iota
+	// LoadingStateInProgress means resource groups are being loaded asynchronously
+	LoadingStateInProgress
+	// LoadingStateCompleted means all resource groups have been loaded
+	LoadingStateCompleted
+)
+
 // factoryProvider is a factory provider for the manager, which injects some specialized functions
 // that need to be retrieved from the `bs.Server` instance without interacting with its interface.
 type factoryProvider interface {
@@ -149,6 +164,8 @@ func newManagerBase(controllerConfig *ControllerConfig, writeRole ResourceGroupW
 		keyspaceIDLookup:      make(map[string]uint32),
 		metrics:               newMetrics(),
 		ruCollector:           newRUCollector(),
+		loadingState:          LoadingStateNotStarted,
+		syncLoadedGroups:      make(map[trackerKey]bool),
 	}
 }
 
@@ -194,7 +211,10 @@ func NewMetadataOnlyManager[T metadataFactoryProvider](srv bs.Server) (*Manager,
 		nil,
 	)
 	m.srv = srv
-	if err := m.initMetadata(); err != nil {
+	if err := m.initControllerConfig(); err != nil {
+		return nil, err
+	}
+	if err := m.loadKeyspaceResourceGroups(); err != nil {
 		return nil, err
 	}
 	return m, nil
@@ -325,13 +345,16 @@ func (m *Manager) Init(ctx context.Context) error {
 			m.wg.Wait()
 			return err
 		}
+		atomic.StoreInt32(&m.loadingState, LoadingStateCompleted)
 	} else {
-		if err := m.initMetadata(); err != nil {
-			return err
-		}
 		// This context is derived from the leader/primary context, it will be canceled
 		// from the outside loop when the leader/primary step down.
 		ctx, m.cancel = context.WithCancel(ctx)
+		if err := m.initMetadata(ctx); err != nil {
+			m.cancel()
+			m.wg.Wait()
+			return err
+		}
 	}
 	m.wg.Add(1)
 	// Start the background metrics flusher.
@@ -369,34 +392,140 @@ func (m *Manager) initControllerConfig() error {
 	return nil
 }
 
-func (m *Manager) initMetadata() error {
+func (m *Manager) initMetadata(ctx context.Context) error {
 	if err := m.initControllerConfig(); err != nil {
 		return err
 	}
 
-	// Load keyspace resource groups from the storage.
-	return m.loadKeyspaceResourceGroups()
+	m.Lock()
+	m.krgms = make(map[uint32]*keyspaceResourceGroupManager)
+	m.syncLoadedGroups = make(map[trackerKey]bool)
+	atomic.StoreInt32(&m.loadingState, LoadingStateNotStarted)
+	m.Unlock()
+
+	m.initReserved()
+	if err := m.loadServiceLimits(); err != nil {
+		return err
+	}
+
+	m.wg.Add(1)
+	go m.asyncLoadResourceGroups(ctx)
+	return nil
+}
+
+func (m *Manager) loadServiceLimits() error {
+	return m.storage.LoadServiceLimits(func(keyspaceID uint32, serviceLimit float64) {
+		m.getOrCreateKeyspaceResourceGroupManager(keyspaceID, false).setServiceLimitFromStorage(serviceLimit)
+	})
 }
 
 func (m *Manager) loadKeyspaceResourceGroups() error {
-	// Empty the keyspace resource group manager map before the loading.
+	tempKrgms, err := m.loadKeyspaceResourceGroupsFromStorage()
+	if err != nil {
+		return err
+	}
 	m.Lock()
-	m.krgms = make(map[uint32]*keyspaceResourceGroupManager)
+	m.krgms = tempKrgms
+	m.syncLoadedGroups = nil
+	atomic.StoreInt32(&m.loadingState, LoadingStateCompleted)
 	m.Unlock()
-	// Load keyspace resource group meta info from the storage.
+	m.initReserved()
+	return m.loadServiceLimits()
+}
+
+func (m *Manager) asyncLoadResourceGroups(ctx context.Context) {
+	defer logutil.LogPanic()
+	defer m.wg.Done()
+
+	const retryInterval = 10 * time.Second
+	retry := 0
+	for {
+		select {
+		case <-ctx.Done():
+			log.Info("async loading resource groups cancelled")
+			return
+		default:
+		}
+		if retry > 0 {
+			log.Info("retrying async loading resource groups", zap.Int("retry", retry))
+			timer := time.NewTimer(retryInterval)
+			select {
+			case <-ctx.Done():
+				timer.Stop()
+				log.Info("async loading resource groups cancelled")
+				return
+			case <-timer.C:
+			}
+		}
+
+		atomic.StoreInt32(&m.loadingState, LoadingStateInProgress)
+		startTime := time.Now()
+		tempKrgms, err := m.loadKeyspaceResourceGroupsFromStorage()
+		if err != nil {
+			log.Error("failed to load resource groups", zap.Error(err), zap.Int("retry", retry))
+			atomic.StoreInt32(&m.loadingState, LoadingStateNotStarted)
+			retry++
+			continue
+		}
+
+		loaded := 0
+		m.Lock()
+		for keyspaceID, tempKrgm := range tempKrgms {
+			krgm := m.krgms[keyspaceID]
+			if krgm == nil {
+				krgm = newKeyspaceResourceGroupManager(keyspaceID, m.storage, m.writeRole)
+				m.krgms[keyspaceID] = krgm
+			}
+			groupsToSync := make([]*ResourceGroup, 0)
+			tempKrgm.RLock()
+			krgm.Lock()
+			for name, group := range tempKrgm.groups {
+				key := trackerKey{keyspaceID: keyspaceID, groupName: name}
+				if !m.syncLoadedGroups[key] {
+					krgm.groups[name] = group
+					groupsToSync = append(groupsToSync, group)
+					loaded++
+				}
+			}
+			krgm.Unlock()
+			tempKrgm.RUnlock()
+			for _, group := range groupsToSync {
+				krgm.syncBurstabilityWithServiceLimit(group)
+			}
+		}
+		m.syncLoadedGroups = nil
+		m.Unlock()
+
+		m.initReserved()
+		atomic.StoreInt32(&m.loadingState, LoadingStateCompleted)
+		duration := time.Since(startTime)
+		asyncLoadGroupDuration.Observe(duration.Seconds())
+		log.Info("async loading resource groups completed", zap.Int("loaded-groups", loaded), zap.Duration("duration", duration))
+		return
+	}
+}
+
+func (m *Manager) loadKeyspaceResourceGroupsFromStorage() (map[uint32]*keyspaceResourceGroupManager, error) {
+	tempKrgms := make(map[uint32]*keyspaceResourceGroupManager)
+	getOrCreateTempKrgm := func(keyspaceID uint32) *keyspaceResourceGroupManager {
+		krgm, ok := tempKrgms[keyspaceID]
+		if !ok {
+			krgm = newKeyspaceResourceGroupManager(keyspaceID, m.storage, m.writeRole)
+			tempKrgms[keyspaceID] = krgm
+		}
+		return krgm
+	}
 	if err := m.storage.LoadResourceGroupSettings(func(keyspaceID uint32, name string, rawValue string) {
-		// Since the default resource group might be loaded from the storage, we don't need to initialize it here.
-		err := m.getOrCreateKeyspaceResourceGroupManager(keyspaceID, false).addResourceGroupFromRaw(name, rawValue)
+		err := getOrCreateTempKrgm(keyspaceID).addResourceGroupFromRaw(name, rawValue)
 		if err != nil {
 			log.Error("failed to add resource group to the keyspace resource group manager",
 				zap.Uint32("keyspace-id", keyspaceID), zap.String("group-name", name), zap.Error(err))
 		}
 	}); err != nil {
-		return err
+		return nil, err
 	}
-	// Load keyspace resource group states from the storage.
 	if err := m.storage.LoadResourceGroupStates(func(keyspaceID uint32, name string, rawValue string) {
-		krgm := m.getKeyspaceResourceGroupManager(keyspaceID)
+		krgm := tempKrgms[keyspaceID]
 		if krgm == nil {
 			log.Warn("failed to get the corresponding keyspace resource group manager",
 				zap.Uint32("keyspace-id", keyspaceID), zap.String("group-name", name))
@@ -408,18 +537,81 @@ func (m *Manager) loadKeyspaceResourceGroups() error {
 				zap.Uint32("keyspace-id", keyspaceID), zap.String("group-name", name), zap.Error(err))
 		}
 	}); err != nil {
+		return nil, err
+	}
+	return tempKrgms, nil
+}
+
+func (m *Manager) loadResourceGroup(keyspaceID uint32, name string) (*ResourceGroup, error) {
+	rawValue, err := m.storage.LoadResourceGroupSetting(keyspaceID, name)
+	if err != nil {
+		return nil, err
+	}
+	if rawValue == "" {
+		return nil, errs.ErrResourceGroupNotExists.FastGenByArgs(name)
+	}
+	krgm := newKeyspaceResourceGroupManager(keyspaceID, m.storage, m.writeRole)
+	if err := krgm.addResourceGroupFromRaw(name, rawValue); err != nil {
+		return nil, err
+	}
+	state, err := m.storage.LoadResourceGroupState(keyspaceID, name)
+	if err == nil && state != "" {
+		if err := krgm.setRawStatesIntoResourceGroup(name, state); err != nil {
+			return nil, err
+		}
+	}
+	return krgm.getMutableResourceGroup(name), nil
+}
+
+func (m *Manager) loadResourceGroupIfNeeded(keyspaceID uint32, name string) error {
+	if atomic.LoadInt32(&m.loadingState) == LoadingStateCompleted {
+		return nil
+	}
+	krgm := m.getKeyspaceResourceGroupManager(keyspaceID)
+	if krgm != nil {
+		if group := krgm.getMutableResourceGroup(name); group != nil {
+			return nil
+		}
+	}
+	if name == DefaultResourceGroupName {
+		m.getOrCreateKeyspaceResourceGroupManager(keyspaceID, true)
+		return nil
+	}
+	group, err := m.loadResourceGroup(keyspaceID, name)
+	if err != nil {
 		return err
 	}
-	// Initialize the reserved keyspace resource group manager and default resource groups.
-	m.initReserved()
-	// Load service limits from the storage after all resource groups are loaded.
-	return m.loadServiceLimits()
+	krgm = m.getOrCreateKeyspaceResourceGroupManager(keyspaceID, false)
+	inserted := false
+	krgm.Lock()
+	if _, exists := krgm.groups[name]; !exists {
+		krgm.groups[name] = group
+		inserted = true
+	}
+	krgm.Unlock()
+	if inserted {
+		krgm.syncBurstabilityWithServiceLimit(group)
+	}
+	markKey := trackerKey{keyspaceID: keyspaceID, groupName: name}
+	m.Lock()
+	if m.syncLoadedGroups != nil {
+		m.syncLoadedGroups[markKey] = true
+	}
+	m.Unlock()
+	syncLoadGroupCounter.Inc()
+	return nil
 }
 
-func (m *Manager) loadServiceLimits() error {
-	return m.storage.LoadServiceLimits(func(keyspaceID uint32, serviceLimit float64) {
-		m.getOrCreateKeyspaceResourceGroupManager(keyspaceID, false).setServiceLimitFromStorage(serviceLimit)
-	})
+func (m *Manager) markResourceGroupSyncLoaded(keyspaceID uint32, name string) {
+	m.Lock()
+	defer m.Unlock()
+	if m.syncLoadedGroups != nil {
+		m.syncLoadedGroups[trackerKey{keyspaceID: keyspaceID, groupName: name}] = true
+	}
+}
+
+func (m *Manager) isResourceGroupLoadingComplete() bool {
+	return atomic.LoadInt32(&m.loadingState) == LoadingStateCompleted
 }
 
 func cloneControllerConfig(cfg *ControllerConfig) *ControllerConfig {
@@ -456,6 +648,7 @@ func (m *Manager) applyResourceGroupSettingFromRaw(keyspaceID uint32, name, rawV
 			zap.Error(err))
 		return err
 	}
+	m.markResourceGroupSyncLoaded(keyspaceID, name)
 	return nil
 }
 
@@ -492,6 +685,7 @@ func (m *Manager) applyResourceGroupStatesFromRaw(keyspaceID uint32, name, rawVa
 			zap.Error(err))
 		return err
 	}
+	m.markResourceGroupSyncLoaded(keyspaceID, name)
 	return nil
 }
 
@@ -574,7 +768,14 @@ func (m *Manager) AddResourceGroup(grouppb *rmpb.ResourceGroup) error {
 	if krgm == nil {
 		return errs.ErrKeyspaceNotExists.FastGenByArgs(keyspaceID)
 	}
-	return krgm.addResourceGroup(grouppb)
+	if err := m.loadResourceGroupIfNeeded(keyspaceID, grouppb.Name); err != nil {
+		log.Debug("failed to load resource group", zap.Uint32("keyspace-id", keyspaceID), zap.String("name", grouppb.Name), zap.Error(err))
+	}
+	if err := krgm.addResourceGroup(grouppb); err != nil {
+		return err
+	}
+	m.markResourceGroupSyncLoaded(keyspaceID, grouppb.Name)
+	return nil
 }
 
 // ModifyResourceGroup modifies an existing resource group.
@@ -583,11 +784,19 @@ func (m *Manager) ModifyResourceGroup(grouppb *rmpb.ResourceGroup) error {
 		return errMetadataWriteDisabled
 	}
 	keyspaceID := ExtractKeyspaceID(grouppb.GetKeyspaceId())
+	if err := m.loadResourceGroupIfNeeded(keyspaceID, grouppb.Name); err != nil {
+		log.Debug("failed to load resource group", zap.Uint32("keyspace-id", keyspaceID), zap.String("name", grouppb.Name), zap.Error(err))
+		return err
+	}
 	krgm, err := m.accessKeyspaceResourceGroupManager(keyspaceID, grouppb.Name)
 	if err != nil {
 		return err
 	}
-	return krgm.modifyResourceGroup(grouppb)
+	if err := krgm.modifyResourceGroup(grouppb); err != nil {
+		return err
+	}
+	m.markResourceGroupSyncLoaded(keyspaceID, grouppb.Name)
+	return nil
 }
 
 // DeleteResourceGroup deletes a resource group.
@@ -595,16 +804,28 @@ func (m *Manager) DeleteResourceGroup(keyspaceID uint32, name string) error {
 	if !m.writeRole.AllowsMetadataWrite() {
 		return errMetadataWriteDisabled
 	}
+	if err := m.loadResourceGroupIfNeeded(keyspaceID, name); err != nil {
+		log.Debug("failed to load resource group", zap.Uint32("keyspace-id", keyspaceID), zap.String("name", name), zap.Error(err))
+		return err
+	}
 	// "default" group can't be deleted, so there is not need to call accessKeyspaceResourceGroupManager
 	krgm := m.getKeyspaceResourceGroupManager(keyspaceID)
 	if krgm == nil {
 		return errs.ErrKeyspaceNotExists.FastGenByArgs(keyspaceID)
 	}
-	return krgm.deleteResourceGroup(name)
+	if err := krgm.deleteResourceGroup(name); err != nil {
+		return err
+	}
+	m.markResourceGroupSyncLoaded(keyspaceID, name)
+	return nil
 }
 
 // GetResourceGroup returns a copy of a resource group.
 func (m *Manager) GetResourceGroup(keyspaceID uint32, name string, withStats bool) (*ResourceGroup, error) {
+	if err := m.loadResourceGroupIfNeeded(keyspaceID, name); err != nil {
+		log.Debug("failed to load resource group", zap.Uint32("keyspace-id", keyspaceID), zap.String("name", name), zap.Error(err))
+		return nil, err
+	}
 	krgm, err := m.accessKeyspaceResourceGroupManager(keyspaceID, name)
 	if err != nil {
 		return nil, err
@@ -614,6 +835,10 @@ func (m *Manager) GetResourceGroup(keyspaceID uint32, name string, withStats boo
 
 // GetMutableResourceGroup returns a mutable resource group.
 func (m *Manager) GetMutableResourceGroup(keyspaceID uint32, name string) (*ResourceGroup, error) {
+	if err := m.loadResourceGroupIfNeeded(keyspaceID, name); err != nil {
+		log.Debug("failed to load resource group", zap.Uint32("keyspace-id", keyspaceID), zap.String("name", name), zap.Error(err))
+		return nil, err
+	}
 	krgm, err := m.accessKeyspaceResourceGroupManager(keyspaceID, name)
 	if err != nil {
 		return nil, err
@@ -622,7 +847,12 @@ func (m *Manager) GetMutableResourceGroup(keyspaceID uint32, name string) (*Reso
 }
 
 // GetResourceGroupList returns copies of resource group list.
+// Returns error if resource groups are still being loaded asynchronously.
 func (m *Manager) GetResourceGroupList(keyspaceID uint32, withStats bool) ([]*ResourceGroup, error) {
+	if !m.isResourceGroupLoadingComplete() {
+		log.Debug("resource groups are still being loaded, cannot return list")
+		return nil, errs.ErrResourceGroupsLoading
+	}
 	krgm, err := m.accessKeyspaceResourceGroupManager(keyspaceID, DefaultResourceGroupName)
 	if err != nil {
 		return nil, err
diff --git a/pkg/mcs/resourcemanager/server/manager_async_test.go b/pkg/mcs/resourcemanager/server/manager_async_test.go
new file mode 100644
index 0000000000..7bf7d51ef1
--- /dev/null
+++ b/pkg/mcs/resourcemanager/server/manager_async_test.go
@@ -0,0 +1,150 @@
+// Copyright 2026 TiKV Project Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package server
+
+import (
+	"context"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/pingcap/kvproto/pkg/resource_manager"
+
+	"github.com/tikv/pd/pkg/errs"
+	"github.com/tikv/pd/pkg/storage"
+	"github.com/tikv/pd/pkg/utils/testutil"
+)
+
+type blockingResourceGroupStorage struct {
+	storage.Storage
+
+	once    sync.Once
+	entered chan struct{}
+	release chan struct{}
+}
+
+func newBlockingResourceGroupStorage() *blockingResourceGroupStorage {
+	return &blockingResourceGroupStorage{
+		Storage: storage.NewStorageWithMemoryBackend(),
+		entered: make(chan struct{}),
+		release: make(chan struct{}),
+	}
+}
+
+func (s *blockingResourceGroupStorage) LoadResourceGroupSettings(f func(keyspaceID uint32, name, rawValue string)) error {
+	s.once.Do(func() {
+		close(s.entered)
+		<-s.release
+	})
+	return s.Storage.LoadResourceGroupSettings(f)
+}
+
+func (s *blockingResourceGroupStorage) waitEntered(t *testing.T) {
+	t.Helper()
+	select {
+	case <-s.entered:
+	case <-time.After(time.Second):
+		t.Fatal("timed out waiting for async resource group loading")
+	}
+}
+
+func (s *blockingResourceGroupStorage) unblock() {
+	close(s.release)
+}
+
+func newAsyncTestGroup(name string, fillRate uint64) *resource_manager.ResourceGroup {
+	return &resource_manager.ResourceGroup{
+		Name:     name,
+		Mode:     resource_manager.GroupMode_RUMode,
+		Priority: middlePriority,
+		RUSettings: &resource_manager.GroupRequestUnitSettings{
+			RU: &resource_manager.TokenBucket{
+				Settings: &resource_manager.TokenLimitSettings{
+					FillRate:   fillRate,
+					BurstLimit: int64(fillRate),
+				},
+			},
+		},
+	}
+}
+
+func stopAsyncTestManager(m *Manager) {
+	if m.cancel != nil {
+		m.cancel()
+	}
+	m.wg.Wait()
+}
+
+func TestAsyncLoadResourceGroupsLazyGet(t *testing.T) {
+	re := require.New(t)
+	store := newBlockingResourceGroupStorage()
+	re.NoError(store.SaveResourceGroupSetting(1, "lazy-group", newAsyncTestGroup("lazy-group", 100)))
+
+	m := NewManager[*mockConfigProvider](&mockConfigProvider{})
+	m.storage = store
+	re.NoError(m.Init(context.Background()))
+	defer stopAsyncTestManager(m)
+
+	store.waitEntered(t)
+
+	_, err := m.GetResourceGroupList(1, false)
+	re.ErrorIs(err, errs.ErrResourceGroupsLoading)
+
+	group, err := m.GetResourceGroup(1, "lazy-group", false)
+	re.NoError(err)
+	re.NotNil(group)
+	re.Equal("lazy-group", group.Name)
+	re.Equal(float64(100), group.RUSettings.RU.getFillRate())
+
+	store.unblock()
+	testutil.Eventually(re, func() bool {
+		groups, err := m.GetResourceGroupList(1, false)
+		return err == nil && len(groups) == 2
+	}, testutil.WithTickInterval(20*time.Millisecond))
+}
+
+func TestAsyncLoadResourceGroupsDoesNotRestoreDeletedLazyGroup(t *testing.T) {
+	re := require.New(t)
+	store := newBlockingResourceGroupStorage()
+	re.NoError(store.SaveResourceGroupSetting(1, "deleted-group", newAsyncTestGroup("deleted-group", 100)))
+
+	m := NewManager[*mockConfigProvider](&mockConfigProvider{})
+	m.storage = store
+	re.NoError(m.Init(context.Background()))
+	defer stopAsyncTestManager(m)
+
+	store.waitEntered(t)
+
+	group, err := m.GetResourceGroup(1, "deleted-group", false)
+	re.NoError(err)
+	re.NotNil(group)
+	re.NoError(m.DeleteResourceGroup(1, "deleted-group"))
+
+	store.unblock()
+	testutil.Eventually(re, func() bool {
+		groups, err := m.GetResourceGroupList(1, false)
+		if err != nil {
+			return false
+		}
+		for _, group := range groups {
+			if group.Name == "deleted-group" {
+				return false
+			}
+		}
+		return true
+	}, testutil.WithTickInterval(20*time.Millisecond))
+}
diff --git a/pkg/mcs/resourcemanager/server/manager_test.go b/pkg/mcs/resourcemanager/server/manager_test.go
index 8d98ffc6db..7898fce3ba 100644
--- a/pkg/mcs/resourcemanager/server/manager_test.go
+++ b/pkg/mcs/resourcemanager/server/manager_test.go
@@ -408,7 +408,9 @@ func TestInitManager(t *testing.T) {
 	m.storage = storage
 	err = m.Init(ctx)
 	re.NoError(err)
-	re.Len(m.getKeyspaceResourceGroupManagers(), 2)
+	testutil.Eventually(re, func() bool {
+		return len(m.getKeyspaceResourceGroupManagers()) == 2
+	})
 	// Get the default resource group.
 	rg, err := m.GetResourceGroup(1, DefaultResourceGroupName, true)
 	re.NoError(err)
diff --git a/pkg/mcs/resourcemanager/server/metrics.go b/pkg/mcs/resourcemanager/server/metrics.go
index 5d4907768b..47b507efc8 100644
--- a/pkg/mcs/resourcemanager/server/metrics.go
+++ b/pkg/mcs/resourcemanager/server/metrics.go
@@ -220,6 +220,22 @@ var (
 			Help:      "The duration of pushing RU metrics to Prometheus.",
 			Buckets:   prometheus.DefBuckets,
 		})
+
+	syncLoadGroupCounter = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Namespace: namespace,
+			Subsystem: serverSubsystem,
+			Name:      "sync_load_group_counter",
+			Help:      "The number of the sync load group.",
+		})
+
+	asyncLoadGroupDuration = prometheus.NewHistogram(
+		prometheus.HistogramOpts{
+			Namespace: namespace,
+			Subsystem: serverSubsystem,
+			Name:      "async_load_group_duration_seconds",
+			Help:      "The duration of the async load group.",
+		})
 )
 
 type metrics struct {
@@ -274,6 +290,8 @@ func init() {
 	prometheus.MustRegister(overrideSettings)
 	prometheus.MustRegister(serviceLimit)
 	prometheus.MustRegister(pushRUMetricsDuration)
+	prometheus.MustRegister(syncLoadGroupCounter)
+	prometheus.MustRegister(asyncLoadGroupDuration)
 }
 
 func newMetrics() *metrics {
diff --git a/pkg/storage/endpoint/resource_group.go b/pkg/storage/endpoint/resource_group.go
index 68a6297772..a54eb7a271 100644
--- a/pkg/storage/endpoint/resource_group.go
+++ b/pkg/storage/endpoint/resource_group.go
@@ -30,9 +30,11 @@ import (
 // ResourceGroupStorage defines the storage operations on the resource group.
 type ResourceGroupStorage interface {
 	LoadResourceGroupSettings(f func(keyspaceID uint32, name, rawValue string)) error
+	LoadResourceGroupSetting(keyspaceID uint32, name string) (string, error)
 	SaveResourceGroupSetting(keyspaceID uint32, name string, msg proto.Message) error
 	DeleteResourceGroupSetting(keyspaceID uint32, name string) error
 	LoadResourceGroupStates(f func(keyspaceID uint32, name, rawValue string)) error
+	LoadResourceGroupState(keyspaceID uint32, name string) (string, error)
 	SaveResourceGroupStates(keyspaceID uint32, name string, obj any) error
 	DeleteResourceGroupStates(keyspaceID uint32, name string) error
 	SaveControllerConfig(config any) error
@@ -73,6 +75,11 @@ func (se *StorageEndpoint) LoadResourceGroupSettings(f func(keyspaceID uint32, n
 	})
 }
 
+// LoadResourceGroupSetting loads a specific resource group from storage.
+func (se *StorageEndpoint) LoadResourceGroupSetting(keyspaceID uint32, name string) (string, error) {
+	return se.Load(keypath.KeyspaceResourceGroupSettingPath(keyspaceID, name))
+}
+
 // SaveResourceGroupStates stores a resource group to storage.
 func (se *StorageEndpoint) SaveResourceGroupStates(keyspaceID uint32, name string, obj any) error {
 	return se.saveJSON(keypath.KeyspaceResourceGroupStatePath(keyspaceID, name), obj)
@@ -102,6 +109,11 @@ func (se *StorageEndpoint) LoadResourceGroupStates(f func(keyspaceID uint32, nam
 	})
 }
 
+// LoadResourceGroupState loads a specific resource group state from storage.
+func (se *StorageEndpoint) LoadResourceGroupState(keyspaceID uint32, name string) (string, error) {
+	return se.Load(keypath.KeyspaceResourceGroupStatePath(keyspaceID, name))
+}
+
 // SaveControllerConfig stores the resource controller config to storage.
 func (se *StorageEndpoint) SaveControllerConfig(config any) error {
 	return se.saveJSON(keypath.ControllerConfigPath(), config)
diff --git a/tests/integrations/mcs/resourcemanager/resource_manager_test.go b/tests/integrations/mcs/resourcemanager/resource_manager_test.go
index 1808175f80..79ec16ebd5 100644
--- a/tests/integrations/mcs/resourcemanager/resource_manager_test.go
+++ b/tests/integrations/mcs/resourcemanager/resource_manager_test.go
@@ -193,6 +193,7 @@ func (suite *resourceManagerClientTestSuite) SetupSuite() {
 		// Ensure RM service discovery has picked up the standalone endpoint before running tests.
 		waitResourceManagerServiceURL(re, suite.client, true)
 	}
+	waitAsyncLoadResourceGroups(re, suite.client)
 
 	suite.initGroups = []*rmpb.ResourceGroup{
 		{
@@ -271,6 +272,13 @@ func waitResourceManagerServiceURL(re *require.Assertions, cli pd.Client, wantNo
 	})
 }
 
+func waitAsyncLoadResourceGroups(re *require.Assertions, cli pd.Client) {
+	testutil.Eventually(re, func() bool {
+		_, err := cli.ListResourceGroups(context.TODO())
+		return err == nil
+	}, testutil.WithTickInterval(100*time.Millisecond))
+}
+
 func TestSwitchModeDuringWorkload(t *testing.T) {
 	for _, tc := range []struct {
 		name      string
@@ -545,6 +553,7 @@ func (suite *resourceManagerClientTestSuite) resignAndWaitLeader(re *require.Ass
 	newLeader := suite.cluster.GetServer(suite.cluster.WaitLeader())
 	re.NotNil(newLeader)
 	waitLeaderServingClient(re, suite.client, newLeader.GetAddr())
+	waitAsyncLoadResourceGroups(re, suite.client)
 }
 
 func (suite *resourceManagerClientTestSuite) TestWatchResourceGroup() {

From 966d279550b3f8e5fb06693a45bea01e3353934b Mon Sep 17 00:00:00 2001
From: bufferflies <1045931706@qq.com>
Date: Thu, 11 Jun 2026 04:42:29 +0200
Subject: [PATCH 02/50] tests: wait for resource group reload after restart

Signed-off-by: bufferflies <1045931706@qq.com>
---
 errors.toml                                    |  8 ++++----
 pkg/errs/errno.go                              |  2 +-
 pkg/mcs/resourcemanager/server/manager.go      |  7 ++++++-
 pkg/mcs/resourcemanager/server/metrics.go      |  6 +++---
 .../resourcemanager/resource_manager_test.go   | 18 ++++++++++++------
 5 files changed, 26 insertions(+), 15 deletions(-)

diff --git a/errors.toml b/errors.toml
index 791560c9de..43413f79a0 100644
--- a/errors.toml
+++ b/errors.toml
@@ -921,14 +921,14 @@ error = '''
 keyspace not found with name: %s
 '''
 
-["PD:scatter:ErrEmptyRegion"]
+["PD:resourcemanager:ErrResourceGroupsLoading"]
 error = '''
-empty region
+resource groups are still being loaded, please try again later
 '''
 
-["PD:resourcemanager:ErrResourceGroupsLoading"]
+["PD:scatter:ErrEmptyRegion"]
 error = '''
-resource groups are still being loaded, please try again later
+empty region
 '''
 
 ["PD:schedule:ErrCreateOperator"]
diff --git a/pkg/errs/errno.go b/pkg/errs/errno.go
index 415f51061a..9ddb0b7b40 100644
--- a/pkg/errs/errno.go
+++ b/pkg/errs/errno.go
@@ -541,7 +541,7 @@ var (
 	ErrResourceGroupNotExists  = errors.Normalize("the %s resource group does not exist", errors.RFCCodeText("PD:resourcemanager:ErrGroupNotExists"))
 	ErrDeleteReservedGroup     = errors.Normalize("cannot delete reserved group", errors.RFCCodeText("PD:resourcemanager:ErrDeleteReservedGroup"))
 	ErrInvalidGroup            = errors.Normalize("invalid group settings, please check %s", errors.RFCCodeText("PD:resourcemanager:ErrInvalidGroup"))
-	ErrResourceGroupsLoading  = errors.Normalize("resource groups are still being loaded, please try again later", errors.RFCCodeText("PD:resourcemanager:ErrResourceGroupsLoading"))
+	ErrResourceGroupsLoading   = errors.Normalize("resource groups are still being loaded, please try again later", errors.RFCCodeText("PD:resourcemanager:ErrResourceGroupsLoading"))
 )
 
 // Microservice errors
diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go
index 606223a5b9..329aea39be 100644
--- a/pkg/mcs/resourcemanager/server/manager.go
+++ b/pkg/mcs/resourcemanager/server/manager.go
@@ -555,7 +555,12 @@ func (m *Manager) loadResourceGroup(keyspaceID uint32, name string) (*ResourceGr
 		return nil, err
 	}
 	state, err := m.storage.LoadResourceGroupState(keyspaceID, name)
-	if err == nil && state != "" {
+	if err != nil {
+		log.Warn("failed to load resource group state, continuing without state",
+			zap.Uint32("keyspace-id", keyspaceID),
+			zap.String("group-name", name),
+			zap.Error(err))
+	} else if state != "" {
 		if err := krgm.setRawStatesIntoResourceGroup(name, state); err != nil {
 			return nil, err
 		}
diff --git a/pkg/mcs/resourcemanager/server/metrics.go b/pkg/mcs/resourcemanager/server/metrics.go
index 47b507efc8..2fa373aa59 100644
--- a/pkg/mcs/resourcemanager/server/metrics.go
+++ b/pkg/mcs/resourcemanager/server/metrics.go
@@ -225,8 +225,8 @@ var (
 		prometheus.CounterOpts{
 			Namespace: namespace,
 			Subsystem: serverSubsystem,
-			Name:      "sync_load_group_counter",
-			Help:      "The number of the sync load group.",
+			Name:      "sync_load_groups_total",
+			Help:      "Total number of resource groups loaded synchronously.",
 		})
 
 	asyncLoadGroupDuration = prometheus.NewHistogram(
@@ -234,7 +234,7 @@ var (
 			Namespace: namespace,
 			Subsystem: serverSubsystem,
 			Name:      "async_load_group_duration_seconds",
-			Help:      "The duration of the async load group.",
+			Help:      "Duration of asynchronous resource group loading in seconds.",
 		})
 )
 
diff --git a/tests/integrations/mcs/resourcemanager/resource_manager_test.go b/tests/integrations/mcs/resourcemanager/resource_manager_test.go
index 79ec16ebd5..c3492aa50f 100644
--- a/tests/integrations/mcs/resourcemanager/resource_manager_test.go
+++ b/tests/integrations/mcs/resourcemanager/resource_manager_test.go
@@ -21,6 +21,7 @@ import (
 	"io"
 	"math/rand/v2"
 	"net/http"
+	"reflect"
 	"strconv"
 	"strings"
 	"sync/atomic"
@@ -634,16 +635,21 @@ func (suite *resourceManagerClientTestSuite) TestWatchResourceGroup() {
 	re.NoError(err)
 	re.Contains(resp, "Success!")
 	// Make sure the resource group active
-	meta, err = controller.GetResourceGroup(group.Name)
-	re.NotNil(meta)
-	re.NoError(err)
+	testutil.Eventually(re, func() bool {
+		meta, err = controller.GetResourceGroup(group.Name)
+		if err != nil || meta == nil {
+			return false
+		}
+		meta = controller.GetActiveResourceGroup(group.Name)
+		return meta != nil
+	}, testutil.WithTickInterval(50*time.Millisecond))
 	modifySettings(group, 30000)
 	resp, err = cli.ModifyResourceGroup(suite.ctx, group)
 	re.NoError(err)
 	re.Contains(resp, "Success!")
 	testutil.Eventually(re, func() bool {
 		meta = controller.GetActiveResourceGroup(group.Name)
-		return meta.RUSettings.RU.Settings.FillRate == uint64(30000)
+		return meta != nil && meta.RUSettings.RU.Settings.FillRate == uint64(30000)
 	}, testutil.WithTickInterval(100*time.Millisecond))
 	re.NoError(failpoint.Disable("github.com/tikv/pd/client/resource_group/controller/watchStreamError"))
 
@@ -1500,8 +1506,8 @@ func (suite *resourceManagerClientTestSuite) TestBasicResourceGroupCURD() {
 	testutil.Eventually(re, func() bool {
 		var err error
 		newGroups, err = cli.ListResourceGroups(suite.ctx)
-		return err == nil
-	}, testutil.WithWaitFor(time.Second))
+		return err == nil && reflect.DeepEqual(groups, newGroups)
+	})
 	re.Equal(groups, newGroups)
 }
 

From fae9a3ef251824a5143bdef36818f8798bb6ea06 Mon Sep 17 00:00:00 2001
From: bufferflies <1045931706@qq.com>
Date: Thu, 11 Jun 2026 05:21:56 +0200
Subject: [PATCH 03/50] resource_group: avoid overwriting default during async
 load

Signed-off-by: bufferflies <1045931706@qq.com>
---
 pkg/mcs/resourcemanager/server/manager.go     | 11 ++++++-
 .../resourcemanager/resource_manager_test.go  | 31 +++++++++++++++++--
 2 files changed, 39 insertions(+), 3 deletions(-)

diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go
index 329aea39be..3a06c6fe91 100644
--- a/pkg/mcs/resourcemanager/server/manager.go
+++ b/pkg/mcs/resourcemanager/server/manager.go
@@ -403,7 +403,7 @@ func (m *Manager) initMetadata(ctx context.Context) error {
 	atomic.StoreInt32(&m.loadingState, LoadingStateNotStarted)
 	m.Unlock()
 
-	m.initReserved()
+	m.initReservedInCache()
 	if err := m.loadServiceLimits(); err != nil {
 		return err
 	}
@@ -703,6 +703,15 @@ func (m *Manager) initReserved() {
 	}
 }
 
+func (m *Manager) initReservedInCache() {
+	// Initialize the reserved default group in memory before async loading
+	// without overwriting persisted default group settings.
+	m.getOrCreateKeyspaceResourceGroupManager(constant.NullKeyspaceID, false).ensureReservedDefaultGroupInCache()
+	for _, krgm := range m.getKeyspaceResourceGroupManagers() {
+		krgm.ensureReservedDefaultGroupInCache()
+	}
+}
+
 // UpdateControllerConfigItem updates the controller config item.
 func (m *Manager) UpdateControllerConfigItem(key string, value any) error {
 	if !m.writeRole.AllowsMetadataWrite() {
diff --git a/tests/integrations/mcs/resourcemanager/resource_manager_test.go b/tests/integrations/mcs/resourcemanager/resource_manager_test.go
index c3492aa50f..e03dd2bcc2 100644
--- a/tests/integrations/mcs/resourcemanager/resource_manager_test.go
+++ b/tests/integrations/mcs/resourcemanager/resource_manager_test.go
@@ -22,12 +22,14 @@ import (
 	"math/rand/v2"
 	"net/http"
 	"reflect"
+	"sort"
 	"strconv"
 	"strings"
 	"sync/atomic"
 	"testing"
 	"time"
 
+	"github.com/gogo/protobuf/proto"
 	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
 	"go.uber.org/goleak"
@@ -1502,13 +1504,38 @@ func (suite *resourceManagerClientTestSuite) TestBasicResourceGroupCURD() {
 	// re-connect client as well
 	suite.client = suite.setupPDClient(re)
 	cli = suite.client
+	expectedGroups := normalizeResourceGroupsForSettingsCompare(groups)
 	var newGroups []*rmpb.ResourceGroup
 	testutil.Eventually(re, func() bool {
 		var err error
 		newGroups, err = cli.ListResourceGroups(suite.ctx)
-		return err == nil && reflect.DeepEqual(groups, newGroups)
+		return err == nil && reflect.DeepEqual(expectedGroups, normalizeResourceGroupsForSettingsCompare(newGroups))
 	})
-	re.Equal(groups, newGroups)
+	re.Equal(expectedGroups, normalizeResourceGroupsForSettingsCompare(newGroups))
+}
+
+func normalizeResourceGroupsForSettingsCompare(groups []*rmpb.ResourceGroup) []*rmpb.ResourceGroup {
+	normalized := make([]*rmpb.ResourceGroup, 0, len(groups))
+	for _, group := range groups {
+		cloned := proto.Clone(group).(*rmpb.ResourceGroup)
+		cloned.RUStats = nil
+		resetTokenBucketRuntimeState(cloned.GetRUSettings().GetRU())
+		rawSettings := cloned.GetRawResourceSettings()
+		resetTokenBucketRuntimeState(rawSettings.GetCpu())
+		resetTokenBucketRuntimeState(rawSettings.GetIoRead())
+		resetTokenBucketRuntimeState(rawSettings.GetIoWrite())
+		normalized = append(normalized, cloned)
+	}
+	sort.Slice(normalized, func(i, j int) bool {
+		return normalized[i].GetName() < normalized[j].GetName()
+	})
+	return normalized
+}
+
+func resetTokenBucketRuntimeState(bucket *rmpb.TokenBucket) {
+	if bucket != nil {
+		bucket.Tokens = 0
+	}
 }
 
 func (suite *resourceManagerClientTestSuite) TestResourceGroupRUConsumption() {

From f7cf2cdcaf13aad4af4b49060f9edbd27a557642 Mon Sep 17 00:00:00 2001
From: bufferflies <1045931706@qq.com>
Date: Thu, 11 Jun 2026 06:01:05 +0200
Subject: [PATCH 04/50] tests: avoid direct proto dependency in RM integration

Signed-off-by: bufferflies <1045931706@qq.com>
---
 .../mcs/resourcemanager/resource_manager_test.go             | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/integrations/mcs/resourcemanager/resource_manager_test.go b/tests/integrations/mcs/resourcemanager/resource_manager_test.go
index e03dd2bcc2..675c82c2a4 100644
--- a/tests/integrations/mcs/resourcemanager/resource_manager_test.go
+++ b/tests/integrations/mcs/resourcemanager/resource_manager_test.go
@@ -29,7 +29,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/gogo/protobuf/proto"
 	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
 	"go.uber.org/goleak"
@@ -1517,7 +1516,9 @@ func (suite *resourceManagerClientTestSuite) TestBasicResourceGroupCURD() {
 func normalizeResourceGroupsForSettingsCompare(groups []*rmpb.ResourceGroup) []*rmpb.ResourceGroup {
 	normalized := make([]*rmpb.ResourceGroup, 0, len(groups))
 	for _, group := range groups {
-		cloned := proto.Clone(group).(*rmpb.ResourceGroup)
+		cloned := typeutil.DeepClone(group, func() *rmpb.ResourceGroup {
+			return &rmpb.ResourceGroup{}
+		})
 		cloned.RUStats = nil
 		resetTokenBucketRuntimeState(cloned.GetRUSettings().GetRU())
 		rawSettings := cloned.GetRawResourceSettings()

From a20e813777d94410189363e90fdbd046b8b41777 Mon Sep 17 00:00:00 2001
From: tongjian <1045931706@qq.com>
Date: Wed, 8 Jul 2026 09:50:36 +0800
Subject: [PATCH 05/50] resource_group: fix lazy-load ordering and
 default-group review findings

Address PR #10873 review comments:

- grpc_service: AcquireTokenBuckets checked accessKeyspaceResourceGroupManager
  before GetMutableResourceGroup, bypassing the lazy-load path for non-default
  groups whose keyspace manager wasn't in memory yet. Reorder so the lazy load
  runs first.
- manager: loadResourceGroupIfNeeded synthesized the reserved default group
  unconditionally, which could persist synthetic defaults over a customized
  one still on disk. Try the storage load first and only fall back to the
  synthetic default on an explicit not-found error.
- manager: AddResourceGroup silently swallowed all loadResourceGroupIfNeeded
  errors, including transient storage failures. Only ignore the expected
  not-found case and propagate others.
- manager_async_test: blockingResourceGroupStorage.unblock could hang
  stopAsyncTestManager's wg.Wait() if a test aborted before calling it. Make
  unblock idempotent and defer it right after Init so teardown can't deadlock.

Signed-off-by: tongjian <doufuxiaowangzi@gmail.com>
Signed-off-by: tongjian <1045931706@qq.com>
---
 .../resourcemanager/server/grpc_service.go    | 14 ++++++++------
 pkg/mcs/resourcemanager/server/manager.go     | 16 ++++++++++------
 .../server/manager_async_test.go              | 19 +++++++++++++++----
 3 files changed, 33 insertions(+), 16 deletions(-)

diff --git a/pkg/mcs/resourcemanager/server/grpc_service.go b/pkg/mcs/resourcemanager/server/grpc_service.go
index 875299c602..bbb73b2340 100644
--- a/pkg/mcs/resourcemanager/server/grpc_service.go
+++ b/pkg/mcs/resourcemanager/server/grpc_service.go
@@ -231,18 +231,20 @@ func (s *Service) AcquireTokenBuckets(stream rmpb.ResourceManager_AcquireTokenBu
 				zap.Uint32("keyspace-id", keyspaceID),
 				zap.String("resource-group", resourceGroupName),
 			)
+			// Get the resource group from manager to acquire token buckets. This also
+			// triggers lazy loading of the group if async loading hasn't completed yet,
+			// so it must happen before accessKeyspaceResourceGroupManager below.
+			rg, err := s.manager.GetMutableResourceGroup(keyspaceID, resourceGroupName)
+			if rg == nil {
+				log.Warn("resource group not found", append(requestFields, zap.Error(err))...)
+				continue
+			}
 			// Get keyspace resource group manager to apply service limit later.
 			krgm, err := s.manager.accessKeyspaceResourceGroupManager(keyspaceID, resourceGroupName)
 			if krgm == nil {
 				log.Warn("keyspace resource group manager not found", append(requestFields, zap.Error(err))...)
 				continue
 			}
-			// Get the resource group from manager to acquire token buckets.
-			rg, err := s.manager.GetMutableResourceGroup(keyspaceID, resourceGroupName)
-			if rg == nil {
-				log.Warn("resource group not found", append(requestFields, zap.Error(err))...)
-				continue
-			}
 			// Send the consumption to update the metrics.
 			err = s.manager.dispatchConsumption(req)
 			if err != nil {
diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go
index 3a06c6fe91..b83c89902a 100644
--- a/pkg/mcs/resourcemanager/server/manager.go
+++ b/pkg/mcs/resourcemanager/server/manager.go
@@ -578,12 +578,14 @@ func (m *Manager) loadResourceGroupIfNeeded(keyspaceID uint32, name string) erro
 			return nil
 		}
 	}
-	if name == DefaultResourceGroupName {
-		m.getOrCreateKeyspaceResourceGroupManager(keyspaceID, true)
-		return nil
-	}
 	group, err := m.loadResourceGroup(keyspaceID, name)
 	if err != nil {
+		if name == DefaultResourceGroupName && errors.ErrorEqual(err, errs.ErrResourceGroupNotExists.FastGenByArgs(name)) {
+			// No persisted default group settings exist yet (e.g. a brand-new
+			// keyspace), so it's safe to synthesize the reserved default group.
+			m.getOrCreateKeyspaceResourceGroupManager(keyspaceID, true)
+			return nil
+		}
 		return err
 	}
 	krgm = m.getOrCreateKeyspaceResourceGroupManager(keyspaceID, false)
@@ -782,8 +784,10 @@ func (m *Manager) AddResourceGroup(grouppb *rmpb.ResourceGroup) error {
 	if krgm == nil {
 		return errs.ErrKeyspaceNotExists.FastGenByArgs(keyspaceID)
 	}
-	if err := m.loadResourceGroupIfNeeded(keyspaceID, grouppb.Name); err != nil {
-		log.Debug("failed to load resource group", zap.Uint32("keyspace-id", keyspaceID), zap.String("name", grouppb.Name), zap.Error(err))
+	if err := m.loadResourceGroupIfNeeded(keyspaceID, grouppb.Name); err != nil &&
+		!errors.ErrorEqual(err, errs.ErrResourceGroupNotExists.FastGenByArgs(grouppb.Name)) {
+		log.Warn("failed to load resource group before add", zap.Uint32("keyspace-id", keyspaceID), zap.String("name", grouppb.Name), zap.Error(err))
+		return err
 	}
 	if err := krgm.addResourceGroup(grouppb); err != nil {
 		return err
diff --git a/pkg/mcs/resourcemanager/server/manager_async_test.go b/pkg/mcs/resourcemanager/server/manager_async_test.go
index 7bf7d51ef1..66daf95580 100644
--- a/pkg/mcs/resourcemanager/server/manager_async_test.go
+++ b/pkg/mcs/resourcemanager/server/manager_async_test.go
@@ -32,9 +32,10 @@ import (
 type blockingResourceGroupStorage struct {
 	storage.Storage
 
-	once    sync.Once
-	entered chan struct{}
-	release chan struct{}
+	once        sync.Once
+	releaseOnce sync.Once
+	entered     chan struct{}
+	release     chan struct{}
 }
 
 func newBlockingResourceGroupStorage() *blockingResourceGroupStorage {
@@ -63,7 +64,9 @@ func (s *blockingResourceGroupStorage) waitEntered(t *testing.T) {
 }
 
 func (s *blockingResourceGroupStorage) unblock() {
-	close(s.release)
+	s.releaseOnce.Do(func() {
+		close(s.release)
+	})
 }
 
 func newAsyncTestGroup(name string, fillRate uint64) *resource_manager.ResourceGroup {
@@ -98,6 +101,10 @@ func TestAsyncLoadResourceGroupsLazyGet(t *testing.T) {
 	m.storage = store
 	re.NoError(m.Init(context.Background()))
 	defer stopAsyncTestManager(m)
+	// Unblock the async loader first (LIFO) so stopAsyncTestManager's wg.Wait()
+	// cannot hang if a later assertion aborts the test before the explicit
+	// store.unblock() call below is reached.
+	defer store.unblock()
 
 	store.waitEntered(t)
 
@@ -126,6 +133,10 @@ func TestAsyncLoadResourceGroupsDoesNotRestoreDeletedLazyGroup(t *testing.T) {
 	m.storage = store
 	re.NoError(m.Init(context.Background()))
 	defer stopAsyncTestManager(m)
+	// Unblock the async loader first (LIFO) so stopAsyncTestManager's wg.Wait()
+	// cannot hang if a later assertion aborts the test before the explicit
+	// store.unblock() call below is reached.
+	defer store.unblock()
 
 	store.waitEntered(t)
 

From cbe73bcf816b860d4240a6819ccccb695cb74e71 Mon Sep 17 00:00:00 2001
From: tongjian <1045931706@qq.com>
Date: Wed, 8 Jul 2026 15:13:19 +0800
Subject: [PATCH 06/50] resource_group: propagate non-not-found errors in
 AcquireTokenBuckets

GetMutableResourceGroup can return (nil, err) for real failures (e.g.
ErrKeyspaceNotExists, storage errors during lazy load), not just a
missing group. Treating every rg == nil as "not found" silently
dropped token requests on real errors instead of failing the stream.

Signed-off-by: tongjian <doufuxiaowangzi@gmail.com>
Signed-off-by: tongjian <1045931706@qq.com>
---
 pkg/mcs/resourcemanager/server/grpc_service.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/pkg/mcs/resourcemanager/server/grpc_service.go b/pkg/mcs/resourcemanager/server/grpc_service.go
index bbb73b2340..0945494cdc 100644
--- a/pkg/mcs/resourcemanager/server/grpc_service.go
+++ b/pkg/mcs/resourcemanager/server/grpc_service.go
@@ -236,6 +236,9 @@ func (s *Service) AcquireTokenBuckets(stream rmpb.ResourceManager_AcquireTokenBu
 			// so it must happen before accessKeyspaceResourceGroupManager below.
 			rg, err := s.manager.GetMutableResourceGroup(keyspaceID, resourceGroupName)
 			if rg == nil {
+				if err != nil && !errors.ErrorEqual(err, errs.ErrResourceGroupNotExists.FastGenByArgs(resourceGroupName)) {
+					return err
+				}
 				log.Warn("resource group not found", append(requestFields, zap.Error(err))...)
 				continue
 			}

From 25ec642094e0a5edd17dd85048a2d5a77ed9109c Mon Sep 17 00:00:00 2001
From: tongjian <1045931706@qq.com>
Date: Mon, 13 Jul 2026 18:40:07 +0800
Subject: [PATCH 07/50] resource_group: don't mark lazily-loaded group synced
 on state load failure

When loadResourceGroup's LoadResourceGroupState read fails, it logs a
warning and still returns the group using default state. But
loadResourceGroupIfNeeded then unconditionally marked the group as
sync-loaded, which made asyncLoadResourceGroups's merge permanently
skip it, even if the later bulk load successfully read the real
persisted state. The group would keep its default/fresh state (e.g. a
re-initialized token bucket) for the rest of the manager's lifetime.

Only mark the group sync-loaded when its state was actually read, so
the async bulk merge remains free to fill in the correct state
afterward if nothing else has modified the group in the meantime.

Signed-off-by: tongjian <doufuxiaowangzi@gmail.com>
Signed-off-by: tongjian <1045931706@qq.com>
---
 pkg/mcs/resourcemanager/server/manager.go | 37 +++++++++++++++--------
 1 file changed, 24 insertions(+), 13 deletions(-)

diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go
index b83c89902a..c2c71e131c 100644
--- a/pkg/mcs/resourcemanager/server/manager.go
+++ b/pkg/mcs/resourcemanager/server/manager.go
@@ -542,17 +542,21 @@ func (m *Manager) loadKeyspaceResourceGroupsFromStorage() (map[uint32]*keyspaceR
 	return tempKrgms, nil
 }
 
-func (m *Manager) loadResourceGroup(keyspaceID uint32, name string) (*ResourceGroup, error) {
+// loadResourceGroup loads a single resource group from storage. The returned
+// stateLoaded reports whether the group's persisted state was successfully
+// read; the caller must not mark such a group as sync-loaded, so that a
+// concurrent or later async bulk load can still fill in its real state.
+func (m *Manager) loadResourceGroup(keyspaceID uint32, name string) (group *ResourceGroup, stateLoaded bool, err error) {
 	rawValue, err := m.storage.LoadResourceGroupSetting(keyspaceID, name)
 	if err != nil {
-		return nil, err
+		return nil, false, err
 	}
 	if rawValue == "" {
-		return nil, errs.ErrResourceGroupNotExists.FastGenByArgs(name)
+		return nil, false, errs.ErrResourceGroupNotExists.FastGenByArgs(name)
 	}
 	krgm := newKeyspaceResourceGroupManager(keyspaceID, m.storage, m.writeRole)
 	if err := krgm.addResourceGroupFromRaw(name, rawValue); err != nil {
-		return nil, err
+		return nil, false, err
 	}
 	state, err := m.storage.LoadResourceGroupState(keyspaceID, name)
 	if err != nil {
@@ -560,12 +564,14 @@ func (m *Manager) loadResourceGroup(keyspaceID uint32, name string) (*ResourceGr
 			zap.Uint32("keyspace-id", keyspaceID),
 			zap.String("group-name", name),
 			zap.Error(err))
-	} else if state != "" {
+		return krgm.getMutableResourceGroup(name), false, nil
+	}
+	if state != "" {
 		if err := krgm.setRawStatesIntoResourceGroup(name, state); err != nil {
-			return nil, err
+			return nil, false, err
 		}
 	}
-	return krgm.getMutableResourceGroup(name), nil
+	return krgm.getMutableResourceGroup(name), true, nil
 }
 
 func (m *Manager) loadResourceGroupIfNeeded(keyspaceID uint32, name string) error {
@@ -578,7 +584,7 @@ func (m *Manager) loadResourceGroupIfNeeded(keyspaceID uint32, name string) erro
 			return nil
 		}
 	}
-	group, err := m.loadResourceGroup(keyspaceID, name)
+	group, stateLoaded, err := m.loadResourceGroup(keyspaceID, name)
 	if err != nil {
 		if name == DefaultResourceGroupName && errors.ErrorEqual(err, errs.ErrResourceGroupNotExists.FastGenByArgs(name)) {
 			// No persisted default group settings exist yet (e.g. a brand-new
@@ -599,12 +605,17 @@ func (m *Manager) loadResourceGroupIfNeeded(keyspaceID uint32, name string) erro
 	if inserted {
 		krgm.syncBurstabilityWithServiceLimit(group)
 	}
-	markKey := trackerKey{keyspaceID: keyspaceID, groupName: name}
-	m.Lock()
-	if m.syncLoadedGroups != nil {
-		m.syncLoadedGroups[markKey] = true
+	// Only mark the group as sync-loaded when its persisted state was actually
+	// read; otherwise a later async bulk load must remain free to fill in the
+	// real state instead of being skipped forever.
+	if stateLoaded {
+		markKey := trackerKey{keyspaceID: keyspaceID, groupName: name}
+		m.Lock()
+		if m.syncLoadedGroups != nil {
+			m.syncLoadedGroups[markKey] = true
+		}
+		m.Unlock()
 	}
-	m.Unlock()
 	syncLoadGroupCounter.Inc()
 	return nil
 }

From 306fd57c17bf04fbac9881038ecb1f07d85d8c67 Mon Sep 17 00:00:00 2001
From: tongjian <1045931706@qq.com>
Date: Tue, 14 Jul 2026 10:26:28 +0800
Subject: [PATCH 08/50] resource_group: distinguish reserved default
 placeholder from confirmed data

initReservedInCache installs a synthetic default resource group into
the cache before async loading starts. loadResourceGroupIfNeeded's
early cache-hit check treated any cached entry as already-loaded, so
it never attempted the persisted point load for a keyspace's default
group as long as the synthetic placeholder occupied the slot. A
customized default group would be served with built-in settings for
the whole startup window, and the async bulk merge could be blocked
from correcting it if a concurrent write marked it sync-loaded first.

Track which cache entries are still just placeholders
(keyspaceResourceGroupManager.reservedGroups, guarded by the same lock
as groups) and clear the mark on any real write. loadResourceGroupIfNeeded
now only treats a cached entry as satisfying the call when it isn't a
placeholder, and is allowed to replace a placeholder with the result
of a real storage load.

Signed-off-by: tongjian <doufuxiaowangzi@gmail.com>
Signed-off-by: tongjian <1045931706@qq.com>
---
 .../server/keyspace_manager.go                | 36 +++++++++++++++++--
 pkg/mcs/resourcemanager/server/manager.go     | 13 +++++--
 2 files changed, 45 insertions(+), 4 deletions(-)

diff --git a/pkg/mcs/resourcemanager/server/keyspace_manager.go b/pkg/mcs/resourcemanager/server/keyspace_manager.go
index d3d8a13dae..8aabecedde 100644
--- a/pkg/mcs/resourcemanager/server/keyspace_manager.go
+++ b/pkg/mcs/resourcemanager/server/keyspace_manager.go
@@ -69,7 +69,13 @@ type consumptionItem struct {
 
 type keyspaceResourceGroupManager struct {
 	syncutil.RWMutex
-	groups          map[string]*ResourceGroup
+	groups map[string]*ResourceGroup
+	// reservedGroups tracks names whose entry in groups is still just the
+	// synthetic placeholder inserted by ensureReservedDefaultGroupInCache or
+	// restoreDefaultResourceGroupFromReserved, not yet confirmed by a storage
+	// load or a real write. It shares the same lock as groups so a lazy load
+	// can atomically decide whether it's safe to replace the placeholder.
+	reservedGroups  map[string]struct{}
 	groupRUTrackers map[string]*groupRUTracker
 	serviceLimiter  *serviceLimiter
 
@@ -89,6 +95,7 @@ func newKeyspaceResourceGroupManager(
 	}
 	return &keyspaceResourceGroupManager{
 		groups:          make(map[string]*ResourceGroup),
+		reservedGroups:  make(map[string]struct{}),
 		groupRUTrackers: make(map[string]*groupRUTracker),
 		keyspaceID:      keyspaceID,
 		storage:         storage,
@@ -159,6 +166,9 @@ func (krgm *keyspaceResourceGroupManager) upsertResourceGroupFromRaw(name string
 				zap.Uint32("keyspace-id", krgm.keyspaceID), zap.String("name", name), zap.String("raw-value", rawValue), zap.Error(err))
 			return err
 		}
+		krgm.Lock()
+		delete(krgm.reservedGroups, group.Name)
+		krgm.Unlock()
 		krgm.syncBurstabilityWithServiceLimit(existing)
 		return nil
 	}
@@ -166,6 +176,7 @@ func (krgm *keyspaceResourceGroupManager) upsertResourceGroupFromRaw(name string
 	resourceGroup := FromProtoResourceGroup(group)
 	krgm.Lock()
 	krgm.groups[group.Name] = resourceGroup
+	delete(krgm.reservedGroups, group.Name)
 	krgm.Unlock()
 	krgm.syncBurstabilityWithServiceLimit(resourceGroup)
 	return nil
@@ -175,6 +186,7 @@ func (krgm *keyspaceResourceGroupManager) deleteResourceGroupFromCache(name stri
 	krgm.Lock()
 	delete(krgm.groups, name)
 	delete(krgm.groupRUTrackers, name)
+	delete(krgm.reservedGroups, name)
 	krgm.Unlock()
 }
 
@@ -218,6 +230,7 @@ func (krgm *keyspaceResourceGroupManager) ensureReservedDefaultGroupInCache() {
 	krgm.Lock()
 	if _, ok := krgm.groups[DefaultResourceGroupName]; !ok {
 		krgm.groups[DefaultResourceGroupName] = defaultGroup
+		krgm.reservedGroups[DefaultResourceGroupName] = struct{}{}
 		inserted = true
 	}
 	krgm.Unlock()
@@ -245,6 +258,7 @@ func (krgm *keyspaceResourceGroupManager) restoreDefaultResourceGroupFromReserve
 	defaultGroup := newDefaultResourceGroup()
 	krgm.Lock()
 	krgm.groups[DefaultResourceGroupName] = defaultGroup
+	krgm.reservedGroups[DefaultResourceGroupName] = struct{}{}
 	krgm.Unlock()
 	krgm.syncBurstabilityWithServiceLimit(defaultGroup)
 }
@@ -266,6 +280,7 @@ func (krgm *keyspaceResourceGroupManager) addResourceGroup(grouppb *rmpb.Resourc
 	}
 	krgm.Lock()
 	krgm.groups[group.Name] = group
+	delete(krgm.reservedGroups, group.Name)
 	krgm.Unlock()
 	krgm.syncBurstabilityWithServiceLimit(group)
 	return nil
@@ -288,7 +303,13 @@ func (krgm *keyspaceResourceGroupManager) modifyResourceGroup(group *rmpb.Resour
 	if err != nil {
 		return err
 	}
-	return curGroup.persistSettings(krgm.keyspaceID, krgm.storage)
+	if err := curGroup.persistSettings(krgm.keyspaceID, krgm.storage); err != nil {
+		return err
+	}
+	krgm.Lock()
+	delete(krgm.reservedGroups, group.Name)
+	krgm.Unlock()
+	return nil
 }
 
 func (krgm *keyspaceResourceGroupManager) deleteResourceGroup(name string) error {
@@ -330,6 +351,17 @@ func (krgm *keyspaceResourceGroupManager) deleteResourceGroup(name string) error
 	return nil
 }
 
+// isReserved reports whether name's cached entry is still just the synthetic
+// placeholder inserted by ensureReservedDefaultGroupInCache or
+// restoreDefaultResourceGroupFromReserved, not yet confirmed by a storage
+// load or a real write.
+func (krgm *keyspaceResourceGroupManager) isReserved(name string) bool {
+	krgm.RLock()
+	defer krgm.RUnlock()
+	_, ok := krgm.reservedGroups[name]
+	return ok
+}
+
 func (krgm *keyspaceResourceGroupManager) getResourceGroup(name string, withStats bool) *ResourceGroup {
 	krgm.RLock()
 	defer krgm.RUnlock()
diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go
index c2c71e131c..8f46501919 100644
--- a/pkg/mcs/resourcemanager/server/manager.go
+++ b/pkg/mcs/resourcemanager/server/manager.go
@@ -580,7 +580,10 @@ func (m *Manager) loadResourceGroupIfNeeded(keyspaceID uint32, name string) erro
 	}
 	krgm := m.getKeyspaceResourceGroupManager(keyspaceID)
 	if krgm != nil {
-		if group := krgm.getMutableResourceGroup(name); group != nil {
+		// A cached entry only satisfies this call if it's confirmed data, not
+		// just a synthetic placeholder (e.g. from ensureReservedDefaultGroupInCache)
+		// installed before async loading had a chance to run.
+		if group := krgm.getMutableResourceGroup(name); group != nil && !krgm.isReserved(name) {
 			return nil
 		}
 	}
@@ -588,7 +591,7 @@ func (m *Manager) loadResourceGroupIfNeeded(keyspaceID uint32, name string) erro
 	if err != nil {
 		if name == DefaultResourceGroupName && errors.ErrorEqual(err, errs.ErrResourceGroupNotExists.FastGenByArgs(name)) {
 			// No persisted default group settings exist yet (e.g. a brand-new
-			// keyspace), so it's safe to synthesize the reserved default group.
+			// keyspace), so it's safe to keep serving the reserved placeholder.
 			m.getOrCreateKeyspaceResourceGroupManager(keyspaceID, true)
 			return nil
 		}
@@ -600,7 +603,13 @@ func (m *Manager) loadResourceGroupIfNeeded(keyspaceID uint32, name string) erro
 	if _, exists := krgm.groups[name]; !exists {
 		krgm.groups[name] = group
 		inserted = true
+	} else if _, reserved := krgm.reservedGroups[name]; reserved {
+		// The existing entry is just an unconfirmed placeholder; the freshly
+		// loaded group is the real, confirmed data, so replacing it is safe.
+		krgm.groups[name] = group
+		inserted = true
 	}
+	delete(krgm.reservedGroups, name)
 	krgm.Unlock()
 	if inserted {
 		krgm.syncBurstabilityWithServiceLimit(group)

From 042d6a77b12d5b6bd6111e6bba28c424d0bf0677 Mon Sep 17 00:00:00 2001
From: tongjian <1045931706@qq.com>
Date: Tue, 14 Jul 2026 15:24:34 +0800
Subject: [PATCH 09/50] resource_group: keep placeholder state out of the
 persist loop and reserved marker

persistResourceGroupRunningState persisted every entry in krgm.groups
regardless of reservedGroups, so the persist loop (running concurrently
with async loading) could write a synthetic default's fresh token
state back to storage before the real persisted state was ever loaded,
permanently overwriting it. Skip reserved entries there.

Also fix an oversight in the previous placeholder-tracking commit:
loadResourceGroupIfNeeded cleared a group's reserved mark unconditionally
after a lazy load, even when LoadResourceGroupState had failed and the
group only carries default state. Only clear the mark when the state
was actually read, otherwise keep it reserved so a later call or the
async bulk merge can still fill in the real state instead of the
partial result being treated as final.

Signed-off-by: tongjian <doufuxiaowangzi@gmail.com>
Signed-off-by: tongjian <1045931706@qq.com>
---
 pkg/mcs/resourcemanager/server/keyspace_manager.go |  8 ++++++++
 pkg/mcs/resourcemanager/server/manager.go          | 10 +++++++++-
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/pkg/mcs/resourcemanager/server/keyspace_manager.go b/pkg/mcs/resourcemanager/server/keyspace_manager.go
index 8aabecedde..0396f556b3 100644
--- a/pkg/mcs/resourcemanager/server/keyspace_manager.go
+++ b/pkg/mcs/resourcemanager/server/keyspace_manager.go
@@ -416,6 +416,14 @@ func (krgm *keyspaceResourceGroupManager) persistResourceGroupRunningState() {
 	for idx := range keys {
 		krgm.RLock()
 		group, ok := krgm.groups[keys[idx]]
+		_, reserved := krgm.reservedGroups[keys[idx]]
+		if ok && reserved {
+			// The entry is still just an unconfirmed placeholder (e.g. the
+			// synthetic default installed before async loading completes);
+			// persisting its fresh state would permanently overwrite any
+			// real persisted state still waiting to be loaded.
+			ok = false
+		}
 		if ok {
 			if err := group.persistStates(krgm.keyspaceID, krgm.storage); err != nil {
 				log.Error("persist keyspace resource group state failed",
diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go
index 8f46501919..940135ea04 100644
--- a/pkg/mcs/resourcemanager/server/manager.go
+++ b/pkg/mcs/resourcemanager/server/manager.go
@@ -609,7 +609,15 @@ func (m *Manager) loadResourceGroupIfNeeded(keyspaceID uint32, name string) erro
 		krgm.groups[name] = group
 		inserted = true
 	}
-	delete(krgm.reservedGroups, name)
+	// Only clear the placeholder mark once the state was actually read; a
+	// metadata-only group (state load failed) must stay reserved so a later
+	// call or the async bulk merge remains free to fill in the real state,
+	// instead of this partial result being treated as final forever.
+	if stateLoaded {
+		delete(krgm.reservedGroups, name)
+	} else {
+		krgm.reservedGroups[name] = struct{}{}
+	}
 	krgm.Unlock()
 	if inserted {
 		krgm.syncBurstabilityWithServiceLimit(group)

From d2cddc91a5d342bfe762cfa7494bab29c6662cbf Mon Sep 17 00:00:00 2001
From: tongjian <1045931706@qq.com>
Date: Wed, 15 Jul 2026 10:58:51 +0800
Subject: [PATCH 10/50] resource_group: route default-group creation through
 the safe load path

getOrCreateKeyspaceResourceGroupManager(id, true) is used by both
AddResourceGroup and SetKeyspaceServiceLimit to make sure a keyspace's
default resource group exists. It called initDefaultResourceGroup
directly, which persists a synthetic default whenever one isn't cached
yet, with no attempt to check storage first. For a keyspace touched for
the first time while async loading is still in progress, this could
overwrite a customized default group's stored settings before the real
data had a chance to load - the same class of bug fixed earlier in
loadResourceGroupIfNeeded, reachable via two more entry points.

Make initDefault=true go through loadResourceGroupIfNeeded (storage
point load first, synthesize only on confirmed not-found) while async
loading is in progress. Once loading has completed, any default still
missing from the cache is confirmed absent from storage, so it's
synthesized directly as before. loadResourceGroupIfNeeded's own
not-found fallback now calls initDefaultResourceGroup directly instead
of going through getOrCreateKeyspaceResourceGroupManager(id, true), to
avoid recursing back into itself.

Signed-off-by: tongjian <doufuxiaowangzi@gmail.com>
Signed-off-by: tongjian <1045931706@qq.com>
---
 pkg/mcs/resourcemanager/server/manager.go | 24 +++++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go
index 940135ea04..a2c59bfd21 100644
--- a/pkg/mcs/resourcemanager/server/manager.go
+++ b/pkg/mcs/resourcemanager/server/manager.go
@@ -295,6 +295,13 @@ func (m *Manager) GetRUVersionPolicy() *RUVersionPolicy {
 	return m.controllerConfig.RUVersionPolicy.Clone()
 }
 
+// getOrCreateKeyspaceResourceGroupManager returns the keyspace resource group
+// manager for keyspaceID, creating it if needed. When initDefault is true, it
+// also ensures the default resource group is present. While async loading is
+// still in progress this goes through loadResourceGroupIfNeeded, which tries
+// a storage point load first so a customized default is never clobbered by a
+// blindly synthesized one. Once loading has completed, any default missing
+// from the cache truly doesn't exist anywhere, so it's synthesized directly.
 func (m *Manager) getOrCreateKeyspaceResourceGroupManager(keyspaceID uint32, initDefault bool) *keyspaceResourceGroupManager {
 	m.Lock()
 	krgm, ok := m.krgms[keyspaceID]
@@ -303,9 +310,15 @@ func (m *Manager) getOrCreateKeyspaceResourceGroupManager(keyspaceID uint32, ini
 		m.krgms[keyspaceID] = krgm
 	}
 	m.Unlock()
-	// Init the default resource group if needed.
 	if initDefault {
-		krgm.initDefaultResourceGroup()
+		if atomic.LoadInt32(&m.loadingState) == LoadingStateCompleted {
+			// Async loading (if any) has already finished, so if the default
+			// group isn't cached yet it truly doesn't exist anywhere; it's
+			// safe to synthesize and persist it directly.
+			krgm.initDefaultResourceGroup()
+		} else if err := m.loadResourceGroupIfNeeded(keyspaceID, DefaultResourceGroupName); err != nil {
+			log.Debug("failed to load default resource group", zap.Uint32("keyspace-id", keyspaceID), zap.Error(err))
+		}
 	}
 	return krgm
 }
@@ -591,8 +604,11 @@ func (m *Manager) loadResourceGroupIfNeeded(keyspaceID uint32, name string) erro
 	if err != nil {
 		if name == DefaultResourceGroupName && errors.ErrorEqual(err, errs.ErrResourceGroupNotExists.FastGenByArgs(name)) {
 			// No persisted default group settings exist yet (e.g. a brand-new
-			// keyspace), so it's safe to keep serving the reserved placeholder.
-			m.getOrCreateKeyspaceResourceGroupManager(keyspaceID, true)
+			// keyspace), so it's safe to synthesize the reserved default group.
+			// This calls initDefaultResourceGroup directly instead of going
+			// through getOrCreateKeyspaceResourceGroupManager(id, true), which
+			// now routes back into this same function and would recurse.
+			m.getOrCreateKeyspaceResourceGroupManager(keyspaceID, false).initDefaultResourceGroup()
 			return nil
 		}
 		return err

From 8f743d909575deddbd22927a69c690b2eefeb99a Mon Sep 17 00:00:00 2001
From: tongjian <1045931706@qq.com>
Date: Wed, 15 Jul 2026 11:02:53 +0800
Subject: [PATCH 11/50] resource_group: don't let Modify confirm a
 state-unconfirmed group

krgm.modifyResourceGroup unconditionally cleared reservedGroups after
patching settings, and Manager.ModifyResourceGroup unconditionally
called markResourceGroupSyncLoaded afterwards. Modify only patches
settings, it never loads or establishes a group's state, so if the
preceding lazy load had failed to read the persisted state
(stateLoaded=false), a Modify call would incorrectly promote the entry
to fully confirmed at both the keyspace-manager level (reservedGroups)
and the manager level (syncLoadedGroups). The async bulk merge would
then skip it forever, so the real persisted running state would never
get applied and the cached token bucket could remain initialized with
empty state indefinitely.

Leave reservedGroups untouched in modifyResourceGroup, and only mark
the group sync-loaded in ModifyResourceGroup when it isn't still
reserved. This lets the existing retry-on-next-access path (and the
async bulk merge) keep trying to fill in the real state instead of the
settings-only patch being treated as a full confirmation.

Signed-off-by: tongjian <doufuxiaowangzi@gmail.com>
Signed-off-by: tongjian <1045931706@qq.com>
---
 pkg/mcs/resourcemanager/server/keyspace_manager.go | 11 ++++-------
 pkg/mcs/resourcemanager/server/manager.go          |  8 +++++++-
 2 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/pkg/mcs/resourcemanager/server/keyspace_manager.go b/pkg/mcs/resourcemanager/server/keyspace_manager.go
index 0396f556b3..8c900c5479 100644
--- a/pkg/mcs/resourcemanager/server/keyspace_manager.go
+++ b/pkg/mcs/resourcemanager/server/keyspace_manager.go
@@ -303,13 +303,10 @@ func (krgm *keyspaceResourceGroupManager) modifyResourceGroup(group *rmpb.Resour
 	if err != nil {
 		return err
 	}
-	if err := curGroup.persistSettings(krgm.keyspaceID, krgm.storage); err != nil {
-		return err
-	}
-	krgm.Lock()
-	delete(krgm.reservedGroups, group.Name)
-	krgm.Unlock()
-	return nil
+	// Deliberately not clearing reservedGroups here: modifying only patches
+	// settings, it never establishes the group's state, so it must not make
+	// a state-unconfirmed entry look fully confirmed.
+	return curGroup.persistSettings(krgm.keyspaceID, krgm.storage)
 }
 
 func (krgm *keyspaceResourceGroupManager) deleteResourceGroup(name string) error {
diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go
index a2c59bfd21..01636ee2de 100644
--- a/pkg/mcs/resourcemanager/server/manager.go
+++ b/pkg/mcs/resourcemanager/server/manager.go
@@ -857,7 +857,13 @@ func (m *Manager) ModifyResourceGroup(grouppb *rmpb.ResourceGroup) error {
 	if err := krgm.modifyResourceGroup(grouppb); err != nil {
 		return err
 	}
-	m.markResourceGroupSyncLoaded(keyspaceID, grouppb.Name)
+	// Modifying only patches settings, it never establishes the group's
+	// state. If the state still hasn't been confirmed (isReserved), marking
+	// it sync-loaded here would make the async bulk merge skip it forever,
+	// so the persisted running state would never get applied.
+	if !krgm.isReserved(grouppb.Name) {
+		m.markResourceGroupSyncLoaded(keyspaceID, grouppb.Name)
+	}
 	return nil
 }
 

From eb3e774035369f01a0f2b7f5d63858338db11141 Mon Sep 17 00:00:00 2001
From: tongjian <1045931706@qq.com>
Date: Wed, 15 Jul 2026 16:12:50 +0800
Subject: [PATCH 12/50] resource_group: add lazy-load coverage for legacy
 null-keyspace groups

Review feedback questioned whether the point loaders
(LoadResourceGroupSetting/LoadResourceGroupState) miss the legacy,
pre-keyspace resource_group/* path that the bulk loaders fall back to
for constant.NullKeyspaceID. They don't: KeyspaceResourceGroupSettingPath
and KeyspaceResourceGroupStatePath already resolve to the legacy path
for NullKeyspaceID, the same helper both point and bulk loaders use.

Add a regression test exercising the actual lazy-load path
(GetResourceGroup during blocked async loading) against a legacy group
saved under NullKeyspaceID, to make this guarantee explicit and catch
any future divergence between the point and bulk loaders.

Signed-off-by: tongjian <doufuxiaowangzi@gmail.com>
Signed-off-by: tongjian <1045931706@qq.com>
---
 .../server/manager_async_test.go              | 32 +++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/pkg/mcs/resourcemanager/server/manager_async_test.go b/pkg/mcs/resourcemanager/server/manager_async_test.go
index 66daf95580..af67e55112 100644
--- a/pkg/mcs/resourcemanager/server/manager_async_test.go
+++ b/pkg/mcs/resourcemanager/server/manager_async_test.go
@@ -25,6 +25,7 @@ import (
 	"github.com/pingcap/kvproto/pkg/resource_manager"
 
 	"github.com/tikv/pd/pkg/errs"
+	"github.com/tikv/pd/pkg/keyspace/constant"
 	"github.com/tikv/pd/pkg/storage"
 	"github.com/tikv/pd/pkg/utils/testutil"
 )
@@ -159,3 +160,34 @@ func TestAsyncLoadResourceGroupsDoesNotRestoreDeletedLazyGroup(t *testing.T) {
 		return true
 	}, testutil.WithTickInterval(20*time.Millisecond))
 }
+
+// TestAsyncLoadResourceGroupsLazyGetLegacyKeyspace guards against the point
+// loaders (LoadResourceGroupSetting/LoadResourceGroupState) diverging from
+// the bulk loaders on legacy, pre-keyspace resource groups: those are saved
+// under constant.NullKeyspaceID, and a lazy Get during async loading must be
+// able to find one the same way the bulk scan would once it completes.
+func TestAsyncLoadResourceGroupsLazyGetLegacyKeyspace(t *testing.T) {
+	re := require.New(t)
+	store := newBlockingResourceGroupStorage()
+	re.NoError(store.SaveResourceGroupSetting(constant.NullKeyspaceID, "legacy-group", newAsyncTestGroup("legacy-group", 100)))
+
+	m := NewManager[*mockConfigProvider](&mockConfigProvider{})
+	m.storage = store
+	re.NoError(m.Init(context.Background()))
+	defer stopAsyncTestManager(m)
+	defer store.unblock()
+
+	store.waitEntered(t)
+
+	group, err := m.GetResourceGroup(constant.NullKeyspaceID, "legacy-group", false)
+	re.NoError(err)
+	re.NotNil(group)
+	re.Equal("legacy-group", group.Name)
+	re.Equal(float64(100), group.RUSettings.RU.getFillRate())
+
+	store.unblock()
+	testutil.Eventually(re, func() bool {
+		group, err := m.GetResourceGroup(constant.NullKeyspaceID, "legacy-group", false)
+		return err == nil && group != nil
+	}, testutil.WithTickInterval(20*time.Millisecond))
+}

From a32358a51102619087950904da20de7816fcccc0 Mon Sep 17 00:00:00 2001
From: tongjian <1045931706@qq.com>
Date: Thu, 16 Jul 2026 15:35:11 +0800
Subject: [PATCH 13/50] resource_group: clear reserved marker when async merge
 installs confirmed data

asyncLoadResourceGroups's merge loop installed the fully-loaded
(settings and state) confirmed group into krgm.groups but never
cleared reservedGroups for it. A group that got marked reserved by an
earlier lazy-load state-read failure would stay reserved forever even
after the bulk load correctly recovered it, permanently excluding it
from the state persist loop and from loadResourceGroupIfNeeded's fast
path.

Clear the reserved marker in the same merge step that installs the
confirmed data. Add a regression test that injects a one-time
LoadResourceGroupState failure during a lazy Get, then lets the async
bulk load complete, and asserts the reserved marker is cleared once
the confirmed data lands; verified the test fails without the fix.

Signed-off-by: tongjian <doufuxiaowangzi@gmail.com>
Signed-off-by: tongjian <1045931706@qq.com>
---
 pkg/mcs/resourcemanager/server/manager.go     |  5 ++
 .../server/manager_async_test.go              | 54 +++++++++++++++++++
 2 files changed, 59 insertions(+)

diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go
index 01636ee2de..e45be2cad3 100644
--- a/pkg/mcs/resourcemanager/server/manager.go
+++ b/pkg/mcs/resourcemanager/server/manager.go
@@ -496,6 +496,11 @@ func (m *Manager) asyncLoadResourceGroups(ctx context.Context) {
 				key := trackerKey{keyspaceID: keyspaceID, groupName: name}
 				if !m.syncLoadedGroups[key] {
 					krgm.groups[name] = group
+					// This group is now confirmed, fully-loaded data (settings
+					// and state); it must no longer be treated as an
+					// unconfirmed placeholder by loadResourceGroupIfNeeded or
+					// skipped by the state persist loop.
+					delete(krgm.reservedGroups, name)
 					groupsToSync = append(groupsToSync, group)
 					loaded++
 				}
diff --git a/pkg/mcs/resourcemanager/server/manager_async_test.go b/pkg/mcs/resourcemanager/server/manager_async_test.go
index af67e55112..73db769b88 100644
--- a/pkg/mcs/resourcemanager/server/manager_async_test.go
+++ b/pkg/mcs/resourcemanager/server/manager_async_test.go
@@ -16,7 +16,9 @@ package server
 
 import (
 	"context"
+	"errors"
 	"sync"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -37,6 +39,10 @@ type blockingResourceGroupStorage struct {
 	releaseOnce sync.Once
 	entered     chan struct{}
 	release     chan struct{}
+
+	// failNextState, when true, makes the very next LoadResourceGroupState
+	// call fail once, then resets itself.
+	failNextState atomic.Bool
 }
 
 func newBlockingResourceGroupStorage() *blockingResourceGroupStorage {
@@ -55,6 +61,13 @@ func (s *blockingResourceGroupStorage) LoadResourceGroupSettings(f func(keyspace
 	return s.Storage.LoadResourceGroupSettings(f)
 }
 
+func (s *blockingResourceGroupStorage) LoadResourceGroupState(keyspaceID uint32, name string) (string, error) {
+	if s.failNextState.CompareAndSwap(true, false) {
+		return "", errors.New("injected resource group state load failure")
+	}
+	return s.Storage.LoadResourceGroupState(keyspaceID, name)
+}
+
 func (s *blockingResourceGroupStorage) waitEntered(t *testing.T) {
 	t.Helper()
 	select {
@@ -191,3 +204,44 @@ func TestAsyncLoadResourceGroupsLazyGetLegacyKeyspace(t *testing.T) {
 		return err == nil && group != nil
 	}, testutil.WithTickInterval(20*time.Millisecond))
 }
+
+// TestAsyncLoadResourceGroupsRecoversFromStateLoadFailure guards against a
+// group getting stuck marked reserved forever after a transient
+// LoadResourceGroupState failure during lazy loading: once the async bulk
+// load subsequently installs the fully-loaded (settings and state)
+// confirmed data for the same group, the reserved marker must be cleared,
+// otherwise loadResourceGroupIfNeeded and the state persist loop would keep
+// treating already-recovered, correct data as an unconfirmed placeholder.
+func TestAsyncLoadResourceGroupsRecoversFromStateLoadFailure(t *testing.T) {
+	re := require.New(t)
+	store := newBlockingResourceGroupStorage()
+	group := newAsyncTestGroup("flaky-group", 100)
+	re.NoError(store.SaveResourceGroupSetting(1, "flaky-group", group))
+	re.NoError(store.SaveResourceGroupStates(1, "flaky-group", FromProtoResourceGroup(group).GetGroupStates()))
+
+	m := NewManager[*mockConfigProvider](&mockConfigProvider{})
+	m.storage = store
+	re.NoError(m.Init(context.Background()))
+	defer stopAsyncTestManager(m)
+	defer store.unblock()
+
+	store.waitEntered(t)
+
+	// Make the lazy load's own state read fail once, so the group is cached
+	// as a metadata-only, still-reserved entry.
+	store.failNextState.Store(true)
+	fetched, err := m.GetResourceGroup(1, "flaky-group", false)
+	re.NoError(err)
+	re.NotNil(fetched)
+
+	krgm := m.getKeyspaceResourceGroupManager(1)
+	re.NotNil(krgm)
+	re.True(krgm.isReserved("flaky-group"), "group should still be reserved after a failed state load")
+
+	// Let the async bulk load proceed; its own state read is unaffected
+	// (failNextState was already consumed) and should install confirmed data.
+	store.unblock()
+	testutil.Eventually(re, func() bool {
+		return !krgm.isReserved("flaky-group")
+	}, testutil.WithTickInterval(20*time.Millisecond))
+}

From 44e1d75943090c9fd7b385456f44feff0370143b Mon Sep 17 00:00:00 2001
From: tongjian <1045931706@qq.com>
Date: Mon, 20 Jul 2026 14:13:50 +0800
Subject: [PATCH 14/50] resource_group: fix unparam lint in async loading tests

newAsyncTestGroup's fillRate parameter always received 100, which
tripped the unparam linter in the statics check. Drop the constant
parameter and hoist the value into a named asyncTestGroupFillRate
constant shared by the helper and the fill-rate assertions.

Signed-off-by: tongjian <doufuxiaowangzi@gmail.com>
Signed-off-by: tongjian <1045931706@qq.com>
---
 .../server/manager_async_test.go              | 23 +++++++++++--------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/pkg/mcs/resourcemanager/server/manager_async_test.go b/pkg/mcs/resourcemanager/server/manager_async_test.go
index 73db769b88..fdcf0704ff 100644
--- a/pkg/mcs/resourcemanager/server/manager_async_test.go
+++ b/pkg/mcs/resourcemanager/server/manager_async_test.go
@@ -83,7 +83,12 @@ func (s *blockingResourceGroupStorage) unblock() {
 	})
 }
 
-func newAsyncTestGroup(name string, fillRate uint64) *resource_manager.ResourceGroup {
+// asyncTestGroupFillRate is the fill rate used by all async-loading test
+// groups; kept as a named constant so the setup and the assertions stay in
+// sync.
+const asyncTestGroupFillRate = 100
+
+func newAsyncTestGroup(name string) *resource_manager.ResourceGroup {
 	return &resource_manager.ResourceGroup{
 		Name:     name,
 		Mode:     resource_manager.GroupMode_RUMode,
@@ -91,8 +96,8 @@ func newAsyncTestGroup(name string, fillRate uint64) *resource_manager.ResourceG
 		RUSettings: &resource_manager.GroupRequestUnitSettings{
 			RU: &resource_manager.TokenBucket{
 				Settings: &resource_manager.TokenLimitSettings{
-					FillRate:   fillRate,
-					BurstLimit: int64(fillRate),
+					FillRate:   asyncTestGroupFillRate,
+					BurstLimit: asyncTestGroupFillRate,
 				},
 			},
 		},
@@ -109,7 +114,7 @@ func stopAsyncTestManager(m *Manager) {
 func TestAsyncLoadResourceGroupsLazyGet(t *testing.T) {
 	re := require.New(t)
 	store := newBlockingResourceGroupStorage()
-	re.NoError(store.SaveResourceGroupSetting(1, "lazy-group", newAsyncTestGroup("lazy-group", 100)))
+	re.NoError(store.SaveResourceGroupSetting(1, "lazy-group", newAsyncTestGroup("lazy-group")))
 
 	m := NewManager[*mockConfigProvider](&mockConfigProvider{})
 	m.storage = store
@@ -129,7 +134,7 @@ func TestAsyncLoadResourceGroupsLazyGet(t *testing.T) {
 	re.NoError(err)
 	re.NotNil(group)
 	re.Equal("lazy-group", group.Name)
-	re.Equal(float64(100), group.RUSettings.RU.getFillRate())
+	re.Equal(float64(asyncTestGroupFillRate), group.RUSettings.RU.getFillRate())
 
 	store.unblock()
 	testutil.Eventually(re, func() bool {
@@ -141,7 +146,7 @@ func TestAsyncLoadResourceGroupsLazyGet(t *testing.T) {
 func TestAsyncLoadResourceGroupsDoesNotRestoreDeletedLazyGroup(t *testing.T) {
 	re := require.New(t)
 	store := newBlockingResourceGroupStorage()
-	re.NoError(store.SaveResourceGroupSetting(1, "deleted-group", newAsyncTestGroup("deleted-group", 100)))
+	re.NoError(store.SaveResourceGroupSetting(1, "deleted-group", newAsyncTestGroup("deleted-group")))
 
 	m := NewManager[*mockConfigProvider](&mockConfigProvider{})
 	m.storage = store
@@ -182,7 +187,7 @@ func TestAsyncLoadResourceGroupsDoesNotRestoreDeletedLazyGroup(t *testing.T) {
 func TestAsyncLoadResourceGroupsLazyGetLegacyKeyspace(t *testing.T) {
 	re := require.New(t)
 	store := newBlockingResourceGroupStorage()
-	re.NoError(store.SaveResourceGroupSetting(constant.NullKeyspaceID, "legacy-group", newAsyncTestGroup("legacy-group", 100)))
+	re.NoError(store.SaveResourceGroupSetting(constant.NullKeyspaceID, "legacy-group", newAsyncTestGroup("legacy-group")))
 
 	m := NewManager[*mockConfigProvider](&mockConfigProvider{})
 	m.storage = store
@@ -196,7 +201,7 @@ func TestAsyncLoadResourceGroupsLazyGetLegacyKeyspace(t *testing.T) {
 	re.NoError(err)
 	re.NotNil(group)
 	re.Equal("legacy-group", group.Name)
-	re.Equal(float64(100), group.RUSettings.RU.getFillRate())
+	re.Equal(float64(asyncTestGroupFillRate), group.RUSettings.RU.getFillRate())
 
 	store.unblock()
 	testutil.Eventually(re, func() bool {
@@ -215,7 +220,7 @@ func TestAsyncLoadResourceGroupsLazyGetLegacyKeyspace(t *testing.T) {
 func TestAsyncLoadResourceGroupsRecoversFromStateLoadFailure(t *testing.T) {
 	re := require.New(t)
 	store := newBlockingResourceGroupStorage()
-	group := newAsyncTestGroup("flaky-group", 100)
+	group := newAsyncTestGroup("flaky-group")
 	re.NoError(store.SaveResourceGroupSetting(1, "flaky-group", group))
 	re.NoError(store.SaveResourceGroupStates(1, "flaky-group", FromProtoResourceGroup(group).GetGroupStates()))
 

From 5fe46baff337295e3d11a506e259c5fae21f4c3b Mon Sep 17 00:00:00 2001
From: tongjian <1045931706@qq.com>
Date: Mon, 20 Jul 2026 15:41:35 +0800
Subject: [PATCH 15/50] resource_group: don't let a racing lazy load resurrect
 a deleted group

loadResourceGroupIfNeeded reads a group from storage without holding the
keyspace lock, then inserts it under the lock. A concurrent Delete that
completed between the read and the insert would leave the stale insert
observing an empty cache and re-adding the just-deleted group. Because
the async bulk scan no longer contains that entry, the resurrected group
stayed visible for the rest of the manager's lifetime.

Add a per-keyspace deleteGen counter, bumped under the write lock on
every cache removal. The lazy load snapshots it before its lock-free
storage read and re-checks it under the insert lock; if it changed, a
Delete raced and the stale result is dropped instead of inserted. A
monotonic counter is used rather than a per-name tombstone map so there
is no unbounded state to clear and no lifecycle window where a late
in-flight reader could still slip through.

Add a deterministic regression test that pauses a lazy load right after
its storage read, deletes the group, then releases the lazy load and
asserts it is not resurrected (verified to fail without the fix).

Signed-off-by: tongjian <doufuxiaowangzi@gmail.com>
Signed-off-by: tongjian <1045931706@qq.com>
---
 .../server/keyspace_manager.go                | 15 +++
 pkg/mcs/resourcemanager/server/manager.go     | 16 +++-
 .../server/manager_async_test.go              | 91 ++++++++++++++++++-
 3 files changed, 117 insertions(+), 5 deletions(-)

diff --git a/pkg/mcs/resourcemanager/server/keyspace_manager.go b/pkg/mcs/resourcemanager/server/keyspace_manager.go
index 8c900c5479..f9638d850e 100644
--- a/pkg/mcs/resourcemanager/server/keyspace_manager.go
+++ b/pkg/mcs/resourcemanager/server/keyspace_manager.go
@@ -78,6 +78,11 @@ type keyspaceResourceGroupManager struct {
 	reservedGroups  map[string]struct{}
 	groupRUTrackers map[string]*groupRUTracker
 	serviceLimiter  *serviceLimiter
+	// deleteGen is bumped under the write lock every time a group is removed
+	// from the cache. A lazy load snapshots it before its lock-free storage
+	// read and re-checks it under the lock before inserting, so a group
+	// deleted after the read is never resurrected by the now-stale result.
+	deleteGen uint64
 
 	keyspaceID uint32
 	storage    endpoint.ResourceGroupStorage
@@ -187,9 +192,19 @@ func (krgm *keyspaceResourceGroupManager) deleteResourceGroupFromCache(name stri
 	delete(krgm.groups, name)
 	delete(krgm.groupRUTrackers, name)
 	delete(krgm.reservedGroups, name)
+	// Signal any in-flight lazy load that a delete happened, so it won't
+	// reinsert a copy read from storage before this deletion.
+	krgm.deleteGen++
 	krgm.Unlock()
 }
 
+// loadDeleteGen returns the current delete generation counter.
+func (krgm *keyspaceResourceGroupManager) loadDeleteGen() uint64 {
+	krgm.RLock()
+	defer krgm.RUnlock()
+	return krgm.deleteGen
+}
+
 func (krgm *keyspaceResourceGroupManager) setRawStatesIntoResourceGroup(name string, rawValue string) error {
 	tokens := &GroupStates{}
 	if err := json.Unmarshal([]byte(rawValue), tokens); err != nil {
diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go
index e45be2cad3..aaa22d6a5b 100644
--- a/pkg/mcs/resourcemanager/server/manager.go
+++ b/pkg/mcs/resourcemanager/server/manager.go
@@ -605,6 +605,12 @@ func (m *Manager) loadResourceGroupIfNeeded(keyspaceID uint32, name string) erro
 			return nil
 		}
 	}
+	// Ensure the keyspace manager exists and snapshot its delete generation
+	// before the lock-free storage read below, so a concurrent Delete that
+	// lands after the read is detected under the insert lock and can't be
+	// undone by this now-stale result.
+	krgm = m.getOrCreateKeyspaceResourceGroupManager(keyspaceID, false)
+	deleteGen := krgm.loadDeleteGen()
 	group, stateLoaded, err := m.loadResourceGroup(keyspaceID, name)
 	if err != nil {
 		if name == DefaultResourceGroupName && errors.ErrorEqual(err, errs.ErrResourceGroupNotExists.FastGenByArgs(name)) {
@@ -613,14 +619,20 @@ func (m *Manager) loadResourceGroupIfNeeded(keyspaceID uint32, name string) erro
 			// This calls initDefaultResourceGroup directly instead of going
 			// through getOrCreateKeyspaceResourceGroupManager(id, true), which
 			// now routes back into this same function and would recurse.
-			m.getOrCreateKeyspaceResourceGroupManager(keyspaceID, false).initDefaultResourceGroup()
+			krgm.initDefaultResourceGroup()
 			return nil
 		}
 		return err
 	}
-	krgm = m.getOrCreateKeyspaceResourceGroupManager(keyspaceID, false)
 	inserted := false
 	krgm.Lock()
+	if krgm.deleteGen != deleteGen {
+		// A Delete raced with our storage read; the result may be stale, so
+		// don't insert it. A later request or the async bulk merge will reload
+		// the group if it still exists.
+		krgm.Unlock()
+		return nil
+	}
 	if _, exists := krgm.groups[name]; !exists {
 		krgm.groups[name] = group
 		inserted = true
diff --git a/pkg/mcs/resourcemanager/server/manager_async_test.go b/pkg/mcs/resourcemanager/server/manager_async_test.go
index fdcf0704ff..f29e22e2fd 100644
--- a/pkg/mcs/resourcemanager/server/manager_async_test.go
+++ b/pkg/mcs/resourcemanager/server/manager_async_test.go
@@ -43,13 +43,22 @@ type blockingResourceGroupStorage struct {
 	// failNextState, when true, makes the very next LoadResourceGroupState
 	// call fail once, then resets itself.
 	failNextState atomic.Bool
+
+	// pausePointState, when true, makes the very next LoadResourceGroupState
+	// call signal pointReached and then block on pointRelease, so a test can
+	// hold a lazy load right after its storage read but before it inserts.
+	pausePointState atomic.Bool
+	pointReached    chan struct{}
+	pointRelease    chan struct{}
 }
 
 func newBlockingResourceGroupStorage() *blockingResourceGroupStorage {
 	return &blockingResourceGroupStorage{
-		Storage: storage.NewStorageWithMemoryBackend(),
-		entered: make(chan struct{}),
-		release: make(chan struct{}),
+		Storage:      storage.NewStorageWithMemoryBackend(),
+		entered:      make(chan struct{}),
+		release:      make(chan struct{}),
+		pointReached: make(chan struct{}),
+		pointRelease: make(chan struct{}),
 	}
 }
 
@@ -65,6 +74,10 @@ func (s *blockingResourceGroupStorage) LoadResourceGroupState(keyspaceID uint32,
 	if s.failNextState.CompareAndSwap(true, false) {
 		return "", errors.New("injected resource group state load failure")
 	}
+	if s.pausePointState.CompareAndSwap(true, false) {
+		close(s.pointReached)
+		<-s.pointRelease
+	}
 	return s.Storage.LoadResourceGroupState(keyspaceID, name)
 }
 
@@ -250,3 +263,75 @@ func TestAsyncLoadResourceGroupsRecoversFromStateLoadFailure(t *testing.T) {
 		return !krgm.isReserved("flaky-group")
 	}, testutil.WithTickInterval(20*time.Millisecond))
 }
+
+// TestAsyncLoadResourceGroupsDeleteRaceDoesNotResurrect reproduces the
+// lazy-load vs concurrent Delete race deterministically: a lazy load reads a
+// group from storage, then a Delete removes it before the lazy load inserts.
+// The stale insert must be rejected (via the delete-generation check) so the
+// deleted group is not resurrected for the rest of the manager's lifetime.
+func TestAsyncLoadResourceGroupsDeleteRaceDoesNotResurrect(t *testing.T) {
+	re := require.New(t)
+	store := newBlockingResourceGroupStorage()
+	group := newAsyncTestGroup("race-group")
+	re.NoError(store.SaveResourceGroupSetting(1, "race-group", group))
+	re.NoError(store.SaveResourceGroupStates(1, "race-group", FromProtoResourceGroup(group).GetGroupStates()))
+
+	m := NewManager[*mockConfigProvider](&mockConfigProvider{})
+	m.storage = store
+	re.NoError(m.Init(context.Background()))
+	defer stopAsyncTestManager(m)
+	defer store.unblock()
+
+	// Async bulk load is blocked, so loadingState stays in progress and lazy
+	// loading is active.
+	store.waitEntered(t)
+
+	// Start a lazy Get that will pause inside its state read, i.e. after it has
+	// read the group from storage but before it inserts into the cache.
+	store.pausePointState.Store(true)
+	var (
+		gotGroup *ResourceGroup
+		gotErr   error
+	)
+	getDone := make(chan struct{})
+	go func() {
+		defer close(getDone)
+		gotGroup, gotErr = m.GetResourceGroup(1, "race-group", false)
+	}()
+
+	select {
+	case <-store.pointReached:
+	case <-time.After(time.Second):
+		t.Fatal("timed out waiting for the lazy load to reach its state read")
+	}
+
+	// While the lazy load is paused, delete the group. Delete does its own
+	// (unpaused) load-then-delete, removing it from storage and cache and
+	// bumping the delete generation.
+	re.NoError(m.DeleteResourceGroup(1, "race-group"))
+
+	// Release the paused lazy load; its now-stale insert must be rejected.
+	close(store.pointRelease)
+	<-getDone
+	re.NoError(gotErr)
+	re.Nil(gotGroup, "the racing lazy load must observe the group as deleted")
+
+	krgm := m.getKeyspaceResourceGroupManager(1)
+	re.NotNil(krgm)
+	re.Nil(krgm.getMutableResourceGroup("race-group"), "deleted group must not be resurrected by the racing lazy load")
+
+	// Finishing async loading must not bring the deleted group back either.
+	store.unblock()
+	testutil.Eventually(re, func() bool {
+		groups, err := m.GetResourceGroupList(1, false)
+		if err != nil {
+			return false
+		}
+		for _, g := range groups {
+			if g.Name == "race-group" {
+				return false
+			}
+		}
+		return true
+	}, testutil.WithTickInterval(20*time.Millisecond))
+}

From 229cd5dd5a08d7d9a58d12d98fef3eccb3a07451 Mon Sep 17 00:00:00 2001
From: tongjian <1045931706@qq.com>
Date: Wed, 22 Jul 2026 15:27:39 +0800
Subject: [PATCH 16/50] resource_group: prevent a stale async loader from
 polluting a new term

asyncLoadResourceGroups only checked its context at the top of the retry
loop. A loader blocked in a storage scan across a leadership change
would, after Init ran again for a new term, wake up and merge its stale
scan into the new term's maps, clear the new term's syncLoadedGroups,
and publish LoadingStateCompleted while the new loader was still
running.

Add a loadEpoch counter to the manager, bumped by initMetadata under the
manager lock and captured by each loader at start. Every shared-state
mutation the loader performs (loading-state transitions and the merge)
now re-verifies the epoch inside the same critical section, so a stale
loader exits instead of touching the newer term's state. Also re-check
context cancellation right after the scans return, and make
initControllerConfig publish the config via clone-and-swap under the
lock, since a re-initialization can race with the previous term's
background goroutines still reading it.

Add a deterministic regression test that blocks a term-1 loader in its
states scan, reinitializes the manager for term 2, deletes the group,
then releases the stale loader and asserts it does not resurrect the
group or disturb the new term (verified to fail without the fix).

Signed-off-by: tongjian <doufuxiaowangzi@gmail.com>
Signed-off-by: tongjian <1045931706@qq.com>
---
 pkg/mcs/resourcemanager/server/manager.go     | 70 ++++++++++++--
 .../server/manager_async_test.go              | 96 ++++++++++++++++++-
 2 files changed, 154 insertions(+), 12 deletions(-)

diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go
index aaa22d6a5b..1d84f775fe 100644
--- a/pkg/mcs/resourcemanager/server/manager.go
+++ b/pkg/mcs/resourcemanager/server/manager.go
@@ -119,6 +119,12 @@ type Manager struct {
 	loadingState int32 // atomic access
 	// syncLoadedGroups records groups that were loaded synchronously (e.g., by lazy loading)
 	syncLoadedGroups map[trackerKey]bool
+	// loadEpoch is bumped (under the manager lock) every time initMetadata
+	// resets the loading state for a new term. The async loader captures it at
+	// start and re-checks it before every shared-state mutation, so a loader
+	// from a previous term that was blocked in a storage scan can never merge
+	// stale data into, or publish completion for, a newer term.
+	loadEpoch uint64
 }
 
 // LoadingState represents the current loading state of resource groups
@@ -392,13 +398,22 @@ func (m *Manager) initControllerConfig() error {
 		log.Error("resource controller config load failed", zap.Error(err), zap.String("v", v))
 		return err
 	}
-	if err = json.Unmarshal([]byte(v), &m.controllerConfig); err != nil {
+	// Unmarshal into a clone and publish it under the lock: on a
+	// re-initialization after a leadership change, the previous term's
+	// background goroutines may still be reading the current config.
+	m.RLock()
+	controllerConfig := cloneControllerConfig(m.controllerConfig)
+	m.RUnlock()
+	if err = json.Unmarshal([]byte(v), &controllerConfig); err != nil {
 		log.Warn("un-marshall controller config failed, fallback to default", zap.Error(err), zap.String("v", v))
 	}
+	m.Lock()
+	m.controllerConfig = controllerConfig
+	m.Unlock()
 
 	// re-save the config to make sure the config has been persisted.
 	if m.writeRole.AllowsMetadataWrite() {
-		if err := m.storage.SaveControllerConfig(m.controllerConfig); err != nil {
+		if err := m.storage.SaveControllerConfig(controllerConfig); err != nil {
 			return err
 		}
 	}
@@ -413,6 +428,8 @@ func (m *Manager) initMetadata(ctx context.Context) error {
 	m.Lock()
 	m.krgms = make(map[uint32]*keyspaceResourceGroupManager)
 	m.syncLoadedGroups = make(map[trackerKey]bool)
+	m.loadEpoch++
+	epoch := m.loadEpoch
 	atomic.StoreInt32(&m.loadingState, LoadingStateNotStarted)
 	m.Unlock()
 
@@ -422,7 +439,7 @@ func (m *Manager) initMetadata(ctx context.Context) error {
 	}
 
 	m.wg.Add(1)
-	go m.asyncLoadResourceGroups(ctx)
+	go m.asyncLoadResourceGroups(ctx, epoch)
 	return nil
 }
 
@@ -446,7 +463,21 @@ func (m *Manager) loadKeyspaceResourceGroups() error {
 	return m.loadServiceLimits()
 }
 
-func (m *Manager) asyncLoadResourceGroups(ctx context.Context) {
+// storeLoadingStateIfCurrent stores the loading state only if the manager has
+// not been reinitialized since the loader with the given epoch started. It
+// returns false when the loader is stale and must exit without touching any
+// further shared state.
+func (m *Manager) storeLoadingStateIfCurrent(epoch uint64, state int32) bool {
+	m.Lock()
+	defer m.Unlock()
+	if m.loadEpoch != epoch {
+		return false
+	}
+	atomic.StoreInt32(&m.loadingState, state)
+	return true
+}
+
+func (m *Manager) asyncLoadResourceGroups(ctx context.Context, epoch uint64) {
 	defer logutil.LogPanic()
 	defer m.wg.Done()
 
@@ -471,18 +502,40 @@ func (m *Manager) asyncLoadResourceGroups(ctx context.Context) {
 			}
 		}
 
-		atomic.StoreInt32(&m.loadingState, LoadingStateInProgress)
+		if !m.storeLoadingStateIfCurrent(epoch, LoadingStateInProgress) {
+			log.Info("async loading resource groups aborted: manager was reinitialized")
+			return
+		}
 		startTime := time.Now()
 		tempKrgms, err := m.loadKeyspaceResourceGroupsFromStorage()
+		// The storage scans above can block for a long time; re-check for
+		// cancellation before touching any shared state, so a loader whose
+		// term already ended doesn't pollute a newer term's state.
+		select {
+		case <-ctx.Done():
+			log.Info("async loading resource groups cancelled")
+			return
+		default:
+		}
 		if err != nil {
 			log.Error("failed to load resource groups", zap.Error(err), zap.Int("retry", retry))
-			atomic.StoreInt32(&m.loadingState, LoadingStateNotStarted)
+			if !m.storeLoadingStateIfCurrent(epoch, LoadingStateNotStarted) {
+				log.Info("async loading resource groups aborted: manager was reinitialized")
+				return
+			}
 			retry++
 			continue
 		}
 
 		loaded := 0
 		m.Lock()
+		if m.loadEpoch != epoch {
+			// The manager was reinitialized for a new term while this loader
+			// was scanning; its result is stale and must not be merged.
+			m.Unlock()
+			log.Info("async loading resource groups aborted: manager was reinitialized")
+			return
+		}
 		for keyspaceID, tempKrgm := range tempKrgms {
 			krgm := m.krgms[keyspaceID]
 			if krgm == nil {
@@ -515,7 +568,10 @@ func (m *Manager) asyncLoadResourceGroups(ctx context.Context) {
 		m.Unlock()
 
 		m.initReserved()
-		atomic.StoreInt32(&m.loadingState, LoadingStateCompleted)
+		if !m.storeLoadingStateIfCurrent(epoch, LoadingStateCompleted) {
+			log.Info("async loading resource groups aborted: manager was reinitialized")
+			return
+		}
 		duration := time.Since(startTime)
 		asyncLoadGroupDuration.Observe(duration.Seconds())
 		log.Info("async loading resource groups completed", zap.Int("loaded-groups", loaded), zap.Duration("duration", duration))
diff --git a/pkg/mcs/resourcemanager/server/manager_async_test.go b/pkg/mcs/resourcemanager/server/manager_async_test.go
index f29e22e2fd..cf1562f66a 100644
--- a/pkg/mcs/resourcemanager/server/manager_async_test.go
+++ b/pkg/mcs/resourcemanager/server/manager_async_test.go
@@ -50,15 +50,26 @@ type blockingResourceGroupStorage struct {
 	pausePointState atomic.Bool
 	pointReached    chan struct{}
 	pointRelease    chan struct{}
+
+	// pauseNextStates, when true, makes the very next bulk
+	// LoadResourceGroupStates call signal statesReached and then block on
+	// statesRelease, so a test can hold an async loader after it has captured
+	// the settings scan but before it merges.
+	pauseNextStates   atomic.Bool
+	statesReached     chan struct{}
+	statesRelease     chan struct{}
+	statesReleaseOnce sync.Once
 }
 
 func newBlockingResourceGroupStorage() *blockingResourceGroupStorage {
 	return &blockingResourceGroupStorage{
-		Storage:      storage.NewStorageWithMemoryBackend(),
-		entered:      make(chan struct{}),
-		release:      make(chan struct{}),
-		pointReached: make(chan struct{}),
-		pointRelease: make(chan struct{}),
+		Storage:       storage.NewStorageWithMemoryBackend(),
+		entered:       make(chan struct{}),
+		release:       make(chan struct{}),
+		pointReached:  make(chan struct{}),
+		pointRelease:  make(chan struct{}),
+		statesReached: make(chan struct{}),
+		statesRelease: make(chan struct{}),
 	}
 }
 
@@ -81,6 +92,14 @@ func (s *blockingResourceGroupStorage) LoadResourceGroupState(keyspaceID uint32,
 	return s.Storage.LoadResourceGroupState(keyspaceID, name)
 }
 
+func (s *blockingResourceGroupStorage) LoadResourceGroupStates(f func(keyspaceID uint32, name, rawValue string)) error {
+	if s.pauseNextStates.CompareAndSwap(true, false) {
+		close(s.statesReached)
+		<-s.statesRelease
+	}
+	return s.Storage.LoadResourceGroupStates(f)
+}
+
 func (s *blockingResourceGroupStorage) waitEntered(t *testing.T) {
 	t.Helper()
 	select {
@@ -96,6 +115,12 @@ func (s *blockingResourceGroupStorage) unblock() {
 	})
 }
 
+func (s *blockingResourceGroupStorage) unblockStates() {
+	s.statesReleaseOnce.Do(func() {
+		close(s.statesRelease)
+	})
+}
+
 // asyncTestGroupFillRate is the fill rate used by all async-loading test
 // groups; kept as a named constant so the setup and the assertions stay in
 // sync.
@@ -335,3 +360,64 @@ func TestAsyncLoadResourceGroupsDeleteRaceDoesNotResurrect(t *testing.T) {
 		return true
 	}, testutil.WithTickInterval(20*time.Millisecond))
 }
+
+// TestAsyncLoadResourceGroupsStaleLoaderDoesNotPolluteNewTerm reproduces the
+// stale-loader race: a loader from an old term is blocked in its storage scan
+// while the leadership changes and Init runs again for a new term. When the
+// old loader finally wakes up, it must not merge its stale scan into the new
+// term's maps, clear the new term's syncLoadedGroups, or publish completion.
+func TestAsyncLoadResourceGroupsStaleLoaderDoesNotPolluteNewTerm(t *testing.T) {
+	re := require.New(t)
+	store := newBlockingResourceGroupStorage()
+	group := newAsyncTestGroup("stale-group")
+	re.NoError(store.SaveResourceGroupSetting(1, "stale-group", group))
+
+	m := NewManager[*mockConfigProvider](&mockConfigProvider{})
+	m.storage = store
+	// Term 1: the loader blocks at the start of its settings scan.
+	re.NoError(m.Init(context.Background()))
+	cancelTerm1 := m.cancel
+	defer stopAsyncTestManager(m)
+	defer store.unblock()
+	defer store.unblockStates()
+
+	store.waitEntered(t)
+
+	// Let the term-1 loader run its settings scan (capturing stale-group into
+	// its temp result) and then block again in the states scan, i.e. after it
+	// has read storage but before it merges.
+	store.pauseNextStates.Store(true)
+	store.unblock()
+	select {
+	case <-store.statesReached:
+	case <-time.After(time.Second):
+		t.Fatal("timed out waiting for the term-1 loader to reach its states scan")
+	}
+
+	// Leadership changes: cancel term 1 and reinitialize for term 2. The
+	// term-2 loader hits neither block (both were consumed) and completes.
+	cancelTerm1()
+	re.NoError(m.Init(context.Background()))
+	testutil.Eventually(re, func() bool {
+		groups, err := m.GetResourceGroupList(1, false)
+		return err == nil && len(groups) == 2
+	}, testutil.WithTickInterval(20*time.Millisecond))
+
+	// Delete the group in term 2, after loading completed.
+	re.NoError(m.DeleteResourceGroup(1, "stale-group"))
+
+	// Release the stale term-1 loader. It must observe its cancelled context /
+	// stale epoch and exit without resurrecting the deleted group or touching
+	// the new term's loading state.
+	store.unblockStates()
+	time.Sleep(200 * time.Millisecond)
+
+	krgm := m.getKeyspaceResourceGroupManager(1)
+	re.NotNil(krgm)
+	re.Nil(krgm.getMutableResourceGroup("stale-group"), "stale loader must not merge into the new term")
+	groups, err := m.GetResourceGroupList(1, false)
+	re.NoError(err)
+	for _, g := range groups {
+		re.NotEqual("stale-group", g.Name)
+	}
+}

From 10b6c2abfc33fb6eb007fbd66e454aabafb174d6 Mon Sep 17 00:00:00 2001
From: tongjian <1045931706@qq.com>
Date: Wed, 22 Jul 2026 15:34:00 +0800
Subject: [PATCH 17/50] resource_group: don't let the bulk merge clobber
 modified settings

When a lazy load reads a group's settings but fails to read its state,
the entry is cached as reserved and deliberately kept out of
syncLoadedGroups so the async bulk merge can still recover the real
state. But the merge replaced the cache entry wholesale, so a
modification persisted after the bulk scan captured its (older)
settings was silently lost from the serving cache for the rest of the
manager's lifetime, while storage kept the new values.

Split the reserved marker into two kinds: reservedPlaceholder (settings
and state both synthetic, e.g. the pre-inserted default group), which
the merge may still replace wholesale, and reservedStateOnly (settings
confirmed from storage, state missing), for which the merge now adopts
only the scanned state into the existing entry and keeps its settings.

Add a regression test that captures the bulk scan before a Modify,
fails the lazy load's state reads, modifies the group, then releases
the merge and asserts the modified settings survive while the scanned
state is adopted (verified to fail without the fix).

Signed-off-by: tongjian <doufuxiaowangzi@gmail.com>
Signed-off-by: tongjian <1045931706@qq.com>
---
 .../server/keyspace_manager.go                | 35 ++++++---
 pkg/mcs/resourcemanager/server/manager.go     | 36 +++++++---
 .../server/manager_async_test.go              | 72 +++++++++++++++++++
 3 files changed, 124 insertions(+), 19 deletions(-)

diff --git a/pkg/mcs/resourcemanager/server/keyspace_manager.go b/pkg/mcs/resourcemanager/server/keyspace_manager.go
index f9638d850e..d37983a787 100644
--- a/pkg/mcs/resourcemanager/server/keyspace_manager.go
+++ b/pkg/mcs/resourcemanager/server/keyspace_manager.go
@@ -67,15 +67,32 @@ type consumptionItem struct {
 	isTiFlash    bool
 }
 
+// reservedKind describes why a cached resource group entry is still
+// considered unconfirmed.
+type reservedKind int
+
+const (
+	// reservedPlaceholder marks an entry whose settings and state are both
+	// synthetic, e.g. the default group pre-inserted by
+	// ensureReservedDefaultGroupInCache before async loading runs. The async
+	// bulk merge may replace such an entry wholesale.
+	reservedPlaceholder reservedKind = iota
+	// reservedStateOnly marks an entry whose settings were confirmed from
+	// storage (and may have been modified and persisted since) but whose
+	// state failed to load. The async bulk merge must only adopt the scanned
+	// state into it, never replace its settings with the scan's older copy.
+	reservedStateOnly
+)
+
 type keyspaceResourceGroupManager struct {
 	syncutil.RWMutex
 	groups map[string]*ResourceGroup
-	// reservedGroups tracks names whose entry in groups is still just the
-	// synthetic placeholder inserted by ensureReservedDefaultGroupInCache or
-	// restoreDefaultResourceGroupFromReserved, not yet confirmed by a storage
-	// load or a real write. It shares the same lock as groups so a lazy load
-	// can atomically decide whether it's safe to replace the placeholder.
-	reservedGroups  map[string]struct{}
+	// reservedGroups tracks names whose entry in groups is not yet fully
+	// confirmed by a storage load or a real write, together with how much of
+	// it is unconfirmed (see reservedKind). It shares the same lock as groups
+	// so a lazy load can atomically decide whether it's safe to replace the
+	// entry.
+	reservedGroups  map[string]reservedKind
 	groupRUTrackers map[string]*groupRUTracker
 	serviceLimiter  *serviceLimiter
 	// deleteGen is bumped under the write lock every time a group is removed
@@ -100,7 +117,7 @@ func newKeyspaceResourceGroupManager(
 	}
 	return &keyspaceResourceGroupManager{
 		groups:          make(map[string]*ResourceGroup),
-		reservedGroups:  make(map[string]struct{}),
+		reservedGroups:  make(map[string]reservedKind),
 		groupRUTrackers: make(map[string]*groupRUTracker),
 		keyspaceID:      keyspaceID,
 		storage:         storage,
@@ -245,7 +262,7 @@ func (krgm *keyspaceResourceGroupManager) ensureReservedDefaultGroupInCache() {
 	krgm.Lock()
 	if _, ok := krgm.groups[DefaultResourceGroupName]; !ok {
 		krgm.groups[DefaultResourceGroupName] = defaultGroup
-		krgm.reservedGroups[DefaultResourceGroupName] = struct{}{}
+		krgm.reservedGroups[DefaultResourceGroupName] = reservedPlaceholder
 		inserted = true
 	}
 	krgm.Unlock()
@@ -273,7 +290,7 @@ func (krgm *keyspaceResourceGroupManager) restoreDefaultResourceGroupFromReserve
 	defaultGroup := newDefaultResourceGroup()
 	krgm.Lock()
 	krgm.groups[DefaultResourceGroupName] = defaultGroup
-	krgm.reservedGroups[DefaultResourceGroupName] = struct{}{}
+	krgm.reservedGroups[DefaultResourceGroupName] = reservedPlaceholder
 	krgm.Unlock()
 	krgm.syncBurstabilityWithServiceLimit(defaultGroup)
 }
diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go
index 1d84f775fe..53bfbc7334 100644
--- a/pkg/mcs/resourcemanager/server/manager.go
+++ b/pkg/mcs/resourcemanager/server/manager.go
@@ -547,16 +547,32 @@ func (m *Manager) asyncLoadResourceGroups(ctx context.Context, epoch uint64) {
 			krgm.Lock()
 			for name, group := range tempKrgm.groups {
 				key := trackerKey{keyspaceID: keyspaceID, groupName: name}
-				if !m.syncLoadedGroups[key] {
-					krgm.groups[name] = group
-					// This group is now confirmed, fully-loaded data (settings
-					// and state); it must no longer be treated as an
-					// unconfirmed placeholder by loadResourceGroupIfNeeded or
-					// skipped by the state persist loop.
-					delete(krgm.reservedGroups, name)
-					groupsToSync = append(groupsToSync, group)
-					loaded++
+				if m.syncLoadedGroups[key] {
+					continue
+				}
+				if kind, reserved := krgm.reservedGroups[name]; reserved && kind == reservedStateOnly {
+					if existing, ok := krgm.groups[name]; ok {
+						// The cached entry's settings are confirmed and may
+						// carry a modification persisted after this scan
+						// started; only its state is missing. Adopt the
+						// scanned state into the existing entry instead of
+						// replacing it, so the newer settings aren't
+						// clobbered by the scan's older copy.
+						existing.SetStatesIntoResourceGroup(group.GetGroupStates())
+						delete(krgm.reservedGroups, name)
+						groupsToSync = append(groupsToSync, existing)
+						loaded++
+						continue
+					}
 				}
+				krgm.groups[name] = group
+				// This group is now confirmed, fully-loaded data (settings
+				// and state); it must no longer be treated as an
+				// unconfirmed placeholder by loadResourceGroupIfNeeded or
+				// skipped by the state persist loop.
+				delete(krgm.reservedGroups, name)
+				groupsToSync = append(groupsToSync, group)
+				loaded++
 			}
 			krgm.Unlock()
 			tempKrgm.RUnlock()
@@ -705,7 +721,7 @@ func (m *Manager) loadResourceGroupIfNeeded(keyspaceID uint32, name string) erro
 	if stateLoaded {
 		delete(krgm.reservedGroups, name)
 	} else {
-		krgm.reservedGroups[name] = struct{}{}
+		krgm.reservedGroups[name] = reservedStateOnly
 	}
 	krgm.Unlock()
 	if inserted {
diff --git a/pkg/mcs/resourcemanager/server/manager_async_test.go b/pkg/mcs/resourcemanager/server/manager_async_test.go
index cf1562f66a..98f8f4d2dd 100644
--- a/pkg/mcs/resourcemanager/server/manager_async_test.go
+++ b/pkg/mcs/resourcemanager/server/manager_async_test.go
@@ -421,3 +421,75 @@ func TestAsyncLoadResourceGroupsStaleLoaderDoesNotPolluteNewTerm(t *testing.T) {
 		re.NotEqual("stale-group", g.Name)
 	}
 }
+
+// TestAsyncLoadResourceGroupsMergeKeepsModifiedSettings reproduces the
+// stale-settings clobber: a group's lazy load reads its settings but fails to
+// read its state, then the group is modified (and the modification persisted)
+// while the async bulk scan still holds the pre-modification settings. The
+// bulk merge must adopt only the scanned state into the cached entry, not
+// replace it wholesale, so the modified settings survive in the serving cache.
+func TestAsyncLoadResourceGroupsMergeKeepsModifiedSettings(t *testing.T) {
+	re := require.New(t)
+	store := newBlockingResourceGroupStorage()
+	group := newAsyncTestGroup("mod-group")
+	re.NoError(store.SaveResourceGroupSetting(1, "mod-group", group))
+	// Persist a state with a recognizable consumption so the test can tell
+	// that the merge really adopted the scanned state.
+	states := FromProtoResourceGroup(group).GetGroupStates()
+	states.RUConsumption.RRU = 123
+	re.NoError(store.SaveResourceGroupStates(1, "mod-group", states))
+
+	m := NewManager[*mockConfigProvider](&mockConfigProvider{})
+	m.storage = store
+	re.NoError(m.Init(context.Background()))
+	defer stopAsyncTestManager(m)
+	defer store.unblock()
+	defer store.unblockStates()
+
+	store.waitEntered(t)
+
+	// Let the bulk loader capture the pre-modification settings, then hold it
+	// right before its states scan (i.e. before it merges).
+	store.pauseNextStates.Store(true)
+	store.unblock()
+	select {
+	case <-store.statesReached:
+	case <-time.After(time.Second):
+		t.Fatal("timed out waiting for the bulk loader to reach its states scan")
+	}
+
+	// Lazily load the group with a failing state read: it is cached with
+	// confirmed settings but unconfirmed (fresh) state.
+	store.failNextState.Store(true)
+	fetched, err := m.GetResourceGroup(1, "mod-group", false)
+	re.NoError(err)
+	re.NotNil(fetched)
+	krgm := m.getKeyspaceResourceGroupManager(1)
+	re.NotNil(krgm)
+	re.True(krgm.isReserved("mod-group"))
+
+	// Modify the group (fill rate 100 -> 200) and persist it. The state read
+	// of Modify's own lazy load fails again, so the entry stays state-only
+	// reserved and out of syncLoadedGroups.
+	store.failNextState.Store(true)
+	modified := newAsyncTestGroup("mod-group")
+	modified.RUSettings.RU.Settings.FillRate = 200
+	modified.KeyspaceId = &resource_manager.KeyspaceIDValue{Value: 1}
+	re.NoError(m.ModifyResourceGroup(modified))
+
+	// Release the bulk loader; its merge must keep the modified settings and
+	// only adopt the scanned state.
+	store.unblockStates()
+	testutil.Eventually(re, func() bool {
+		_, err := m.GetResourceGroupList(1, false)
+		return err == nil
+	}, testutil.WithTickInterval(20*time.Millisecond))
+
+	got, err := m.GetResourceGroup(1, "mod-group", false)
+	re.NoError(err)
+	re.NotNil(got)
+	re.Equal(float64(200), got.RUSettings.RU.getFillRate(), "modified settings must survive the bulk merge")
+	re.False(krgm.isReserved("mod-group"), "state adoption must clear the reserved marker")
+	re.Equal(float64(123), krgm.getMutableResourceGroup("mod-group").GetGroupStates().RUConsumption.RRU,
+		"the scanned state must be adopted into the cached entry")
+}

From 64e13136909eaa21d5655d7faa3784118c9fb26c Mon Sep 17 00:00:00 2001
From: tongjian <1045931706@qq.com>
Date: Wed, 22 Jul 2026 15:41:45 +0800
Subject: [PATCH 18/50] resource_group: persist fresh-store default and retry
 lazy loads on delete races

Two fixes for the async loading path:

initDefaultResourceGroup bailed out whenever any cache entry existed
for the default group, including the synthetic placeholder pre-inserted
by initReservedInCache. On a fresh store, the confirmed-not-found
fallback therefore never created or persisted the default group, and
the entry stayed an unconfirmed placeholder for the manager lifetime:
its settings were never stored and the persist loop permanently skipped
its token and consumption state. Treat a reservedPlaceholder entry as
absent (synthesize and persist), while still returning early for
confirmed entries and reservedStateOnly ones, whose real settings must
not be overwritten with synthetic values.

The per-keyspace deleteGen is shared by every group, so deleting group
B while group A was being lazily loaded discarded A's valid result and
made its request spuriously report the group as missing. Retry the
storage read (up to 3 attempts) on a generation mismatch: the re-read
observes post-delete storage, so an unrelated deletion just reloads
successfully, and a deletion of the group itself now correctly reports
not-found instead of silently returning nothing.

Add regression tests for both (each verified to fail without its fix).

Signed-off-by: tongjian <doufuxiaowangzi@gmail.com>
Signed-off-by: tongjian <1045931706@qq.com>
---
 .../server/keyspace_manager.go                |  10 +-
 pkg/mcs/resourcemanager/server/manager.go     | 124 ++++++++++--------
 .../server/manager_async_test.go              | 113 +++++++++++++++-
 3 files changed, 187 insertions(+), 60 deletions(-)

diff --git a/pkg/mcs/resourcemanager/server/keyspace_manager.go b/pkg/mcs/resourcemanager/server/keyspace_manager.go
index d37983a787..15c9e0af82 100644
--- a/pkg/mcs/resourcemanager/server/keyspace_manager.go
+++ b/pkg/mcs/resourcemanager/server/keyspace_manager.go
@@ -240,8 +240,16 @@ func (krgm *keyspaceResourceGroupManager) setRawStatesIntoResourceGroup(name str
 func (krgm *keyspaceResourceGroupManager) initDefaultResourceGroup() {
 	krgm.RLock()
 	_, ok := krgm.groups[DefaultResourceGroupName]
+	kind, reserved := krgm.reservedGroups[DefaultResourceGroupName]
 	krgm.RUnlock()
-	if ok {
+	// A cached entry only makes initialization unnecessary if it's confirmed
+	// data, or at least has confirmed settings (reservedStateOnly), which must
+	// not be overwritten with synthetic ones. A reservedPlaceholder means
+	// nothing is persisted for the default group (e.g. a fresh store): it must
+	// still be created and persisted here, otherwise it would stay an
+	// unconfirmed placeholder forever, with its settings never stored and its
+	// state persistence permanently skipped.
+	if ok && (!reserved || kind == reservedStateOnly) {
 		return
 	}
 	defaultGroup := newDefaultResourceGroup()
diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go
index 53bfbc7334..f41e7a1d9f 100644
--- a/pkg/mcs/resourcemanager/server/manager.go
+++ b/pkg/mcs/resourcemanager/server/manager.go
@@ -677,69 +677,77 @@ func (m *Manager) loadResourceGroupIfNeeded(keyspaceID uint32, name string) erro
 			return nil
 		}
 	}
-	// Ensure the keyspace manager exists and snapshot its delete generation
-	// before the lock-free storage read below, so a concurrent Delete that
-	// lands after the read is detected under the insert lock and can't be
-	// undone by this now-stale result.
 	krgm = m.getOrCreateKeyspaceResourceGroupManager(keyspaceID, false)
-	deleteGen := krgm.loadDeleteGen()
-	group, stateLoaded, err := m.loadResourceGroup(keyspaceID, name)
-	if err != nil {
-		if name == DefaultResourceGroupName && errors.ErrorEqual(err, errs.ErrResourceGroupNotExists.FastGenByArgs(name)) {
-			// No persisted default group settings exist yet (e.g. a brand-new
-			// keyspace), so it's safe to synthesize the reserved default group.
-			// This calls initDefaultResourceGroup directly instead of going
-			// through getOrCreateKeyspaceResourceGroupManager(id, true), which
-			// now routes back into this same function and would recurse.
-			krgm.initDefaultResourceGroup()
-			return nil
+	// deleteGen is shared by every group in the keyspace, so a concurrent
+	// Delete of any group invalidates the lock-free storage read below.
+	// Deletes are rare: retry the read a few times so an unrelated deletion
+	// doesn't make this request spuriously report the group as missing; if it
+	// keeps racing, give up without inserting and let a later request or the
+	// async bulk merge reload the group.
+	const maxLoadAttempts = 3
+	for attempt := 1; ; attempt++ {
+		// Snapshot the delete generation before the lock-free storage read,
+		// so a Delete that lands after the read is detected under the insert
+		// lock and can't be undone by the now-stale result.
+		deleteGen := krgm.loadDeleteGen()
+		group, stateLoaded, err := m.loadResourceGroup(keyspaceID, name)
+		if err != nil {
+			if name == DefaultResourceGroupName && errors.ErrorEqual(err, errs.ErrResourceGroupNotExists.FastGenByArgs(name)) {
+				// No persisted default group settings exist yet (e.g. a brand-new
+				// keyspace), so it's safe to synthesize the reserved default group.
+				// This calls initDefaultResourceGroup directly instead of going
+				// through getOrCreateKeyspaceResourceGroupManager(id, true), which
+				// now routes back into this same function and would recurse.
+				krgm.initDefaultResourceGroup()
+				return nil
+			}
+			return err
+		}
+		inserted := false
+		krgm.Lock()
+		if krgm.deleteGen != deleteGen {
+			krgm.Unlock()
+			if attempt >= maxLoadAttempts {
+				return nil
+			}
+			continue
+		}
+		if _, exists := krgm.groups[name]; !exists {
+			krgm.groups[name] = group
+			inserted = true
+		} else if _, reserved := krgm.reservedGroups[name]; reserved {
+			// The existing entry is just an unconfirmed placeholder; the freshly
+			// loaded group is the real, confirmed data, so replacing it is safe.
+			krgm.groups[name] = group
+			inserted = true
+		}
+		// Only clear the placeholder mark once the state was actually read; a
+		// metadata-only group (state load failed) must stay reserved so a later
+		// call or the async bulk merge remains free to fill in the real state,
+		// instead of this partial result being treated as final forever.
+		if stateLoaded {
+			delete(krgm.reservedGroups, name)
+		} else {
+			krgm.reservedGroups[name] = reservedStateOnly
 		}
-		return err
-	}
-	inserted := false
-	krgm.Lock()
-	if krgm.deleteGen != deleteGen {
-		// A Delete raced with our storage read; the result may be stale, so
-		// don't insert it. A later request or the async bulk merge will reload
-		// the group if it still exists.
 		krgm.Unlock()
-		return nil
-	}
-	if _, exists := krgm.groups[name]; !exists {
-		krgm.groups[name] = group
-		inserted = true
-	} else if _, reserved := krgm.reservedGroups[name]; reserved {
-		// The existing entry is just an unconfirmed placeholder; the freshly
-		// loaded group is the real, confirmed data, so replacing it is safe.
-		krgm.groups[name] = group
-		inserted = true
-	}
-	// Only clear the placeholder mark once the state was actually read; a
-	// metadata-only group (state load failed) must stay reserved so a later
-	// call or the async bulk merge remains free to fill in the real state,
-	// instead of this partial result being treated as final forever.
-	if stateLoaded {
-		delete(krgm.reservedGroups, name)
-	} else {
-		krgm.reservedGroups[name] = reservedStateOnly
-	}
-	krgm.Unlock()
-	if inserted {
-		krgm.syncBurstabilityWithServiceLimit(group)
-	}
-	// Only mark the group as sync-loaded when its persisted state was actually
-	// read; otherwise a later async bulk load must remain free to fill in the
-	// real state instead of being skipped forever.
-	if stateLoaded {
-		markKey := trackerKey{keyspaceID: keyspaceID, groupName: name}
-		m.Lock()
-		if m.syncLoadedGroups != nil {
-			m.syncLoadedGroups[markKey] = true
+		if inserted {
+			krgm.syncBurstabilityWithServiceLimit(group)
 		}
-		m.Unlock()
+		// Only mark the group as sync-loaded when its persisted state was actually
+		// read; otherwise a later async bulk load must remain free to fill in the
+		// real state instead of being skipped forever.
+		if stateLoaded {
+			markKey := trackerKey{keyspaceID: keyspaceID, groupName: name}
+			m.Lock()
+			if m.syncLoadedGroups != nil {
+				m.syncLoadedGroups[markKey] = true
+			}
+			m.Unlock()
+		}
+		syncLoadGroupCounter.Inc()
+		return nil
 	}
-	syncLoadGroupCounter.Inc()
-	return nil
 }
 
 func (m *Manager) markResourceGroupSyncLoaded(keyspaceID uint32, name string) {
diff --git a/pkg/mcs/resourcemanager/server/manager_async_test.go b/pkg/mcs/resourcemanager/server/manager_async_test.go
index 98f8f4d2dd..7dd44f521c 100644
--- a/pkg/mcs/resourcemanager/server/manager_async_test.go
+++ b/pkg/mcs/resourcemanager/server/manager_async_test.go
@@ -338,7 +338,9 @@ func TestAsyncLoadResourceGroupsDeleteRaceDoesNotResurrect(t *testing.T) {
 	// Release the paused lazy load; its now-stale insert must be rejected.
 	close(store.pointRelease)
 	<-getDone
-	re.NoError(gotErr)
+	// The generation mismatch makes the lazy load retry its storage read,
+	// which now correctly observes the group as deleted.
+	re.ErrorContains(gotErr, "does not exist")
 	re.Nil(gotGroup, "the racing lazy load must observe the group as deleted")
 
 	krgm := m.getKeyspaceResourceGroupManager(1)
@@ -493,3 +495,112 @@ func TestAsyncLoadResourceGroupsMergeKeepsModifiedSettings(t *testing.T) {
 	re.Equal(float64(123), krgm.getMutableResourceGroup("mod-group").GetGroupStates().RUConsumption.RRU,
 		"the scanned state must be adopted into the cached entry")
 }
+
+// TestAsyncLoadResourceGroupsFreshStoreDefaultPersisted guards against the
+// fresh-store dead end: initReservedInCache pre-inserts a synthetic default
+// placeholder, and on a store with nothing persisted, the confirmed-not-found
+// fallback used to bail out on its cache-exists check, leaving the default
+// group an unconfirmed placeholder forever — settings never persisted and its
+// state persistence permanently skipped.
+func TestAsyncLoadResourceGroupsFreshStoreDefaultPersisted(t *testing.T) {
+	re := require.New(t)
+	// A completely fresh store: nothing persisted at all.
+	store := newBlockingResourceGroupStorage()
+
+	m := NewManager[*mockConfigProvider](&mockConfigProvider{})
+	m.storage = store
+	re.NoError(m.Init(context.Background()))
+	defer stopAsyncTestManager(m)
+	defer store.unblock()
+
+	store.waitEntered(t)
+
+	// Fetch the default group while async loading is still in progress: the
+	// point load confirms nothing is persisted, so the placeholder must be
+	// promoted to a real, persisted default group.
+	group, err := m.GetResourceGroup(constant.NullKeyspaceID, DefaultResourceGroupName, false)
+	re.NoError(err)
+	re.NotNil(group)
+	krgm := m.getKeyspaceResourceGroupManager(constant.NullKeyspaceID)
+	re.NotNil(krgm)
+	re.False(krgm.isReserved(DefaultResourceGroupName), "the default group must be confirmed after synthesis")
+	raw, err := store.LoadResourceGroupSetting(constant.NullKeyspaceID, DefaultResourceGroupName)
+	re.NoError(err)
+	re.NotEmpty(raw, "the synthesized default group settings must be persisted")
+
+	// Loading completion must keep it confirmed.
+	store.unblock()
+	testutil.Eventually(re, func() bool {
+		_, err := m.GetResourceGroupList(constant.NullKeyspaceID, false)
+		return err == nil
+	}, testutil.WithTickInterval(20*time.Millisecond))
+	re.False(krgm.isReserved(DefaultResourceGroupName))
+}
+
+// TestAsyncLoadResourceGroupsUnrelatedDeleteDoesNotFailLazyLoad guards against
+// the delete-generation check being too coarse: deleting group B while group A
+// is being lazily loaded must not make A's request spuriously report the group
+// as missing — the lazy load retries its storage read and succeeds.
+func TestAsyncLoadResourceGroupsUnrelatedDeleteDoesNotFailLazyLoad(t *testing.T) {
+	re := require.New(t)
+	store := newBlockingResourceGroupStorage()
+	re.NoError(store.SaveResourceGroupSetting(1, "group-a", newAsyncTestGroup("group-a")))
+	re.NoError(store.SaveResourceGroupSetting(1, "group-b", newAsyncTestGroup("group-b")))
+
+	m := NewManager[*mockConfigProvider](&mockConfigProvider{})
+	m.storage = store
+	re.NoError(m.Init(context.Background()))
+	defer stopAsyncTestManager(m)
+	defer store.unblock()
+
+	store.waitEntered(t)
+
+	// Start a lazy Get of group-a and pause it inside its state read, i.e.
+	// after it has read the group from storage but before it inserts.
+	store.pausePointState.Store(true)
+	var (
+		gotGroup *ResourceGroup
+		gotErr   error
+	)
+	getDone := make(chan struct{})
+	go func() {
+		defer close(getDone)
+		gotGroup, gotErr = m.GetResourceGroup(1, "group-a", false)
+	}()
+	select {
+	case <-store.pointReached:
+	case <-time.After(time.Second):
+		t.Fatal("timed out waiting for the lazy load to reach its state read")
+	}
+
+	// Delete the unrelated group-b while group-a's lazy load is paused; this
+	// bumps the keyspace's delete generation.
+	re.NoError(m.DeleteResourceGroup(1, "group-b"))
+
+	// Release group-a's lazy load: the generation mismatch must make it retry
+	// and succeed, not report group-a as missing.
+	close(store.pointRelease)
+	<-getDone
+	re.NoError(gotErr)
+	re.NotNil(gotGroup, "an unrelated delete must not fail the lazy load")
+	re.Equal("group-a", gotGroup.Name)
+
+	// After loading completes, group-a is present and group-b stays deleted.
+	store.unblock()
+	testutil.Eventually(re, func() bool {
+		groups, err := m.GetResourceGroupList(1, false)
+		if err != nil {
+			return false
+		}
+		foundA := false
+		for _, g := range groups {
+			if g.Name == "group-b" {
+				return false
+			}
+			if g.Name == "group-a" {
+				foundA = true
+			}
+		}
+		return foundA
+	}, testutil.WithTickInterval(20*time.Millisecond))
+}

From 57475ed3e2669ab94d02505472dc15f441d294b9 Mon Sep 17 00:00:00 2001
From: tongjian <1045931706@qq.com>
Date: Wed, 22 Jul 2026 15:46:17 +0800
Subject: [PATCH 19/50] resource_group: guard token bucket state writes with
 the group lock

SetStatesIntoResourceGroup wrote the token bucket fields (Tokens,
LastUpdate, Initialized) via setState with no lock, while RequestRU
mutates the same fields under the group lock. Both the async bulk
merge's state adoption and the metadata watcher's runtime state sync
call SetStatesIntoResourceGroup on groups that are already serving
token requests, racing with them.

Take the group lock around setState; UpdateRUConsumption already locks
internally. The lock order stays keyspace-manager lock -> group lock,
consistent with every other path (resource_group.go and
token_buckets.go never reference the keyspace manager).

Signed-off-by: tongjian <doufuxiaowangzi@gmail.com>
Signed-off-by: tongjian <1045931706@qq.com>
---
 pkg/mcs/resourcemanager/server/resource_group.go | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/pkg/mcs/resourcemanager/server/resource_group.go b/pkg/mcs/resourcemanager/server/resource_group.go
index 37fdd8bdb8..f1ab18adfd 100644
--- a/pkg/mcs/resourcemanager/server/resource_group.go
+++ b/pkg/mcs/resourcemanager/server/resource_group.go
@@ -379,7 +379,13 @@ func (rg *ResourceGroup) SetStatesIntoResourceGroup(states *GroupStates) {
 	switch rg.Mode {
 	case rmpb.GroupMode_RUMode:
 		if state := states.RU; state != nil {
+			// The group may already be serving requests: setState writes the
+			// token bucket fields directly, so guard it with the group lock
+			// the same way RequestRU does. UpdateRUConsumption below locks
+			// internally.
+			rg.Lock()
 			rg.RUSettings.RU.setState(state)
+			rg.Unlock()
 			log.Debug("update group token bucket state", zap.String("name", rg.Name), zap.Any("state", state))
 		}
 		if states.RUConsumption != nil {

From 78e9e69389534883c96c9048eadef248d10785bf Mon Sep 17 00:00:00 2001
From: tongjian <1045931706@qq.com>
Date: Thu, 23 Jul 2026 15:41:12 +0800
Subject: [PATCH 20/50] address comment

Signed-off-by: tongjian <1045931706@qq.com>
---
 .../server/keyspace_manager.go                |  45 ++------
 pkg/mcs/resourcemanager/server/manager.go     |  67 ++++-------
 .../server/manager_async_test.go              | 104 +++++++++---------
 3 files changed, 80 insertions(+), 136 deletions(-)

diff --git a/pkg/mcs/resourcemanager/server/keyspace_manager.go b/pkg/mcs/resourcemanager/server/keyspace_manager.go
index 15c9e0af82..5c2d61b440 100644
--- a/pkg/mcs/resourcemanager/server/keyspace_manager.go
+++ b/pkg/mcs/resourcemanager/server/keyspace_manager.go
@@ -67,32 +67,12 @@ type consumptionItem struct {
 	isTiFlash    bool
 }
 
-// reservedKind describes why a cached resource group entry is still
-// considered unconfirmed.
-type reservedKind int
-
-const (
-	// reservedPlaceholder marks an entry whose settings and state are both
-	// synthetic, e.g. the default group pre-inserted by
-	// ensureReservedDefaultGroupInCache before async loading runs. The async
-	// bulk merge may replace such an entry wholesale.
-	reservedPlaceholder reservedKind = iota
-	// reservedStateOnly marks an entry whose settings were confirmed from
-	// storage (and may have been modified and persisted since) but whose
-	// state failed to load. The async bulk merge must only adopt the scanned
-	// state into it, never replace its settings with the scan's older copy.
-	reservedStateOnly
-)
-
 type keyspaceResourceGroupManager struct {
 	syncutil.RWMutex
 	groups map[string]*ResourceGroup
-	// reservedGroups tracks names whose entry in groups is not yet fully
-	// confirmed by a storage load or a real write, together with how much of
-	// it is unconfirmed (see reservedKind). It shares the same lock as groups
-	// so a lazy load can atomically decide whether it's safe to replace the
-	// entry.
-	reservedGroups  map[string]reservedKind
+	// reservedGroups tracks names whose entry in groups is still a synthetic
+	// placeholder and not yet confirmed by a storage load or a real write.
+	reservedGroups  map[string]struct{}
 	groupRUTrackers map[string]*groupRUTracker
 	serviceLimiter  *serviceLimiter
 	// deleteGen is bumped under the write lock every time a group is removed
@@ -117,7 +97,7 @@ func newKeyspaceResourceGroupManager(
 	}
 	return &keyspaceResourceGroupManager{
 		groups:          make(map[string]*ResourceGroup),
-		reservedGroups:  make(map[string]reservedKind),
+		reservedGroups:  make(map[string]struct{}),
 		groupRUTrackers: make(map[string]*groupRUTracker),
 		keyspaceID:      keyspaceID,
 		storage:         storage,
@@ -240,16 +220,13 @@ func (krgm *keyspaceResourceGroupManager) setRawStatesIntoResourceGroup(name str
 func (krgm *keyspaceResourceGroupManager) initDefaultResourceGroup() {
 	krgm.RLock()
 	_, ok := krgm.groups[DefaultResourceGroupName]
-	kind, reserved := krgm.reservedGroups[DefaultResourceGroupName]
+	_, reserved := krgm.reservedGroups[DefaultResourceGroupName]
 	krgm.RUnlock()
 	// A cached entry only makes initialization unnecessary if it's confirmed
-	// data, or at least has confirmed settings (reservedStateOnly), which must
-	// not be overwritten with synthetic ones. A reservedPlaceholder means
-	// nothing is persisted for the default group (e.g. a fresh store): it must
-	// still be created and persisted here, otherwise it would stay an
-	// unconfirmed placeholder forever, with its settings never stored and its
-	// state persistence permanently skipped.
-	if ok && (!reserved || kind == reservedStateOnly) {
+	// data. A reserved placeholder means nothing is persisted for the default
+	// group (e.g. a fresh store): it must still be created and persisted here,
+	// otherwise its settings are never stored and state persistence stays skipped.
+	if ok && !reserved {
 		return
 	}
 	defaultGroup := newDefaultResourceGroup()
@@ -270,7 +247,7 @@ func (krgm *keyspaceResourceGroupManager) ensureReservedDefaultGroupInCache() {
 	krgm.Lock()
 	if _, ok := krgm.groups[DefaultResourceGroupName]; !ok {
 		krgm.groups[DefaultResourceGroupName] = defaultGroup
-		krgm.reservedGroups[DefaultResourceGroupName] = reservedPlaceholder
+		krgm.reservedGroups[DefaultResourceGroupName] = struct{}{}
 		inserted = true
 	}
 	krgm.Unlock()
@@ -298,7 +275,7 @@ func (krgm *keyspaceResourceGroupManager) restoreDefaultResourceGroupFromReserve
 	defaultGroup := newDefaultResourceGroup()
 	krgm.Lock()
 	krgm.groups[DefaultResourceGroupName] = defaultGroup
-	krgm.reservedGroups[DefaultResourceGroupName] = reservedPlaceholder
+	krgm.reservedGroups[DefaultResourceGroupName] = struct{}{}
 	krgm.Unlock()
 	krgm.syncBurstabilityWithServiceLimit(defaultGroup)
 }
diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go
index f41e7a1d9f..dc574b69f0 100644
--- a/pkg/mcs/resourcemanager/server/manager.go
+++ b/pkg/mcs/resourcemanager/server/manager.go
@@ -550,21 +550,6 @@ func (m *Manager) asyncLoadResourceGroups(ctx context.Context, epoch uint64) {
 				if m.syncLoadedGroups[key] {
 					continue
 				}
-				if kind, reserved := krgm.reservedGroups[name]; reserved && kind == reservedStateOnly {
-					if existing, ok := krgm.groups[name]; ok {
-						// The cached entry's settings are confirmed and may
-						// carry a modification persisted after this scan
-						// started; only its state is missing. Adopt the
-						// scanned state into the existing entry instead of
-						// replacing it, so the newer settings aren't
-						// clobbered by the scan's older copy.
-						existing.SetStatesIntoResourceGroup(group.GetGroupStates())
-						delete(krgm.reservedGroups, name)
-						groupsToSync = append(groupsToSync, existing)
-						loaded++
-						continue
-					}
-				}
 				krgm.groups[name] = group
 				// This group is now confirmed, fully-loaded data (settings
 				// and state); it must no longer be treated as an
@@ -632,36 +617,33 @@ func (m *Manager) loadKeyspaceResourceGroupsFromStorage() (map[uint32]*keyspaceR
 	return tempKrgms, nil
 }
 
-// loadResourceGroup loads a single resource group from storage. The returned
-// stateLoaded reports whether the group's persisted state was successfully
-// read; the caller must not mark such a group as sync-loaded, so that a
-// concurrent or later async bulk load can still fill in its real state.
-func (m *Manager) loadResourceGroup(keyspaceID uint32, name string) (group *ResourceGroup, stateLoaded bool, err error) {
+// loadResourceGroup loads a single resource group from storage.
+func (m *Manager) loadResourceGroup(keyspaceID uint32, name string) (*ResourceGroup, error) {
 	rawValue, err := m.storage.LoadResourceGroupSetting(keyspaceID, name)
 	if err != nil {
-		return nil, false, err
+		return nil, err
 	}
 	if rawValue == "" {
-		return nil, false, errs.ErrResourceGroupNotExists.FastGenByArgs(name)
+		return nil, errs.ErrResourceGroupNotExists.FastGenByArgs(name)
 	}
 	krgm := newKeyspaceResourceGroupManager(keyspaceID, m.storage, m.writeRole)
 	if err := krgm.addResourceGroupFromRaw(name, rawValue); err != nil {
-		return nil, false, err
+		return nil, err
 	}
 	state, err := m.storage.LoadResourceGroupState(keyspaceID, name)
 	if err != nil {
-		log.Warn("failed to load resource group state, continuing without state",
+		log.Warn("failed to load resource group state",
 			zap.Uint32("keyspace-id", keyspaceID),
 			zap.String("group-name", name),
 			zap.Error(err))
-		return krgm.getMutableResourceGroup(name), false, nil
+		return nil, err
 	}
 	if state != "" {
 		if err := krgm.setRawStatesIntoResourceGroup(name, state); err != nil {
-			return nil, false, err
+			return nil, err
 		}
 	}
-	return krgm.getMutableResourceGroup(name), true, nil
+	return krgm.getMutableResourceGroup(name), nil
 }
 
 func (m *Manager) loadResourceGroupIfNeeded(keyspaceID uint32, name string) error {
@@ -690,7 +672,7 @@ func (m *Manager) loadResourceGroupIfNeeded(keyspaceID uint32, name string) erro
 		// so a Delete that lands after the read is detected under the insert
 		// lock and can't be undone by the now-stale result.
 		deleteGen := krgm.loadDeleteGen()
-		group, stateLoaded, err := m.loadResourceGroup(keyspaceID, name)
+		group, err := m.loadResourceGroup(keyspaceID, name)
 		if err != nil {
 			if name == DefaultResourceGroupName && errors.ErrorEqual(err, errs.ErrResourceGroupNotExists.FastGenByArgs(name)) {
 				// No persisted default group settings exist yet (e.g. a brand-new
@@ -704,9 +686,12 @@ func (m *Manager) loadResourceGroupIfNeeded(keyspaceID uint32, name string) erro
 			return err
 		}
 		inserted := false
+		markKey := trackerKey{keyspaceID: keyspaceID, groupName: name}
+		m.Lock()
 		krgm.Lock()
 		if krgm.deleteGen != deleteGen {
 			krgm.Unlock()
+			m.Unlock()
 			if attempt >= maxLoadAttempts {
 				return nil
 			}
@@ -721,30 +706,16 @@ func (m *Manager) loadResourceGroupIfNeeded(keyspaceID uint32, name string) erro
 			krgm.groups[name] = group
 			inserted = true
 		}
-		// Only clear the placeholder mark once the state was actually read; a
-		// metadata-only group (state load failed) must stay reserved so a later
-		// call or the async bulk merge remains free to fill in the real state,
-		// instead of this partial result being treated as final forever.
-		if stateLoaded {
-			delete(krgm.reservedGroups, name)
-		} else {
-			krgm.reservedGroups[name] = reservedStateOnly
-		}
+		delete(krgm.reservedGroups, name)
 		krgm.Unlock()
+		if m.syncLoadedGroups != nil {
+			m.syncLoadedGroups[markKey] = true
+		}
+		m.Unlock()
+		failpoint.Inject("lazyLoadAfterCachePublish", func() {})
 		if inserted {
 			krgm.syncBurstabilityWithServiceLimit(group)
 		}
-		// Only mark the group as sync-loaded when its persisted state was actually
-		// read; otherwise a later async bulk load must remain free to fill in the
-		// real state instead of being skipped forever.
-		if stateLoaded {
-			markKey := trackerKey{keyspaceID: keyspaceID, groupName: name}
-			m.Lock()
-			if m.syncLoadedGroups != nil {
-				m.syncLoadedGroups[markKey] = true
-			}
-			m.Unlock()
-		}
 		syncLoadGroupCounter.Inc()
 		return nil
 	}
diff --git a/pkg/mcs/resourcemanager/server/manager_async_test.go b/pkg/mcs/resourcemanager/server/manager_async_test.go
index 7dd44f521c..a59f7da05b 100644
--- a/pkg/mcs/resourcemanager/server/manager_async_test.go
+++ b/pkg/mcs/resourcemanager/server/manager_async_test.go
@@ -24,6 +24,7 @@ import (
 
 	"github.com/stretchr/testify/require"
 
+	"github.com/pingcap/failpoint"
 	"github.com/pingcap/kvproto/pkg/resource_manager"
 
 	"github.com/tikv/pd/pkg/errs"
@@ -248,14 +249,11 @@ func TestAsyncLoadResourceGroupsLazyGetLegacyKeyspace(t *testing.T) {
 	}, testutil.WithTickInterval(20*time.Millisecond))
 }
 
-// TestAsyncLoadResourceGroupsRecoversFromStateLoadFailure guards against a
-// group getting stuck marked reserved forever after a transient
-// LoadResourceGroupState failure during lazy loading: once the async bulk
-// load subsequently installs the fully-loaded (settings and state)
-// confirmed data for the same group, the reserved marker must be cleared,
-// otherwise loadResourceGroupIfNeeded and the state persist loop would keep
-// treating already-recovered, correct data as an unconfirmed placeholder.
-func TestAsyncLoadResourceGroupsRecoversFromStateLoadFailure(t *testing.T) {
+// TestAsyncLoadResourceGroupsDoesNotServeStateLoadFailure guards against a
+// group with confirmed settings but failed state loading being exposed with a
+// fresh token bucket before the async bulk loader can recover its persisted
+// state.
+func TestAsyncLoadResourceGroupsDoesNotServeStateLoadFailure(t *testing.T) {
 	re := require.New(t)
 	store := newBlockingResourceGroupStorage()
 	group := newAsyncTestGroup("flaky-group")
@@ -270,22 +268,22 @@ func TestAsyncLoadResourceGroupsRecoversFromStateLoadFailure(t *testing.T) {
 
 	store.waitEntered(t)
 
-	// Make the lazy load's own state read fail once, so the group is cached
-	// as a metadata-only, still-reserved entry.
+	// Make the lazy load's own state read fail once. The group must remain
+	// unavailable rather than being returned with a fresh token bucket state.
 	store.failNextState.Store(true)
 	fetched, err := m.GetResourceGroup(1, "flaky-group", false)
-	re.NoError(err)
-	re.NotNil(fetched)
+	re.Error(err)
+	re.Nil(fetched)
 
 	krgm := m.getKeyspaceResourceGroupManager(1)
 	re.NotNil(krgm)
-	re.True(krgm.isReserved("flaky-group"), "group should still be reserved after a failed state load")
+	re.Nil(krgm.getMutableResourceGroup("flaky-group"), "failed state load must not publish the group")
 
 	// Let the async bulk load proceed; its own state read is unaffected
 	// (failNextState was already consumed) and should install confirmed data.
 	store.unblock()
 	testutil.Eventually(re, func() bool {
-		return !krgm.isReserved("flaky-group")
+		return krgm.getMutableResourceGroup("flaky-group") != nil && !krgm.isReserved("flaky-group")
 	}, testutil.WithTickInterval(20*time.Millisecond))
 }
 
@@ -424,22 +422,16 @@ func TestAsyncLoadResourceGroupsStaleLoaderDoesNotPolluteNewTerm(t *testing.T) {
 	}
 }
 
-// TestAsyncLoadResourceGroupsMergeKeepsModifiedSettings reproduces the
-// stale-settings clobber: a group's lazy load reads its settings but fails to
-// read its state, then the group is modified (and the modification persisted)
-// while the async bulk scan still holds the pre-modification settings. The
-// bulk merge must adopt only the scanned state into the cached entry, not
-// replace it wholesale, so the modified settings survive in the serving cache.
-func TestAsyncLoadResourceGroupsMergeKeepsModifiedSettings(t *testing.T) {
+// TestAsyncLoadResourceGroupsLazyPublishAndMarkAreAtomic reproduces the race
+// where a lazy load publishes a cache entry before recording it in
+// syncLoadedGroups. A bulk merge entering that gap can overwrite mutable state
+// updated by token-bucket handling with its older scan result.
+func TestAsyncLoadResourceGroupsLazyPublishAndMarkAreAtomic(t *testing.T) {
 	re := require.New(t)
 	store := newBlockingResourceGroupStorage()
-	group := newAsyncTestGroup("mod-group")
-	re.NoError(store.SaveResourceGroupSetting(1, "mod-group", group))
-	// Persist a state with a recognizable consumption so the test can tell
-	// that the merge really adopted the scanned state.
-	states := FromProtoResourceGroup(group).GetGroupStates()
-	states.RUConsumption.RRU = 123
-	re.NoError(store.SaveResourceGroupStates(1, "mod-group", states))
+	group := newAsyncTestGroup("atomic-group")
+	re.NoError(store.SaveResourceGroupSetting(1, "atomic-group", group))
+	re.NoError(store.SaveResourceGroupStates(1, "atomic-group", FromProtoResourceGroup(group).GetGroupStates()))
 
 	m := NewManager[*mockConfigProvider](&mockConfigProvider{})
 	m.storage = store
@@ -450,8 +442,7 @@ func TestAsyncLoadResourceGroupsMergeKeepsModifiedSettings(t *testing.T) {
 
 	store.waitEntered(t)
 
-	// Let the bulk loader capture the pre-modification settings, then hold it
-	// right before its states scan (i.e. before it merges).
+	// Let the bulk loader capture fill rate 100, then hold it before merge.
 	store.pauseNextStates.Store(true)
 	store.unblock()
 	select {
@@ -460,40 +451,45 @@ func TestAsyncLoadResourceGroupsMergeKeepsModifiedSettings(t *testing.T) {
 		t.Fatal("timed out waiting for the bulk loader to reach its states scan")
 	}
 
-	// Lazily load the group with a failing state read: it is cached with
-	// confirmed settings but unconfirmed (fresh) state.
-	store.failNextState.Store(true)
-	fetched, err := m.GetResourceGroup(1, "mod-group", false)
-	re.NoError(err)
-	re.NotNil(fetched)
-	krgm := m.getKeyspaceResourceGroupManager(1)
-	re.NotNil(krgm)
-	re.True(krgm.isReserved("mod-group"))
+	re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/mcs/resourcemanager/server/lazyLoadAfterCachePublish", `pause`))
+	defer func() {
+		re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/mcs/resourcemanager/server/lazyLoadAfterCachePublish"))
+	}()
 
-	// Modify the group (fill rate 100 -> 200) and persist it. The state read
-	// of Modify's own lazy load fails again, so the entry stays state-only
-	// reserved and out of syncLoadedGroups.
-	store.failNextState.Store(true)
-	modified := newAsyncTestGroup("mod-group")
-	modified.RUSettings.RU.Settings.FillRate = 200
-	modified.KeyspaceId = &resource_manager.KeyspaceIDValue{Value: 1}
-	re.NoError(m.ModifyResourceGroup(modified))
+	getDone := make(chan struct{})
+	go func() {
+		defer close(getDone)
+		_, err := m.GetResourceGroup(1, "atomic-group", false)
+		re.NoError(err)
+	}()
 
-	// Release the bulk loader; its merge must keep the modified settings and
-	// only adopt the scanned state.
+	var krgm *keyspaceResourceGroupManager
+	testutil.Eventually(re, func() bool {
+		krgm = m.getKeyspaceResourceGroupManager(1)
+		if krgm == nil {
+			return false
+		}
+		return krgm.getMutableResourceGroup("atomic-group") != nil
+	}, testutil.WithTickInterval(20*time.Millisecond))
+
+	krgm.getMutableResourceGroup("atomic-group").UpdateRUConsumption(&resource_manager.Consumption{RRU: 10})
+
+	// With the lazy load paused at the cache-publish hook, the bulk merge must
+	// not be able to overwrite the updated cache entry.
 	store.unblockStates()
 	testutil.Eventually(re, func() bool {
 		_, err := m.GetResourceGroupList(1, false)
 		return err == nil
 	}, testutil.WithTickInterval(20*time.Millisecond))
 
-	got, err := m.GetResourceGroup(1, "mod-group", false)
+	re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/mcs/resourcemanager/server/lazyLoadAfterCachePublish"))
+	<-getDone
+
+	got, err := m.GetResourceGroup(1, "atomic-group", false)
 	re.NoError(err)
 	re.NotNil(got)
-	re.Equal(float64(200), got.RUSettings.RU.getFillRate(), "modified settings must survive the bulk merge")
-	re.False(krgm.isReserved("mod-group"), "state adoption must clear the reserved marker")
-	re.Equal(float64(123), krgm.getMutableResourceGroup("mod-group").GetGroupStates().RUConsumption.RRU,
-		"the scanned state must be adopted into the cached entry")
+	re.Equal(float64(10), krgm.getMutableResourceGroup("atomic-group").GetGroupStates().RUConsumption.RRU,
+		"bulk merge must not overwrite the published lazy-loaded group")
 }
 
 // TestAsyncLoadResourceGroupsFreshStoreDefaultPersisted guards against the

From 3766698423d00c2f7190572a3d245c18a89d34b0 Mon Sep 17 00:00:00 2001
From: tongjian <1045931706@qq.com>
Date: Thu, 23 Jul 2026 17:00:04 +0800
Subject: [PATCH 21/50] resource_group: use warn level for retryable bulk load
 failures

The async loader retries indefinitely until the scan succeeds, so a
failed attempt is not a terminal condition and warn level fits better.
This also satisfies the error-log-review check on new error-level logs.

Signed-off-by: tongjian <1045931706@qq.com>
---
 pkg/mcs/resourcemanager/server/manager.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go
index dc574b69f0..aff2700d3f 100644
--- a/pkg/mcs/resourcemanager/server/manager.go
+++ b/pkg/mcs/resourcemanager/server/manager.go
@@ -518,7 +518,8 @@ func (m *Manager) asyncLoadResourceGroups(ctx context.Context, epoch uint64) {
 		default:
 		}
 		if err != nil {
-			log.Error("failed to load resource groups", zap.Error(err), zap.Int("retry", retry))
+			// Use warn level since the loader retries indefinitely until it succeeds.
+			log.Warn("failed to load resource groups", zap.Error(err), zap.Int("retry", retry))
 			if !m.storeLoadingStateIfCurrent(epoch, LoadingStateNotStarted) {
 				log.Info("async loading resource groups aborted: manager was reinitialized")
 				return

From 7fe5201f158632b9d595db8a640a82d6e6918e29 Mon Sep 17 00:00:00 2001
From: tongjian <1045931706@qq.com>
Date: Thu, 23 Jul 2026 20:01:41 +0800
Subject: [PATCH 22/50] resource_group: keep cross-term lazy loads out of the
 new term's state

A lazy load captured its keyspace manager before the lock-free storage
read. If the read blocked across a leadership change, Init replaced
m.krgms and syncLoadedGroups in the meantime, so the resumed call
inserted the group into a detached manager while marking it in the new
term's map - making the new bulk merge skip a group its cache doesn't
contain, leaving it unavailable for the whole term.

Capture the load epoch and the current keyspace manager atomically
under the manager lock at the start of every load attempt, and
re-validate both in the same critical section that publishes the cache
entry and the sync-loaded marker (and before the confirmed-not-found
default synthesis). On a mismatch the load retries against the freshly
captured state, so a request that straddles a re-election still
succeeds against the new term. markResourceGroupSyncLoaded now also
takes the mutated keyspace manager and skips marking when it is no
longer the live one, closing the same window for Add/Modify/Delete and
the watcher paths.

Add a regression test that pauses a term-1 lazy load inside its storage
read, reinitializes the manager for term 2, then releases the load and
asserts it retries and publishes into the new term (verified to fail
without the fix).

Signed-off-by: tongjian <1045931706@qq.com>
---
 pkg/mcs/resourcemanager/server/manager.go     | 65 +++++++++++++++----
 .../server/manager_async_test.go              | 62 ++++++++++++++++++
 2 files changed, 114 insertions(+), 13 deletions(-)

diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go
index aff2700d3f..a811b6d042 100644
--- a/pkg/mcs/resourcemanager/server/manager.go
+++ b/pkg/mcs/resourcemanager/server/manager.go
@@ -660,15 +660,27 @@ func (m *Manager) loadResourceGroupIfNeeded(keyspaceID uint32, name string) erro
 			return nil
 		}
 	}
-	krgm = m.getOrCreateKeyspaceResourceGroupManager(keyspaceID, false)
-	// deleteGen is shared by every group in the keyspace, so a concurrent
-	// Delete of any group invalidates the lock-free storage read below.
-	// Deletes are rare: retry the read a few times so an unrelated deletion
-	// doesn't make this request spuriously report the group as missing; if it
-	// keeps racing, give up without inserting and let a later request or the
-	// async bulk merge reload the group.
+	// The lock-free storage read below can be invalidated while it runs: a
+	// concurrent Delete of any group in the keyspace bumps deleteGen, and a
+	// leadership change replaces m.krgms and m.syncLoadedGroups (bumping
+	// loadEpoch). Both are rare: retry the read a few times against freshly
+	// captured state so neither makes this request spuriously fail or publish
+	// into a detached manager; if it keeps racing, give up without inserting
+	// and let a later request or the async bulk merge reload the group.
 	const maxLoadAttempts = 3
 	for attempt := 1; ; attempt++ {
+		// Capture the load epoch and the current keyspace manager atomically,
+		// and re-capture them on every retry: publishing into a previous
+		// term's detached manager while marking the new term's map would make
+		// the new bulk merge skip a group its cache doesn't contain.
+		m.Lock()
+		epoch := m.loadEpoch
+		krgm = m.krgms[keyspaceID]
+		if krgm == nil {
+			krgm = newKeyspaceResourceGroupManager(keyspaceID, m.storage, m.writeRole)
+			m.krgms[keyspaceID] = krgm
+		}
+		m.Unlock()
 		// Snapshot the delete generation before the lock-free storage read,
 		// so a Delete that lands after the read is detected under the insert
 		// lock and can't be undone by the now-stale result.
@@ -676,6 +688,15 @@ func (m *Manager) loadResourceGroupIfNeeded(keyspaceID uint32, name string) erro
 		group, err := m.loadResourceGroup(keyspaceID, name)
 		if err != nil {
 			if name == DefaultResourceGroupName && errors.ErrorEqual(err, errs.ErrResourceGroupNotExists.FastGenByArgs(name)) {
+				m.RLock()
+				stale := m.loadEpoch != epoch || m.krgms[keyspaceID] != krgm
+				m.RUnlock()
+				if stale {
+					if attempt >= maxLoadAttempts {
+						return nil
+					}
+					continue
+				}
 				// No persisted default group settings exist yet (e.g. a brand-new
 				// keyspace), so it's safe to synthesize the reserved default group.
 				// This calls initDefaultResourceGroup directly instead of going
@@ -689,6 +710,15 @@ func (m *Manager) loadResourceGroupIfNeeded(keyspaceID uint32, name string) erro
 		inserted := false
 		markKey := trackerKey{keyspaceID: keyspaceID, groupName: name}
 		m.Lock()
+		if m.loadEpoch != epoch || m.krgms[keyspaceID] != krgm {
+			// The manager was reinitialized for a new term while the storage
+			// read was in flight; retry against the new term's state.
+			m.Unlock()
+			if attempt >= maxLoadAttempts {
+				return nil
+			}
+			continue
+		}
 		krgm.Lock()
 		if krgm.deleteGen != deleteGen {
 			krgm.Unlock()
@@ -722,9 +752,18 @@ func (m *Manager) loadResourceGroupIfNeeded(keyspaceID uint32, name string) erro
 	}
 }
 
-func (m *Manager) markResourceGroupSyncLoaded(keyspaceID uint32, name string) {
+// markResourceGroupSyncLoaded records that the group in krgm was written or
+// fully loaded synchronously. The caller passes the keyspace manager it
+// actually mutated: if that manager is no longer the live one (the manager was
+// reinitialized for a new term while the caller was blocked on storage I/O),
+// the marker is skipped, since marking the new term's map for a group its
+// cache doesn't contain would make the bulk merge skip loading it.
+func (m *Manager) markResourceGroupSyncLoaded(keyspaceID uint32, krgm *keyspaceResourceGroupManager, name string) {
 	m.Lock()
 	defer m.Unlock()
+	if m.krgms[keyspaceID] != krgm {
+		return
+	}
 	if m.syncLoadedGroups != nil {
 		m.syncLoadedGroups[trackerKey{keyspaceID: keyspaceID, groupName: name}] = true
 	}
@@ -768,7 +807,7 @@ func (m *Manager) applyResourceGroupSettingFromRaw(keyspaceID uint32, name, rawV
 			zap.Error(err))
 		return err
 	}
-	m.markResourceGroupSyncLoaded(keyspaceID, name)
+	m.markResourceGroupSyncLoaded(keyspaceID, krgm, name)
 	return nil
 }
 
@@ -805,7 +844,7 @@ func (m *Manager) applyResourceGroupStatesFromRaw(keyspaceID uint32, name, rawVa
 			zap.Error(err))
 		return err
 	}
-	m.markResourceGroupSyncLoaded(keyspaceID, name)
+	m.markResourceGroupSyncLoaded(keyspaceID, krgm, name)
 	return nil
 }
 
@@ -905,7 +944,7 @@ func (m *Manager) AddResourceGroup(grouppb *rmpb.ResourceGroup) error {
 	if err := krgm.addResourceGroup(grouppb); err != nil {
 		return err
 	}
-	m.markResourceGroupSyncLoaded(keyspaceID, grouppb.Name)
+	m.markResourceGroupSyncLoaded(keyspaceID, krgm, grouppb.Name)
 	return nil
 }
 
@@ -931,7 +970,7 @@ func (m *Manager) ModifyResourceGroup(grouppb *rmpb.ResourceGroup) error {
 	// it sync-loaded here would make the async bulk merge skip it forever,
 	// so the persisted running state would never get applied.
 	if !krgm.isReserved(grouppb.Name) {
-		m.markResourceGroupSyncLoaded(keyspaceID, grouppb.Name)
+		m.markResourceGroupSyncLoaded(keyspaceID, krgm, grouppb.Name)
 	}
 	return nil
 }
@@ -953,7 +992,7 @@ func (m *Manager) DeleteResourceGroup(keyspaceID uint32, name string) error {
 	if err := krgm.deleteResourceGroup(name); err != nil {
 		return err
 	}
-	m.markResourceGroupSyncLoaded(keyspaceID, name)
+	m.markResourceGroupSyncLoaded(keyspaceID, krgm, name)
 	return nil
 }
 
diff --git a/pkg/mcs/resourcemanager/server/manager_async_test.go b/pkg/mcs/resourcemanager/server/manager_async_test.go
index a59f7da05b..6304e19cc8 100644
--- a/pkg/mcs/resourcemanager/server/manager_async_test.go
+++ b/pkg/mcs/resourcemanager/server/manager_async_test.go
@@ -600,3 +600,65 @@ func TestAsyncLoadResourceGroupsUnrelatedDeleteDoesNotFailLazyLoad(t *testing.T)
 		return foundA
 	}, testutil.WithTickInterval(20*time.Millisecond))
 }
+
+// TestAsyncLoadResourceGroupsStaleLazyLoadRetriesNewTerm reproduces the
+// cross-term lazy load: the load captures the keyspace manager, then blocks in
+// its storage read while the leadership changes and Init replaces m.krgms and
+// syncLoadedGroups. On resume it must not publish into the detached old
+// manager while marking the group in the new term's map (which would make the
+// new bulk merge skip a group its cache doesn't contain); instead it retries
+// against the freshly captured state and publishes into the new term.
+func TestAsyncLoadResourceGroupsStaleLazyLoadRetriesNewTerm(t *testing.T) {
+	re := require.New(t)
+	store := newBlockingResourceGroupStorage()
+	re.NoError(store.SaveResourceGroupSetting(1, "cross-term", newAsyncTestGroup("cross-term")))
+
+	m := NewManager[*mockConfigProvider](&mockConfigProvider{})
+	m.storage = store
+	re.NoError(m.Init(context.Background()))
+	cancelTerm1 := m.cancel
+	defer stopAsyncTestManager(m)
+	defer store.unblock()
+
+	store.waitEntered(t)
+
+	// The term-1 lazy load pauses inside its state read, holding the old
+	// term's keyspace manager.
+	store.pausePointState.Store(true)
+	var (
+		gotGroup *ResourceGroup
+		gotErr   error
+	)
+	getDone := make(chan struct{})
+	go func() {
+		defer close(getDone)
+		gotGroup, gotErr = m.GetResourceGroup(1, "cross-term", false)
+	}()
+	select {
+	case <-store.pointReached:
+	case <-time.After(time.Second):
+		t.Fatal("timed out waiting for the lazy load to reach its state read")
+	}
+
+	// Leadership changes: reinitialize the manager for term 2 while the
+	// term-1 lazy load is still blocked. Term 2's bulk loader parks on the
+	// same settings-scan block until store.unblock().
+	cancelTerm1()
+	re.NoError(m.Init(context.Background()))
+
+	// Release the stale lazy load: it must detect the term change, retry, and
+	// publish into the new term's manager, so the request still succeeds.
+	close(store.pointRelease)
+	<-getDone
+	re.NoError(gotErr)
+	re.NotNil(gotGroup, "the cross-term lazy load must retry and succeed against the new term")
+	re.Equal("cross-term", gotGroup.Name)
+
+	// Finish loading; the group must remain present after the term-2 bulk
+	// merge (it was correctly marked in the same term it was published in).
+	store.unblock()
+	testutil.Eventually(re, func() bool {
+		g, err := m.GetResourceGroup(1, "cross-term", false)
+		return err == nil && g != nil
+	}, testutil.WithTickInterval(20*time.Millisecond))
+}

From 017899a7ee49975da689f4a0b75206e59c47612a Mon Sep 17 00:00:00 2001
From: tongjian <1045931706@qq.com>
Date: Fri, 24 Jul 2026 10:57:59 +0800
Subject: [PATCH 23/50] resource_group: publish metadata mutations against the
 current manager

A metadata mutation (Add/Modify/Delete) applied its cache effect to the
keyspace manager captured before its storage I/O and recorded the
sync-loaded marker in a separate lock section. Skipping the marker when
that manager turned out detached (previous fix) was not enough: the
mutation had already run against the old manager, so e.g. a Delete
straddling a leadership change removed the group from storage while the
new term's bulk merge reinstalled its pre-deletion snapshot - the API
reported success but the group stayed in the live cache.

Split each mutation into a storage phase (idempotent, done once) and a
publish phase: publishResourceGroupMutation re-resolves the current
keyspace manager inside one manager-lock critical section and applies
the cache effect and the sync-loaded marker there atomically. Since the
bulk merge holds the manager lock across its whole merge step, the two
can only be fully ordered - publish first and the merge skips the
marked group, merge first and the publish overrides its stale snapshot.
Because the current manager is resolved inside the critical section, a
mutation straddling a leadership change publishes into the live term by
construction, with no retry loop. Modify keeps its reserved-placeholder
marking gate, and republishes its patched settings if the merge
installed a pre-modification snapshot in between.

Also make the lazy load's exhausted-retry paths return
ErrResourceGroupsLoading instead of nil: reporting success without
publishing the group made callers surface an existing group as
nonexistent or silently skip token acquisition for it.

Add deterministic regression tests for both (each verified to fail
without its fix), and make the test storage's state-read pause hook
re-armable and filtered by group name.

Signed-off-by: tongjian <1045931706@qq.com>
---
 .../server/keyspace_manager.go                |  62 +++--
 .../server/keyspace_manager_test.go           |   4 +-
 pkg/mcs/resourcemanager/server/manager.go     | 114 +++++++--
 .../server/manager_async_test.go              | 221 +++++++++++++++---
 .../resourcemanager/server/manager_test.go    |   3 +-
 5 files changed, 340 insertions(+), 64 deletions(-)

diff --git a/pkg/mcs/resourcemanager/server/keyspace_manager.go b/pkg/mcs/resourcemanager/server/keyspace_manager.go
index 5c2d61b440..2cc83efd66 100644
--- a/pkg/mcs/resourcemanager/server/keyspace_manager.go
+++ b/pkg/mcs/resourcemanager/server/keyspace_manager.go
@@ -186,13 +186,19 @@ func (krgm *keyspaceResourceGroupManager) upsertResourceGroupFromRaw(name string
 
 func (krgm *keyspaceResourceGroupManager) deleteResourceGroupFromCache(name string) {
 	krgm.Lock()
+	krgm.removeResourceGroupLocked(name)
+	krgm.Unlock()
+}
+
+// removeResourceGroupLocked removes every cache trace of the group. The
+// caller must hold the write lock.
+func (krgm *keyspaceResourceGroupManager) removeResourceGroupLocked(name string) {
 	delete(krgm.groups, name)
 	delete(krgm.groupRUTrackers, name)
 	delete(krgm.reservedGroups, name)
 	// Signal any in-flight lazy load that a delete happened, so it won't
 	// reinsert a copy read from storage before this deletion.
 	krgm.deleteGen++
-	krgm.Unlock()
 }
 
 // loadDeleteGen returns the current delete generation counter.
@@ -280,21 +286,33 @@ func (krgm *keyspaceResourceGroupManager) restoreDefaultResourceGroupFromReserve
 	krgm.syncBurstabilityWithServiceLimit(defaultGroup)
 }
 
-func (krgm *keyspaceResourceGroupManager) addResourceGroup(grouppb *rmpb.ResourceGroup) error {
+// persistResourceGroup validates grouppb, builds the in-memory group, and
+// persists its settings and states to storage. It's the storage-only phase of
+// an Add: the caller is responsible for publishing the returned group into
+// whichever keyspace manager is current at publish time.
+func (krgm *keyspaceResourceGroupManager) persistResourceGroup(grouppb *rmpb.ResourceGroup) (*ResourceGroup, error) {
 	if err := validateResourceGroupProto(grouppb); err != nil {
-		return err
+		return nil, err
 	}
 	group := FromProtoResourceGroup(grouppb)
 	if krgm.writeRole.AllowsMetadataWrite() {
 		if err := group.persistSettings(krgm.keyspaceID, krgm.storage); err != nil {
-			return err
+			return nil, err
 		}
 	}
 	if krgm.writeRole.AllowsStateWrite() {
 		if err := group.persistStates(krgm.keyspaceID, krgm.storage); err != nil {
-			return err
+			return nil, err
 		}
 	}
+	return group, nil
+}
+
+func (krgm *keyspaceResourceGroupManager) addResourceGroup(grouppb *rmpb.ResourceGroup) error {
+	group, err := krgm.persistResourceGroup(grouppb)
+	if err != nil {
+		return err
+	}
 	krgm.Lock()
 	krgm.groups[group.Name] = group
 	delete(krgm.reservedGroups, group.Name)
@@ -303,30 +321,39 @@ func (krgm *keyspaceResourceGroupManager) addResourceGroup(grouppb *rmpb.Resourc
 	return nil
 }
 
-func (krgm *keyspaceResourceGroupManager) modifyResourceGroup(group *rmpb.ResourceGroup) error {
+// modifyResourceGroup patches the cached group's settings and persists them,
+// returning the patched group so the caller can republish it if the cache it
+// came from is no longer the live one.
+func (krgm *keyspaceResourceGroupManager) modifyResourceGroup(group *rmpb.ResourceGroup) (*ResourceGroup, error) {
 	if group == nil || group.Name == "" {
-		return errs.ErrInvalidGroup.FastGenByArgs("the group name")
+		return nil, errs.ErrInvalidGroup.FastGenByArgs("the group name")
 	}
 	krgm.RLock()
 	curGroup, ok := krgm.groups[group.Name]
 	krgm.RUnlock()
 	if !ok {
-		return errs.ErrResourceGroupNotExists.FastGenByArgs(group.Name)
+		return nil, errs.ErrResourceGroupNotExists.FastGenByArgs(group.Name)
 	}
 	if !krgm.writeRole.AllowsMetadataWrite() {
-		return errMetadataWriteDisabled
+		return nil, errMetadataWriteDisabled
 	}
-	err := curGroup.PatchSettings(group)
-	if err != nil {
-		return err
+	if err := curGroup.PatchSettings(group); err != nil {
+		return nil, err
 	}
 	// Deliberately not clearing reservedGroups here: modifying only patches
 	// settings, it never establishes the group's state, so it must not make
 	// a state-unconfirmed entry look fully confirmed.
-	return curGroup.persistSettings(krgm.keyspaceID, krgm.storage)
+	if err := curGroup.persistSettings(krgm.keyspaceID, krgm.storage); err != nil {
+		return nil, err
+	}
+	return curGroup, nil
 }
 
-func (krgm *keyspaceResourceGroupManager) deleteResourceGroup(name string) error {
+// deleteResourceGroupFromStorage validates the request and removes the
+// group's settings and states from storage. It's the storage-only phase of a
+// Delete: the caller is responsible for removing the group from whichever
+// keyspace manager cache is current at publish time.
+func (krgm *keyspaceResourceGroupManager) deleteResourceGroupFromStorage(name string) error {
 	if name == DefaultResourceGroupName {
 		return errs.ErrDeleteReservedGroup
 	}
@@ -361,6 +388,13 @@ func (krgm *keyspaceResourceGroupManager) deleteResourceGroup(name string) error
 				zap.Error(err))
 		}
 	}
+	return nil
+}
+
+func (krgm *keyspaceResourceGroupManager) deleteResourceGroup(name string) error {
+	if err := krgm.deleteResourceGroupFromStorage(name); err != nil {
+		return err
+	}
 	krgm.deleteResourceGroupFromCache(name)
 	return nil
 }
diff --git a/pkg/mcs/resourcemanager/server/keyspace_manager_test.go b/pkg/mcs/resourcemanager/server/keyspace_manager_test.go
index 5dad0672e4..7cf5ad4ca1 100644
--- a/pkg/mcs/resourcemanager/server/keyspace_manager_test.go
+++ b/pkg/mcs/resourcemanager/server/keyspace_manager_test.go
@@ -187,7 +187,7 @@ func TestModifyResourceGroup(t *testing.T) {
 			},
 		},
 	}
-	err = krgm.modifyResourceGroup(modifiedGroup)
+	_, err = krgm.modifyResourceGroup(modifiedGroup)
 	re.NoError(err)
 
 	// Verify the group was modified.
@@ -206,7 +206,7 @@ func TestModifyResourceGroup(t *testing.T) {
 		Name: "non_existent",
 		Mode: rmpb.GroupMode_RUMode,
 	}
-	err = krgm.modifyResourceGroup(nonExistentGroup)
+	_, err = krgm.modifyResourceGroup(nonExistentGroup)
 	re.Error(err)
 }
 
diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go
index a811b6d042..39629cac7c 100644
--- a/pkg/mcs/resourcemanager/server/manager.go
+++ b/pkg/mcs/resourcemanager/server/manager.go
@@ -693,7 +693,10 @@ func (m *Manager) loadResourceGroupIfNeeded(keyspaceID uint32, name string) erro
 				m.RUnlock()
 				if stale {
 					if attempt >= maxLoadAttempts {
-						return nil
+						// Exhausted retries without confirming absence; report
+						// a retryable loading error rather than a bogus
+						// success the caller would mistake for a load.
+						return errs.ErrResourceGroupsLoading
 					}
 					continue
 				}
@@ -715,7 +718,10 @@ func (m *Manager) loadResourceGroupIfNeeded(keyspaceID uint32, name string) erro
 			// read was in flight; retry against the new term's state.
 			m.Unlock()
 			if attempt >= maxLoadAttempts {
-				return nil
+				// Exhausted retries without publishing the group; report a
+				// retryable loading error rather than a bogus success that
+				// would surface an existing group as nonexistent.
+				return errs.ErrResourceGroupsLoading
 			}
 			continue
 		}
@@ -724,7 +730,10 @@ func (m *Manager) loadResourceGroupIfNeeded(keyspaceID uint32, name string) erro
 			krgm.Unlock()
 			m.Unlock()
 			if attempt >= maxLoadAttempts {
-				return nil
+				// Exhausted retries without publishing the group; report a
+				// retryable loading error rather than a bogus success that
+				// would surface an existing group as nonexistent.
+				return errs.ErrResourceGroupsLoading
 			}
 			continue
 		}
@@ -769,6 +778,42 @@ func (m *Manager) markResourceGroupSyncLoaded(keyspaceID uint32, krgm *keyspaceR
 	}
 }
 
+// publishResourceGroupMutation applies a metadata mutation's cache effect and
+// its sync-loaded marker atomically with respect to the async bulk merge,
+// against whichever keyspace manager is current at publish time. The merge
+// holds the manager lock across its whole merge step, so the two can only be
+// fully ordered: publish first and the merge skips the marked group; merge
+// first and the publish overrides its stale snapshot. Because the current
+// manager is re-resolved inside this critical section, a mutation whose
+// storage phase straddled a leadership change still publishes into the live
+// term (and marks the same term's map) instead of a detached manager, so no
+// retry is needed.
+//
+// fn runs with the keyspace manager write lock held and must not do I/O; it
+// returns whether to record the sync-loaded marker and, when a group was
+// (re)installed, the group to sync burstability for.
+func (m *Manager) publishResourceGroupMutation(
+	keyspaceID uint32, name string,
+	fn func(krgm *keyspaceResourceGroupManager) (mark bool, synced *ResourceGroup),
+) {
+	m.Lock()
+	defer m.Unlock()
+	krgm, ok := m.krgms[keyspaceID]
+	if !ok {
+		krgm = newKeyspaceResourceGroupManager(keyspaceID, m.storage, m.writeRole)
+		m.krgms[keyspaceID] = krgm
+	}
+	krgm.Lock()
+	mark, synced := fn(krgm)
+	krgm.Unlock()
+	if synced != nil {
+		krgm.syncBurstabilityWithServiceLimit(synced)
+	}
+	if mark && m.syncLoadedGroups != nil {
+		m.syncLoadedGroups[trackerKey{keyspaceID: keyspaceID, groupName: name}] = true
+	}
+}
+
 func (m *Manager) isResourceGroupLoadingComplete() bool {
 	return atomic.LoadInt32(&m.loadingState) == LoadingStateCompleted
 }
@@ -941,10 +986,18 @@ func (m *Manager) AddResourceGroup(grouppb *rmpb.ResourceGroup) error {
 		log.Warn("failed to load resource group before add", zap.Uint32("keyspace-id", keyspaceID), zap.String("name", grouppb.Name), zap.Error(err))
 		return err
 	}
-	if err := krgm.addResourceGroup(grouppb); err != nil {
+	// Storage phase: validate and persist. Publishing the cache effect is done
+	// separately below, against whichever keyspace manager is current then.
+	group, err := krgm.persistResourceGroup(grouppb)
+	if err != nil {
 		return err
 	}
-	m.markResourceGroupSyncLoaded(keyspaceID, krgm, grouppb.Name)
+	failpoint.InjectCall("addResourceGroupBeforePublish")
+	m.publishResourceGroupMutation(keyspaceID, grouppb.Name, func(cur *keyspaceResourceGroupManager) (bool, *ResourceGroup) {
+		cur.groups[group.Name] = group
+		delete(cur.reservedGroups, group.Name)
+		return true, group
+	})
 	return nil
 }
 
@@ -962,16 +1015,38 @@ func (m *Manager) ModifyResourceGroup(grouppb *rmpb.ResourceGroup) error {
 	if err != nil {
 		return err
 	}
-	if err := krgm.modifyResourceGroup(grouppb); err != nil {
+	patched, err := krgm.modifyResourceGroup(grouppb)
+	if err != nil {
 		return err
 	}
-	// Modifying only patches settings, it never establishes the group's
-	// state. If the state still hasn't been confirmed (isReserved), marking
-	// it sync-loaded here would make the async bulk merge skip it forever,
-	// so the persisted running state would never get applied.
-	if !krgm.isReserved(grouppb.Name) {
-		m.markResourceGroupSyncLoaded(keyspaceID, krgm, grouppb.Name)
-	}
+	failpoint.InjectCall("modifyResourceGroupBeforePublish")
+	m.publishResourceGroupMutation(keyspaceID, grouppb.Name, func(cur *keyspaceResourceGroupManager) (bool, *ResourceGroup) {
+		existing, ok := cur.groups[grouppb.Name]
+		if !ok {
+			// The patch ran against a manager that is no longer live (the
+			// manager was reinitialized while we were persisting); republish
+			// the patched group into the current term so the persisted
+			// settings aren't lost from the serving cache.
+			cur.groups[grouppb.Name] = patched
+			delete(cur.reservedGroups, grouppb.Name)
+			return true, patched
+		}
+		if existing != patched {
+			// A different object (e.g. installed by the new term's bulk
+			// merge from a pre-modification snapshot); re-apply the settings
+			// patch so the persisted settings win.
+			if err := existing.PatchSettings(grouppb); err != nil {
+				log.Warn("failed to re-apply resource group settings on republish",
+					zap.Uint32("keyspace-id", keyspaceID), zap.String("name", grouppb.Name), zap.Error(err))
+			}
+		}
+		// Modifying only patches settings, it never establishes the group's
+		// state. If the state still hasn't been confirmed (reserved), marking
+		// it sync-loaded would make the async bulk merge skip it forever, so
+		// the persisted running state would never get applied.
+		_, reserved := cur.reservedGroups[grouppb.Name]
+		return !reserved, nil
+	})
 	return nil
 }
 
@@ -989,10 +1064,19 @@ func (m *Manager) DeleteResourceGroup(keyspaceID uint32, name string) error {
 	if krgm == nil {
 		return errs.ErrKeyspaceNotExists.FastGenByArgs(keyspaceID)
 	}
-	if err := krgm.deleteResourceGroup(name); err != nil {
+	failpoint.InjectCall("deleteResourceGroupBeforeStorage")
+	// Storage phase: validate and remove from storage. Publishing the cache
+	// effect is done separately below, against whichever keyspace manager is
+	// current then, so a delete straddling a leadership change still removes
+	// the group from the live cache and marks the live term's map (making the
+	// new bulk merge skip its pre-deletion snapshot of the group).
+	if err := krgm.deleteResourceGroupFromStorage(name); err != nil {
 		return err
 	}
-	m.markResourceGroupSyncLoaded(keyspaceID, krgm, name)
+	m.publishResourceGroupMutation(keyspaceID, name, func(cur *keyspaceResourceGroupManager) (bool, *ResourceGroup) {
+		cur.removeResourceGroupLocked(name)
+		return true, nil
+	})
 	return nil
 }
 
diff --git a/pkg/mcs/resourcemanager/server/manager_async_test.go b/pkg/mcs/resourcemanager/server/manager_async_test.go
index 6304e19cc8..b797a289f8 100644
--- a/pkg/mcs/resourcemanager/server/manager_async_test.go
+++ b/pkg/mcs/resourcemanager/server/manager_async_test.go
@@ -45,12 +45,12 @@ type blockingResourceGroupStorage struct {
 	// call fail once, then resets itself.
 	failNextState atomic.Bool
 
-	// pausePointState, when true, makes the very next LoadResourceGroupState
-	// call signal pointReached and then block on pointRelease, so a test can
-	// hold a lazy load right after its storage read but before it inserts.
-	pausePointState atomic.Bool
-	pointReached    chan struct{}
-	pointRelease    chan struct{}
+	// statePause, when armed via armStatePause, makes the next
+	// LoadResourceGroupState call for the armed group name signal reached and
+	// then block on release, so a test can hold a lazy load right after its
+	// storage read but before it inserts. Re-armable, and filtered by name so
+	// unrelated groups' loads pass through undisturbed.
+	statePause atomic.Pointer[statePause]
 
 	// pauseNextStates, when true, makes the very next bulk
 	// LoadResourceGroupStates call signal statesReached and then block on
@@ -62,18 +62,40 @@ type blockingResourceGroupStorage struct {
 	statesReleaseOnce sync.Once
 }
 
+type statePause struct {
+	name    string
+	reached chan struct{}
+	release chan struct{}
+}
+
 func newBlockingResourceGroupStorage() *blockingResourceGroupStorage {
 	return &blockingResourceGroupStorage{
 		Storage:       storage.NewStorageWithMemoryBackend(),
 		entered:       make(chan struct{}),
 		release:       make(chan struct{}),
-		pointReached:  make(chan struct{}),
-		pointRelease:  make(chan struct{}),
 		statesReached: make(chan struct{}),
 		statesRelease: make(chan struct{}),
 	}
 }
 
+// armStatePause arms a one-shot pause on the next LoadResourceGroupState call
+// for the given group name and returns the pause handle. The test must wait
+// on reached and eventually close release.
+func (s *blockingResourceGroupStorage) armStatePause(name string) *statePause {
+	p := &statePause{name: name, reached: make(chan struct{}), release: make(chan struct{})}
+	s.statePause.Store(p)
+	return p
+}
+
+func waitStatePauseReached(t *testing.T, p *statePause) {
+	t.Helper()
+	select {
+	case <-p.reached:
+	case <-time.After(time.Second):
+		t.Fatal("timed out waiting for the lazy load to reach its state read")
+	}
+}
+
 func (s *blockingResourceGroupStorage) LoadResourceGroupSettings(f func(keyspaceID uint32, name, rawValue string)) error {
 	s.once.Do(func() {
 		close(s.entered)
@@ -86,9 +108,9 @@ func (s *blockingResourceGroupStorage) LoadResourceGroupState(keyspaceID uint32,
 	if s.failNextState.CompareAndSwap(true, false) {
 		return "", errors.New("injected resource group state load failure")
 	}
-	if s.pausePointState.CompareAndSwap(true, false) {
-		close(s.pointReached)
-		<-s.pointRelease
+	if p := s.statePause.Load(); p != nil && p.name == name && s.statePause.CompareAndSwap(p, nil) {
+		close(p.reached)
+		<-p.release
 	}
 	return s.Storage.LoadResourceGroupState(keyspaceID, name)
 }
@@ -311,7 +333,7 @@ func TestAsyncLoadResourceGroupsDeleteRaceDoesNotResurrect(t *testing.T) {
 
 	// Start a lazy Get that will pause inside its state read, i.e. after it has
 	// read the group from storage but before it inserts into the cache.
-	store.pausePointState.Store(true)
+	pause := store.armStatePause("race-group")
 	var (
 		gotGroup *ResourceGroup
 		gotErr   error
@@ -322,11 +344,7 @@ func TestAsyncLoadResourceGroupsDeleteRaceDoesNotResurrect(t *testing.T) {
 		gotGroup, gotErr = m.GetResourceGroup(1, "race-group", false)
 	}()
 
-	select {
-	case <-store.pointReached:
-	case <-time.After(time.Second):
-		t.Fatal("timed out waiting for the lazy load to reach its state read")
-	}
+	waitStatePauseReached(t, pause)
 
 	// While the lazy load is paused, delete the group. Delete does its own
 	// (unpaused) load-then-delete, removing it from storage and cache and
@@ -334,7 +352,7 @@ func TestAsyncLoadResourceGroupsDeleteRaceDoesNotResurrect(t *testing.T) {
 	re.NoError(m.DeleteResourceGroup(1, "race-group"))
 
 	// Release the paused lazy load; its now-stale insert must be rejected.
-	close(store.pointRelease)
+	close(pause.release)
 	<-getDone
 	// The generation mismatch makes the lazy load retry its storage read,
 	// which now correctly observes the group as deleted.
@@ -553,7 +571,7 @@ func TestAsyncLoadResourceGroupsUnrelatedDeleteDoesNotFailLazyLoad(t *testing.T)
 
 	// Start a lazy Get of group-a and pause it inside its state read, i.e.
 	// after it has read the group from storage but before it inserts.
-	store.pausePointState.Store(true)
+	pause := store.armStatePause("group-a")
 	var (
 		gotGroup *ResourceGroup
 		gotErr   error
@@ -563,11 +581,7 @@ func TestAsyncLoadResourceGroupsUnrelatedDeleteDoesNotFailLazyLoad(t *testing.T)
 		defer close(getDone)
 		gotGroup, gotErr = m.GetResourceGroup(1, "group-a", false)
 	}()
-	select {
-	case <-store.pointReached:
-	case <-time.After(time.Second):
-		t.Fatal("timed out waiting for the lazy load to reach its state read")
-	}
+	waitStatePauseReached(t, pause)
 
 	// Delete the unrelated group-b while group-a's lazy load is paused; this
 	// bumps the keyspace's delete generation.
@@ -575,7 +589,7 @@ func TestAsyncLoadResourceGroupsUnrelatedDeleteDoesNotFailLazyLoad(t *testing.T)
 
 	// Release group-a's lazy load: the generation mismatch must make it retry
 	// and succeed, not report group-a as missing.
-	close(store.pointRelease)
+	close(pause.release)
 	<-getDone
 	re.NoError(gotErr)
 	re.NotNil(gotGroup, "an unrelated delete must not fail the lazy load")
@@ -624,7 +638,7 @@ func TestAsyncLoadResourceGroupsStaleLazyLoadRetriesNewTerm(t *testing.T) {
 
 	// The term-1 lazy load pauses inside its state read, holding the old
 	// term's keyspace manager.
-	store.pausePointState.Store(true)
+	pause := store.armStatePause("cross-term")
 	var (
 		gotGroup *ResourceGroup
 		gotErr   error
@@ -634,11 +648,7 @@ func TestAsyncLoadResourceGroupsStaleLazyLoadRetriesNewTerm(t *testing.T) {
 		defer close(getDone)
 		gotGroup, gotErr = m.GetResourceGroup(1, "cross-term", false)
 	}()
-	select {
-	case <-store.pointReached:
-	case <-time.After(time.Second):
-		t.Fatal("timed out waiting for the lazy load to reach its state read")
-	}
+	waitStatePauseReached(t, pause)
 
 	// Leadership changes: reinitialize the manager for term 2 while the
 	// term-1 lazy load is still blocked. Term 2's bulk loader parks on the
@@ -648,7 +658,7 @@ func TestAsyncLoadResourceGroupsStaleLazyLoadRetriesNewTerm(t *testing.T) {
 
 	// Release the stale lazy load: it must detect the term change, retry, and
 	// publish into the new term's manager, so the request still succeeds.
-	close(store.pointRelease)
+	close(pause.release)
 	<-getDone
 	re.NoError(gotErr)
 	re.NotNil(gotGroup, "the cross-term lazy load must retry and succeed against the new term")
@@ -662,3 +672,150 @@ func TestAsyncLoadResourceGroupsStaleLazyLoadRetriesNewTerm(t *testing.T) {
 		return err == nil && g != nil
 	}, testutil.WithTickInterval(20*time.Millisecond))
 }
+
+// TestAsyncLoadResourceGroupsCrossTermDeletePublishesToNewTerm reproduces the
+// cross-term delete race: a Delete resolves its keyspace manager, then stalls
+// before its storage phase while the leadership changes and the new term's
+// bulk loader snapshots storage (still containing the group). When the Delete
+// resumes, it removes the group from storage but its cache effect and
+// sync-loaded marker must land in the *current* term — otherwise the new
+// merge would reinstall its pre-deletion snapshot and the API would report
+// success while the group stays in the live cache.
+func TestAsyncLoadResourceGroupsCrossTermDeletePublishesToNewTerm(t *testing.T) {
+	re := require.New(t)
+	store := newBlockingResourceGroupStorage()
+	re.NoError(store.SaveResourceGroupSetting(1, "ct-del", newAsyncTestGroup("ct-del")))
+
+	m := NewManager[*mockConfigProvider](&mockConfigProvider{})
+	m.storage = store
+	re.NoError(m.Init(context.Background()))
+	cancelTerm1 := m.cancel
+	defer stopAsyncTestManager(m)
+	defer store.unblock()
+	defer store.unblockStates()
+
+	// Let term 1 load fully so the Delete starts against a settled term.
+	store.waitEntered(t)
+	store.unblock()
+	testutil.Eventually(re, func() bool {
+		groups, err := m.GetResourceGroupList(1, false)
+		return err == nil && len(groups) == 2
+	}, testutil.WithTickInterval(20*time.Millisecond))
+
+	// Park the Delete between resolving its keyspace manager and its storage
+	// phase.
+	reached := make(chan struct{})
+	release := make(chan struct{})
+	re.NoError(failpoint.EnableCall("github.com/tikv/pd/pkg/mcs/resourcemanager/server/deleteResourceGroupBeforeStorage", func() {
+		close(reached)
+		<-release
+	}))
+	defer func() {
+		re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/mcs/resourcemanager/server/deleteResourceGroupBeforeStorage"))
+	}()
+	var delErr error
+	delDone := make(chan struct{})
+	go func() {
+		defer close(delDone)
+		delErr = m.DeleteResourceGroup(1, "ct-del")
+	}()
+	select {
+	case <-reached:
+	case <-time.After(time.Second):
+		t.Fatal("timed out waiting for the delete to reach its storage phase")
+	}
+
+	// Leadership changes: term 2's loader snapshots storage (the group is
+	// still there) and parks before merging.
+	cancelTerm1()
+	store.pauseNextStates.Store(true)
+	re.NoError(m.Init(context.Background()))
+	select {
+	case <-store.statesReached:
+	case <-time.After(time.Second):
+		t.Fatal("timed out waiting for the term-2 loader to snapshot storage")
+	}
+
+	// Resume the Delete: storage removal proceeds, and the cache effect and
+	// marker must be published into term 2, not the detached term-1 manager.
+	close(release)
+	<-delDone
+	re.NoError(delErr)
+
+	// Let term 2 merge its pre-deletion snapshot; the marker must make it
+	// skip the deleted group.
+	store.unblockStates()
+	testutil.Eventually(re, func() bool {
+		groups, err := m.GetResourceGroupList(1, false)
+		if err != nil {
+			return false
+		}
+		for _, g := range groups {
+			if g.Name == "ct-del" {
+				return false
+			}
+		}
+		return true
+	}, testutil.WithTickInterval(20*time.Millisecond))
+	g, err := m.GetResourceGroup(1, "ct-del", false)
+	re.NoError(err)
+	re.Nil(g, "the deleted group must not be resurrected by the new term's merge")
+}
+
+// TestAsyncLoadResourceGroupsExhaustedRetriesReturnLoadingError guards the
+// exhausted-retry path of the lazy load: when every attempt loses the
+// delete-generation race, the load must fail with a retryable loading error
+// instead of reporting success without publishing the group, which callers
+// would misread as the group not existing.
+func TestAsyncLoadResourceGroupsExhaustedRetriesReturnLoadingError(t *testing.T) {
+	re := require.New(t)
+	store := newBlockingResourceGroupStorage()
+	re.NoError(store.SaveResourceGroupSetting(1, "keep-a", newAsyncTestGroup("keep-a")))
+	for _, name := range []string{"del-b1", "del-b2", "del-b3"} {
+		re.NoError(store.SaveResourceGroupSetting(1, name, newAsyncTestGroup(name)))
+	}
+
+	m := NewManager[*mockConfigProvider](&mockConfigProvider{})
+	m.storage = store
+	re.NoError(m.Init(context.Background()))
+	defer stopAsyncTestManager(m)
+	defer store.unblock()
+
+	// Keep the bulk loader parked so lazy loading stays active.
+	store.waitEntered(t)
+
+	// Park keep-a's lazy load inside each of its three read attempts, and
+	// delete an unrelated group while it's parked so every attempt observes a
+	// delete-generation change.
+	pause := store.armStatePause("keep-a")
+	var (
+		gotGroup *ResourceGroup
+		gotErr   error
+	)
+	getDone := make(chan struct{})
+	go func() {
+		defer close(getDone)
+		gotGroup, gotErr = m.GetResourceGroup(1, "keep-a", false)
+	}()
+	for _, victim := range []string{"del-b1", "del-b2", "del-b3"} {
+		waitStatePauseReached(t, pause)
+		re.NoError(m.DeleteResourceGroup(1, victim))
+		next := store.armStatePause("keep-a")
+		close(pause.release)
+		pause = next
+	}
+	<-getDone
+	// The last arm is left unconsumed; drop it so later loads pass through.
+	store.statePause.Store(nil)
+
+	re.ErrorIs(gotErr, errs.ErrResourceGroupsLoading,
+		"exhausted retries must surface a retryable loading error, not a bogus success")
+	re.Nil(gotGroup)
+
+	// The group still exists; once loading completes it must be served again.
+	store.unblock()
+	testutil.Eventually(re, func() bool {
+		g, err := m.GetResourceGroup(1, "keep-a", false)
+		return err == nil && g != nil
+	}, testutil.WithTickInterval(20*time.Millisecond))
+}
diff --git a/pkg/mcs/resourcemanager/server/manager_test.go b/pkg/mcs/resourcemanager/server/manager_test.go
index c045597240..8dba975aa0 100644
--- a/pkg/mcs/resourcemanager/server/manager_test.go
+++ b/pkg/mcs/resourcemanager/server/manager_test.go
@@ -803,7 +803,8 @@ func TestKeyspaceResourceGroupManagerWriteRoleGates(t *testing.T) {
 			RU: &rmpb.TokenBucket{Settings: &rmpb.TokenLimitSettings{FillRate: 200}},
 		},
 	}
-	re.ErrorIs(tokenOnlyKRGM.modifyResourceGroup(modifiedGroup), errMetadataWriteDisabled)
+	_, err := tokenOnlyKRGM.modifyResourceGroup(modifiedGroup)
+	re.ErrorIs(err, errMetadataWriteDisabled)
 	re.Equal(float64(100), tokenOnlyKRGM.getResourceGroup(group.GetName(), false).getFillRate())
 	re.ErrorIs(tokenOnlyKRGM.deleteResourceGroup(group.GetName()), errMetadataWriteDisabled)
 	re.NotNil(tokenOnlyKRGM.getResourceGroup(group.GetName(), false))

From 58186a03ee09e7fe261e63ec87cffa9a5da6cd60 Mon Sep 17 00:00:00 2001
From: tongjian <1045931706@qq.com>
Date: Fri, 24 Jul 2026 14:42:51 +0800
Subject: [PATCH 24/50] resource_group: confirm a modified default instead of
 leaving it reserved

After reservedStateOnly was removed, reservedGroups only ever holds the
synthetic default placeholder. The Modify publish path still gated its
sync-loaded marker on !reserved and never cleared the marker, so a
Modify of a reserved default placeholder (e.g. the fresh placeholder a
new term installs while the Modify's publish straddled the leadership
change) persisted the new settings but left the entry a reserved,
unmarked placeholder. The bulk merge could then revert it to a
pre-modification snapshot, or initReserved could re-synthesize a fresh
default over it, silently dropping the modified settings from the
serving cache while storage kept the new value.

Since a persisted-settings group is confirmed data, clear the reserved
marker and record it as sync-loaded on every Modify publish. There is
no state-only reserved kind anymore, so nothing pending is skipped by
marking.

Add a cross-term regression test that stalls a Modify of the default
before publishing, reinitializes the manager for a new term, then
resumes the publish and asserts the default is confirmed with the
modified settings (verified to fail without the fix).

Signed-off-by: tongjian <1045931706@qq.com>
---
 pkg/mcs/resourcemanager/server/manager.go     | 15 ++--
 .../server/manager_async_test.go              | 88 +++++++++++++++++++
 2 files changed, 97 insertions(+), 6 deletions(-)

diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go
index 39629cac7c..d391a760bf 100644
--- a/pkg/mcs/resourcemanager/server/manager.go
+++ b/pkg/mcs/resourcemanager/server/manager.go
@@ -1040,12 +1040,15 @@ func (m *Manager) ModifyResourceGroup(grouppb *rmpb.ResourceGroup) error {
 					zap.Uint32("keyspace-id", keyspaceID), zap.String("name", grouppb.Name), zap.Error(err))
 			}
 		}
-		// Modifying only patches settings, it never establishes the group's
-		// state. If the state still hasn't been confirmed (reserved), marking
-		// it sync-loaded would make the async bulk merge skip it forever, so
-		// the persisted running state would never get applied.
-		_, reserved := cur.reservedGroups[grouppb.Name]
-		return !reserved, nil
+		// The settings have now been persisted, so the entry is confirmed
+		// data even if it started as a reserved default placeholder (the only
+		// thing reservedGroups ever holds). Clear the marker and record it as
+		// sync-loaded: otherwise the bulk merge could revert it to a
+		// pre-modification snapshot, or initReserved could re-synthesize a
+		// fresh default over it, silently dropping the just-modified settings
+		// from the serving cache while storage keeps the new value.
+		delete(cur.reservedGroups, grouppb.Name)
+		return true, nil
 	})
 	return nil
 }
diff --git a/pkg/mcs/resourcemanager/server/manager_async_test.go b/pkg/mcs/resourcemanager/server/manager_async_test.go
index b797a289f8..c0dcdf7c4e 100644
--- a/pkg/mcs/resourcemanager/server/manager_async_test.go
+++ b/pkg/mcs/resourcemanager/server/manager_async_test.go
@@ -819,3 +819,91 @@ func TestAsyncLoadResourceGroupsExhaustedRetriesReturnLoadingError(t *testing.T)
 		return err == nil && g != nil
 	}, testutil.WithTickInterval(20*time.Millisecond))
 }
+
+// TestAsyncLoadResourceGroupsCrossTermModifyDefaultStaysConfirmed guards the
+// Modify-of-default publish path across a leadership change. A Modify patches
+// and persists the default group in term 1, then stalls before publishing.
+// Term 2 reinitializes the manager, giving it a fresh reserved default
+// placeholder. When the Modify resumes, publishing must leave the group
+// confirmed (not reserved) in the live term with the modified settings -
+// otherwise it stays a reserved placeholder that the bulk merge or
+// initReserved can revert to a pre-modification/synthetic default while
+// storage keeps the new value.
+func TestAsyncLoadResourceGroupsCrossTermModifyDefaultStaysConfirmed(t *testing.T) {
+	re := require.New(t)
+	store := newBlockingResourceGroupStorage()
+
+	m := NewManager[*mockConfigProvider](&mockConfigProvider{})
+	m.storage = store
+	re.NoError(m.Init(context.Background()))
+	cancelTerm1 := m.cancel
+	defer stopAsyncTestManager(m)
+	defer store.unblock()
+	defer store.unblockStates()
+
+	// Let term 1 load fully so the default group is confirmed and persisted.
+	store.waitEntered(t)
+	store.unblock()
+	testutil.Eventually(re, func() bool {
+		_, err := m.GetResourceGroupList(constant.NullKeyspaceID, false)
+		return err == nil
+	}, testutil.WithTickInterval(20*time.Millisecond))
+
+	// Park the Modify after it patched and persisted, before it publishes.
+	reached := make(chan struct{})
+	release := make(chan struct{})
+	re.NoError(failpoint.EnableCall("github.com/tikv/pd/pkg/mcs/resourcemanager/server/modifyResourceGroupBeforePublish", func() {
+		close(reached)
+		<-release
+	}))
+	defer func() {
+		re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/mcs/resourcemanager/server/modifyResourceGroupBeforePublish"))
+	}()
+	modified := newAsyncTestGroup(DefaultResourceGroupName)
+	modified.RUSettings.RU.Settings.FillRate = 4242
+	var modErr error
+	modDone := make(chan struct{})
+	go func() {
+		defer close(modDone)
+		modErr = m.ModifyResourceGroup(modified)
+	}()
+	select {
+	case <-reached:
+	case <-time.After(time.Second):
+		t.Fatal("timed out waiting for the modify to reach its publish phase")
+	}
+
+	// Leadership changes: term 2 gets a fresh reserved default placeholder,
+	// with its loader parked before merging.
+	cancelTerm1()
+	store.pauseNextStates.Store(true)
+	re.NoError(m.Init(context.Background()))
+	select {
+	case <-store.statesReached:
+	case <-time.After(time.Second):
+		t.Fatal("timed out waiting for the term-2 loader to snapshot storage")
+	}
+
+	// Resume the Modify's publish into term 2.
+	close(release)
+	<-modDone
+	re.NoError(modErr)
+
+	// The default in term 2 must be confirmed (not a reserved placeholder),
+	// so neither the merge nor initReserved reverts the modified settings.
+	krgm := m.getKeyspaceResourceGroupManager(constant.NullKeyspaceID)
+	re.NotNil(krgm)
+	re.False(krgm.isReserved(DefaultResourceGroupName),
+		"a modified default must be published as confirmed data, not left reserved")
+
+	store.unblockStates()
+	testutil.Eventually(re, func() bool {
+		_, err := m.GetResourceGroupList(constant.NullKeyspaceID, false)
+		return err == nil
+	}, testutil.WithTickInterval(20*time.Millisecond))
+	g, err := m.GetResourceGroup(constant.NullKeyspaceID, DefaultResourceGroupName, false)
+	re.NoError(err)
+	re.NotNil(g)
+	re.Equal(float64(4242), g.RUSettings.RU.getFillRate(),
+		"the modified default settings must survive into the new term")
+}

From 2b7cbefd40ff44b76b17d9c1ecd0d5c5a89b779e Mon Sep 17 00:00:00 2001
From: tongjian <1045931706@qq.com>
Date: Fri, 24 Jul 2026 15:50:48 +0800
Subject: [PATCH 25/50] resource_group: preserve confirmed state when
 republishing a modified group

The previous fix cleared the reserved marker and marked a modified
group sync-loaded, but when the current term held a different object
(a reserved default placeholder or a bulk-merge snapshot) it only
re-applied the settings patch onto that object. The placeholder carries
synthetic token/consumption state, so confirming it froze the fresh
state: the bulk merge then skipped the persisted state and the persist
loop could write the synthetic state back over the real value.

Install the patched group wholesale instead of patching the in-place
object. modifyResourceGroup patches an object that was loaded/confirmed
first, so it carries both the modified settings and the confirmed
running state; publishing it replaces the placeholder's synthetic
state with the real one, and burstability is synced like any freshly
installed group.

Extend the cross-term modify-default regression test to seed a
recognizable persisted running state and assert it survives the
republish (verified to fail when only settings are re-applied).

Signed-off-by: tongjian <1045931706@qq.com>
---
 pkg/mcs/resourcemanager/server/manager.go     | 38 +++++++++----------
 .../server/manager_async_test.go              | 12 ++++++
 2 files changed, 29 insertions(+), 21 deletions(-)

diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go
index d391a760bf..a0ef3b6e9e 100644
--- a/pkg/mcs/resourcemanager/server/manager.go
+++ b/pkg/mcs/resourcemanager/server/manager.go
@@ -1021,34 +1021,30 @@ func (m *Manager) ModifyResourceGroup(grouppb *rmpb.ResourceGroup) error {
 	}
 	failpoint.InjectCall("modifyResourceGroupBeforePublish")
 	m.publishResourceGroupMutation(keyspaceID, grouppb.Name, func(cur *keyspaceResourceGroupManager) (bool, *ResourceGroup) {
-		existing, ok := cur.groups[grouppb.Name]
-		if !ok {
-			// The patch ran against a manager that is no longer live (the
-			// manager was reinitialized while we were persisting); republish
-			// the patched group into the current term so the persisted
-			// settings aren't lost from the serving cache.
+		var synced *ResourceGroup
+		if existing := cur.groups[grouppb.Name]; existing != patched {
+			// A different object sits in the current term's cache: either the
+			// group is missing, a reserved default placeholder (with synthetic
+			// token/consumption state), or a pre-modification bulk-merge
+			// snapshot. Install `patched` wholesale rather than only patching
+			// settings onto it. `patched` was loaded/confirmed before it was
+			// modified, so it carries both the modified settings and the
+			// group's confirmed running state - patching the placeholder in
+			// place would keep its synthetic state, which the marker below
+			// would then freeze as confirmed and the persist loop would write
+			// back over the real state.
 			cur.groups[grouppb.Name] = patched
-			delete(cur.reservedGroups, grouppb.Name)
-			return true, patched
-		}
-		if existing != patched {
-			// A different object (e.g. installed by the new term's bulk
-			// merge from a pre-modification snapshot); re-apply the settings
-			// patch so the persisted settings win.
-			if err := existing.PatchSettings(grouppb); err != nil {
-				log.Warn("failed to re-apply resource group settings on republish",
-					zap.Uint32("keyspace-id", keyspaceID), zap.String("name", grouppb.Name), zap.Error(err))
-			}
+			synced = patched
 		}
-		// The settings have now been persisted, so the entry is confirmed
-		// data even if it started as a reserved default placeholder (the only
-		// thing reservedGroups ever holds). Clear the marker and record it as
+		// The settings and state are now confirmed data, even if the entry
+		// started as a reserved default placeholder (the only thing
+		// reservedGroups ever holds). Clear the marker and record it as
 		// sync-loaded: otherwise the bulk merge could revert it to a
 		// pre-modification snapshot, or initReserved could re-synthesize a
 		// fresh default over it, silently dropping the just-modified settings
 		// from the serving cache while storage keeps the new value.
 		delete(cur.reservedGroups, grouppb.Name)
-		return true, nil
+		return true, synced
 	})
 	return nil
 }
diff --git a/pkg/mcs/resourcemanager/server/manager_async_test.go b/pkg/mcs/resourcemanager/server/manager_async_test.go
index c0dcdf7c4e..358e8b5ec9 100644
--- a/pkg/mcs/resourcemanager/server/manager_async_test.go
+++ b/pkg/mcs/resourcemanager/server/manager_async_test.go
@@ -832,6 +832,14 @@ func TestAsyncLoadResourceGroupsExhaustedRetriesReturnLoadingError(t *testing.T)
 func TestAsyncLoadResourceGroupsCrossTermModifyDefaultStaysConfirmed(t *testing.T) {
 	re := require.New(t)
 	store := newBlockingResourceGroupStorage()
+	// Seed a persisted default with a recognizable running state so the test
+	// can tell a confirmed republish (state preserved) from a synthetic
+	// placeholder (state reset).
+	seed := newAsyncTestGroup(DefaultResourceGroupName)
+	re.NoError(store.SaveResourceGroupSetting(constant.NullKeyspaceID, DefaultResourceGroupName, seed))
+	seedStates := FromProtoResourceGroup(seed).GetGroupStates()
+	seedStates.RUConsumption.RRU = 777
+	re.NoError(store.SaveResourceGroupStates(constant.NullKeyspaceID, DefaultResourceGroupName, seedStates))
 
 	m := NewManager[*mockConfigProvider](&mockConfigProvider{})
 	m.storage = store
@@ -906,4 +914,8 @@ func TestAsyncLoadResourceGroupsCrossTermModifyDefaultStaysConfirmed(t *testing.
 	re.NotNil(g)
 	re.Equal(float64(4242), g.RUSettings.RU.getFillRate(),
 		"the modified default settings must survive into the new term")
+	// The confirmed running state must survive too, not revert to the fresh
+	// synthetic placeholder state.
+	re.Equal(float64(777), krgm.getMutableResourceGroup(DefaultResourceGroupName).GetGroupStates().RUConsumption.RRU,
+		"the confirmed running state must be preserved, not reset to a synthetic placeholder")
 }

From 308788ca71968b60038092e4328406afa47dca49 Mon Sep 17 00:00:00 2001
From: tongjian <1045931706@qq.com>
Date: Fri, 24 Jul 2026 20:26:34 +0800
Subject: [PATCH 26/50] resource_group: merge loaded groups in bounded batches

The async bulk merge held the manager lock across installing every
loaded group and synchronizing its burstability - O(total groups) work.
Concurrent point and token requests take the same lock to resolve a
keyspace manager, so on a cluster with many resource groups the merge
stalled all requests until loading completed, a large latency spike
exactly when async loading finished.

Flatten the loaded groups and merge them in bounded batches, releasing
the manager lock between batches and re-validating the load epoch each
batch (a term change can land between batches). Burstability sync now
runs outside the manager lock. Correctness is unchanged: a concurrent
write's publish is still atomic under the manager lock and each batch
still checks syncLoadedGroups, so publish-before-batch skips the group
and batch-before-publish is overridden.

Add BenchmarkAsyncLoadMergeReaderStall, which measures the worst-case
manager-read-lock stall of a concurrent reader during the merge. At
500k groups the worst-case stall drops from ~169ms (single lock over
all groups) to ~41ms (bounded batches), and stays bounded as the group
count grows.

Signed-off-by: tongjian <1045931706@qq.com>
---
 pkg/mcs/resourcemanager/server/manager.go     | 92 +++++++++++++------
 .../server/manager_async_test.go              | 80 +++++++++++++++-
 2 files changed, 142 insertions(+), 30 deletions(-)

diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go
index a0ef3b6e9e..e349e9f78f 100644
--- a/pkg/mcs/resourcemanager/server/manager.go
+++ b/pkg/mcs/resourcemanager/server/manager.go
@@ -528,44 +528,84 @@ func (m *Manager) asyncLoadResourceGroups(ctx context.Context, epoch uint64) {
 			continue
 		}
 
-		loaded := 0
-		m.Lock()
-		if m.loadEpoch != epoch {
-			// The manager was reinitialized for a new term while this loader
-			// was scanning; its result is stale and must not be merged.
-			m.Unlock()
-			log.Info("async loading resource groups aborted: manager was reinitialized")
-			return
+		// Flatten the loaded groups so the merge below can run in bounded
+		// batches. Holding m.Lock across the whole O(total groups) merge would
+		// block every concurrent point/token request (they need the lock to
+		// resolve a keyspace manager) for the entire merge, causing a large
+		// latency spike right when async loading completes on a cluster with
+		// many resource groups.
+		type mergeItem struct {
+			keyspaceID uint32
+			name       string
+			group      *ResourceGroup
 		}
+		pending := make([]mergeItem, 0)
 		for keyspaceID, tempKrgm := range tempKrgms {
-			krgm := m.krgms[keyspaceID]
-			if krgm == nil {
-				krgm = newKeyspaceResourceGroupManager(keyspaceID, m.storage, m.writeRole)
-				m.krgms[keyspaceID] = krgm
-			}
-			groupsToSync := make([]*ResourceGroup, 0)
 			tempKrgm.RLock()
-			krgm.Lock()
 			for name, group := range tempKrgm.groups {
-				key := trackerKey{keyspaceID: keyspaceID, groupName: name}
+				pending = append(pending, mergeItem{keyspaceID: keyspaceID, name: name, group: group})
+			}
+			tempKrgm.RUnlock()
+		}
+
+		const mergeBatchSize = 1024
+		loaded := 0
+		aborted := false
+		for start := 0; start < len(pending); start += mergeBatchSize {
+			end := min(start+mergeBatchSize, len(pending))
+			type syncItem struct {
+				krgm  *keyspaceResourceGroupManager
+				group *ResourceGroup
+			}
+			toSync := make([]syncItem, 0, end-start)
+			m.Lock()
+			if m.loadEpoch != epoch {
+				// The manager was reinitialized for a new term while this
+				// loader was scanning; its result is stale and must not be
+				// merged. Re-checked every batch since a term change can land
+				// between batches.
+				m.Unlock()
+				aborted = true
+				break
+			}
+			for _, it := range pending[start:end] {
+				key := trackerKey{keyspaceID: it.keyspaceID, groupName: it.name}
 				if m.syncLoadedGroups[key] {
 					continue
 				}
-				krgm.groups[name] = group
+				krgm := m.krgms[it.keyspaceID]
+				if krgm == nil {
+					krgm = newKeyspaceResourceGroupManager(it.keyspaceID, m.storage, m.writeRole)
+					m.krgms[it.keyspaceID] = krgm
+				}
+				krgm.Lock()
+				krgm.groups[it.name] = it.group
 				// This group is now confirmed, fully-loaded data (settings
-				// and state); it must no longer be treated as an
-				// unconfirmed placeholder by loadResourceGroupIfNeeded or
-				// skipped by the state persist loop.
-				delete(krgm.reservedGroups, name)
-				groupsToSync = append(groupsToSync, group)
+				// and state); it must no longer be treated as an unconfirmed
+				// placeholder by loadResourceGroupIfNeeded or skipped by the
+				// state persist loop.
+				delete(krgm.reservedGroups, it.name)
+				krgm.Unlock()
+				toSync = append(toSync, syncItem{krgm: krgm, group: it.group})
 				loaded++
 			}
-			krgm.Unlock()
-			tempKrgm.RUnlock()
-			for _, group := range groupsToSync {
-				krgm.syncBurstabilityWithServiceLimit(group)
+			m.Unlock()
+			// Sync burstability outside m.Lock; it only needs the keyspace and
+			// group locks.
+			for _, s := range toSync {
+				s.krgm.syncBurstabilityWithServiceLimit(s.group)
 			}
 		}
+		if aborted {
+			log.Info("async loading resource groups aborted: manager was reinitialized")
+			return
+		}
+		m.Lock()
+		if m.loadEpoch != epoch {
+			m.Unlock()
+			log.Info("async loading resource groups aborted: manager was reinitialized")
+			return
+		}
 		m.syncLoadedGroups = nil
 		m.Unlock()
 
diff --git a/pkg/mcs/resourcemanager/server/manager_async_test.go b/pkg/mcs/resourcemanager/server/manager_async_test.go
index 358e8b5ec9..4a78975432 100644
--- a/pkg/mcs/resourcemanager/server/manager_async_test.go
+++ b/pkg/mcs/resourcemanager/server/manager_async_test.go
@@ -17,6 +17,7 @@ package server
 import (
 	"context"
 	"errors"
+	"fmt"
 	"sync"
 	"sync/atomic"
 	"testing"
@@ -123,12 +124,12 @@ func (s *blockingResourceGroupStorage) LoadResourceGroupStates(f func(keyspaceID
 	return s.Storage.LoadResourceGroupStates(f)
 }
 
-func (s *blockingResourceGroupStorage) waitEntered(t *testing.T) {
-	t.Helper()
+func (s *blockingResourceGroupStorage) waitEntered(tb testing.TB) {
+	tb.Helper()
 	select {
 	case <-s.entered:
-	case <-time.After(time.Second):
-		t.Fatal("timed out waiting for async resource group loading")
+	case <-time.After(5 * time.Second):
+		tb.Fatal("timed out waiting for async resource group loading")
 	}
 }
 
@@ -919,3 +920,74 @@ func TestAsyncLoadResourceGroupsCrossTermModifyDefaultStaysConfirmed(t *testing.
 	re.Equal(float64(777), krgm.getMutableResourceGroup(DefaultResourceGroupName).GetGroupStates().RUConsumption.RRU,
 		"the confirmed running state must be preserved, not reset to a synthetic placeholder")
 }
+
+// BenchmarkAsyncLoadMergeReaderStall measures the worst-case time a concurrent
+// reader is blocked while the async bulk merge installs a large number of
+// resource groups. The probe uses GetControllerConfig, whose only cost is the
+// manager read lock - the same lock the merge takes - so it isolates how long
+// the merge stalls readers. It guards against the merge holding that lock
+// across the whole O(total groups) work, which would stall every point and
+// token request until loading completes on a cluster with many groups.
+//
+// Run with:
+//
+//	go test -run '^$' -bench BenchmarkAsyncLoadMergeReaderStall ./pkg/mcs/resourcemanager/server/
+func BenchmarkAsyncLoadMergeReaderStall(b *testing.B) {
+	const groupCount = 500000
+	store := newBlockingResourceGroupStorage()
+	for i := range groupCount {
+		name := fmt.Sprintf("bench-group-%06d", i)
+		if err := store.SaveResourceGroupSetting(1, name, newAsyncTestGroup(name)); err != nil {
+			b.Fatal(err)
+		}
+	}
+
+	m := NewManager[*mockConfigProvider](&mockConfigProvider{})
+	m.storage = store
+	if err := m.Init(context.Background()); err != nil {
+		b.Fatal(err)
+	}
+	defer stopAsyncTestManager(m)
+	defer store.unblock()
+
+	store.waitEntered(b)
+
+	// Concurrent readers take only the manager read lock and record the
+	// longest single acquisition seen while the merge runs.
+	stop := make(chan struct{})
+	var maxStall atomic.Int64
+	var wg sync.WaitGroup
+	for range 8 {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for {
+				select {
+				case <-stop:
+					return
+				default:
+				}
+				start := time.Now()
+				_ = m.GetControllerConfig()
+				if d := time.Since(start).Nanoseconds(); d > maxStall.Load() {
+					maxStall.Store(d)
+				}
+			}
+		}()
+	}
+
+	b.ResetTimer()
+	store.unblock()
+	deadline := time.Now().Add(30 * time.Second)
+	for !m.isResourceGroupLoadingComplete() {
+		if time.Now().After(deadline) {
+			b.Fatal("timed out waiting for async loading to complete")
+		}
+		time.Sleep(time.Millisecond)
+	}
+	b.StopTimer()
+
+	close(stop)
+	wg.Wait()
+	b.ReportMetric(float64(maxStall.Load())/1e6, "max-reader-stall-ms")
+}

From 29488804c4801ba89278d35b760860421651e645 Mon Sep 17 00:00:00 2001
From: tongjian <1045931706@qq.com>
Date: Fri, 24 Jul 2026 20:58:02 +0800
Subject: [PATCH 27/50] resource_group: gate reserved-default backfill on the
 load epoch

The async-load completion tail set syncLoadedGroups=nil, ran
initReserved (which re-resolves keyspace managers and persists
synthetic defaults), then published LoadingStateCompleted. Every
other shared-state mutation in this file re-checks loadEpoch, but
initReserved did not, so a re-election landing between clearing the
sync map and publishing completion could let a stale loader synthesize
and persist a default into the new term, clobbering a customized
default the new term had not loaded yet.

Publish completion first via the epoch-guarded storeLoadingStateIfCurrent
and only backfill reserved defaults when it succeeds, so a stale loader
returns without touching the new term. Keyspaces whose default the
loader would have backfilled remain covered: once loading is complete,
getOrCreateKeyspaceResourceGroupManager synthesizes the default on
demand.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Signed-off-by: tongjian <1045931706@qq.com>
---
 pkg/mcs/resourcemanager/server/manager.go | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go
index e349e9f78f..4a877bb597 100644
--- a/pkg/mcs/resourcemanager/server/manager.go
+++ b/pkg/mcs/resourcemanager/server/manager.go
@@ -609,11 +609,20 @@ func (m *Manager) asyncLoadResourceGroups(ctx context.Context, epoch uint64) {
 		m.syncLoadedGroups = nil
 		m.Unlock()
 
-		m.initReserved()
+		// Publish completion before backfilling reserved defaults. Unlike every
+		// other shared-state mutation here, initReserved re-resolves managers and
+		// persists synthetic defaults without an epoch guard, so a re-election
+		// landing in this window could make a stale loader clobber the new term's
+		// not-yet-loaded default. storeLoadingStateIfCurrent is epoch-guarded, so
+		// gating on it first makes a stale loader return without ever running
+		// initReserved. A keyspace whose default this loader would have backfilled
+		// is still covered: once loading is complete, getOrCreateKeyspaceResource-
+		// GroupManager synthesizes the default directly on demand.
 		if !m.storeLoadingStateIfCurrent(epoch, LoadingStateCompleted) {
 			log.Info("async loading resource groups aborted: manager was reinitialized")
 			return
 		}
+		m.initReserved()
 		duration := time.Since(startTime)
 		asyncLoadGroupDuration.Observe(duration.Seconds())
 		log.Info("async loading resource groups completed", zap.Int("loaded-groups", loaded), zap.Duration("duration", duration))

From 4a9a31d2d5416674c89a7e606e27e25fdb8837d4 Mon Sep 17 00:00:00 2001
From: tongjian <1045931706@qq.com>
Date: Sun, 26 Jul 2026 22:08:58 +0800
Subject: [PATCH 28/50] resource_group: harden async loading error paths and
 observability

Address review findings on the async resource group loading change.

- AcquireTokenBuckets: a transient lazy-load failure (storage read error, or
  retries exhausted while async loading is in progress) no longer returns from
  the stream handler. The error belongs to a single resource group, while
  returning tore down the whole token bucket stream and forced every client
  multiplexed on it to reconnect.
- asyncLoadGroupDuration used the default buckets, whose top finite bucket is
  10s, so the slow loads this metric exists to measure all landed in +Inf. Give
  it an explicit exponential range spanning sub-second to ~27min.
- Add async_load_group_failures_total and resource_group_loading_state so a load
  that never succeeds is alertable: the loader retries indefinitely, so it no
  longer fails Init loudly.
- Metadata watcher mode never ran an async loader, so nothing consumed the
  sync-loaded markers while every watch event kept adding entries that were
  never removed. Drop the map once the watcher is initialized.
- Report ErrResourceGroupsLoading as retryable: 503 over HTTP instead of 500,
  and codes.Unavailable over gRPC instead of an opaque codes.Unknown.
- Collapse the lazy-loading fast path onto a single read lock, size the merge
  slice up front, use errs.ErrResourceGroupNotExists.Equal instead of building
  an error just to compare it, and switch loadingState to atomic.Int32.

Signed-off-by: tongjian <1045931706@qq.com>
---
 .../metadataapi/config_service.go             | 11 +++
 .../metadataapi/config_service_test.go        | 26 +++++-
 .../resourcemanager/server/grpc_service.go    | 30 ++++--
 .../server/keyspace_manager.go                | 14 +++
 pkg/mcs/resourcemanager/server/manager.go     | 57 +++++++++---
 .../server/manager_async_test.go              | 92 +++++++++++++++++++
 .../server/metadata_watcher_test.go           | 49 ++++++++++
 pkg/mcs/resourcemanager/server/metrics.go     | 27 ++++++
 8 files changed, 282 insertions(+), 24 deletions(-)

diff --git a/pkg/mcs/resourcemanager/metadataapi/config_service.go b/pkg/mcs/resourcemanager/metadataapi/config_service.go
index 900d8056a5..8c9387c8ad 100644
--- a/pkg/mcs/resourcemanager/metadataapi/config_service.go
+++ b/pkg/mcs/resourcemanager/metadataapi/config_service.go
@@ -309,6 +309,13 @@ func (*ConfigService) respondStoreReadError(c *gin.Context, err error) {
 		c.String(http.StatusNotFound, err.Error())
 		return
 	}
+	// Resource groups are still being loaded asynchronously: the request can
+	// succeed once loading completes, so report it as retryable rather than as
+	// an internal error.
+	if errs.ErrResourceGroupsLoading.Equal(err) {
+		c.String(http.StatusServiceUnavailable, err.Error())
+		return
+	}
 	c.String(http.StatusInternalServerError, err.Error())
 }
 
@@ -321,5 +328,9 @@ func (*ConfigService) respondStoreWriteError(c *gin.Context, err error) {
 		c.String(http.StatusNotFound, err.Error())
 		return
 	}
+	if errs.ErrResourceGroupsLoading.Equal(err) {
+		c.String(http.StatusServiceUnavailable, err.Error())
+		return
+	}
 	c.String(http.StatusInternalServerError, err.Error())
 }
diff --git a/pkg/mcs/resourcemanager/metadataapi/config_service_test.go b/pkg/mcs/resourcemanager/metadataapi/config_service_test.go
index 0a44c826ff..2f31315b22 100644
--- a/pkg/mcs/resourcemanager/metadataapi/config_service_test.go
+++ b/pkg/mcs/resourcemanager/metadataapi/config_service_test.go
@@ -90,6 +90,26 @@ func TestConfigServiceGroupCRUDAndErrorCodes(t *testing.T) {
 	re.Equal(http.StatusBadRequest, resp.Code)
 }
 
+// TestConfigServiceLoadingReturns503 asserts the "resource groups are still
+// being loaded" error is reported as retryable. It is a transient startup state,
+// not an internal error, so callers and load balancers must be able to tell the
+// difference and retry.
+func TestConfigServiceLoadingReturns503(t *testing.T) {
+	t.Parallel()
+
+	re := require.New(t)
+	store := newTestStore()
+	handler := newTestHTTPHandler(store)
+
+	store.listErr = pderrors.ErrResourceGroupsLoading
+	resp := doJSONRequest(re, handler, http.MethodGet, "/resource-manager/api/v1/config/groups", nil)
+	re.Equal(http.StatusServiceUnavailable, resp.Code)
+
+	store.listErr = errors.New("boom")
+	resp = doJSONRequest(re, handler, http.MethodGet, "/resource-manager/api/v1/config/groups", nil)
+	re.Equal(http.StatusInternalServerError, resp.Code)
+}
+
 func TestConfigServiceControllerAllOrNothing(t *testing.T) {
 	t.Parallel()
 
@@ -215,6 +235,7 @@ type testStore struct {
 	serviceLimits                map[uint32]float64
 	addErr                       error
 	setServiceLimitErr           error
+	listErr                      error
 	updatedControllerConfigItems []string
 }
 
@@ -270,7 +291,10 @@ func (s *testStore) GetResourceGroup(keyspaceID uint32, name string, withStats b
 	return group.Clone(withStats), nil
 }
 
-func (*testStore) GetResourceGroupList(_ uint32, _ bool) ([]*rmserver.ResourceGroup, error) {
+func (s *testStore) GetResourceGroupList(_ uint32, _ bool) ([]*rmserver.ResourceGroup, error) {
+	if s.listErr != nil {
+		return nil, s.listErr
+	}
 	return []*rmserver.ResourceGroup{}, nil
 }
 
diff --git a/pkg/mcs/resourcemanager/server/grpc_service.go b/pkg/mcs/resourcemanager/server/grpc_service.go
index 0945494cdc..f6c95224ab 100644
--- a/pkg/mcs/resourcemanager/server/grpc_service.go
+++ b/pkg/mcs/resourcemanager/server/grpc_service.go
@@ -102,6 +102,16 @@ func (s *Service) checkServing() error {
 	return nil
 }
 
+// wrapLoadingError converts the retryable "resource groups are still loading"
+// error into codes.Unavailable, so generic client-side retry logic can act on
+// it instead of seeing an opaque codes.Unknown. Other errors pass through.
+func wrapLoadingError(err error) error {
+	if errs.ErrResourceGroupsLoading.Equal(err) {
+		return status.Error(codes.Unavailable, err.Error())
+	}
+	return err
+}
+
 // GetResourceGroup implements ResourceManagerServer.GetResourceGroup.
 func (s *Service) GetResourceGroup(_ context.Context, req *rmpb.GetResourceGroupRequest) (*rmpb.GetResourceGroupResponse, error) {
 	if err := s.checkServing(); err != nil {
@@ -110,7 +120,7 @@ func (s *Service) GetResourceGroup(_ context.Context, req *rmpb.GetResourceGroup
 	keyspaceID := ExtractKeyspaceID(req.GetKeyspaceId())
 	rg, err := s.manager.GetResourceGroup(keyspaceID, req.ResourceGroupName, req.WithRuStats)
 	if err != nil {
-		return nil, err
+		return nil, wrapLoadingError(err)
 	}
 	if rg == nil {
 		return nil, errs.ErrResourceGroupNotExists.FastGenByArgs(req.ResourceGroupName)
@@ -129,7 +139,7 @@ func (s *Service) ListResourceGroups(_ context.Context, req *rmpb.ListResourceGr
 	keyspaceID := ExtractKeyspaceID(req.GetKeyspaceId())
 	groups, err := s.manager.GetResourceGroupList(keyspaceID, req.WithRuStats)
 	if err != nil {
-		return nil, err
+		return nil, wrapLoadingError(err)
 	}
 	resps := &rmpb.ListResourceGroupsResponse{
 		Groups: make([]*rmpb.ResourceGroup, 0, len(groups)),
@@ -151,7 +161,7 @@ func (s *Service) AddResourceGroup(_ context.Context, req *rmpb.PutResourceGroup
 	}
 	err := s.manager.AddResourceGroup(req.GetGroup())
 	if err != nil {
-		return nil, err
+		return nil, wrapLoadingError(err)
 	}
 	return &rmpb.PutResourceGroupResponse{Body: "Success!"}, nil
 }
@@ -166,7 +176,7 @@ func (s *Service) DeleteResourceGroup(_ context.Context, req *rmpb.DeleteResourc
 	}
 	err := s.manager.DeleteResourceGroup(ExtractKeyspaceID(req.GetKeyspaceId()), req.ResourceGroupName)
 	if err != nil {
-		return nil, err
+		return nil, wrapLoadingError(err)
 	}
 	return &rmpb.DeleteResourceGroupResponse{Body: "Success!"}, nil
 }
@@ -181,7 +191,7 @@ func (s *Service) ModifyResourceGroup(_ context.Context, req *rmpb.PutResourceGr
 	}
 	err := s.manager.ModifyResourceGroup(req.GetGroup())
 	if err != nil {
-		return nil, err
+		return nil, wrapLoadingError(err)
 	}
 	return &rmpb.PutResourceGroupResponse{Body: "Success!"}, nil
 }
@@ -234,12 +244,14 @@ func (s *Service) AcquireTokenBuckets(stream rmpb.ResourceManager_AcquireTokenBu
 			// Get the resource group from manager to acquire token buckets. This also
 			// triggers lazy loading of the group if async loading hasn't completed yet,
 			// so it must happen before accessKeyspaceResourceGroupManager below.
+			// Lazy loading can fail transiently (a storage read error, or retries
+			// exhausted while async loading is still in progress). Skip just this
+			// request instead of returning: the error belongs to one resource group,
+			// while returning would tear down the whole stream and force every
+			// client multiplexed on it to reconnect.
 			rg, err := s.manager.GetMutableResourceGroup(keyspaceID, resourceGroupName)
 			if rg == nil {
-				if err != nil && !errors.ErrorEqual(err, errs.ErrResourceGroupNotExists.FastGenByArgs(resourceGroupName)) {
-					return err
-				}
-				log.Warn("resource group not found", append(requestFields, zap.Error(err))...)
+				log.Warn("resource group is unavailable", append(requestFields, zap.Error(err))...)
 				continue
 			}
 			// Get keyspace resource group manager to apply service limit later.
diff --git a/pkg/mcs/resourcemanager/server/keyspace_manager.go b/pkg/mcs/resourcemanager/server/keyspace_manager.go
index 2cc83efd66..b716796f35 100644
--- a/pkg/mcs/resourcemanager/server/keyspace_manager.go
+++ b/pkg/mcs/resourcemanager/server/keyspace_manager.go
@@ -410,6 +410,20 @@ func (krgm *keyspaceResourceGroupManager) isReserved(name string) bool {
 	return ok
 }
 
+// hasConfirmedResourceGroup reports whether name is cached as confirmed data,
+// i.e. present and not a reserved placeholder. It answers both questions under
+// a single read lock, since it sits on the lazy-loading fast path that every
+// point and token request takes while async loading is still in progress.
+func (krgm *keyspaceResourceGroupManager) hasConfirmedResourceGroup(name string) bool {
+	krgm.RLock()
+	defer krgm.RUnlock()
+	if _, ok := krgm.groups[name]; !ok {
+		return false
+	}
+	_, reserved := krgm.reservedGroups[name]
+	return !reserved
+}
+
 func (krgm *keyspaceResourceGroupManager) getResourceGroup(name string, withStats bool) *ResourceGroup {
 	krgm.RLock()
 	defer krgm.RUnlock()
diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go
index 4a877bb597..fd79af970e 100644
--- a/pkg/mcs/resourcemanager/server/manager.go
+++ b/pkg/mcs/resourcemanager/server/manager.go
@@ -116,7 +116,7 @@ type Manager struct {
 	// ruCollector is used to collect the RU metering data.
 	ruCollector *ruCollector
 	// async loading state management
-	loadingState int32 // atomic access
+	loadingState atomic.Int32
 	// syncLoadedGroups records groups that were loaded synchronously (e.g., by lazy loading)
 	syncLoadedGroups map[trackerKey]bool
 	// loadEpoch is bumped (under the manager lock) every time initMetadata
@@ -161,7 +161,7 @@ type metadataWatcherProvider interface {
 }
 
 func newManagerBase(controllerConfig *ControllerConfig, writeRole ResourceGroupWriteRole) *Manager {
-	return &Manager{
+	m := &Manager{
 		writeRole:             writeRole,
 		controllerConfig:      controllerConfig,
 		krgms:                 make(map[uint32]*keyspaceResourceGroupManager),
@@ -170,9 +170,21 @@ func newManagerBase(controllerConfig *ControllerConfig, writeRole ResourceGroupW
 		keyspaceIDLookup:      make(map[string]uint32),
 		metrics:               newMetrics(),
 		ruCollector:           newRUCollector(),
-		loadingState:          LoadingStateNotStarted,
 		syncLoadedGroups:      make(map[trackerKey]bool),
 	}
+	m.setLoadingState(LoadingStateNotStarted)
+	return m
+}
+
+// setLoadingState publishes the loading state both to the atomic field the
+// serving paths read and to the gauge operators alert on.
+func (m *Manager) setLoadingState(state int32) {
+	m.loadingState.Store(state)
+	resourceGroupLoadingStateGauge.Set(float64(state))
+}
+
+func (m *Manager) getLoadingState() int32 {
+	return m.loadingState.Load()
 }
 
 // NewManager returns a new manager base on the given server,
@@ -317,7 +329,7 @@ func (m *Manager) getOrCreateKeyspaceResourceGroupManager(keyspaceID uint32, ini
 	}
 	m.Unlock()
 	if initDefault {
-		if atomic.LoadInt32(&m.loadingState) == LoadingStateCompleted {
+		if m.getLoadingState() == LoadingStateCompleted {
 			// Async loading (if any) has already finished, so if the default
 			// group isn't cached yet it truly doesn't exist anywhere; it's
 			// safe to synthesize and persist it directly.
@@ -364,7 +376,13 @@ func (m *Manager) Init(ctx context.Context) error {
 			m.wg.Wait()
 			return err
 		}
-		atomic.StoreInt32(&m.loadingState, LoadingStateCompleted)
+		// No async loader runs in this mode, so nothing will ever consume the
+		// sync-loaded markers. Drop the map: otherwise every watcher event keeps
+		// adding entries that are never removed, including for deleted groups.
+		m.Lock()
+		m.syncLoadedGroups = nil
+		m.Unlock()
+		m.setLoadingState(LoadingStateCompleted)
 	} else {
 		// This context is derived from the leader/primary context, it will be canceled
 		// from the outside loop when the leader/primary step down.
@@ -430,7 +448,7 @@ func (m *Manager) initMetadata(ctx context.Context) error {
 	m.syncLoadedGroups = make(map[trackerKey]bool)
 	m.loadEpoch++
 	epoch := m.loadEpoch
-	atomic.StoreInt32(&m.loadingState, LoadingStateNotStarted)
+	m.setLoadingState(LoadingStateNotStarted)
 	m.Unlock()
 
 	m.initReservedInCache()
@@ -457,7 +475,7 @@ func (m *Manager) loadKeyspaceResourceGroups() error {
 	m.Lock()
 	m.krgms = tempKrgms
 	m.syncLoadedGroups = nil
-	atomic.StoreInt32(&m.loadingState, LoadingStateCompleted)
+	m.setLoadingState(LoadingStateCompleted)
 	m.Unlock()
 	m.initReserved()
 	return m.loadServiceLimits()
@@ -473,7 +491,7 @@ func (m *Manager) storeLoadingStateIfCurrent(epoch uint64, state int32) bool {
 	if m.loadEpoch != epoch {
 		return false
 	}
-	atomic.StoreInt32(&m.loadingState, state)
+	m.setLoadingState(state)
 	return true
 }
 
@@ -519,6 +537,9 @@ func (m *Manager) asyncLoadResourceGroups(ctx context.Context, epoch uint64) {
 		}
 		if err != nil {
 			// Use warn level since the loader retries indefinitely until it succeeds.
+			// The failure counter and the loading state gauge are what make a load
+			// that never succeeds alertable, since it no longer fails `Init` loudly.
+			asyncLoadGroupFailureCounter.Inc()
 			log.Warn("failed to load resource groups", zap.Error(err), zap.Int("retry", retry))
 			if !m.storeLoadingStateIfCurrent(epoch, LoadingStateNotStarted) {
 				log.Info("async loading resource groups aborted: manager was reinitialized")
@@ -539,7 +560,15 @@ func (m *Manager) asyncLoadResourceGroups(ctx context.Context, epoch uint64) {
 			name       string
 			group      *ResourceGroup
 		}
-		pending := make([]mergeItem, 0)
+		// Size the slice up front: it holds every loaded group, which is exactly
+		// the scale this loader exists to handle.
+		totalGroups := 0
+		for _, tempKrgm := range tempKrgms {
+			tempKrgm.RLock()
+			totalGroups += len(tempKrgm.groups)
+			tempKrgm.RUnlock()
+		}
+		pending := make([]mergeItem, 0, totalGroups)
 		for keyspaceID, tempKrgm := range tempKrgms {
 			tempKrgm.RLock()
 			for name, group := range tempKrgm.groups {
@@ -697,7 +726,7 @@ func (m *Manager) loadResourceGroup(keyspaceID uint32, name string) (*ResourceGr
 }
 
 func (m *Manager) loadResourceGroupIfNeeded(keyspaceID uint32, name string) error {
-	if atomic.LoadInt32(&m.loadingState) == LoadingStateCompleted {
+	if m.getLoadingState() == LoadingStateCompleted {
 		return nil
 	}
 	krgm := m.getKeyspaceResourceGroupManager(keyspaceID)
@@ -705,7 +734,7 @@ func (m *Manager) loadResourceGroupIfNeeded(keyspaceID uint32, name string) erro
 		// A cached entry only satisfies this call if it's confirmed data, not
 		// just a synthetic placeholder (e.g. from ensureReservedDefaultGroupInCache)
 		// installed before async loading had a chance to run.
-		if group := krgm.getMutableResourceGroup(name); group != nil && !krgm.isReserved(name) {
+		if krgm.hasConfirmedResourceGroup(name) {
 			return nil
 		}
 	}
@@ -736,7 +765,7 @@ func (m *Manager) loadResourceGroupIfNeeded(keyspaceID uint32, name string) erro
 		deleteGen := krgm.loadDeleteGen()
 		group, err := m.loadResourceGroup(keyspaceID, name)
 		if err != nil {
-			if name == DefaultResourceGroupName && errors.ErrorEqual(err, errs.ErrResourceGroupNotExists.FastGenByArgs(name)) {
+			if name == DefaultResourceGroupName && errs.ErrResourceGroupNotExists.Equal(err) {
 				m.RLock()
 				stale := m.loadEpoch != epoch || m.krgms[keyspaceID] != krgm
 				m.RUnlock()
@@ -864,7 +893,7 @@ func (m *Manager) publishResourceGroupMutation(
 }
 
 func (m *Manager) isResourceGroupLoadingComplete() bool {
-	return atomic.LoadInt32(&m.loadingState) == LoadingStateCompleted
+	return m.getLoadingState() == LoadingStateCompleted
 }
 
 func cloneControllerConfig(cfg *ControllerConfig) *ControllerConfig {
@@ -1031,7 +1060,7 @@ func (m *Manager) AddResourceGroup(grouppb *rmpb.ResourceGroup) error {
 		return errs.ErrKeyspaceNotExists.FastGenByArgs(keyspaceID)
 	}
 	if err := m.loadResourceGroupIfNeeded(keyspaceID, grouppb.Name); err != nil &&
-		!errors.ErrorEqual(err, errs.ErrResourceGroupNotExists.FastGenByArgs(grouppb.Name)) {
+		!errs.ErrResourceGroupNotExists.Equal(err) {
 		log.Warn("failed to load resource group before add", zap.Uint32("keyspace-id", keyspaceID), zap.String("name", grouppb.Name), zap.Error(err))
 		return err
 	}
diff --git a/pkg/mcs/resourcemanager/server/manager_async_test.go b/pkg/mcs/resourcemanager/server/manager_async_test.go
index 4a78975432..661c456917 100644
--- a/pkg/mcs/resourcemanager/server/manager_async_test.go
+++ b/pkg/mcs/resourcemanager/server/manager_async_test.go
@@ -18,12 +18,14 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"sync"
 	"sync/atomic"
 	"testing"
 	"time"
 
 	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
 
 	"github.com/pingcap/failpoint"
 	"github.com/pingcap/kvproto/pkg/resource_manager"
@@ -991,3 +993,93 @@ func BenchmarkAsyncLoadMergeReaderStall(b *testing.B) {
 	wg.Wait()
 	b.ReportMetric(float64(maxStall.Load())/1e6, "max-reader-stall-ms")
 }
+
+// fakeTokenBucketsStream feeds a fixed set of requests to AcquireTokenBuckets
+// and records what it sends back. grpc.ServerStream stays nil since
+// AcquireTokenBuckets only ever calls Send/Recv on it.
+type fakeTokenBucketsStream struct {
+	grpc.ServerStream
+
+	requests []*resource_manager.TokenBucketsRequest
+	recvCnt  int
+	sent     []*resource_manager.TokenBucketsResponse
+}
+
+// Context is needed by the metrics stream wrapper, which resolves the peer IP
+// for its labels.
+func (*fakeTokenBucketsStream) Context() context.Context { return context.Background() }
+
+func (s *fakeTokenBucketsStream) Recv() (*resource_manager.TokenBucketsRequest, error) {
+	if s.recvCnt >= len(s.requests) {
+		return nil, io.EOF
+	}
+	req := s.requests[s.recvCnt]
+	s.recvCnt++
+	return req, nil
+}
+
+func (s *fakeTokenBucketsStream) Send(resp *resource_manager.TokenBucketsResponse) error {
+	s.sent = append(s.sent, resp)
+	return nil
+}
+
+func newRUTokenBucketRequest(keyspaceID uint32, name string, ru float64) *resource_manager.TokenBucketRequest {
+	return &resource_manager.TokenBucketRequest{
+		ResourceGroupName: name,
+		KeyspaceId:        &resource_manager.KeyspaceIDValue{Value: keyspaceID},
+		Request: &resource_manager.TokenBucketRequest_RuItems{
+			RuItems: &resource_manager.TokenBucketRequest_RequestRU{
+				RequestRU: []*resource_manager.RequestUnitItem{
+					{Type: resource_manager.RequestUnitType_RU, Value: ru},
+				},
+			},
+		},
+		ConsumptionSinceLastRequest: &resource_manager.Consumption{},
+	}
+}
+
+// TestAcquireTokenBucketsSurvivesLazyLoadFailure guards against a transient
+// lazy-load failure tearing down the whole token bucket stream: the error
+// belongs to a single resource group, so the other groups multiplexed on the
+// same stream must still be served instead of every client being forced to
+// reconnect.
+func TestAcquireTokenBucketsSurvivesLazyLoadFailure(t *testing.T) {
+	re := require.New(t)
+	store := newBlockingResourceGroupStorage()
+	for _, name := range []string{"bad-group", "good-group"} {
+		group := newAsyncTestGroup(name)
+		re.NoError(store.SaveResourceGroupSetting(1, name, group))
+		re.NoError(store.SaveResourceGroupStates(1, name, FromProtoResourceGroup(group).GetGroupStates()))
+	}
+
+	m := NewManager[*mockConfigProvider](&mockConfigProvider{})
+	m.storage = store
+	m.srv = &testBasicServer{}
+	re.NoError(m.Init(context.Background()))
+	defer stopAsyncTestManager(m)
+	defer store.unblock()
+
+	// Keep the bulk loader parked so lazy loading stays active.
+	store.waitEntered(t)
+
+	// Make bad-group's lazy load fail on its state read. good-group is
+	// requested in the same batch and must still get its tokens.
+	store.failNextState.Store(true)
+	stream := &fakeTokenBucketsStream{
+		requests: []*resource_manager.TokenBucketsRequest{{
+			TargetRequestPeriodMs: 1000,
+			ClientUniqueId:        1,
+			Requests: []*resource_manager.TokenBucketRequest{
+				newRUTokenBucketRequest(1, "bad-group", 10),
+				newRUTokenBucketRequest(1, "good-group", 10),
+			},
+		}},
+	}
+	svc := &Service{ctx: context.Background(), manager: m}
+
+	re.NoError(svc.AcquireTokenBuckets(stream),
+		"a single group's lazy-load failure must not fail the stream")
+	re.Len(stream.sent, 1)
+	re.Len(stream.sent[0].Responses, 1, "only the loadable group should be answered")
+	re.Equal("good-group", stream.sent[0].Responses[0].ResourceGroupName)
+}
diff --git a/pkg/mcs/resourcemanager/server/metadata_watcher_test.go b/pkg/mcs/resourcemanager/server/metadata_watcher_test.go
index 1335492a9b..ce0cc762b3 100644
--- a/pkg/mcs/resourcemanager/server/metadata_watcher_test.go
+++ b/pkg/mcs/resourcemanager/server/metadata_watcher_test.go
@@ -388,3 +388,52 @@ func TestInitializeMetadataWatcher(t *testing.T) {
 		re.InDelta(123.5, m.GetKeyspaceServiceLimiter(10).ServiceLimit, 0.00001)
 	})
 }
+
+// TestMetadataWatcherModeReleasesSyncLoadedGroups guards against the
+// sync-loaded markers accumulating for the process lifetime in metadata watcher
+// mode: no async loader runs there, so nothing ever consumes them, and every
+// watch event would otherwise add an entry that is never removed.
+func TestMetadataWatcherModeReleasesSyncLoadedGroups(t *testing.T) {
+	re := require.New(t)
+
+	m := newManagerBase(&ControllerConfig{}, ResourceGroupWriteRoleLegacyAll)
+	m.storage = storage.NewStorageWithMemoryBackend()
+	m.srv = &testBasicServer{}
+	m.enableMetadataWatcher = true
+	re.NotNil(m.syncLoadedGroups)
+
+	originalFactory := newMetadataLoopWatcher
+	defer func() { newMetadataLoopWatcher = originalFactory }()
+	newMetadataLoopWatcher = func(
+		_ context.Context,
+		_ *sync.WaitGroup,
+		_ *clientv3.Client,
+		_, _ string,
+		_ func([]*clientv3.Event) error,
+		_, _ func(*mvccpb.KeyValue) error,
+		_ func([]*clientv3.Event) error,
+		_ bool,
+	) metadataLoopWatcher {
+		return &fakeMetadataLoopWatcher{waitLoadFn: func() error { return nil }}
+	}
+
+	re.NoError(m.Init(context.Background()))
+	defer m.close()
+
+	re.Equal(LoadingStateCompleted, m.getLoadingState())
+	m.RLock()
+	syncLoadedGroups := m.syncLoadedGroups
+	m.RUnlock()
+	re.Nil(syncLoadedGroups, "watcher mode must not retain sync-loaded markers")
+
+	// A watch event after initialization must stay a no-op for the markers.
+	group := newMetadataWatcherResourceGroup("watched", middlePriority, 100, 100)
+	rawValue, err := proto.Marshal(group)
+	re.NoError(err)
+	re.NoError(m.applyResourceGroupSettingFromRaw(10, "watched", string(rawValue)))
+	m.RLock()
+	syncLoadedGroups = m.syncLoadedGroups
+	m.RUnlock()
+	re.Nil(syncLoadedGroups)
+	re.NotNil(m.getKeyspaceResourceGroupManager(10).getResourceGroup("watched", false))
+}
diff --git a/pkg/mcs/resourcemanager/server/metrics.go b/pkg/mcs/resourcemanager/server/metrics.go
index 3bdac0f9fd..59bc2cf054 100644
--- a/pkg/mcs/resourcemanager/server/metrics.go
+++ b/pkg/mcs/resourcemanager/server/metrics.go
@@ -235,6 +235,31 @@ var (
 			Subsystem: serverSubsystem,
 			Name:      "async_load_group_duration_seconds",
 			Help:      "Duration of asynchronous resource group loading in seconds.",
+			// The whole point of the async loading is that a cluster with many
+			// resource groups can take far longer than the default buckets' 10s
+			// ceiling, so use a coarse but wide exponential range instead: it
+			// spans sub-second up to ~27min, which keeps the slow loads this
+			// metric exists to observe out of the +Inf bucket.
+			Buckets: prometheus.ExponentialBuckets(0.1, 4, 8),
+		})
+
+	asyncLoadGroupFailureCounter = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Namespace: namespace,
+			Subsystem: serverSubsystem,
+			Name:      "async_load_group_failures_total",
+			Help:      "Total number of failed attempts to load resource groups asynchronously.",
+		})
+
+	// resourceGroupLoadingStateGauge exposes the loading state so a load that
+	// keeps failing is alertable: the loader retries indefinitely, so without
+	// this the only symptom is a stuck state plus a periodic warning log.
+	resourceGroupLoadingStateGauge = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: serverSubsystem,
+			Name:      "resource_group_loading_state",
+			Help:      "Current resource group loading state: 0 - not started, 1 - in progress, 2 - completed.",
 		})
 )
 
@@ -292,6 +317,8 @@ func init() {
 	prometheus.MustRegister(pushRUMetricsDuration)
 	prometheus.MustRegister(syncLoadGroupCounter)
 	prometheus.MustRegister(asyncLoadGroupDuration)
+	prometheus.MustRegister(asyncLoadGroupFailureCounter)
+	prometheus.MustRegister(resourceGroupLoadingStateGauge)
 }
 
 func newMetrics() *metrics {

From 458e69f2d4574e81b6faaff80829ad63f32dead0 Mon Sep 17 00:00:00 2001
From: bufferflies <1045931706@qq.com>
Date: Mon, 27 Jul 2026 13:42:02 +0200
Subject: [PATCH 29/50] resource_group: close remaining async-load
 epoch/watcher races

Address the two still-open review comments on the async resource
group loading change.

- loadResourceGroupIfNeeded treated LoadingStateCompleted as proof the
  cache is authoritative, but in metadata-watcher mode the cache is
  only eventually consistent: PD writes metadata directly and the
  watcher applies it asynchronously, so a write can outrace its own
  watcher event. Gate the early return on !enableMetadataWatcher so
  watcher mode always falls through to a storage point load.
- storeLoadingStateIfCurrent releases the manager lock as soon as it
  verifies the epoch, so that check did not cover the initReserved
  call that followed it: a re-election landing in the gap could let a
  stale loader synthesize and persist a default into the new term.
  initReserved now takes the epoch and re-verifies it under the
  manager lock immediately before touching krgms.

Signed-off-by: bufferflies <1045931706@qq.com>
---
 pkg/mcs/resourcemanager/server/manager.go     | 32 ++++++++++++++++---
 .../server/metadata_watcher.go                |  5 ++-
 2 files changed, 32 insertions(+), 5 deletions(-)

diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go
index fd79af970e..9256e6b402 100644
--- a/pkg/mcs/resourcemanager/server/manager.go
+++ b/pkg/mcs/resourcemanager/server/manager.go
@@ -476,8 +476,9 @@ func (m *Manager) loadKeyspaceResourceGroups() error {
 	m.krgms = tempKrgms
 	m.syncLoadedGroups = nil
 	m.setLoadingState(LoadingStateCompleted)
+	epoch := m.loadEpoch
 	m.Unlock()
-	m.initReserved()
+	m.initReserved(epoch)
 	return m.loadServiceLimits()
 }
 
@@ -651,7 +652,12 @@ func (m *Manager) asyncLoadResourceGroups(ctx context.Context, epoch uint64) {
 			log.Info("async loading resource groups aborted: manager was reinitialized")
 			return
 		}
-		m.initReserved()
+		// storeLoadingStateIfCurrent releases m.Lock as soon as it verifies the
+		// epoch, so that check alone does not cover initReserved below: a
+		// re-election could still land in the gap between the check returning
+		// and initReserved running. Re-verify the epoch immediately before
+		// initReserved touches krgms, closing that window.
+		m.initReserved(epoch)
 		duration := time.Since(startTime)
 		asyncLoadGroupDuration.Observe(duration.Seconds())
 		log.Info("async loading resource groups completed", zap.Int("loaded-groups", loaded), zap.Duration("duration", duration))
@@ -726,7 +732,13 @@ func (m *Manager) loadResourceGroup(keyspaceID uint32, name string) (*ResourceGr
 }
 
 func (m *Manager) loadResourceGroupIfNeeded(keyspaceID uint32, name string) error {
-	if m.getLoadingState() == LoadingStateCompleted {
+	// In metadata-watcher mode the cache is only eventually consistent with
+	// storage: PD writes metadata directly and the watcher applies it
+	// asynchronously, so LoadingStateCompleted here only means the initial
+	// watcher bootstrap finished, not that every subsequent write has been
+	// observed yet. Always fall through to a point load in that mode so a
+	// write that outraces its own watcher event is still visible.
+	if !m.enableMetadataWatcher && m.getLoadingState() == LoadingStateCompleted {
 		return nil
 	}
 	krgm := m.getKeyspaceResourceGroupManager(keyspaceID)
@@ -971,7 +983,19 @@ func (m *Manager) applyResourceGroupStatesFromRaw(keyspaceID uint32, name, rawVa
 	return nil
 }
 
-func (m *Manager) initReserved() {
+// initReserved backfills the default resource group for every keyspace whose
+// default wasn't confirmed by loading. The caller must have just verified
+// epoch via storeLoadingStateIfCurrent; re-verify it here under the manager
+// lock immediately before touching krgms, since that earlier check alone
+// does not cover this call once its lock is released.
+func (m *Manager) initReserved(epoch uint64) {
+	m.Lock()
+	if m.loadEpoch != epoch {
+		m.Unlock()
+		log.Info("skip initReserved: manager was reinitialized")
+		return
+	}
+	m.Unlock()
 	// Initialize the null keyspace resource group manager if it doesn't exist.
 	m.getOrCreateKeyspaceResourceGroupManager(constant.NullKeyspaceID, true)
 	// Initialize the default resource group respectively for each keyspace if it doesn't exist.
diff --git a/pkg/mcs/resourcemanager/server/metadata_watcher.go b/pkg/mcs/resourcemanager/server/metadata_watcher.go
index 0f8a97d697..eb8e7c133f 100644
--- a/pkg/mcs/resourcemanager/server/metadata_watcher.go
+++ b/pkg/mcs/resourcemanager/server/metadata_watcher.go
@@ -180,7 +180,10 @@ func (m *Manager) initializeMetadataWatcher(ctx context.Context) error {
 		return err
 	}
 	// Ensure reserved default groups exist even if settings were missing in storage.
-	m.initReserved()
+	m.RLock()
+	epoch := m.loadEpoch
+	m.RUnlock()
+	m.initReserved(epoch)
 	return nil
 }
 

From 1d7726fc330c925f22607058589f94d995f21000 Mon Sep 17 00:00:00 2001
From: bufferflies <1045931706@qq.com>
Date: Tue, 28 Jul 2026 14:18:17 +0200
Subject: [PATCH 30/50] resource_group: stop racy default backfill and fix
 retry status code

The async loader ran initReserved after publishing LoadingStateCompleted,
so its unconditional synthesize-and-persist of missing default groups
could race a concurrent Add/ModifyResourceGroup for the same keyspace and
clobber the confirmed write in both storage and cache. Live requests
already synthesize a missing default on demand, sequenced with their own
subsequent write, so drop the async loader's eager backfill entirely and
keep it only on the two construction-time paths that run before the
manager serves any request (loadKeyspaceResourceGroups and
initializeMetadataWatcher), where no concurrent writer can race it.

Also export wrapLoadingError as WrapLoadingError and apply it in the
standalone-mode resourceGroupProxyServer's local Add/Modify/Delete
handlers. That mapping previously only ran in the RM microservice's gRPC
service; the proxy's handlers returned metadataManager errors directly,
so ErrResourceGroupsLoading surfaced as codes.Unknown there instead of
the retryable codes.Unavailable.

Signed-off-by: bufferflies <1045931706@qq.com>
---
 .../resourcemanager/server/grpc_service.go    | 17 +++++---
 pkg/mcs/resourcemanager/server/manager.go     | 43 ++++++++++++-------
 .../server/metadata_watcher.go                |  5 ++-
 server/resource_group_proxy_service.go        |  6 +--
 4 files changed, 45 insertions(+), 26 deletions(-)

diff --git a/pkg/mcs/resourcemanager/server/grpc_service.go b/pkg/mcs/resourcemanager/server/grpc_service.go
index f6c95224ab..14fab1dfc7 100644
--- a/pkg/mcs/resourcemanager/server/grpc_service.go
+++ b/pkg/mcs/resourcemanager/server/grpc_service.go
@@ -102,10 +102,13 @@ func (s *Service) checkServing() error {
 	return nil
 }
 
-// wrapLoadingError converts the retryable "resource groups are still loading"
+// WrapLoadingError converts the retryable "resource groups are still loading"
 // error into codes.Unavailable, so generic client-side retry logic can act on
 // it instead of seeing an opaque codes.Unknown. Other errors pass through.
-func wrapLoadingError(err error) error {
+// Exported because the standalone-mode metadata proxy (server.resourceGroup-
+// ProxyServer) calls the local Manager directly instead of going through
+// this gRPC service, and must apply the same mapping itself.
+func WrapLoadingError(err error) error {
 	if errs.ErrResourceGroupsLoading.Equal(err) {
 		return status.Error(codes.Unavailable, err.Error())
 	}
@@ -120,7 +123,7 @@ func (s *Service) GetResourceGroup(_ context.Context, req *rmpb.GetResourceGroup
 	keyspaceID := ExtractKeyspaceID(req.GetKeyspaceId())
 	rg, err := s.manager.GetResourceGroup(keyspaceID, req.ResourceGroupName, req.WithRuStats)
 	if err != nil {
-		return nil, wrapLoadingError(err)
+		return nil, WrapLoadingError(err)
 	}
 	if rg == nil {
 		return nil, errs.ErrResourceGroupNotExists.FastGenByArgs(req.ResourceGroupName)
@@ -139,7 +142,7 @@ func (s *Service) ListResourceGroups(_ context.Context, req *rmpb.ListResourceGr
 	keyspaceID := ExtractKeyspaceID(req.GetKeyspaceId())
 	groups, err := s.manager.GetResourceGroupList(keyspaceID, req.WithRuStats)
 	if err != nil {
-		return nil, wrapLoadingError(err)
+		return nil, WrapLoadingError(err)
 	}
 	resps := &rmpb.ListResourceGroupsResponse{
 		Groups: make([]*rmpb.ResourceGroup, 0, len(groups)),
@@ -161,7 +164,7 @@ func (s *Service) AddResourceGroup(_ context.Context, req *rmpb.PutResourceGroup
 	}
 	err := s.manager.AddResourceGroup(req.GetGroup())
 	if err != nil {
-		return nil, wrapLoadingError(err)
+		return nil, WrapLoadingError(err)
 	}
 	return &rmpb.PutResourceGroupResponse{Body: "Success!"}, nil
 }
@@ -176,7 +179,7 @@ func (s *Service) DeleteResourceGroup(_ context.Context, req *rmpb.DeleteResourc
 	}
 	err := s.manager.DeleteResourceGroup(ExtractKeyspaceID(req.GetKeyspaceId()), req.ResourceGroupName)
 	if err != nil {
-		return nil, wrapLoadingError(err)
+		return nil, WrapLoadingError(err)
 	}
 	return &rmpb.DeleteResourceGroupResponse{Body: "Success!"}, nil
 }
@@ -191,7 +194,7 @@ func (s *Service) ModifyResourceGroup(_ context.Context, req *rmpb.PutResourceGr
 	}
 	err := s.manager.ModifyResourceGroup(req.GetGroup())
 	if err != nil {
-		return nil, wrapLoadingError(err)
+		return nil, WrapLoadingError(err)
 	}
 	return &rmpb.PutResourceGroupResponse{Body: "Success!"}, nil
 }
diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go
index 9256e6b402..8974f26f56 100644
--- a/pkg/mcs/resourcemanager/server/manager.go
+++ b/pkg/mcs/resourcemanager/server/manager.go
@@ -478,6 +478,10 @@ func (m *Manager) loadKeyspaceResourceGroups() error {
 	m.setLoadingState(LoadingStateCompleted)
 	epoch := m.loadEpoch
 	m.Unlock()
+	// This runs to completion before the manager is exposed to any request
+	// (NewMetadataOnlyManager's caller has not returned yet), so there is no
+	// live writer to race against; eagerly confirming every loaded keyspace's
+	// default here is safe, unlike the same backfill from the async loader.
 	m.initReserved(epoch)
 	return m.loadServiceLimits()
 }
@@ -639,25 +643,23 @@ func (m *Manager) asyncLoadResourceGroups(ctx context.Context, epoch uint64) {
 		m.syncLoadedGroups = nil
 		m.Unlock()
 
-		// Publish completion before backfilling reserved defaults. Unlike every
-		// other shared-state mutation here, initReserved re-resolves managers and
-		// persists synthetic defaults without an epoch guard, so a re-election
-		// landing in this window could make a stale loader clobber the new term's
-		// not-yet-loaded default. storeLoadingStateIfCurrent is epoch-guarded, so
-		// gating on it first makes a stale loader return without ever running
-		// initReserved. A keyspace whose default this loader would have backfilled
-		// is still covered: once loading is complete, getOrCreateKeyspaceResource-
-		// GroupManager synthesizes the default directly on demand.
+		// No eager reserved-default backfill runs here. An eager pass would
+		// re-resolve every keyspace manager and persist a synthetic default
+		// concurrently with live requests: once completion is published below,
+		// a request for a keyspace whose default was never customized can
+		// synthesize and publish it (via getOrCreateKeyspaceResourceGroupManager
+		// or loadResourceGroupIfNeeded's confirmed-not-found path) in the same
+		// goroutine as its own subsequent write, with no independent writer
+		// racing it. An out-of-band backfill loop has no such ordering guarantee
+		// against those on-demand writers, so it can persist a synthetic default
+		// after a concurrent customized write and silently discard it. A
+		// keyspace nobody ever queries needs no persisted default: the persist
+		// loop already skips unconfirmed reserved placeholders, so leaving one
+		// reserved indefinitely is a normal, accepted state, not a leak.
 		if !m.storeLoadingStateIfCurrent(epoch, LoadingStateCompleted) {
 			log.Info("async loading resource groups aborted: manager was reinitialized")
 			return
 		}
-		// storeLoadingStateIfCurrent releases m.Lock as soon as it verifies the
-		// epoch, so that check alone does not cover initReserved below: a
-		// re-election could still land in the gap between the check returning
-		// and initReserved running. Re-verify the epoch immediately before
-		// initReserved touches krgms, closing that window.
-		m.initReserved(epoch)
 		duration := time.Since(startTime)
 		asyncLoadGroupDuration.Observe(duration.Seconds())
 		log.Info("async loading resource groups completed", zap.Int("loaded-groups", loaded), zap.Duration("duration", duration))
@@ -988,6 +990,17 @@ func (m *Manager) applyResourceGroupStatesFromRaw(keyspaceID uint32, name, rawVa
 // epoch via storeLoadingStateIfCurrent; re-verify it here under the manager
 // lock immediately before touching krgms, since that earlier check alone
 // does not cover this call once its lock is released.
+//
+// Only call this from a path that runs before the manager serves any
+// request (construction-time full loads), not from the async loader: once
+// requests are flowing, a concurrent Add/ModifyResourceGroup can confirm a
+// keyspace's default between this function's existence check and its
+// unconditional persist, and this backfill's synthetic write would then
+// silently clobber that write in both storage and the live cache. Live
+// requests already synthesize a missing default on demand (via
+// getOrCreateKeyspaceResourceGroupManager or loadResourceGroupIfNeeded's
+// confirmed-not-found path) sequenced with their own subsequent write, so no
+// out-of-band backfill is needed once serving has started.
 func (m *Manager) initReserved(epoch uint64) {
 	m.Lock()
 	if m.loadEpoch != epoch {
diff --git a/pkg/mcs/resourcemanager/server/metadata_watcher.go b/pkg/mcs/resourcemanager/server/metadata_watcher.go
index eb8e7c133f..3a2410600e 100644
--- a/pkg/mcs/resourcemanager/server/metadata_watcher.go
+++ b/pkg/mcs/resourcemanager/server/metadata_watcher.go
@@ -179,7 +179,10 @@ func (m *Manager) initializeMetadataWatcher(ctx context.Context) error {
 	if err := watcher.WaitLoad(); err != nil {
 		return err
 	}
-	// Ensure reserved default groups exist even if settings were missing in storage.
+	// This runs synchronously before Init() returns and before
+	// LoadingStateCompleted is published, so no request can be in flight yet
+	// to race against; ensure reserved default groups exist even if settings
+	// were missing in storage.
 	m.RLock()
 	epoch := m.loadEpoch
 	m.RUnlock()
diff --git a/server/resource_group_proxy_service.go b/server/resource_group_proxy_service.go
index 72f1f249ae..71eead8d63 100644
--- a/server/resource_group_proxy_service.go
+++ b/server/resource_group_proxy_service.go
@@ -134,7 +134,7 @@ func (s *resourceGroupProxyServer) AddResourceGroup(ctx context.Context, req *re
 		return nil, status.Error(codes.Internal, "resource group metadata manager is not initialized")
 	}
 	if err := s.metadataManager.AddResourceGroup(req.GetGroup()); err != nil {
-		return nil, err
+		return nil, rm_server.WrapLoadingError(err)
 	}
 	return &resource_manager.PutResourceGroupResponse{Body: "Success!"}, nil
 }
@@ -162,7 +162,7 @@ func (s *resourceGroupProxyServer) ModifyResourceGroup(ctx context.Context, req
 		return nil, status.Error(codes.Internal, "resource group metadata manager is not initialized")
 	}
 	if err := s.metadataManager.ModifyResourceGroup(req.GetGroup()); err != nil {
-		return nil, err
+		return nil, rm_server.WrapLoadingError(err)
 	}
 	return &resource_manager.PutResourceGroupResponse{Body: "Success!"}, nil
 }
@@ -193,7 +193,7 @@ func (s *resourceGroupProxyServer) DeleteResourceGroup(ctx context.Context, req
 		rm_server.ExtractKeyspaceID(req.GetKeyspaceId()),
 		req.GetResourceGroupName(),
 	); err != nil {
-		return nil, err
+		return nil, rm_server.WrapLoadingError(err)
 	}
 	return &resource_manager.DeleteResourceGroupResponse{Body: "Success!"}, nil
 }

From f8d99b2a4ca73c7f16cae5361ca5a312d1d8e327 Mon Sep 17 00:00:00 2001
From: bufferflies <1045931706@qq.com>
Date: Thu, 30 Jul 2026 14:07:33 +0200
Subject: [PATCH 31/50] resource_group: serialize default group synthesis
 against real writes

initDefaultResourceGroup's check-then-persist wasn't serialized against
itself or against a real Add/ModifyResourceGroup(default), so a synthetic
write could commit to storage and cache after a concurrent customized
write, silently discarding it. A defaultGroupMu now serializes every path
that can create or persist the default group, with a re-check for
confirmed data after acquiring it.

The lazy-load confirmed-not-found synthesis also bypassed
publishResourceGroupMutation, so it never set a sync-loaded marker; a
still-in-progress bulk merge could then replace the newly published
default (and any live consumption update after it) with a stale storage
snapshot. initDefaultResourceGroup now reports whether it performed a
synthesis so that path can mark it sync-loaded.

Signed-off-by: bufferflies <1045931706@qq.com>
---
 .../server/keyspace_manager.go                | 45 ++++++++++++++-----
 pkg/mcs/resourcemanager/server/manager.go     | 30 ++++++++++++-
 2 files changed, 63 insertions(+), 12 deletions(-)

diff --git a/pkg/mcs/resourcemanager/server/keyspace_manager.go b/pkg/mcs/resourcemanager/server/keyspace_manager.go
index b716796f35..48393bdc0e 100644
--- a/pkg/mcs/resourcemanager/server/keyspace_manager.go
+++ b/pkg/mcs/resourcemanager/server/keyspace_manager.go
@@ -80,6 +80,15 @@ type keyspaceResourceGroupManager struct {
 	// read and re-checks it under the lock before inserting, so a group
 	// deleted after the read is never resurrected by the now-stale result.
 	deleteGen uint64
+	// defaultGroupMu serializes every path that can create or persist the
+	// default group: on-demand synthesis (initDefaultResourceGroup) and a
+	// real Add/ModifyResourceGroup targeting "default" both hold it across
+	// their persist step. Without this, a synthetic write and a concurrent
+	// customized write race independently of krgm's RWMutex (which is only
+	// held for the cache mutation, not the storage I/O before it), so
+	// whichever one's storage/cache write lands last wins even if it
+	// started first - silently discarding a successful customized write.
+	defaultGroupMu syncutil.Mutex
 
 	keyspaceID uint32
 	storage    endpoint.ResourceGroupStorage
@@ -223,22 +232,36 @@ func (krgm *keyspaceResourceGroupManager) setRawStatesIntoResourceGroup(name str
 	return nil
 }
 
-func (krgm *keyspaceResourceGroupManager) initDefaultResourceGroup() {
-	krgm.RLock()
-	_, ok := krgm.groups[DefaultResourceGroupName]
-	_, reserved := krgm.reservedGroups[DefaultResourceGroupName]
-	krgm.RUnlock()
-	// A cached entry only makes initialization unnecessary if it's confirmed
-	// data. A reserved placeholder means nothing is persisted for the default
-	// group (e.g. a fresh store): it must still be created and persisted here,
-	// otherwise its settings are never stored and state persistence stays skipped.
-	if ok && !reserved {
-		return
+// initDefaultResourceGroup synthesizes and persists the built-in default
+// group if nothing confirmed exists yet. It reports whether it actually
+// performed a synthesis, so callers that participate in the async bulk-load
+// merge (loadResourceGroupIfNeeded) know when they must publish a
+// sync-loaded marker for what this call just wrote.
+func (krgm *keyspaceResourceGroupManager) initDefaultResourceGroup() bool {
+	// A confirmed cached entry means initialization is unnecessary; a missing
+	// or reserved-placeholder entry means nothing is persisted for the
+	// default group (e.g. a fresh store), so it must still be created and
+	// persisted, otherwise its settings are never stored and state
+	// persistence stays skipped.
+	if krgm.hasConfirmedResourceGroup(DefaultResourceGroupName) {
+		return false
+	}
+	// Serialize against every other synthesis or real Add/ModifyResourceGroup
+	// targeting "default": see the defaultGroupMu doc comment on the struct.
+	krgm.defaultGroupMu.Lock()
+	defer krgm.defaultGroupMu.Unlock()
+	// Re-check under defaultGroupMu: while this goroutine waited for the
+	// lock, a real write may have already confirmed the default group, in
+	// which case synthesizing here would silently clobber it.
+	if krgm.hasConfirmedResourceGroup(DefaultResourceGroupName) {
+		return false
 	}
 	defaultGroup := newDefaultResourceGroup()
 	if err := krgm.addResourceGroup(defaultGroup.IntoProtoResourceGroup(krgm.keyspaceID)); err != nil {
 		log.Warn("init default group failed", zap.Uint32("keyspace-id", krgm.keyspaceID), zap.Error(err))
+		return false
 	}
+	return true
 }
 
 func (krgm *keyspaceResourceGroupManager) ensureReservedDefaultGroupInCache() {
diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go
index 8974f26f56..ea90661c9c 100644
--- a/pkg/mcs/resourcemanager/server/manager.go
+++ b/pkg/mcs/resourcemanager/server/manager.go
@@ -797,7 +797,19 @@ func (m *Manager) loadResourceGroupIfNeeded(keyspaceID uint32, name string) erro
 				// This calls initDefaultResourceGroup directly instead of going
 				// through getOrCreateKeyspaceResourceGroupManager(id, true), which
 				// now routes back into this same function and would recurse.
-				krgm.initDefaultResourceGroup()
+				if krgm.initDefaultResourceGroup() {
+					// This synthesis bypassed publishResourceGroupMutation, so the
+					// sync-loaded marker was never set. Set it now: otherwise a
+					// bulk merge still in progress for this term doesn't know this
+					// group is already confirmed, and can replace it - including
+					// any live consumption update applied after this point - with
+					// a possibly-stale snapshot taken by the storage scan.
+					m.Lock()
+					if m.loadEpoch == epoch && m.krgms[keyspaceID] == krgm && m.syncLoadedGroups != nil {
+						m.syncLoadedGroups[trackerKey{keyspaceID: keyspaceID, groupName: DefaultResourceGroupName}] = true
+					}
+					m.Unlock()
+				}
 				return nil
 			}
 			return err
@@ -1101,6 +1113,16 @@ func (m *Manager) AddResourceGroup(grouppb *rmpb.ResourceGroup) error {
 		log.Warn("failed to load resource group before add", zap.Uint32("keyspace-id", keyspaceID), zap.String("name", grouppb.Name), zap.Error(err))
 		return err
 	}
+	if grouppb.Name == DefaultResourceGroupName {
+		// Serialize against a concurrent on-demand synthesis of the same
+		// default group (initDefaultResourceGroup, e.g. from another
+		// request's getOrCreateKeyspaceResourceGroupManager/
+		// loadResourceGroupIfNeeded call): without this, the synthetic
+		// write's storage/cache commit can land after this real write's,
+		// silently discarding these customized settings.
+		krgm.defaultGroupMu.Lock()
+		defer krgm.defaultGroupMu.Unlock()
+	}
 	// Storage phase: validate and persist. Publishing the cache effect is done
 	// separately below, against whichever keyspace manager is current then.
 	group, err := krgm.persistResourceGroup(grouppb)
@@ -1130,6 +1152,12 @@ func (m *Manager) ModifyResourceGroup(grouppb *rmpb.ResourceGroup) error {
 	if err != nil {
 		return err
 	}
+	if grouppb.Name == DefaultResourceGroupName {
+		// Serialize against a concurrent on-demand synthesis of the same
+		// default group; see the matching guard in AddResourceGroup.
+		krgm.defaultGroupMu.Lock()
+		defer krgm.defaultGroupMu.Unlock()
+	}
 	patched, err := krgm.modifyResourceGroup(grouppb)
 	if err != nil {
 		return err

From ba8534d05a1c9bc8e7203a8eadb7aba77871b6c1 Mon Sep 17 00:00:00 2001
From: bufferflies <1045931706@qq.com>
Date: Mon, 3 Aug 2026 09:14:04 +0200
Subject: [PATCH 32/50] resource_group: guard cross-term mutation publish and
 narrow default-synthesis race

publishResourceGroupMutation re-resolved whichever keyspace manager was
current at publish time and always applied the mutation's result into
it. A mutation parked between its storage phase and this publish (e.g.
across a leadership change) could resume after a newer term's own
Add/Modify/Delete for the same group had already published confirmed
data, silently overwriting it with the older result and marking it
sync-loaded - hiding the newer data from the bulk merge too. The publish
is now skipped when the current manager differs from the one the caller
persisted against AND that manager already has confirmed (non-reserved)
data for the group; when the new term hasn't confirmed the group yet,
the mutation still publishes into it, since there is nothing newer to
protect - this keeps the existing cross-term Delete/Modify tests passing.

initDefaultResourceGroup's defaultGroupMu only serializes callers that
share the same keyspaceResourceGroupManager instance; it does nothing
across a term change, since Init gives the new term an entirely separate
krgm with its own mutex. initDefaultResourceGroup now accepts an
optional stillCurrent callback, checked immediately after acquiring
defaultGroupMu, so a caller with access to the Manager can detect a term
change that completed before the persist starts and skip synthesizing
against a detached manager. This narrows, but does not eliminate, the
window: a term change landing while the persist's storage write is
already in flight still can't be caught by an in-memory check alone -
closing that residual gap needs a conditional (CAS) storage write.

Signed-off-by: bufferflies <1045931706@qq.com>
---
 .../server/keyspace_manager.go                | 21 ++++-
 .../server/keyspace_manager_test.go           |  6 +-
 pkg/mcs/resourcemanager/server/manager.go     | 79 +++++++++++++------
 3 files changed, 77 insertions(+), 29 deletions(-)

diff --git a/pkg/mcs/resourcemanager/server/keyspace_manager.go b/pkg/mcs/resourcemanager/server/keyspace_manager.go
index 48393bdc0e..01b629329c 100644
--- a/pkg/mcs/resourcemanager/server/keyspace_manager.go
+++ b/pkg/mcs/resourcemanager/server/keyspace_manager.go
@@ -237,7 +237,20 @@ func (krgm *keyspaceResourceGroupManager) setRawStatesIntoResourceGroup(name str
 // performed a synthesis, so callers that participate in the async bulk-load
 // merge (loadResourceGroupIfNeeded) know when they must publish a
 // sync-loaded marker for what this call just wrote.
-func (krgm *keyspaceResourceGroupManager) initDefaultResourceGroup() bool {
+// initDefaultResourceGroup synthesizes and persists the built-in default
+// group if nothing confirmed exists yet. defaultGroupMu only serializes
+// callers that share this krgm instance; it does nothing across a term
+// change, since Init gives the new term an entirely new krgm object with its
+// own, separate defaultGroupMu. stillCurrent, when non-nil, is checked
+// immediately before the persist (right after defaultGroupMu is acquired) so
+// a caller with access to the owning Manager can catch a term change that
+// completed before this point - typically by comparing this krgm against the
+// manager's live entry for its keyspace ID. It cannot catch a term change
+// that lands while the persist's storage write is already in flight; closing
+// that residual window needs a conditional (CAS) storage write, which this
+// does not implement. Pass nil when no such coordination is needed or
+// available (e.g. in tests that exercise krgm without a Manager).
+func (krgm *keyspaceResourceGroupManager) initDefaultResourceGroup(stillCurrent func() bool) bool {
 	// A confirmed cached entry means initialization is unnecessary; a missing
 	// or reserved-placeholder entry means nothing is persisted for the
 	// default group (e.g. a fresh store), so it must still be created and
@@ -247,7 +260,8 @@ func (krgm *keyspaceResourceGroupManager) initDefaultResourceGroup() bool {
 		return false
 	}
 	// Serialize against every other synthesis or real Add/ModifyResourceGroup
-	// targeting "default": see the defaultGroupMu doc comment on the struct.
+	// targeting "default" that shares this krgm instance: see the
+	// defaultGroupMu doc comment on the struct.
 	krgm.defaultGroupMu.Lock()
 	defer krgm.defaultGroupMu.Unlock()
 	// Re-check under defaultGroupMu: while this goroutine waited for the
@@ -256,6 +270,9 @@ func (krgm *keyspaceResourceGroupManager) initDefaultResourceGroup() bool {
 	if krgm.hasConfirmedResourceGroup(DefaultResourceGroupName) {
 		return false
 	}
+	if stillCurrent != nil && !stillCurrent() {
+		return false
+	}
 	defaultGroup := newDefaultResourceGroup()
 	if err := krgm.addResourceGroup(defaultGroup.IntoProtoResourceGroup(krgm.keyspaceID)); err != nil {
 		log.Warn("init default group failed", zap.Uint32("keyspace-id", krgm.keyspaceID), zap.Error(err))
diff --git a/pkg/mcs/resourcemanager/server/keyspace_manager_test.go b/pkg/mcs/resourcemanager/server/keyspace_manager_test.go
index 7cf5ad4ca1..ee2562fffd 100644
--- a/pkg/mcs/resourcemanager/server/keyspace_manager_test.go
+++ b/pkg/mcs/resourcemanager/server/keyspace_manager_test.go
@@ -87,7 +87,7 @@ func TestInitDefaultResourceGroup(t *testing.T) {
 	re.False(exists)
 
 	// Initialize the default resource group.
-	krgm.initDefaultResourceGroup()
+	krgm.initDefaultResourceGroup(nil)
 
 	// Verify the default resource group is created.
 	defaultGroup, exists := krgm.groups[DefaultResourceGroupName]
@@ -225,7 +225,7 @@ func TestDeleteResourceGroupBehavior(t *testing.T) {
 		_, ok := krgm.groupRUTrackers[group.GetName()]
 		re.False(ok)
 
-		krgm.initDefaultResourceGroup()
+		krgm.initDefaultResourceGroup(nil)
 		re.Error(krgm.deleteResourceGroup(DefaultResourceGroupName))
 		re.NotNil(krgm.getResourceGroup(DefaultResourceGroupName, false))
 	})
@@ -361,7 +361,7 @@ func TestGetResourceGroupList(t *testing.T) {
 	re.Equal("group2", groups[1].Name)
 	re.Equal("group3", groups[2].Name)
 
-	krgm.initDefaultResourceGroup()
+	krgm.initDefaultResourceGroup(nil)
 	groups = krgm.getResourceGroupList(false, true)
 	re.Len(groups, 4)
 	groups = krgm.getResourceGroupList(false, false)
diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go
index ea90661c9c..b6d1f1c886 100644
--- a/pkg/mcs/resourcemanager/server/manager.go
+++ b/pkg/mcs/resourcemanager/server/manager.go
@@ -333,7 +333,11 @@ func (m *Manager) getOrCreateKeyspaceResourceGroupManager(keyspaceID uint32, ini
 			// Async loading (if any) has already finished, so if the default
 			// group isn't cached yet it truly doesn't exist anywhere; it's
 			// safe to synthesize and persist it directly.
-			krgm.initDefaultResourceGroup()
+			krgm.initDefaultResourceGroup(func() bool {
+				m.RLock()
+				defer m.RUnlock()
+				return m.krgms[keyspaceID] == krgm
+			})
 		} else if err := m.loadResourceGroupIfNeeded(keyspaceID, DefaultResourceGroupName); err != nil {
 			log.Debug("failed to load default resource group", zap.Uint32("keyspace-id", keyspaceID), zap.Error(err))
 		}
@@ -797,7 +801,12 @@ func (m *Manager) loadResourceGroupIfNeeded(keyspaceID uint32, name string) erro
 				// This calls initDefaultResourceGroup directly instead of going
 				// through getOrCreateKeyspaceResourceGroupManager(id, true), which
 				// now routes back into this same function and would recurse.
-				if krgm.initDefaultResourceGroup() {
+				stillCurrent := func() bool {
+					m.RLock()
+					defer m.RUnlock()
+					return m.loadEpoch == epoch && m.krgms[keyspaceID] == krgm
+				}
+				if krgm.initDefaultResourceGroup(stillCurrent) {
 					// This synthesis bypassed publishResourceGroupMutation, so the
 					// sync-loaded marker was never set. Set it now: otherwise a
 					// bulk merge still in progress for this term doesn't know this
@@ -887,31 +896,48 @@ func (m *Manager) markResourceGroupSyncLoaded(keyspaceID uint32, krgm *keyspaceR
 // against whichever keyspace manager is current at publish time. The merge
 // holds the manager lock across its whole merge step, so the two can only be
 // fully ordered: publish first and the merge skips the marked group; merge
-// first and the publish overrides its stale snapshot. Because the current
-// manager is re-resolved inside this critical section, a mutation whose
-// storage phase straddled a leadership change still publishes into the live
-// term (and marks the same term's map) instead of a detached manager, so no
-// retry is needed.
+// first and the publish overrides its stale snapshot.
+//
+// krgm is the keyspace manager the caller actually persisted the mutation's
+// storage phase against. If it's no longer the live manager for keyspaceID
+// (Init replaced the whole krgms map for a new term while the caller was
+// blocked on storage I/O) AND the new term already has confirmed data for
+// this group - installed by its own bulk merge, lazy load, or a live
+// Add/Modify/Delete that raced ahead of this delayed one - fn's result was
+// computed from data read in the old term and is stale relative to that
+// confirmed data; applying it here would silently overwrite it in the cache
+// and, via the sync-loaded marker, hide the group from the new term's bulk
+// merge too, so the mutation is dropped instead. If the new term hasn't
+// confirmed this group yet (missing, or still a reserved placeholder), there
+// is nothing newer to protect, so fn's result is applied into the new term's
+// manager - the two async-load tests exercise exactly this case (a mutation
+// straddling a leadership change with no competing new-term write) and
+// require it to still take effect.
 //
 // fn runs with the keyspace manager write lock held and must not do I/O; it
 // returns whether to record the sync-loaded marker and, when a group was
 // (re)installed, the group to sync burstability for.
 func (m *Manager) publishResourceGroupMutation(
-	keyspaceID uint32, name string,
+	keyspaceID uint32, name string, krgm *keyspaceResourceGroupManager,
 	fn func(krgm *keyspaceResourceGroupManager) (mark bool, synced *ResourceGroup),
 ) {
 	m.Lock()
 	defer m.Unlock()
-	krgm, ok := m.krgms[keyspaceID]
+	cur, ok := m.krgms[keyspaceID]
 	if !ok {
-		krgm = newKeyspaceResourceGroupManager(keyspaceID, m.storage, m.writeRole)
-		m.krgms[keyspaceID] = krgm
+		cur = newKeyspaceResourceGroupManager(keyspaceID, m.storage, m.writeRole)
+		m.krgms[keyspaceID] = cur
+	}
+	if cur != krgm && cur.hasConfirmedResourceGroup(name) {
+		log.Info("skip publishing resource group mutation: a newer confirmed write already exists",
+			zap.Uint32("keyspace-id", keyspaceID), zap.String("group-name", name))
+		return
 	}
-	krgm.Lock()
-	mark, synced := fn(krgm)
-	krgm.Unlock()
+	cur.Lock()
+	mark, synced := fn(cur)
+	cur.Unlock()
 	if synced != nil {
-		krgm.syncBurstabilityWithServiceLimit(synced)
+		cur.syncBurstabilityWithServiceLimit(synced)
 	}
 	if mark && m.syncLoadedGroups != nil {
 		m.syncLoadedGroups[trackerKey{keyspaceID: keyspaceID, groupName: name}] = true
@@ -1024,8 +1050,12 @@ func (m *Manager) initReserved(epoch uint64) {
 	// Initialize the null keyspace resource group manager if it doesn't exist.
 	m.getOrCreateKeyspaceResourceGroupManager(constant.NullKeyspaceID, true)
 	// Initialize the default resource group respectively for each keyspace if it doesn't exist.
+	// No stillCurrent check is needed: this whole function only runs before
+	// the manager serves any request (see the doc comment above), so there is
+	// no concurrent Add/ModifyResourceGroup or other initDefaultResourceGroup
+	// call to race against.
 	for _, krgm := range m.getKeyspaceResourceGroupManagers() {
-		krgm.initDefaultResourceGroup()
+		krgm.initDefaultResourceGroup(nil)
 	}
 }
 
@@ -1124,13 +1154,14 @@ func (m *Manager) AddResourceGroup(grouppb *rmpb.ResourceGroup) error {
 		defer krgm.defaultGroupMu.Unlock()
 	}
 	// Storage phase: validate and persist. Publishing the cache effect is done
-	// separately below, against whichever keyspace manager is current then.
+	// separately below, against krgm if it's still the live manager for
+	// keyspaceID by then, or dropped otherwise - see publishResourceGroupMutation.
 	group, err := krgm.persistResourceGroup(grouppb)
 	if err != nil {
 		return err
 	}
 	failpoint.InjectCall("addResourceGroupBeforePublish")
-	m.publishResourceGroupMutation(keyspaceID, grouppb.Name, func(cur *keyspaceResourceGroupManager) (bool, *ResourceGroup) {
+	m.publishResourceGroupMutation(keyspaceID, grouppb.Name, krgm, func(cur *keyspaceResourceGroupManager) (bool, *ResourceGroup) {
 		cur.groups[group.Name] = group
 		delete(cur.reservedGroups, group.Name)
 		return true, group
@@ -1163,7 +1194,7 @@ func (m *Manager) ModifyResourceGroup(grouppb *rmpb.ResourceGroup) error {
 		return err
 	}
 	failpoint.InjectCall("modifyResourceGroupBeforePublish")
-	m.publishResourceGroupMutation(keyspaceID, grouppb.Name, func(cur *keyspaceResourceGroupManager) (bool, *ResourceGroup) {
+	m.publishResourceGroupMutation(keyspaceID, grouppb.Name, krgm, func(cur *keyspaceResourceGroupManager) (bool, *ResourceGroup) {
 		var synced *ResourceGroup
 		if existing := cur.groups[grouppb.Name]; existing != patched {
 			// A different object sits in the current term's cache: either the
@@ -1208,14 +1239,14 @@ func (m *Manager) DeleteResourceGroup(keyspaceID uint32, name string) error {
 	}
 	failpoint.InjectCall("deleteResourceGroupBeforeStorage")
 	// Storage phase: validate and remove from storage. Publishing the cache
-	// effect is done separately below, against whichever keyspace manager is
-	// current then, so a delete straddling a leadership change still removes
-	// the group from the live cache and marks the live term's map (making the
-	// new bulk merge skip its pre-deletion snapshot of the group).
+	// effect is done separately below, against krgm if it's still the live
+	// manager for keyspaceID by then, or dropped otherwise (a delete
+	// straddling a leadership change) - see publishResourceGroupMutation. The
+	// storage removal above already took effect regardless.
 	if err := krgm.deleteResourceGroupFromStorage(name); err != nil {
 		return err
 	}
-	m.publishResourceGroupMutation(keyspaceID, name, func(cur *keyspaceResourceGroupManager) (bool, *ResourceGroup) {
+	m.publishResourceGroupMutation(keyspaceID, name, krgm, func(cur *keyspaceResourceGroupManager) (bool, *ResourceGroup) {
 		cur.removeResourceGroupLocked(name)
 		return true, nil
 	})

From d16df5d937f3b29c5131f8230009aa67f714fc6a Mon Sep 17 00:00:00 2001
From: bufferflies <1045931706@qq.com>
Date: Mon, 3 Aug 2026 09:38:38 +0200
Subject: [PATCH 33/50] resource_group: document a known gap in cross-term
 Delete publish

publishResourceGroupMutation's confirmed-write guard assumes the write
that confirmed a group is the storage-latest one, which holds for
Add/Modify (parked only after their persist) but not for Delete (parked
before its persist, via deleteResourceGroupBeforeStorage). A Delete
parked there that resumes after a same-named Add confirms in the new
term becomes the genuine last storage writer but has its publish
skipped, leaving the cache stale with no self-correction path. No test
exercises this interleaving yet; documented as a known limitation
pending a storage-side revision check, same class of gap as
initDefaultResourceGroup's stillCurrent check.

Signed-off-by: bufferflies <1045931706@qq.com>
---
 pkg/mcs/resourcemanager/server/manager.go | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go
index b6d1f1c886..c3c39187be 100644
--- a/pkg/mcs/resourcemanager/server/manager.go
+++ b/pkg/mcs/resourcemanager/server/manager.go
@@ -917,6 +917,25 @@ func (m *Manager) markResourceGroupSyncLoaded(keyspaceID uint32, krgm *keyspaceR
 // fn runs with the keyspace manager write lock held and must not do I/O; it
 // returns whether to record the sync-loaded marker and, when a group was
 // (re)installed, the group to sync burstability for.
+//
+// Known gap: this "skip if the new term already confirmed the group" rule
+// assumes the confirming write's storage effect is genuinely the latest one,
+// which holds for Add/Modify (both persist before they can be parked - see
+// addResourceGroupBeforePublish/modifyResourceGroupBeforePublish) but not for
+// Delete, which can be parked before its storage phase
+// (deleteResourceGroupBeforeStorage). If an old-term Delete is parked there,
+// a new-term Add for the same group persists and publishes (getting
+// confirmed), and only then does the old Delete's storage removal finally
+// run, it deletes that newer write in storage - making Delete genuinely the
+// last writer - but this function skips its publish because the group is
+// already "confirmed", leaving the cache showing the deleted group as
+// present. Nothing re-syncs it afterward, since it reads as confirmed to
+// every other path too. This mirrors, in the opposite direction, a gap the
+// old unconditional-apply code had (a delayed Delete publish could instead
+// wipe a newer confirmed Add). Neither direction is fully closed without a
+// storage-side revision to tell which write actually landed last; that's the
+// same class of gap tracked for initDefaultResourceGroup's stillCurrent
+// check. No regression test exercises this interleaving yet.
 func (m *Manager) publishResourceGroupMutation(
 	keyspaceID uint32, name string, krgm *keyspaceResourceGroupManager,
 	fn func(krgm *keyspaceResourceGroupManager) (mark bool, synced *ResourceGroup),

From 3042dde47b3faf45ece4a6ac5df5be89243c0707 Mon Sep 17 00:00:00 2001
From: bufferflies <1045931706@qq.com>
Date: Tue, 4 Aug 2026 08:52:06 +0200
Subject: [PATCH 34/50] resource_group: adapt KeyspaceIDValue literal to the
 oneof API

KeyspaceIDValue moved to a oneof (Keyspace) as part of the kvproto
apiv3 migration merged from master (e61f72770). Every other call
site in this package already existed on master and was migrated
there; this one is in a file the merge brought in only from this
branch, so it needed the same one-line update by hand.

Signed-off-by: bufferflies <1045931706@qq.com>
---
 pkg/mcs/resourcemanager/server/manager_async_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/mcs/resourcemanager/server/manager_async_test.go b/pkg/mcs/resourcemanager/server/manager_async_test.go
index 661c456917..1afac83e26 100644
--- a/pkg/mcs/resourcemanager/server/manager_async_test.go
+++ b/pkg/mcs/resourcemanager/server/manager_async_test.go
@@ -1026,7 +1026,7 @@ func (s *fakeTokenBucketsStream) Send(resp *resource_manager.TokenBucketsRespons
 func newRUTokenBucketRequest(keyspaceID uint32, name string, ru float64) *resource_manager.TokenBucketRequest {
 	return &resource_manager.TokenBucketRequest{
 		ResourceGroupName: name,
-		KeyspaceId:        &resource_manager.KeyspaceIDValue{Value: keyspaceID},
+		KeyspaceId:        &resource_manager.KeyspaceIDValue{Keyspace: &resource_manager.KeyspaceIDValue_Value{Value: keyspaceID}},
 		Request: &resource_manager.TokenBucketRequest_RuItems{
 			RuItems: &resource_manager.TokenBucketRequest_RequestRU{
 				RequestRU: []*resource_manager.RequestUnitItem{

From 3069a85dd1da88338975b12f037d7bf5730c0ee9 Mon Sep 17 00:00:00 2001
From: bufferflies <1045931706@qq.com>
Date: Tue, 4 Aug 2026 08:53:21 +0200
Subject: [PATCH 35/50] resource_group: link the cross-term Delete publish gap
 to tikv/pd#11105

Signed-off-by: bufferflies <1045931706@qq.com>
---
 pkg/mcs/resourcemanager/server/manager.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go
index 8ec2c3702a..319cc89698 100644
--- a/pkg/mcs/resourcemanager/server/manager.go
+++ b/pkg/mcs/resourcemanager/server/manager.go
@@ -918,6 +918,8 @@ func (m *Manager) markResourceGroupSyncLoaded(keyspaceID uint32, krgm *keyspaceR
 // returns whether to record the sync-loaded marker and, when a group was
 // (re)installed, the group to sync burstability for.
 //
+// TODO(#11105): close this gap with a storage-side revision/CAS check.
+//
 // Known gap: this "skip if the new term already confirmed the group" rule
 // assumes the confirming write's storage effect is genuinely the latest one,
 // which holds for Add/Modify (both persist before they can be parked - see

From ee42b555e83f48fc0c69610248696fe65d778acf Mon Sep 17 00:00:00 2001
From: bufferflies <1045931706@qq.com>
Date: Wed, 5 Aug 2026 04:13:02 +0200
Subject: [PATCH 36/50] resource_group: dedupe stale initDefaultResourceGroup
 doc comment, reorder TODO

initDefaultResourceGroup's doc comment had two overlapping paragraphs
stacked on top of each other, left over from when stillCurrent was
added on top of an earlier version of the comment - merge them into
one. Also move the TODO(#11105) marker after the explanation it
refers to instead of before it.

Signed-off-by: bufferflies <1045931706@qq.com>
---
 .../server/keyspace_manager.go                | 26 +++++++++----------
 pkg/mcs/resourcemanager/server/manager.go     |  4 +--
 2 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/pkg/mcs/resourcemanager/server/keyspace_manager.go b/pkg/mcs/resourcemanager/server/keyspace_manager.go
index 01b629329c..5030722ca8 100644
--- a/pkg/mcs/resourcemanager/server/keyspace_manager.go
+++ b/pkg/mcs/resourcemanager/server/keyspace_manager.go
@@ -237,19 +237,19 @@ func (krgm *keyspaceResourceGroupManager) setRawStatesIntoResourceGroup(name str
 // performed a synthesis, so callers that participate in the async bulk-load
 // merge (loadResourceGroupIfNeeded) know when they must publish a
 // sync-loaded marker for what this call just wrote.
-// initDefaultResourceGroup synthesizes and persists the built-in default
-// group if nothing confirmed exists yet. defaultGroupMu only serializes
-// callers that share this krgm instance; it does nothing across a term
-// change, since Init gives the new term an entirely new krgm object with its
-// own, separate defaultGroupMu. stillCurrent, when non-nil, is checked
-// immediately before the persist (right after defaultGroupMu is acquired) so
-// a caller with access to the owning Manager can catch a term change that
-// completed before this point - typically by comparing this krgm against the
-// manager's live entry for its keyspace ID. It cannot catch a term change
-// that lands while the persist's storage write is already in flight; closing
-// that residual window needs a conditional (CAS) storage write, which this
-// does not implement. Pass nil when no such coordination is needed or
-// available (e.g. in tests that exercise krgm without a Manager).
+//
+// defaultGroupMu only serializes callers that share this krgm instance; it
+// does nothing across a term change, since Init gives the new term an
+// entirely new krgm object with its own, separate defaultGroupMu.
+// stillCurrent, when non-nil, is checked immediately before the persist
+// (right after defaultGroupMu is acquired) so a caller with access to the
+// owning Manager can catch a term change that completed before this point -
+// typically by comparing this krgm against the manager's live entry for its
+// keyspace ID. It cannot catch a term change that lands while the persist's
+// storage write is already in flight; closing that residual window needs a
+// conditional (CAS) storage write, which this does not implement. Pass nil
+// when no such coordination is needed or available (e.g. in tests that
+// exercise krgm without a Manager).
 func (krgm *keyspaceResourceGroupManager) initDefaultResourceGroup(stillCurrent func() bool) bool {
 	// A confirmed cached entry means initialization is unnecessary; a missing
 	// or reserved-placeholder entry means nothing is persisted for the
diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go
index 319cc89698..4a1a12649f 100644
--- a/pkg/mcs/resourcemanager/server/manager.go
+++ b/pkg/mcs/resourcemanager/server/manager.go
@@ -918,8 +918,6 @@ func (m *Manager) markResourceGroupSyncLoaded(keyspaceID uint32, krgm *keyspaceR
 // returns whether to record the sync-loaded marker and, when a group was
 // (re)installed, the group to sync burstability for.
 //
-// TODO(#11105): close this gap with a storage-side revision/CAS check.
-//
 // Known gap: this "skip if the new term already confirmed the group" rule
 // assumes the confirming write's storage effect is genuinely the latest one,
 // which holds for Add/Modify (both persist before they can be parked - see
@@ -938,6 +936,8 @@ func (m *Manager) markResourceGroupSyncLoaded(keyspaceID uint32, krgm *keyspaceR
 // storage-side revision to tell which write actually landed last; that's the
 // same class of gap tracked for initDefaultResourceGroup's stillCurrent
 // check. No regression test exercises this interleaving yet.
+//
+// TODO(#11105): close this gap with a storage-side revision/CAS check.
 func (m *Manager) publishResourceGroupMutation(
 	keyspaceID uint32, name string, krgm *keyspaceResourceGroupManager,
 	fn func(krgm *keyspaceResourceGroupManager) (mark bool, synced *ResourceGroup),

From 49d5f108cc22396d9552c56b6e4ddb5eda041aad Mon Sep 17 00:00:00 2001
From: bufferflies <1045931706@qq.com>
Date: Wed, 5 Aug 2026 04:31:05 +0200
Subject: [PATCH 37/50] resource_group: distinguish stale-term and real errors
 in initDefaultResourceGroup

initDefaultResourceGroup returned a single bool, collapsing three
distinct outcomes into one false: the group was already confirmed
(fine), stillCurrent caught a term change before the persist started
(nothing was created anywhere, caller must retry against the fresh
term), or the persist itself failed with a real storage error
(caller must propagate it). loadResourceGroupIfNeeded's only branching
call site treated all three as success, so a leadership change
landing in the narrow window between its own staleness check and
initDefaultResourceGroup's internal stillCurrent recheck - or a
genuine storage error synthesizing the default group - was reported
to the RPC caller as a successful load/modify of a group that was
never actually created anywhere, surfacing as a spurious
ErrResourceGroupNotExists on the next real access instead.

Return (created bool, err error) instead: nil error means confirmed-
or-created (safe to treat as success), errs.ErrResourceGroupsLoading
means stale (retry, matching the two sibling term-change checks
already in loadResourceGroupIfNeeded), any other error is a real
failure to propagate. The other three call sites (two best-effort
pre-warm paths, three test setups) don't branch on the outcome and
now discard it explicitly.

Signed-off-by: bufferflies <1045931706@qq.com>
---
 .../server/keyspace_manager.go                | 24 +++++++++-----
 .../server/keyspace_manager_test.go           | 10 ++++--
 pkg/mcs/resourcemanager/server/manager.go     | 32 ++++++++++++++++---
 3 files changed, 51 insertions(+), 15 deletions(-)

diff --git a/pkg/mcs/resourcemanager/server/keyspace_manager.go b/pkg/mcs/resourcemanager/server/keyspace_manager.go
index 5030722ca8..466a5aaee5 100644
--- a/pkg/mcs/resourcemanager/server/keyspace_manager.go
+++ b/pkg/mcs/resourcemanager/server/keyspace_manager.go
@@ -233,10 +233,18 @@ func (krgm *keyspaceResourceGroupManager) setRawStatesIntoResourceGroup(name str
 }
 
 // initDefaultResourceGroup synthesizes and persists the built-in default
-// group if nothing confirmed exists yet. It reports whether it actually
+// group if nothing confirmed exists yet. created reports whether it actually
 // performed a synthesis, so callers that participate in the async bulk-load
 // merge (loadResourceGroupIfNeeded) know when they must publish a
-// sync-loaded marker for what this call just wrote.
+// sync-loaded marker for what this call just wrote. The three (created, err)
+// outcomes need different handling and must not be collapsed into a single
+// bool by the caller: (false, nil) means confirmed data already existed -
+// nothing to do, safe to treat as success; (false, errs.ErrResourceGroupsLoading)
+// means stillCurrent caught a term change before the persist started - the
+// group was not created anywhere, so the caller must retry against the fresh
+// term rather than treat this as success; (false, any other non-nil error)
+// means the persist itself failed (e.g. a storage write error) - the caller
+// must propagate it rather than silently swallow a real failure as success.
 //
 // defaultGroupMu only serializes callers that share this krgm instance; it
 // does nothing across a term change, since Init gives the new term an
@@ -250,14 +258,14 @@ func (krgm *keyspaceResourceGroupManager) setRawStatesIntoResourceGroup(name str
 // conditional (CAS) storage write, which this does not implement. Pass nil
 // when no such coordination is needed or available (e.g. in tests that
 // exercise krgm without a Manager).
-func (krgm *keyspaceResourceGroupManager) initDefaultResourceGroup(stillCurrent func() bool) bool {
+func (krgm *keyspaceResourceGroupManager) initDefaultResourceGroup(stillCurrent func() bool) (created bool, err error) {
 	// A confirmed cached entry means initialization is unnecessary; a missing
 	// or reserved-placeholder entry means nothing is persisted for the
 	// default group (e.g. a fresh store), so it must still be created and
 	// persisted, otherwise its settings are never stored and state
 	// persistence stays skipped.
 	if krgm.hasConfirmedResourceGroup(DefaultResourceGroupName) {
-		return false
+		return false, nil
 	}
 	// Serialize against every other synthesis or real Add/ModifyResourceGroup
 	// targeting "default" that shares this krgm instance: see the
@@ -268,17 +276,17 @@ func (krgm *keyspaceResourceGroupManager) initDefaultResourceGroup(stillCurrent
 	// lock, a real write may have already confirmed the default group, in
 	// which case synthesizing here would silently clobber it.
 	if krgm.hasConfirmedResourceGroup(DefaultResourceGroupName) {
-		return false
+		return false, nil
 	}
 	if stillCurrent != nil && !stillCurrent() {
-		return false
+		return false, errs.ErrResourceGroupsLoading
 	}
 	defaultGroup := newDefaultResourceGroup()
 	if err := krgm.addResourceGroup(defaultGroup.IntoProtoResourceGroup(krgm.keyspaceID)); err != nil {
 		log.Warn("init default group failed", zap.Uint32("keyspace-id", krgm.keyspaceID), zap.Error(err))
-		return false
+		return false, err
 	}
-	return true
+	return true, nil
 }
 
 func (krgm *keyspaceResourceGroupManager) ensureReservedDefaultGroupInCache() {
diff --git a/pkg/mcs/resourcemanager/server/keyspace_manager_test.go b/pkg/mcs/resourcemanager/server/keyspace_manager_test.go
index 76cd168b0f..49f58d4400 100644
--- a/pkg/mcs/resourcemanager/server/keyspace_manager_test.go
+++ b/pkg/mcs/resourcemanager/server/keyspace_manager_test.go
@@ -87,7 +87,9 @@ func TestInitDefaultResourceGroup(t *testing.T) {
 	re.False(exists)
 
 	// Initialize the default resource group.
-	krgm.initDefaultResourceGroup(nil)
+	created, err := krgm.initDefaultResourceGroup(nil)
+	re.NoError(err)
+	re.True(created)
 
 	// Verify the default resource group is created.
 	defaultGroup, exists := krgm.groups[DefaultResourceGroupName]
@@ -225,7 +227,8 @@ func TestDeleteResourceGroupBehavior(t *testing.T) {
 		_, ok := krgm.groupRUTrackers[group.GetName()]
 		re.False(ok)
 
-		krgm.initDefaultResourceGroup(nil)
+		_, err := krgm.initDefaultResourceGroup(nil)
+		re.NoError(err)
 		re.Error(krgm.deleteResourceGroup(DefaultResourceGroupName))
 		re.NotNil(krgm.getResourceGroup(DefaultResourceGroupName, false))
 	})
@@ -361,7 +364,8 @@ func TestGetResourceGroupList(t *testing.T) {
 	re.Equal("group2", groups[1].Name)
 	re.Equal("group3", groups[2].Name)
 
-	krgm.initDefaultResourceGroup(nil)
+	_, err := krgm.initDefaultResourceGroup(nil)
+	re.NoError(err)
 	groups = krgm.getResourceGroupList(false, true)
 	re.Len(groups, 4)
 	groups = krgm.getResourceGroupList(false, false)
diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go
index 4a1a12649f..c728506cbd 100644
--- a/pkg/mcs/resourcemanager/server/manager.go
+++ b/pkg/mcs/resourcemanager/server/manager.go
@@ -332,8 +332,12 @@ func (m *Manager) getOrCreateKeyspaceResourceGroupManager(keyspaceID uint32, ini
 		if m.getLoadingState() == LoadingStateCompleted {
 			// Async loading (if any) has already finished, so if the default
 			// group isn't cached yet it truly doesn't exist anywhere; it's
-			// safe to synthesize and persist it directly.
-			krgm.initDefaultResourceGroup(func() bool {
+			// safe to synthesize and persist it directly. This is a
+			// best-effort pre-warm: any failure (stale term or a real
+			// storage error) is already logged inside initDefaultResourceGroup,
+			// and a later request for the default group will retry through
+			// loadResourceGroupIfNeeded, which does surface such errors.
+			_, _ = krgm.initDefaultResourceGroup(func() bool {
 				m.RLock()
 				defer m.RUnlock()
 				return m.krgms[keyspaceID] == krgm
@@ -806,7 +810,24 @@ func (m *Manager) loadResourceGroupIfNeeded(keyspaceID uint32, name string) erro
 					defer m.RUnlock()
 					return m.loadEpoch == epoch && m.krgms[keyspaceID] == krgm
 				}
-				if krgm.initDefaultResourceGroup(stillCurrent) {
+				created, initErr := krgm.initDefaultResourceGroup(stillCurrent)
+				if initErr != nil {
+					if errs.ErrResourceGroupsLoading.Equal(initErr) {
+						// stillCurrent caught a term change that landed after the
+						// stale check above but before the persist started; the
+						// group was not created anywhere, so retry against the
+						// fresh term instead of reporting a bogus success.
+						if attempt >= maxLoadAttempts {
+							return errs.ErrResourceGroupsLoading
+						}
+						continue
+					}
+					// A real failure persisting the default group (e.g. a storage
+					// write error): propagate it rather than silently reporting
+					// success for a group that was never actually created.
+					return initErr
+				}
+				if created {
 					// This synthesis bypassed publishResourceGroupMutation, so the
 					// sync-loaded marker was never set. Set it now: otherwise a
 					// bulk merge still in progress for this term doesn't know this
@@ -1076,7 +1097,10 @@ func (m *Manager) initReserved(epoch uint64) {
 	// no concurrent Add/ModifyResourceGroup or other initDefaultResourceGroup
 	// call to race against.
 	for _, krgm := range m.getKeyspaceResourceGroupManagers() {
-		krgm.initDefaultResourceGroup(nil)
+		// Any failure is already logged inside initDefaultResourceGroup; a
+		// later request for the default group retries through
+		// loadResourceGroupIfNeeded once serving starts.
+		_, _ = krgm.initDefaultResourceGroup(nil)
 	}
 }
 

From 4efec590b8482690daf73adad56633713829b197 Mon Sep 17 00:00:00 2001
From: bufferflies <1045931706@qq.com>
Date: Wed, 5 Aug 2026 05:03:00 +0200
Subject: [PATCH 38/50] resource_group: fix SetKeyspaceServiceLimit to survive
 a cross-term leadership change

SetKeyspaceServiceLimit resolved a keyspace manager once and mutated
it directly, with no epoch or identity check. If Init replaced the
whole krgms map for a new term while the call was still resolving or
persisting, the write still landed in storage but only updated the
now-detached old-term krgm's in-memory limiter - the live serving
cache (and GetKeyspaceServiceLimiter) kept returning the stale
pre-write value indefinitely, with no self-correction path short of
the next full reload. This predates the async-loading work, but
became unsafe as a side effect of it: krgms didn't used to be wholly
replaced across a leadership change.

Split into the same storage-then-publish shape already used by
Add/Modify/DeleteResourceGroup: persist against the originally
resolved krgm (the write itself doesn't depend on krgm identity),
then re-resolve whichever keyspace manager is current and mirror the
persisted value into it if it differs. Unlike the resource-group
case there's no reserved/confirmed distinction to protect - a
service limit is a single scalar with no CAS at the storage layer
either - so the persisted value is simply mirrored in.

Adds a new setServiceLimitBeforeStorage failpoint (mirroring
deleteResourceGroupBeforeStorage's position) and a regression test
that parks the call before its persist, changes leadership so the new
term's synchronous loadServiceLimits reads the old value, then resumes
the call and checks the new term ends up with the written value
instead of the detached term's. Verified the test fails (asserting 0
instead of 4242) with the publish step reverted.

Signed-off-by: bufferflies <1045931706@qq.com>
---
 pkg/mcs/resourcemanager/server/manager.go     | 27 ++++++-
 .../server/manager_async_test.go              | 74 +++++++++++++++++++
 2 files changed, 100 insertions(+), 1 deletion(-)

diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go
index c728506cbd..1a4ccac7aa 100644
--- a/pkg/mcs/resourcemanager/server/manager.go
+++ b/pkg/mcs/resourcemanager/server/manager.go
@@ -276,7 +276,32 @@ func (m *Manager) SetKeyspaceServiceLimit(keyspaceID uint32, serviceLimit float6
 		return errMetadataWriteDisabled
 	}
 	// If the keyspace is not found, create a new keyspace resource group manager.
-	m.getOrCreateKeyspaceResourceGroupManager(keyspaceID, true).setServiceLimit(serviceLimit)
+	krgm := m.getOrCreateKeyspaceResourceGroupManager(keyspaceID, true)
+	failpoint.InjectCall("setServiceLimitBeforeStorage")
+	// Storage phase: this persists synchronously inside setServiceLimit.
+	krgm.setServiceLimit(serviceLimit)
+	// Publish phase: mirror the persisted value into whichever keyspace
+	// manager is current now, in case Init replaced the whole krgms map for
+	// a new term while the storage write above was in flight - without
+	// this, the write still lands in storage but only updates the detached
+	// krgm's in-memory limiter, leaving the live serving cache (and
+	// GetKeyspaceServiceLimiter) showing the stale pre-write value until
+	// the next full reload. This mirrors the storage-then-publish shape of
+	// publishResourceGroupMutation, but there's no reserved/confirmed
+	// distinction to protect here: a service limit is a single scalar with
+	// no CAS at the storage layer either (the same known gap tracked
+	// elsewhere in this file), so whatever value ends up winning in storage
+	// is simply mirrored into the live cache without persisting again.
+	m.Lock()
+	cur, ok := m.krgms[keyspaceID]
+	if !ok {
+		cur = newKeyspaceResourceGroupManager(keyspaceID, m.storage, m.writeRole)
+		m.krgms[keyspaceID] = cur
+	}
+	m.Unlock()
+	if cur != krgm {
+		cur.setServiceLimitFromStorage(serviceLimit)
+	}
 	return nil
 }
 
diff --git a/pkg/mcs/resourcemanager/server/manager_async_test.go b/pkg/mcs/resourcemanager/server/manager_async_test.go
index 1afac83e26..ad61e730f7 100644
--- a/pkg/mcs/resourcemanager/server/manager_async_test.go
+++ b/pkg/mcs/resourcemanager/server/manager_async_test.go
@@ -923,6 +923,80 @@ func TestAsyncLoadResourceGroupsCrossTermModifyDefaultStaysConfirmed(t *testing.
 		"the confirmed running state must be preserved, not reset to a synthetic placeholder")
 }
 
+// TestAsyncLoadResourceGroupsCrossTermSetServiceLimitPublishesToNewTerm
+// reproduces a leadership change straddling SetKeyspaceServiceLimit: it
+// resolves a keyspace manager, then stalls before its storage phase while the
+// leadership change replaces m.krgms and the new term's synchronous
+// loadServiceLimits reads the still-unwritten old value. When the call
+// resumes, its storage write must still land, and its cache effect must be
+// mirrored into the *current* term's keyspace manager, not just the detached
+// term-1 one - otherwise the new term keeps serving the stale (default zero)
+// limit indefinitely, with nothing to ever re-sync it.
+func TestAsyncLoadResourceGroupsCrossTermSetServiceLimitPublishesToNewTerm(t *testing.T) {
+	re := require.New(t)
+	store := newBlockingResourceGroupStorage()
+
+	m := NewManager[*mockConfigProvider](&mockConfigProvider{})
+	m.storage = store
+	re.NoError(m.Init(context.Background()))
+	cancelTerm1 := m.cancel
+	defer stopAsyncTestManager(m)
+	defer store.unblock()
+
+	// Let term 1 load fully so the call starts against a settled term.
+	store.waitEntered(t)
+	store.unblock()
+	testutil.Eventually(re, func() bool {
+		_, err := m.GetResourceGroupList(constant.NullKeyspaceID, false)
+		return err == nil
+	}, testutil.WithTickInterval(20*time.Millisecond))
+
+	// Park SetKeyspaceServiceLimit between resolving its keyspace manager and
+	// its storage phase.
+	reached := make(chan struct{})
+	release := make(chan struct{})
+	re.NoError(failpoint.EnableCall("github.com/tikv/pd/pkg/mcs/resourcemanager/server/setServiceLimitBeforeStorage", func() {
+		close(reached)
+		<-release
+	}))
+	defer func() {
+		re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/mcs/resourcemanager/server/setServiceLimitBeforeStorage"))
+	}()
+	var setErr error
+	setDone := make(chan struct{})
+	go func() {
+		defer close(setDone)
+		setErr = m.SetKeyspaceServiceLimit(constant.NullKeyspaceID, 4242)
+	}()
+	select {
+	case <-reached:
+	case <-time.After(time.Second):
+		t.Fatal("timed out waiting for SetKeyspaceServiceLimit to reach its storage phase")
+	}
+
+	// Leadership changes while the write is parked: term 2's synchronous
+	// loadServiceLimits runs and completes here, reading storage before the
+	// parked write has persisted anything.
+	cancelTerm1()
+	re.NoError(m.Init(context.Background()))
+
+	// Resume the write: it persists into storage now, then must publish into
+	// whichever keyspace manager is current (term 2), not the detached term-1
+	// one.
+	close(release)
+	<-setDone
+	re.NoError(setErr)
+
+	limiter := m.GetKeyspaceServiceLimiter(constant.NullKeyspaceID)
+	re.NotNil(limiter)
+	re.Equal(float64(4242), limiter.ServiceLimit,
+		"the service limit set during the leadership change must be visible in the new term, not stuck on the detached old one")
+
+	raw, err := store.LoadServiceLimit(constant.NullKeyspaceID)
+	re.NoError(err)
+	re.Equal(float64(4242), raw, "the service limit must be persisted regardless of the leadership change")
+}
+
 // BenchmarkAsyncLoadMergeReaderStall measures the worst-case time a concurrent
 // reader is blocked while the async bulk merge installs a large number of
 // resource groups. The probe uses GetControllerConfig, whose only cost is the

From ab938b64409ad1adc238ca784f54255b4061e374 Mon Sep 17 00:00:00 2001
From: bufferflies <1045931706@qq.com>
Date: Wed, 5 Aug 2026 05:30:39 +0200
Subject: [PATCH 39/50] resource_group: fix two data races between config
 writes and a leadership change

SetKeyspaceRUVersion mutated m.controllerConfig under the lock, then
re-read the m.controllerConfig field itself after unlocking to persist
it. initControllerConfig (run on every leadership change) can reassign
that same field wholesale, also under the lock - the unlocked re-read
raced with that reassignment. Fixed by capturing the object to save as
a local variable while still holding the lock, matching the pattern
initControllerConfig itself already used for its own save call.

That comparison surfaced a second, independent race: after publishing
its freshly loaded config into m.controllerConfig, initControllerConfig
went on to marshal-and-save that same (now live, shared) object without
holding any lock. Once published, any concurrent config mutator holding
m.Lock() - SetKeyspaceRUVersion included - can be mutating the same
RUVersionPolicy.Overrides map this unlocked save is concurrently
marshaling. Fixed by reordering: persist first (while the config is
still a private, unpublished local value nobody else can reach), then
publish it under the lock.

Adds TestManagerSetKeyspaceRUVersionConcurrentWithLeadershipChange,
which runs SetKeyspaceRUVersion and repeated Init cycles concurrently
under -race. It reliably reproduced both races before the corresponding
fix (confirmed by reverting each fix individually and observing the
race detector flag the specific access it protects) and is clean across
10 repeated runs with both in place.

Signed-off-by: bufferflies <1045931706@qq.com>
---
 pkg/mcs/resourcemanager/server/manager.go     | 23 +++++++++----
 .../server/ru_version_policy_test.go          | 34 +++++++++++++++++++
 2 files changed, 51 insertions(+), 6 deletions(-)

diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go
index 1a4ccac7aa..1c74ea3c74 100644
--- a/pkg/mcs/resourcemanager/server/manager.go
+++ b/pkg/mcs/resourcemanager/server/manager.go
@@ -326,8 +326,16 @@ func (m *Manager) SetKeyspaceRUVersion(keyspaceID uint32, ruVersion int32) error
 	} else {
 		m.controllerConfig.RUVersionPolicy.Overrides[keyspaceID] = ruVersion
 	}
+	// Capture the config object to save while still holding the lock, instead
+	// of re-reading the m.controllerConfig field after unlocking below:
+	// initControllerConfig can reassign that field wholesale (under the same
+	// lock) on a leadership change, so an unlocked re-read races with it and
+	// can end up saving a different config object than the one just mutated
+	// above. Matches the pattern initControllerConfig itself already uses -
+	// save a locally captured reference, never the field.
+	controllerConfig := m.controllerConfig
 	m.Unlock()
-	return m.storage.SaveControllerConfig(m.controllerConfig)
+	return m.storage.SaveControllerConfig(controllerConfig)
 }
 
 // GetRUVersionPolicy returns a deep copy of the current RU version policy from the controller config.
@@ -458,16 +466,19 @@ func (m *Manager) initControllerConfig() error {
 	if err = json.Unmarshal([]byte(v), &controllerConfig); err != nil {
 		log.Warn("un-marshall controller config failed, fallback to default", zap.Error(err), zap.String("v", v))
 	}
-	m.Lock()
-	m.controllerConfig = controllerConfig
-	m.Unlock()
-
-	// re-save the config to make sure the config has been persisted.
+	// re-save the config to make sure the config has been persisted. This
+	// must run before controllerConfig is published into m.controllerConfig
+	// below: once published, it's reachable (and mutable) by any concurrent
+	// caller holding m.Lock() - e.g. SetKeyspaceRUVersion - which would race
+	// with this unlocked marshal-and-save if it ran after instead.
 	if m.writeRole.AllowsMetadataWrite() {
 		if err := m.storage.SaveControllerConfig(controllerConfig); err != nil {
 			return err
 		}
 	}
+	m.Lock()
+	m.controllerConfig = controllerConfig
+	m.Unlock()
 	return nil
 }
 
diff --git a/pkg/mcs/resourcemanager/server/ru_version_policy_test.go b/pkg/mcs/resourcemanager/server/ru_version_policy_test.go
index 7c711e7c6f..cca2d6b968 100644
--- a/pkg/mcs/resourcemanager/server/ru_version_policy_test.go
+++ b/pkg/mcs/resourcemanager/server/ru_version_policy_test.go
@@ -17,6 +17,7 @@ package server
 import (
 	"context"
 	"encoding/json"
+	"sync"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -279,3 +280,36 @@ func TestManagerSetKeyspaceRUVersionResetToDefault(t *testing.T) {
 	_, exists := policy.Overrides[1]
 	re.False(exists)
 }
+
+// TestManagerSetKeyspaceRUVersionConcurrentWithLeadershipChange guards
+// against a data race between SetKeyspaceRUVersion and a leadership change.
+// SetKeyspaceRUVersion used to mutate m.controllerConfig under the lock, then
+// re-read the m.controllerConfig field itself after unlocking to persist it -
+// racing against initControllerConfig, which can reassign that same field
+// wholesale (also under the lock) when Init runs again for a new term. Run
+// under -race: any regression back to the unlocked re-read fails this test
+// without needing a specific interleaving to be forced.
+func TestManagerSetKeyspaceRUVersionConcurrentWithLeadershipChange(t *testing.T) {
+	re := require.New(t)
+	m := prepareManager()
+	re.NoError(m.Init(context.Background()))
+
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		for i := range 100 {
+			_ = m.SetKeyspaceRUVersion(1, int32(i%2)+1)
+		}
+	}()
+	go func() {
+		defer wg.Done()
+		for range 20 {
+			m.cancel()
+			m.wg.Wait()
+			_ = m.Init(context.Background())
+		}
+	}()
+	wg.Wait()
+	stopAsyncTestManager(m)
+}

From d82f87847a5a06cfab751e98e506645d4cc4525a Mon Sep 17 00:00:00 2001
From: bufferflies <1045931706@qq.com>
Date: Wed, 5 Aug 2026 09:04:54 +0200
Subject: [PATCH 40/50] resource_group: fix watcher-mode default clobbering and
 SetKeyspaceServiceLimit cross-term races

Two independent gaps:

1. getOrCreateKeyspaceResourceGroupManager treated LoadingStateCompleted
   as "cache is authoritative" unconditionally, but in metadata-watcher
   mode that only means the initial bootstrap finished, not that every
   PD write already in storage has had its watch event delivered yet -
   the same caveat loadResourceGroupIfNeeded already documents and
   guards against at its own entry point. A request landing in that gap
   for a keyspace whose default was just customized (write already in
   storage, watch event not yet applied) would find nothing cached and
   persist the built-in default over it. Added the same
   !m.enableMetadataWatcher guard loadResourceGroupIfNeeded uses, so
   watcher mode always falls through to a storage point load instead of
   trusting the cache.

2. SetKeyspaceServiceLimit's publish-phase mirror (added to fix the
   earlier cross-term staleness gap) applied unconditionally. If an
   old-term call persisted and then parked before its mirror step, a
   competing new-term call for the same keyspace could fully persist
   and publish its own value in the meantime; the old call's mirror
   would then overwrite that newer value with its own stale one. Fixed
   by serializing all SetKeyspaceServiceLimit calls per keyspace ID via
   a new serviceLimitLocks field (syncutil.LockGroup, reusing the same
   utility pkg/keyspace already uses for its own per-keyspace metadata
   lock). Unlike a krgm-scoped lock (e.g. defaultGroupMu), this lives on
   Manager and survives Init, so a call parked mid-persist in an old
   term keeps blocking a same-keyspace call in a new term until it
   fully finishes - closing the race without holding the broad
   manager-wide lock across the storage write.

Both come with regression tests that fail against the pre-fix code:
TestGetOrCreateKeyspaceResourceGroupManagerWatcherModeDoesNotClobberPendingDefault
(reverting the guard reproduces the built-in default overwriting the
customized one) and
TestAsyncLoadResourceGroupsCrossTermSetServiceLimitSerializesAgainstCompetingCall
(reverting serviceLimitLocks reproduces the new-term value being
clobbered by the stale old-term mirror in 2 of 3 runs, confirming the
test catches the race).

Signed-off-by: bufferflies <1045931706@qq.com>
---
 pkg/mcs/resourcemanager/server/manager.go     | 54 +++++++++--
 .../server/manager_async_test.go              | 94 +++++++++++++++++++
 .../server/metadata_watcher_test.go           | 45 ++++++++-
 3 files changed, 180 insertions(+), 13 deletions(-)

diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go
index 1c74ea3c74..a296c45635 100644
--- a/pkg/mcs/resourcemanager/server/manager.go
+++ b/pkg/mcs/resourcemanager/server/manager.go
@@ -125,6 +125,18 @@ type Manager struct {
 	// from a previous term that was blocked in a storage scan can never merge
 	// stale data into, or publish completion for, a newer term.
 	loadEpoch uint64
+	// serviceLimitLocks serializes SetKeyspaceServiceLimit calls per keyspace
+	// ID. Unlike krgm-scoped locks (e.g. defaultGroupMu), this lives on the
+	// Manager itself and is never reset by Init, so it keeps serializing a
+	// call parked mid-persist in an old term against a competing call in a
+	// new term for the same keyspace - without it, the old call's eventual
+	// publish could overwrite a newer value the new-term call already both
+	// persisted and published. Entries are intentionally never removed: the
+	// keyspace ID space is bounded, matching krgms/keyspaceNameLookup's own
+	// grow-only lifetime, so there's no working-set pressure to justify the
+	// extra bookkeeping (and its own subtleties - see LockGroup's Unlock)
+	// that WithRemoveEntryOnUnlock would add.
+	serviceLimitLocks *syncutil.LockGroup
 }
 
 // LoadingState represents the current loading state of resource groups
@@ -171,6 +183,7 @@ func newManagerBase(controllerConfig *ControllerConfig, writeRole ResourceGroupW
 		metrics:               newMetrics(),
 		ruCollector:           newRUCollector(),
 		syncLoadedGroups:      make(map[trackerKey]bool),
+		serviceLimitLocks:     syncutil.NewLockGroup(),
 	}
 	m.setLoadingState(LoadingStateNotStarted)
 	return m
@@ -275,6 +288,16 @@ func (m *Manager) SetKeyspaceServiceLimit(keyspaceID uint32, serviceLimit float6
 	if !m.writeRole.AllowsMetadataWrite() {
 		return errMetadataWriteDisabled
 	}
+	// Serialize against every other SetKeyspaceServiceLimit call for this
+	// keyspace, including ones that started in a different term: krgm
+	// identity resets on every Init, but this lock doesn't, so it's what
+	// keeps a call parked mid-persist in an old term from clobbering a
+	// competing new-term call's result once it resumes. See the
+	// serviceLimitLocks field doc for why a krgm-scoped lock (like
+	// defaultGroupMu) can't provide this guarantee on its own.
+	m.serviceLimitLocks.Lock(keyspaceID)
+	defer m.serviceLimitLocks.Unlock(keyspaceID)
+
 	// If the keyspace is not found, create a new keyspace resource group manager.
 	krgm := m.getOrCreateKeyspaceResourceGroupManager(keyspaceID, true)
 	failpoint.InjectCall("setServiceLimitBeforeStorage")
@@ -289,9 +312,11 @@ func (m *Manager) SetKeyspaceServiceLimit(keyspaceID uint32, serviceLimit float6
 	// the next full reload. This mirrors the storage-then-publish shape of
 	// publishResourceGroupMutation, but there's no reserved/confirmed
 	// distinction to protect here: a service limit is a single scalar with
-	// no CAS at the storage layer either (the same known gap tracked
-	// elsewhere in this file), so whatever value ends up winning in storage
-	// is simply mirrored into the live cache without persisting again.
+	// no CAS at the storage layer, so nothing short of the serviceLimitLocks
+	// guard above tells this call whether cur already holds a newer value.
+	// With that guard held for the whole call, no other SetKeyspaceServiceLimit
+	// for this keyspace can run concurrently, so whatever's in storage now is
+	// still what this call itself just wrote - safe to mirror unconditionally.
 	m.Lock()
 	cur, ok := m.krgms[keyspaceID]
 	if !ok {
@@ -362,13 +387,22 @@ func (m *Manager) getOrCreateKeyspaceResourceGroupManager(keyspaceID uint32, ini
 	}
 	m.Unlock()
 	if initDefault {
-		if m.getLoadingState() == LoadingStateCompleted {
-			// Async loading (if any) has already finished, so if the default
-			// group isn't cached yet it truly doesn't exist anywhere; it's
-			// safe to synthesize and persist it directly. This is a
-			// best-effort pre-warm: any failure (stale term or a real
-			// storage error) is already logged inside initDefaultResourceGroup,
-			// and a later request for the default group will retry through
+		// In metadata-watcher mode, LoadingStateCompleted only means the
+		// initial watcher bootstrap finished, not that every subsequent PD
+		// write has been observed yet - same caveat loadResourceGroupIfNeeded
+		// documents at its own entry point. Without this guard, a request
+		// landing right after bootstrap but before the watcher delivers a
+		// just-written customized default would see nothing cached, take
+		// this branch, and persist the built-in default over it. Always fall
+		// through to loadResourceGroupIfNeeded in that mode, which does a
+		// storage point load first instead of trusting the cache.
+		if !m.enableMetadataWatcher && m.getLoadingState() == LoadingStateCompleted {
+			// Async loading has already finished, so if the default group
+			// isn't cached yet it truly doesn't exist anywhere; it's safe to
+			// synthesize and persist it directly. This is a best-effort
+			// pre-warm: any failure (stale term or a real storage error) is
+			// already logged inside initDefaultResourceGroup, and a later
+			// request for the default group will retry through
 			// loadResourceGroupIfNeeded, which does surface such errors.
 			_, _ = krgm.initDefaultResourceGroup(func() bool {
 				m.RLock()
diff --git a/pkg/mcs/resourcemanager/server/manager_async_test.go b/pkg/mcs/resourcemanager/server/manager_async_test.go
index ad61e730f7..0f36cb944d 100644
--- a/pkg/mcs/resourcemanager/server/manager_async_test.go
+++ b/pkg/mcs/resourcemanager/server/manager_async_test.go
@@ -997,6 +997,100 @@ func TestAsyncLoadResourceGroupsCrossTermSetServiceLimitPublishesToNewTerm(t *te
 	re.Equal(float64(4242), raw, "the service limit must be persisted regardless of the leadership change")
 }
 
+// TestAsyncLoadResourceGroupsCrossTermSetServiceLimitSerializesAgainstCompetingCall
+// guards against the unconditional publish-phase mirror in
+// SetKeyspaceServiceLimit clobbering a competing, fully-completed call for
+// the same keyspace. Without serviceLimitLocks, an old-term call parked
+// mid-persist could resume after a new-term call for the same keyspace
+// already persisted and published its own value, and mirror its own older
+// value back in - leaving the live cache stale even though storage (and
+// every other observer) already moved on. serviceLimitLocks makes the
+// new-term call wait for the old one to fully finish, including its own
+// mirror step, before it can even start, so this interleaving can no longer
+// happen.
+func TestAsyncLoadResourceGroupsCrossTermSetServiceLimitSerializesAgainstCompetingCall(t *testing.T) {
+	re := require.New(t)
+	store := newBlockingResourceGroupStorage()
+
+	m := NewManager[*mockConfigProvider](&mockConfigProvider{})
+	m.storage = store
+	re.NoError(m.Init(context.Background()))
+	cancelTerm1 := m.cancel
+	defer stopAsyncTestManager(m)
+	defer store.unblock()
+
+	store.waitEntered(t)
+	store.unblock()
+	testutil.Eventually(re, func() bool {
+		_, err := m.GetResourceGroupList(constant.NullKeyspaceID, false)
+		return err == nil
+	}, testutil.WithTickInterval(20*time.Millisecond))
+
+	// Park the old-term call between resolving its keyspace manager and its
+	// storage phase - it already holds serviceLimitLocks for this keyspace
+	// by this point. The new-term call goes through this same, still-armed
+	// failpoint too once serviceLimitLocks lets it proceed; parkOnce keeps
+	// only the first (old-term) hit actually parking, so the second one
+	// passes straight through instead of trying to close(reached) again.
+	reached := make(chan struct{})
+	release := make(chan struct{})
+	var parkOnce sync.Once
+	re.NoError(failpoint.EnableCall("github.com/tikv/pd/pkg/mcs/resourcemanager/server/setServiceLimitBeforeStorage", func() {
+		parkOnce.Do(func() {
+			close(reached)
+			<-release
+		})
+	}))
+	defer func() {
+		re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/mcs/resourcemanager/server/setServiceLimitBeforeStorage"))
+	}()
+	var oldErr error
+	oldDone := make(chan struct{})
+	go func() {
+		defer close(oldDone)
+		oldErr = m.SetKeyspaceServiceLimit(constant.NullKeyspaceID, 100)
+	}()
+	select {
+	case <-reached:
+	case <-time.After(time.Second):
+		t.Fatal("timed out waiting for the old-term call to reach its storage phase")
+	}
+
+	// Leadership changes while the old-term call is parked.
+	cancelTerm1()
+	re.NoError(m.Init(context.Background()))
+
+	// A new-term call for the same keyspace must not be able to run while the
+	// old-term call still holds serviceLimitLocks for it: it should block
+	// before even reaching the (still-armed) failpoint above, since
+	// serviceLimitLocks is acquired first.
+	var newErr error
+	newDone := make(chan struct{})
+	go func() {
+		defer close(newDone)
+		newErr = m.SetKeyspaceServiceLimit(constant.NullKeyspaceID, 200)
+	}()
+	select {
+	case <-newDone:
+		t.Fatal("the new-term call must be blocked by serviceLimitLocks while the old-term call is still parked")
+	case <-time.After(100 * time.Millisecond):
+	}
+
+	// Resume the old-term call: it persists and publishes 100, then releases
+	// serviceLimitLocks, letting the new-term call finally run and persist
+	// and publish 200.
+	close(release)
+	<-oldDone
+	re.NoError(oldErr)
+	<-newDone
+	re.NoError(newErr)
+
+	limiter := m.GetKeyspaceServiceLimiter(constant.NullKeyspaceID)
+	re.NotNil(limiter)
+	re.Equal(float64(200), limiter.ServiceLimit,
+		"the new-term call must run - and win - only after the old-term call fully finished, not race ahead of or get clobbered by it")
+}
+
 // BenchmarkAsyncLoadMergeReaderStall measures the worst-case time a concurrent
 // reader is blocked while the async bulk merge installs a large number of
 // resource groups. The probe uses GetControllerConfig, whose only cost is the
diff --git a/pkg/mcs/resourcemanager/server/metadata_watcher_test.go b/pkg/mcs/resourcemanager/server/metadata_watcher_test.go
index e846fd5aa4..bb3d8ad105 100644
--- a/pkg/mcs/resourcemanager/server/metadata_watcher_test.go
+++ b/pkg/mcs/resourcemanager/server/metadata_watcher_test.go
@@ -31,6 +31,7 @@ import (
 	"github.com/tikv/pd/pkg/keyspace/constant"
 	"github.com/tikv/pd/pkg/storage"
 	"github.com/tikv/pd/pkg/utils/keypath"
+	"github.com/tikv/pd/pkg/utils/syncutil"
 )
 
 type countingServiceLimitLoadStorage struct {
@@ -45,9 +46,10 @@ func (s *countingServiceLimitLoadStorage) LoadServiceLimits(f func(keyspaceID ui
 
 func newMetadataWatcherTestManager(store storage.Storage) *Manager {
 	return &Manager{
-		storage:          store,
-		krgms:            make(map[uint32]*keyspaceResourceGroupManager),
-		controllerConfig: &ControllerConfig{},
+		storage:           store,
+		krgms:             make(map[uint32]*keyspaceResourceGroupManager),
+		controllerConfig:  &ControllerConfig{},
+		serviceLimitLocks: syncutil.NewLockGroup(),
 	}
 }
 
@@ -437,3 +439,40 @@ func TestMetadataWatcherModeReleasesSyncLoadedGroups(t *testing.T) {
 	re.Nil(syncLoadedGroups)
 	re.NotNil(m.getKeyspaceResourceGroupManager(10).getResourceGroup("watched", false))
 }
+
+// TestGetOrCreateKeyspaceResourceGroupManagerWatcherModeDoesNotClobberPendingDefault
+// guards against trusting LoadingStateCompleted as "cache is authoritative" in
+// metadata-watcher mode. There it only means the initial bootstrap finished,
+// not that every write already in storage has had its watch event delivered
+// to this replica's cache yet. If getOrCreateKeyspaceResourceGroupManager
+// synthesized directly on that signal, a request landing after a customized
+// default was persisted but before its watch event arrived would find
+// nothing cached and persist the built-in default over it.
+func TestGetOrCreateKeyspaceResourceGroupManagerWatcherModeDoesNotClobberPendingDefault(t *testing.T) {
+	re := require.New(t)
+
+	store := storage.NewStorageWithMemoryBackend()
+	// A customized default is already persisted - e.g. by PD - but its watch
+	// event has not been delivered to this replica's cache yet.
+	customized := newMetadataWatcherResourceGroup(DefaultResourceGroupName, middlePriority, 555, 555)
+	re.NoError(store.SaveResourceGroupSetting(1, DefaultResourceGroupName, customized))
+
+	m := newMetadataWatcherTestManager(store)
+	m.enableMetadataWatcher = true
+	m.setLoadingState(LoadingStateCompleted)
+
+	krgm := m.getOrCreateKeyspaceResourceGroupManager(1, true)
+	re.NotNil(krgm)
+
+	group := krgm.getResourceGroup(DefaultResourceGroupName, false)
+	re.NotNil(group)
+	re.Equal(float64(555), group.getFillRate(),
+		"the customized default already in storage must be picked up by a point load, not overwritten by a synthesized built-in default")
+
+	raw, err := store.LoadResourceGroupSetting(1, DefaultResourceGroupName)
+	re.NoError(err)
+	loaded := &rmpb.ResourceGroup{}
+	re.NoError(proto.Unmarshal([]byte(raw), loaded))
+	re.Equal(uint64(555), loaded.RUSettings.RU.Settings.FillRate,
+		"the customized default persisted before the watcher delivered its event must survive in storage")
+}

From 0ec8dabd2550bb2a075698debf993f699a726a61 Mon Sep 17 00:00:00 2001
From: bufferflies <1045931706@qq.com>
Date: Wed, 5 Aug 2026 11:14:29 +0200
Subject: [PATCH 41/50] resource_group: close two more cross-term races found
 in follow-up review

Two more gaps surfaced by review on #10873, both closed here; a third
related gap is documented but deferred to tikv/pd#11105.

1. loadServiceLimits (run on every Init/leadership change) applied the
   value its bulk storage scan had already read before doing any
   locking. serviceLimitLocks alone doesn't protect against this: a
   concurrent SetKeyspaceServiceLimit call can persist and mirror a
   newer value while the replay's callback for that keyspace is still
   in flight, and the replay would then silently clobber the cache
   with its own now-stale snapshot. Fixed by re-reading the value from
   storage under serviceLimitLocks instead of trusting the bulk scan's
   value.

2. Several places inserted or updated a group in an already-published,
   concurrently-readable keyspace manager and only synced its derived
   burst-limit override afterward, outside the lock that made the
   change visible: the async bulk-merge loop, the single-group lazy
   load path, publishResourceGroupMutation's generic publish step
   (shared by AddResourceGroup/ModifyResourceGroup/SetKeyspaceServiceLimit's
   cross-term mirror), the on-demand default-group synthesis path, and
   the reserved-default-group placeholder paths. A concurrent token
   request landing in that window could read a group as unbounded and
   bypass an active keyspace service limit until the sync caught up.
   Fixed by adding a krgm-lock-already-held sync variant
   (syncBurstabilityWithServiceLimitLocked, plus a group-lock-only core
   shared with the merged ApplySettings+sync path added to
   upsertResourceGroupFromRaw's in-place update branch) and moving the
   sync inside the same critical section as the visibility change at
   every affected call site, so a concurrent reader - which needs the
   same lock - can no longer observe the change before its sync.

3. Not fixed here: publishResourceGroupMutation's "skip if the new
   term already confirmed the group" guard assumes a confirmed write
   is always the latest one. That's known to be unsafe for Delete
   already (tracked in #11105); review pointed out it's also unsafe
   for Add/Modify, since their storage write can still be in flight
   when a new-term read confirms an older snapshot. Documented in the
   Known gap comment and folded into #11105's scope; needs the same
   storage-side revision/CAS work already tracked there.

Also filed tikv/pd#11111 for a separate, pre-existing bug found while
auditing this: ModifyResourceGroup never re-syncs a group's burst
override when patching an already-cached group in place (the common,
no-leadership-change case), since publishResourceGroupMutation's sync
step only runs when the cached object's identity changed.

Each fix has a regression test verified to fail with the exact
predicted signature against the pre-fix code and pass against the fix;
full suite green under -race with failpoints enabled.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Signed-off-by: bufferflies <1045931706@qq.com>
---
 .../server/keyspace_manager.go                |  86 ++++++++--
 pkg/mcs/resourcemanager/server/manager.go     |  99 ++++++++----
 .../server/manager_async_test.go              | 151 ++++++++++++++++++
 .../resourcemanager/server/resource_group.go  |   4 +
 4 files changed, 291 insertions(+), 49 deletions(-)

diff --git a/pkg/mcs/resourcemanager/server/keyspace_manager.go b/pkg/mcs/resourcemanager/server/keyspace_manager.go
index 466a5aaee5..80121b2ed0 100644
--- a/pkg/mcs/resourcemanager/server/keyspace_manager.go
+++ b/pkg/mcs/resourcemanager/server/keyspace_manager.go
@@ -172,15 +172,28 @@ func (krgm *keyspaceResourceGroupManager) upsertResourceGroupFromRaw(name string
 	existing := krgm.groups[group.Name]
 	krgm.RUnlock()
 	if existing != nil {
-		if err := existing.ApplySettings(group); err != nil {
+		// Merge the settings patch and the burst-limit sync into a single
+		// rg.Lock() critical section: ApplySettings can flip a group from
+		// bounded to unbounded (e.g. a raw burst limit going negative)
+		// without touching overrideBurstLimit at all, so applying them as two
+		// separate critical sections would let a concurrent token request
+		// observe the now-unbounded settings before the sync catches up and
+		// bypass an active keyspace service limit in between.
+		serviceLimit, isSet := krgm.getServiceLimit()
+		existing.Lock()
+		patchErr := existing.patchSettingsLocked(group, false)
+		if patchErr == nil {
+			applyBurstabilitySyncLocked(existing, serviceLimit, isSet)
+		}
+		existing.Unlock()
+		if patchErr != nil {
 			log.Error("failed to apply the keyspace resource group settings from raw value",
-				zap.Uint32("keyspace-id", krgm.keyspaceID), zap.String("name", name), zap.String("raw-value", rawValue), zap.Error(err))
-			return err
+				zap.Uint32("keyspace-id", krgm.keyspaceID), zap.String("name", name), zap.String("raw-value", rawValue), zap.Error(patchErr))
+			return patchErr
 		}
 		krgm.Lock()
 		delete(krgm.reservedGroups, group.Name)
 		krgm.Unlock()
-		krgm.syncBurstabilityWithServiceLimit(existing)
 		return nil
 	}
 
@@ -188,8 +201,8 @@ func (krgm *keyspaceResourceGroupManager) upsertResourceGroupFromRaw(name string
 	krgm.Lock()
 	krgm.groups[group.Name] = resourceGroup
 	delete(krgm.reservedGroups, group.Name)
+	krgm.syncBurstabilityWithServiceLimitLocked(resourceGroup)
 	krgm.Unlock()
-	krgm.syncBurstabilityWithServiceLimit(resourceGroup)
 	return nil
 }
 
@@ -297,17 +310,13 @@ func (krgm *keyspaceResourceGroupManager) ensureReservedDefaultGroupInCache() {
 		return
 	}
 	defaultGroup := newDefaultResourceGroup()
-	inserted := false
 	krgm.Lock()
 	if _, ok := krgm.groups[DefaultResourceGroupName]; !ok {
 		krgm.groups[DefaultResourceGroupName] = defaultGroup
 		krgm.reservedGroups[DefaultResourceGroupName] = struct{}{}
-		inserted = true
+		krgm.syncBurstabilityWithServiceLimitLocked(defaultGroup)
 	}
 	krgm.Unlock()
-	if inserted {
-		krgm.syncBurstabilityWithServiceLimit(defaultGroup)
-	}
 }
 
 func newDefaultResourceGroup() *ResourceGroup {
@@ -330,8 +339,8 @@ func (krgm *keyspaceResourceGroupManager) restoreDefaultResourceGroupFromReserve
 	krgm.Lock()
 	krgm.groups[DefaultResourceGroupName] = defaultGroup
 	krgm.reservedGroups[DefaultResourceGroupName] = struct{}{}
+	krgm.syncBurstabilityWithServiceLimitLocked(defaultGroup)
 	krgm.Unlock()
-	krgm.syncBurstabilityWithServiceLimit(defaultGroup)
 }
 
 // persistResourceGroup validates grouppb, builds the in-memory group, and
@@ -364,8 +373,8 @@ func (krgm *keyspaceResourceGroupManager) addResourceGroup(grouppb *rmpb.Resourc
 	krgm.Lock()
 	krgm.groups[group.Name] = group
 	delete(krgm.reservedGroups, group.Name)
+	krgm.syncBurstabilityWithServiceLimitLocked(group)
 	krgm.Unlock()
-	krgm.syncBurstabilityWithServiceLimit(group)
 	return nil
 }
 
@@ -582,6 +591,12 @@ func (krgm *keyspaceResourceGroupManager) getServiceLimiter() *serviceLimiter {
 func (krgm *keyspaceResourceGroupManager) getServiceLimit() (float64, bool) {
 	krgm.RLock()
 	defer krgm.RUnlock()
+	return krgm.getServiceLimitLocked()
+}
+
+// getServiceLimitLocked is getServiceLimit for a caller that already holds
+// krgm's lock (read or write).
+func (krgm *keyspaceResourceGroupManager) getServiceLimitLocked() (float64, bool) {
 	if krgm.serviceLimiter == nil {
 		return 0, false
 	}
@@ -1033,14 +1048,55 @@ func (krgm *keyspaceResourceGroupManager) cleanupOverrides() {
 // Newly loaded groups can miss the initial service-limit replay, so apply the
 // same baseline burst invalidation when they enter the cache.
 func (krgm *keyspaceResourceGroupManager) syncBurstabilityWithServiceLimit(group *ResourceGroup) {
-	if group == nil || group.getBurstLimit(true) >= 0 || group.getOverrideBurstLimit() >= 0 {
+	krgm.RLock()
+	defer krgm.RUnlock()
+	krgm.syncBurstabilityWithServiceLimitLocked(group)
+}
+
+// syncBurstabilityWithServiceLimitLocked is syncBurstabilityWithServiceLimit
+// for a caller that already holds krgm's lock (read or write). It must never
+// call back into a krgm-locking helper (e.g. getServiceLimit/syncBurstabilityWithServiceLimit
+// itself): krgm's lock isn't reentrant, so doing so from the same goroutine
+// would deadlock. Taking group's own lock below is safe while holding krgm's:
+// group has no back-reference to krgm, so this one-directional nesting
+// (krgm's lock outer, group's lock inner) can't cycle with any other lock
+// ordering in this package.
+//
+// Callers use this to make a group's cache insertion and its burst-limit
+// sync take effect atomically under a single krgm.Lock() critical section:
+// without it, a concurrent token request could observe the group between the
+// insert and a separately-locked sync call and read its unsynced, possibly
+// unlimited/negative burst setting, letting it bypass an active keyspace
+// service limit until the sync catches up.
+func (krgm *keyspaceResourceGroupManager) syncBurstabilityWithServiceLimitLocked(group *ResourceGroup) {
+	if group == nil {
+		return
+	}
+	serviceLimit, isSet := krgm.getServiceLimitLocked()
+	group.Lock()
+	defer group.Unlock()
+	applyBurstabilitySyncLocked(group, serviceLimit, isSet)
+}
+
+// applyBurstabilitySyncLocked is the group-lock-only core shared by both
+// syncBurstabilityWithServiceLimitLocked (krgm-lock callers) and
+// upsertResourceGroupFromRaw's in-place update path (which merges this into
+// the same rg.Lock() critical section as ApplySettings, so a settings patch
+// that newly makes a group unbounded can never become visible without its
+// burst override already applied). The caller must already hold group's
+// lock; serviceLimit/isSet come from a plain krgm.getServiceLimit() read
+// taken before acquiring it - service-limit staleness of a few instructions
+// here is the same self-healing eventual consistency already accepted
+// elsewhere in this file (e.g. invalidateBurstability), unlike the group's
+// own settings-vs-override visibility this function exists to make atomic.
+func applyBurstabilitySyncLocked(group *ResourceGroup, serviceLimit float64, isSet bool) {
+	if group == nil || group.getBurstLimitLocked(true) >= 0 || group.getOverrideBurstLimitLocked() >= 0 {
 		return
 	}
-	serviceLimit, isSet := krgm.getServiceLimit()
 	if !isSet || serviceLimit <= 0 {
 		return
 	}
-	group.overrideBurstLimit(int64(serviceLimit))
+	group.overrideBurstLimitLocked(int64(serviceLimit))
 }
 
 // Since the burstable resource groups won't require tokens from the server anymore,
diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go
index a296c45635..0612230633 100644
--- a/pkg/mcs/resourcemanager/server/manager.go
+++ b/pkg/mcs/resourcemanager/server/manager.go
@@ -540,7 +540,26 @@ func (m *Manager) initMetadata(ctx context.Context) error {
 }
 
 func (m *Manager) loadServiceLimits() error {
-	return m.storage.LoadServiceLimits(func(keyspaceID uint32, serviceLimit float64) {
+	return m.storage.LoadServiceLimits(func(keyspaceID uint32, _ float64) {
+		failpoint.InjectCall("loadServiceLimitsBeforeApply", keyspaceID)
+		// Serialize against SetKeyspaceServiceLimit for this keyspace, and
+		// re-read the value from storage instead of trusting the one the
+		// bulk scan above already fetched: that value was read before this
+		// lock was acquired, so a concurrent SetKeyspaceServiceLimit call
+		// (from either this term or a still-in-flight previous one) can
+		// persist and mirror a newer value in between, which this callback
+		// would otherwise silently clobber with its now-stale snapshot. A
+		// fresh point read taken under the same lock SetKeyspaceServiceLimit
+		// itself uses is guaranteed to observe whichever write actually
+		// landed last, since setServiceLimit's storage write always
+		// completes before SetKeyspaceServiceLimit releases this lock.
+		m.serviceLimitLocks.Lock(keyspaceID)
+		defer m.serviceLimitLocks.Unlock(keyspaceID)
+		serviceLimit, err := m.storage.LoadServiceLimit(keyspaceID)
+		if err != nil {
+			log.Warn("failed to reload service limit", zap.Uint32("keyspace-id", keyspaceID), zap.Error(err))
+			return
+		}
 		m.getOrCreateKeyspaceResourceGroupManager(keyspaceID, false).setServiceLimitFromStorage(serviceLimit)
 	})
 }
@@ -665,11 +684,6 @@ func (m *Manager) asyncLoadResourceGroups(ctx context.Context, epoch uint64) {
 		aborted := false
 		for start := 0; start < len(pending); start += mergeBatchSize {
 			end := min(start+mergeBatchSize, len(pending))
-			type syncItem struct {
-				krgm  *keyspaceResourceGroupManager
-				group *ResourceGroup
-			}
-			toSync := make([]syncItem, 0, end-start)
 			m.Lock()
 			if m.loadEpoch != epoch {
 				// The manager was reinitialized for a new term while this
@@ -697,16 +711,15 @@ func (m *Manager) asyncLoadResourceGroups(ctx context.Context, epoch uint64) {
 				// placeholder by loadResourceGroupIfNeeded or skipped by the
 				// state persist loop.
 				delete(krgm.reservedGroups, it.name)
+				failpoint.InjectCall("mergeBeforeBurstSync", it.keyspaceID, it.name)
+				// Sync burstability while still holding krgm's lock, so the
+				// group never becomes visible to a concurrent reader with an
+				// unsynced burst setting - see syncBurstabilityWithServiceLimitLocked.
+				krgm.syncBurstabilityWithServiceLimitLocked(it.group)
 				krgm.Unlock()
-				toSync = append(toSync, syncItem{krgm: krgm, group: it.group})
 				loaded++
 			}
 			m.Unlock()
-			// Sync burstability outside m.Lock; it only needs the keyspace and
-			// group locks.
-			for _, s := range toSync {
-				s.krgm.syncBurstabilityWithServiceLimit(s.group)
-			}
 		}
 		if aborted {
 			log.Info("async loading resource groups aborted: manager was reinitialized")
@@ -951,15 +964,18 @@ func (m *Manager) loadResourceGroupIfNeeded(keyspaceID uint32, name string) erro
 			inserted = true
 		}
 		delete(krgm.reservedGroups, name)
+		if inserted {
+			// Sync burstability while krgm's lock is still held, so the group
+			// never becomes visible to a concurrent reader with an unsynced
+			// burst setting - see syncBurstabilityWithServiceLimitLocked.
+			krgm.syncBurstabilityWithServiceLimitLocked(group)
+		}
 		krgm.Unlock()
 		if m.syncLoadedGroups != nil {
 			m.syncLoadedGroups[markKey] = true
 		}
 		m.Unlock()
 		failpoint.Inject("lazyLoadAfterCachePublish", func() {})
-		if inserted {
-			krgm.syncBurstabilityWithServiceLimit(group)
-		}
 		syncLoadGroupCounter.Inc()
 		return nil
 	}
@@ -1010,23 +1026,35 @@ func (m *Manager) markResourceGroupSyncLoaded(keyspaceID uint32, krgm *keyspaceR
 // (re)installed, the group to sync burstability for.
 //
 // Known gap: this "skip if the new term already confirmed the group" rule
-// assumes the confirming write's storage effect is genuinely the latest one,
-// which holds for Add/Modify (both persist before they can be parked - see
-// addResourceGroupBeforePublish/modifyResourceGroupBeforePublish) but not for
-// Delete, which can be parked before its storage phase
-// (deleteResourceGroupBeforeStorage). If an old-term Delete is parked there,
-// a new-term Add for the same group persists and publishes (getting
-// confirmed), and only then does the old Delete's storage removal finally
-// run, it deletes that newer write in storage - making Delete genuinely the
-// last writer - but this function skips its publish because the group is
-// already "confirmed", leaving the cache showing the deleted group as
-// present. Nothing re-syncs it afterward, since it reads as confirmed to
-// every other path too. This mirrors, in the opposite direction, a gap the
-// old unconditional-apply code had (a delayed Delete publish could instead
-// wipe a newer confirmed Add). Neither direction is fully closed without a
-// storage-side revision to tell which write actually landed last; that's the
-// same class of gap tracked for initDefaultResourceGroup's stillCurrent
-// check. No regression test exercises this interleaving yet.
+// assumes the confirming write's storage effect is genuinely the latest one.
+// That assumption can fail for any mutation kind, not just Delete:
+//
+//   - Delete can be parked before its storage phase (deleteResourceGroupBeforeStorage).
+//     If an old-term Delete is parked there, a new-term Add for the same group
+//     persists and publishes (getting confirmed), and only then does the old
+//     Delete's storage removal finally run, it deletes that newer write in
+//     storage - making Delete genuinely the last writer - but this function
+//     skips its publish because the group is already "confirmed", leaving the
+//     cache showing the deleted group as present.
+//   - Add/Modify's own storage write always completes before either of them can
+//     be parked (see addResourceGroupBeforePublish/modifyResourceGroupBeforePublish,
+//     both placed after the storage write) - but that only rules out being
+//     parked *after* persisting, not the storage write itself taking real
+//     wall-clock time to land. A new-term bulk merge or lazy load can read and
+//     confirm an older snapshot while an old-term Add/Modify's storage write is
+//     still in flight; when that write then completes, it's the latest value in
+//     storage, but this function still sees "already confirmed" for the older
+//     snapshot and drops the publish, leaving the cache stale relative to
+//     storage indefinitely.
+//
+// In both cases nothing re-syncs the cache afterward, since it reads as
+// confirmed to every other path too. The Delete case also mirrors, in the
+// opposite direction, a gap the old unconditional-apply code had (a delayed
+// Delete publish could instead wipe a newer confirmed Add). None of this is
+// fully closed without a storage-side revision to tell which write actually
+// landed last; that's the same class of gap tracked for
+// initDefaultResourceGroup's stillCurrent check. No regression test exercises
+// either interleaving yet.
 //
 // TODO(#11105): close this gap with a storage-side revision/CAS check.
 func (m *Manager) publishResourceGroupMutation(
@@ -1047,10 +1075,13 @@ func (m *Manager) publishResourceGroupMutation(
 	}
 	cur.Lock()
 	mark, synced := fn(cur)
-	cur.Unlock()
 	if synced != nil {
-		cur.syncBurstabilityWithServiceLimit(synced)
+		// Sync burstability while cur's lock is still held, so the group
+		// never becomes visible to a concurrent reader with an unsynced
+		// burst setting - see syncBurstabilityWithServiceLimitLocked.
+		cur.syncBurstabilityWithServiceLimitLocked(synced)
 	}
+	cur.Unlock()
 	if mark && m.syncLoadedGroups != nil {
 		m.syncLoadedGroups[trackerKey{keyspaceID: keyspaceID, groupName: name}] = true
 	}
diff --git a/pkg/mcs/resourcemanager/server/manager_async_test.go b/pkg/mcs/resourcemanager/server/manager_async_test.go
index 0f36cb944d..62787746a6 100644
--- a/pkg/mcs/resourcemanager/server/manager_async_test.go
+++ b/pkg/mcs/resourcemanager/server/manager_async_test.go
@@ -1091,6 +1091,157 @@ func TestAsyncLoadResourceGroupsCrossTermSetServiceLimitSerializesAgainstCompeti
 		"the new-term call must run - and win - only after the old-term call fully finished, not race ahead of or get clobbered by it")
 }
 
+// TestLoadServiceLimitsDoesNotClobberConcurrentSet guards against a race
+// between loadServiceLimits' bulk replay (run on every Init/leadership
+// change) and a concurrent SetKeyspaceServiceLimit call for the same
+// keyspace. loadServiceLimits used to apply the value its bulk storage scan
+// had already read before doing any locking; if a concurrent
+// SetKeyspaceServiceLimit call persisted and mirrored a newer value while the
+// replay's callback for that keyspace was still in flight, the replay would
+// silently overwrite the cache with its own now-stale snapshot, leaving the
+// cache stuck behind storage until the next full reload. serviceLimitLocks
+// alone (the fix for the sibling cross-term race above) does not close this:
+// the stale value here was captured before any lock was ever taken, so
+// merely serializing the two callers does not stop the replay from applying
+// data that was already out of date the moment it read it. The fix instead
+// re-reads the keyspace's service limit from storage under the same lock,
+// discarding the bulk scan's value entirely.
+func TestLoadServiceLimitsDoesNotClobberConcurrentSet(t *testing.T) {
+	re := require.New(t)
+	m := prepareManager()
+	const keyspaceID = 1
+	re.NoError(m.storage.SaveServiceLimit(keyspaceID, 100))
+
+	reached := make(chan struct{})
+	release := make(chan struct{})
+	re.NoError(failpoint.EnableCall("github.com/tikv/pd/pkg/mcs/resourcemanager/server/loadServiceLimitsBeforeApply", func(gotKeyspaceID uint32) {
+		if gotKeyspaceID != keyspaceID {
+			return
+		}
+		close(reached)
+		<-release
+	}))
+	defer func() {
+		re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/mcs/resourcemanager/server/loadServiceLimitsBeforeApply"))
+	}()
+
+	loadDone := make(chan struct{})
+	go func() {
+		defer close(loadDone)
+		re.NoError(m.loadServiceLimits())
+	}()
+	select {
+	case <-reached:
+	case <-time.After(time.Second):
+		t.Fatal("timed out waiting for loadServiceLimits to reach its replay callback")
+	}
+
+	// While the replay is parked - holding only the (100) value its scan
+	// already captured - a concurrent SetKeyspaceServiceLimit call persists
+	// and mirrors a newer value into the same, live keyspace manager.
+	re.NoError(m.SetKeyspaceServiceLimit(keyspaceID, 200))
+
+	close(release)
+	<-loadDone
+
+	limiter := m.GetKeyspaceServiceLimiter(keyspaceID)
+	re.NotNil(limiter)
+	re.Equal(float64(200), limiter.ServiceLimit,
+		"the replay must not overwrite a concurrently-set newer value with its own stale snapshot")
+}
+
+// TestAsyncLoadResourceGroupsMergeGroupNeverVisibleUnsynced guards against a
+// window in the bulk merge where a newly-loaded group became visible in
+// krgm.groups (readable by any concurrent GetResourceGroup/token request)
+// before its burst limit was synced against the keyspace's active service
+// limit. A group with no explicit burst limit reads as unbounded until that
+// sync runs, so a request landing in the gap could be granted unlimited
+// burst and bypass the service limit until the merge caught up. The fix
+// moved the sync inside the same krgm.Lock() critical section as the insert,
+// so a concurrent reader - which also needs krgm's lock - can no longer
+// observe the group until both have happened.
+func TestAsyncLoadResourceGroupsMergeGroupNeverVisibleUnsynced(t *testing.T) {
+	re := require.New(t)
+	store := storage.NewStorageWithMemoryBackend()
+	const keyspaceID = 1
+	const groupName = "burstable-rg"
+	unbounded := &resource_manager.ResourceGroup{
+		Name:     groupName,
+		Mode:     resource_manager.GroupMode_RUMode,
+		Priority: middlePriority,
+		RUSettings: &resource_manager.GroupRequestUnitSettings{
+			RU: &resource_manager.TokenBucket{
+				Settings: &resource_manager.TokenLimitSettings{
+					FillRate:   UnlimitedRate,
+					BurstLimit: UnlimitedBurstLimit,
+				},
+			},
+		},
+	}
+	re.NoError(store.SaveResourceGroupSetting(keyspaceID, groupName, unbounded))
+	re.NoError(store.SaveServiceLimit(keyspaceID, 50))
+
+	m := NewManager[*mockConfigProvider](&mockConfigProvider{})
+	m.storage = store
+
+	reached := make(chan struct{})
+	release := make(chan struct{})
+	var releaseOnce sync.Once
+	unblock := func() { releaseOnce.Do(func() { close(release) }) }
+	re.NoError(failpoint.EnableCall("github.com/tikv/pd/pkg/mcs/resourcemanager/server/mergeBeforeBurstSync", func(gotKeyspaceID uint32, gotName string) {
+		if gotKeyspaceID != keyspaceID || gotName != groupName {
+			return
+		}
+		close(reached)
+		<-release
+	}))
+	defer func() {
+		re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/mcs/resourcemanager/server/mergeBeforeBurstSync"))
+	}()
+
+	re.NoError(m.Init(context.Background()))
+	defer stopAsyncTestManager(m)
+	// Unblock the merge first (LIFO), so stopAsyncTestManager's wg.Wait()
+	// cannot hang if a later assertion aborts the test before the explicit
+	// unblock() call below is reached.
+	defer unblock()
+
+	// loadServiceLimits (run synchronously inside Init, before the async
+	// merge starts) already created krgm for keyspaceID via the service
+	// limit saved above - capture it now, before parking, so the reader
+	// below can call krgm.getResourceGroup directly instead of going through
+	// m.getKeyspaceResourceGroupManager. The latter needs m.RLock(), which
+	// the merge holds for its *entire* batch regardless of this fix, so
+	// routing the read through it would block on the wrong lock and the test
+	// would pass even without the fix; reading via the captured krgm
+	// isolates the one lock (krgm's own) this test is actually about.
+	krgm := m.getKeyspaceResourceGroupManager(keyspaceID)
+	re.NotNil(krgm)
+
+	select {
+	case <-reached:
+	case <-time.After(time.Second):
+		t.Fatal("timed out waiting for the merge to reach its burst-sync point")
+	}
+
+	readDone := make(chan *ResourceGroup)
+	go func() {
+		readDone <- krgm.getResourceGroup(groupName, false)
+	}()
+
+	select {
+	case <-readDone:
+		t.Fatal("a concurrent reader must not observe the group while the merge is parked between insert and burst sync - krgm's lock should still be held")
+	case <-time.After(100 * time.Millisecond):
+	}
+
+	unblock()
+	group := <-readDone
+	re.NotNil(group)
+	re.GreaterOrEqual(group.getOverrideBurstLimit(), int64(0),
+		"the group must never become visible without its burst override already synced")
+}
+
 // BenchmarkAsyncLoadMergeReaderStall measures the worst-case time a concurrent
 // reader is blocked while the async bulk merge installs a large number of
 // resource groups. The probe uses GetControllerConfig, whose only cost is the
diff --git a/pkg/mcs/resourcemanager/server/resource_group.go b/pkg/mcs/resourcemanager/server/resource_group.go
index 4a0276c1f4..4ac5aed7c0 100644
--- a/pkg/mcs/resourcemanager/server/resource_group.go
+++ b/pkg/mcs/resourcemanager/server/resource_group.go
@@ -178,6 +178,10 @@ func (rg *ResourceGroup) getBurstLimitLocked(ignoreOverride ...bool) int64 {
 func (rg *ResourceGroup) getOverrideBurstLimit() int64 {
 	rg.RLock()
 	defer rg.RUnlock()
+	return rg.getOverrideBurstLimitLocked()
+}
+
+func (rg *ResourceGroup) getOverrideBurstLimitLocked() int64 {
 	return rg.RUSettings.RU.overrideBurstLimit
 }
 

From e6b896ede149d765039fb1250cf5acecb527a62f Mon Sep 17 00:00:00 2001
From: bufferflies <1045931706@qq.com>
Date: Wed, 5 Aug 2026 11:53:47 +0200
Subject: [PATCH 42/50] resource_group: route default-group synthesis through
 publishResourceGroupMutation

initDefaultResourceGroup used to publish the synthesized default group
through krgm's own lock directly (via addResourceGroup), with the
caller (loadResourceGroupIfNeeded) setting the sync-loaded marker
afterward in a separate, later m.Lock() critical section. That left a
window where a concurrent async bulk-merge batch - which only skips an
item already marked - could run in between and overwrite the
synthesized group, including any live consumption/token update applied
to it in that window, with its own possibly-stale scanned copy.

Fixed by moving initDefaultResourceGroup from a krgm method to a
Manager method and routing its publish step through
publishResourceGroupMutation, the same path a real Add/ModifyResourceGroup
already uses for the default group. That function holds m.Lock() across
both the cache-visibility change and the sync-loaded marker set, so
nothing that also needs m.Lock() - including a merge batch - can ever
observe one without the other. It also already sets the marker itself,
so the caller-side manual marker-setting block in loadResourceGroupIfNeeded
is now dead code and removed.

This reuses the existing defaultGroupMu -> m.Lock() -> krgm.Lock()
ordering AddResourceGroup/ModifyResourceGroup already exercise for the
default group, so it introduces no new lock ordering. stillCurrent
remains as a fail-fast check before the storage write (avoiding a
wasted persist when the caller already knows krgm is stale), but is no
longer the only guard against a term change landing while that write
is in flight: publishResourceGroupMutation's own confirmed-write check
now also covers that window.

Regression test (TestInitDefaultResourceGroupMarksAtomicallyWithPublish)
parks the publish between the cache-visibility change and the marker
set via a new failpoint and confirms a concurrent m.Lock()-holding
operation cannot proceed until both have happened; verified it fails
(cleanly, via timeout - the pre-fix code never reaches this failpoint
at all) against the pre-fix code and passes against the fix. Updated
the three keyspace_manager_test.go callers that exercised krgm.initDefaultResourceGroup
directly to go through a Manager instead.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Signed-off-by: bufferflies <1045931706@qq.com>
---
 .../server/keyspace_manager.go                |  57 ----------
 .../server/keyspace_manager_test.go           |  16 ++-
 pkg/mcs/resourcemanager/server/manager.go     | 100 +++++++++++++++---
 .../server/manager_async_test.go              |  67 ++++++++++++
 4 files changed, 163 insertions(+), 77 deletions(-)

diff --git a/pkg/mcs/resourcemanager/server/keyspace_manager.go b/pkg/mcs/resourcemanager/server/keyspace_manager.go
index 80121b2ed0..1da724e787 100644
--- a/pkg/mcs/resourcemanager/server/keyspace_manager.go
+++ b/pkg/mcs/resourcemanager/server/keyspace_manager.go
@@ -245,63 +245,6 @@ func (krgm *keyspaceResourceGroupManager) setRawStatesIntoResourceGroup(name str
 	return nil
 }
 
-// initDefaultResourceGroup synthesizes and persists the built-in default
-// group if nothing confirmed exists yet. created reports whether it actually
-// performed a synthesis, so callers that participate in the async bulk-load
-// merge (loadResourceGroupIfNeeded) know when they must publish a
-// sync-loaded marker for what this call just wrote. The three (created, err)
-// outcomes need different handling and must not be collapsed into a single
-// bool by the caller: (false, nil) means confirmed data already existed -
-// nothing to do, safe to treat as success; (false, errs.ErrResourceGroupsLoading)
-// means stillCurrent caught a term change before the persist started - the
-// group was not created anywhere, so the caller must retry against the fresh
-// term rather than treat this as success; (false, any other non-nil error)
-// means the persist itself failed (e.g. a storage write error) - the caller
-// must propagate it rather than silently swallow a real failure as success.
-//
-// defaultGroupMu only serializes callers that share this krgm instance; it
-// does nothing across a term change, since Init gives the new term an
-// entirely new krgm object with its own, separate defaultGroupMu.
-// stillCurrent, when non-nil, is checked immediately before the persist
-// (right after defaultGroupMu is acquired) so a caller with access to the
-// owning Manager can catch a term change that completed before this point -
-// typically by comparing this krgm against the manager's live entry for its
-// keyspace ID. It cannot catch a term change that lands while the persist's
-// storage write is already in flight; closing that residual window needs a
-// conditional (CAS) storage write, which this does not implement. Pass nil
-// when no such coordination is needed or available (e.g. in tests that
-// exercise krgm without a Manager).
-func (krgm *keyspaceResourceGroupManager) initDefaultResourceGroup(stillCurrent func() bool) (created bool, err error) {
-	// A confirmed cached entry means initialization is unnecessary; a missing
-	// or reserved-placeholder entry means nothing is persisted for the
-	// default group (e.g. a fresh store), so it must still be created and
-	// persisted, otherwise its settings are never stored and state
-	// persistence stays skipped.
-	if krgm.hasConfirmedResourceGroup(DefaultResourceGroupName) {
-		return false, nil
-	}
-	// Serialize against every other synthesis or real Add/ModifyResourceGroup
-	// targeting "default" that shares this krgm instance: see the
-	// defaultGroupMu doc comment on the struct.
-	krgm.defaultGroupMu.Lock()
-	defer krgm.defaultGroupMu.Unlock()
-	// Re-check under defaultGroupMu: while this goroutine waited for the
-	// lock, a real write may have already confirmed the default group, in
-	// which case synthesizing here would silently clobber it.
-	if krgm.hasConfirmedResourceGroup(DefaultResourceGroupName) {
-		return false, nil
-	}
-	if stillCurrent != nil && !stillCurrent() {
-		return false, errs.ErrResourceGroupsLoading
-	}
-	defaultGroup := newDefaultResourceGroup()
-	if err := krgm.addResourceGroup(defaultGroup.IntoProtoResourceGroup(krgm.keyspaceID)); err != nil {
-		log.Warn("init default group failed", zap.Uint32("keyspace-id", krgm.keyspaceID), zap.Error(err))
-		return false, err
-	}
-	return true, nil
-}
-
 func (krgm *keyspaceResourceGroupManager) ensureReservedDefaultGroupInCache() {
 	krgm.RLock()
 	_, ok := krgm.groups[DefaultResourceGroupName]
diff --git a/pkg/mcs/resourcemanager/server/keyspace_manager_test.go b/pkg/mcs/resourcemanager/server/keyspace_manager_test.go
index 49f58d4400..3f2d82ec80 100644
--- a/pkg/mcs/resourcemanager/server/keyspace_manager_test.go
+++ b/pkg/mcs/resourcemanager/server/keyspace_manager_test.go
@@ -86,8 +86,12 @@ func TestInitDefaultResourceGroup(t *testing.T) {
 	_, exists := krgm.groups[DefaultResourceGroupName]
 	re.False(exists)
 
-	// Initialize the default resource group.
-	created, err := krgm.initDefaultResourceGroup(nil)
+	// Initialize the default resource group. initDefaultResourceGroup now
+	// publishes through Manager.publishResourceGroupMutation, so it needs a
+	// Manager with krgm registered as the live entry for keyspace 1.
+	m := prepareManager()
+	m.krgms[1] = krgm
+	created, err := m.initDefaultResourceGroup(1, krgm, nil)
 	re.NoError(err)
 	re.True(created)
 
@@ -227,7 +231,9 @@ func TestDeleteResourceGroupBehavior(t *testing.T) {
 		_, ok := krgm.groupRUTrackers[group.GetName()]
 		re.False(ok)
 
-		_, err := krgm.initDefaultResourceGroup(nil)
+		m := prepareManager()
+		m.krgms[1] = krgm
+		_, err := m.initDefaultResourceGroup(1, krgm, nil)
 		re.NoError(err)
 		re.Error(krgm.deleteResourceGroup(DefaultResourceGroupName))
 		re.NotNil(krgm.getResourceGroup(DefaultResourceGroupName, false))
@@ -364,7 +370,9 @@ func TestGetResourceGroupList(t *testing.T) {
 	re.Equal("group2", groups[1].Name)
 	re.Equal("group3", groups[2].Name)
 
-	_, err := krgm.initDefaultResourceGroup(nil)
+	m := prepareManager()
+	m.krgms[1] = krgm
+	_, err := m.initDefaultResourceGroup(1, krgm, nil)
 	re.NoError(err)
 	groups = krgm.getResourceGroupList(false, true)
 	re.Len(groups, 4)
diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go
index 0612230633..eb514e08d5 100644
--- a/pkg/mcs/resourcemanager/server/manager.go
+++ b/pkg/mcs/resourcemanager/server/manager.go
@@ -404,7 +404,7 @@ func (m *Manager) getOrCreateKeyspaceResourceGroupManager(keyspaceID uint32, ini
 			// already logged inside initDefaultResourceGroup, and a later
 			// request for the default group will retry through
 			// loadResourceGroupIfNeeded, which does surface such errors.
-			_, _ = krgm.initDefaultResourceGroup(func() bool {
+			_, _ = m.initDefaultResourceGroup(keyspaceID, krgm, func() bool {
 				m.RLock()
 				defer m.RUnlock()
 				return m.krgms[keyspaceID] == krgm
@@ -893,7 +893,11 @@ func (m *Manager) loadResourceGroupIfNeeded(keyspaceID uint32, name string) erro
 					defer m.RUnlock()
 					return m.loadEpoch == epoch && m.krgms[keyspaceID] == krgm
 				}
-				created, initErr := krgm.initDefaultResourceGroup(stillCurrent)
+				// initDefaultResourceGroup publishes through
+				// publishResourceGroupMutation, which sets the sync-loaded
+				// marker itself atomically with the cache effect - no
+				// separate marker-setting step is needed here.
+				_, initErr := m.initDefaultResourceGroup(keyspaceID, krgm, stillCurrent)
 				if initErr != nil {
 					if errs.ErrResourceGroupsLoading.Equal(initErr) {
 						// stillCurrent caught a term change that landed after the
@@ -910,19 +914,6 @@ func (m *Manager) loadResourceGroupIfNeeded(keyspaceID uint32, name string) erro
 					// success for a group that was never actually created.
 					return initErr
 				}
-				if created {
-					// This synthesis bypassed publishResourceGroupMutation, so the
-					// sync-loaded marker was never set. Set it now: otherwise a
-					// bulk merge still in progress for this term doesn't know this
-					// group is already confirmed, and can replace it - including
-					// any live consumption update applied after this point - with
-					// a possibly-stale snapshot taken by the storage scan.
-					m.Lock()
-					if m.loadEpoch == epoch && m.krgms[keyspaceID] == krgm && m.syncLoadedGroups != nil {
-						m.syncLoadedGroups[trackerKey{keyspaceID: keyspaceID, groupName: DefaultResourceGroupName}] = true
-					}
-					m.Unlock()
-				}
 				return nil
 			}
 			return err
@@ -1082,11 +1073,88 @@ func (m *Manager) publishResourceGroupMutation(
 		cur.syncBurstabilityWithServiceLimitLocked(synced)
 	}
 	cur.Unlock()
+	failpoint.InjectCall("publishMutationBeforeMark")
 	if mark && m.syncLoadedGroups != nil {
 		m.syncLoadedGroups[trackerKey{keyspaceID: keyspaceID, groupName: name}] = true
 	}
 }
 
+// initDefaultResourceGroup synthesizes and persists the built-in default
+// group for keyspaceID into krgm if nothing confirmed exists yet, publishing
+// it through publishResourceGroupMutation - the same path a real
+// Add/ModifyResourceGroup uses - so the cache effect and the sync-loaded
+// marker land atomically with respect to the async bulk merge. This used to
+// publish through krgm's own lock only, with the caller setting the marker
+// afterward in a separate critical section; that left a window where the
+// bulk merge could run in between and replace the synthesized group - along
+// with any live consumption/token update applied to it in that window - with
+// its own possibly-stale scanned copy, since it had no way yet to know the
+// group was confirmed. Routing through publishResourceGroupMutation closes
+// that window the same way it already does for Add/Modify/Delete.
+//
+// created reports whether it actually performed a synthesis. The three
+// (created, err) outcomes need different handling and must not be collapsed
+// into a single bool by the caller: (false, nil) means confirmed data
+// already existed - nothing to do, safe to treat as success; (false,
+// errs.ErrResourceGroupsLoading) means stillCurrent caught a term change
+// before the persist started - the group was not created anywhere, so the
+// caller must retry against the fresh term rather than treat this as
+// success; (false, any other non-nil error) means the persist itself failed
+// (e.g. a storage write error) - the caller must propagate it rather than
+// silently swallow a real failure as success.
+//
+// defaultGroupMu only serializes callers that share the krgm instance; it
+// does nothing across a term change, since Init gives the new term an
+// entirely new krgm object with its own, separate defaultGroupMu.
+// stillCurrent, when non-nil, is checked immediately before the persist
+// (right after defaultGroupMu is acquired) to fail fast and avoid a wasted
+// storage write when the caller already knows krgm is stale - typically by
+// comparing it against the manager's live entry for its keyspace ID. It is
+// no longer the only guard against a term change landing while the persist's
+// storage write is in flight: publishResourceGroupMutation's own
+// cur != krgm && cur.hasConfirmedResourceGroup(name) check now also protects
+// that window (and, if the new term hasn't confirmed a default yet, still
+// applies this call's result into the new term's live krgm instead of
+// silently dropping it into a detached one). Pass nil when no such
+// fail-fast check is needed or available (e.g. in tests that exercise a
+// krgm/Manager pair with no concurrent writer to race against).
+func (m *Manager) initDefaultResourceGroup(keyspaceID uint32, krgm *keyspaceResourceGroupManager, stillCurrent func() bool) (created bool, err error) {
+	// A confirmed cached entry means initialization is unnecessary; a missing
+	// or reserved-placeholder entry means nothing is persisted for the
+	// default group (e.g. a fresh store), so it must still be created and
+	// persisted, otherwise its settings are never stored and state
+	// persistence stays skipped.
+	if krgm.hasConfirmedResourceGroup(DefaultResourceGroupName) {
+		return false, nil
+	}
+	// Serialize against every other synthesis or real Add/ModifyResourceGroup
+	// targeting "default" that shares this krgm instance: see the
+	// defaultGroupMu doc comment on the struct.
+	krgm.defaultGroupMu.Lock()
+	defer krgm.defaultGroupMu.Unlock()
+	// Re-check under defaultGroupMu: while this goroutine waited for the
+	// lock, a real write may have already confirmed the default group, in
+	// which case synthesizing here would silently clobber it.
+	if krgm.hasConfirmedResourceGroup(DefaultResourceGroupName) {
+		return false, nil
+	}
+	if stillCurrent != nil && !stillCurrent() {
+		return false, errs.ErrResourceGroupsLoading
+	}
+	defaultGroup := newDefaultResourceGroup()
+	group, err := krgm.persistResourceGroup(defaultGroup.IntoProtoResourceGroup(krgm.keyspaceID))
+	if err != nil {
+		log.Warn("init default group failed", zap.Uint32("keyspace-id", krgm.keyspaceID), zap.Error(err))
+		return false, err
+	}
+	m.publishResourceGroupMutation(keyspaceID, DefaultResourceGroupName, krgm, func(cur *keyspaceResourceGroupManager) (bool, *ResourceGroup) {
+		cur.groups[group.Name] = group
+		delete(cur.reservedGroups, group.Name)
+		return true, group
+	})
+	return true, nil
+}
+
 func (m *Manager) isResourceGroupLoadingComplete() bool {
 	return m.getLoadingState() == LoadingStateCompleted
 }
@@ -1201,7 +1269,7 @@ func (m *Manager) initReserved(epoch uint64) {
 		// Any failure is already logged inside initDefaultResourceGroup; a
 		// later request for the default group retries through
 		// loadResourceGroupIfNeeded once serving starts.
-		_, _ = krgm.initDefaultResourceGroup(nil)
+		_, _ = m.initDefaultResourceGroup(krgm.keyspaceID, krgm, nil)
 	}
 }
 
diff --git a/pkg/mcs/resourcemanager/server/manager_async_test.go b/pkg/mcs/resourcemanager/server/manager_async_test.go
index 62787746a6..b942bc7161 100644
--- a/pkg/mcs/resourcemanager/server/manager_async_test.go
+++ b/pkg/mcs/resourcemanager/server/manager_async_test.go
@@ -923,6 +923,73 @@ func TestAsyncLoadResourceGroupsCrossTermModifyDefaultStaysConfirmed(t *testing.
 		"the confirmed running state must be preserved, not reset to a synthetic placeholder")
 }
 
+// TestInitDefaultResourceGroupMarksAtomicallyWithPublish guards against a
+// window where a synthesized default group became visible in the cache
+// before its sync-loaded marker was set. initDefaultResourceGroup used to
+// publish through krgm's own lock directly, with the caller (loadResourceGroupIfNeeded)
+// setting the sync-loaded marker afterward in a separate, later m.Lock()
+// critical section. A concurrent bulk-merge batch - which only skips an item
+// already marked - could run in that window and overwrite the synthesized
+// group, including any live consumption/token update applied to it in the
+// meantime, with its own possibly-stale scanned copy. initDefaultResourceGroup
+// now publishes through publishResourceGroupMutation, which holds m.Lock()
+// across both the cache-visibility change and the marker set, so nothing
+// that also needs m.Lock() - including a merge batch - can ever observe one
+// without the other.
+func TestInitDefaultResourceGroupMarksAtomicallyWithPublish(t *testing.T) {
+	re := require.New(t)
+	m := prepareManager()
+	const keyspaceID = 1
+	krgm := newKeyspaceResourceGroupManager(keyspaceID, m.storage, m.writeRole)
+	m.krgms[keyspaceID] = krgm
+
+	reached := make(chan struct{})
+	release := make(chan struct{})
+	re.NoError(failpoint.EnableCall("github.com/tikv/pd/pkg/mcs/resourcemanager/server/publishMutationBeforeMark", func() {
+		close(reached)
+		<-release
+	}))
+	defer func() {
+		re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/mcs/resourcemanager/server/publishMutationBeforeMark"))
+	}()
+
+	var initErr error
+	initDone := make(chan struct{})
+	go func() {
+		defer close(initDone)
+		_, initErr = m.initDefaultResourceGroup(keyspaceID, krgm, nil)
+	}()
+
+	select {
+	case <-reached:
+	case <-time.After(time.Second):
+		t.Fatal("timed out waiting for initDefaultResourceGroup to reach its publish")
+	}
+
+	// While parked between the cache-visibility change and the marker set,
+	// nothing that needs m.Lock() (e.g. a concurrent merge batch checking
+	// whether to skip this group) should be able to proceed.
+	checkDone := make(chan bool)
+	go func() {
+		m.RLock()
+		_, marked := m.syncLoadedGroups[trackerKey{keyspaceID: keyspaceID, groupName: DefaultResourceGroupName}]
+		m.RUnlock()
+		checkDone <- marked
+	}()
+
+	select {
+	case <-checkDone:
+		t.Fatal("a concurrent m.Lock()-holding operation must not proceed while the publish is parked between visibility and marking")
+	case <-time.After(100 * time.Millisecond):
+	}
+
+	close(release)
+	<-initDone
+	re.NoError(initErr)
+	marked := <-checkDone
+	re.True(marked, "the marker must already be set by the time any concurrent m.Lock() holder can observe the published group")
+}
+
 // TestAsyncLoadResourceGroupsCrossTermSetServiceLimitPublishesToNewTerm
 // reproduces a leadership change straddling SetKeyspaceServiceLimit: it
 // resolves a keyspace manager, then stalls before its storage phase while the

From 45df73d3dd8ee589d36fff1c7b5fdd448bc1f4e7 Mon Sep 17 00:00:00 2001
From: bufferflies <1045931706@qq.com>
Date: Thu, 6 Aug 2026 02:32:31 +0200
Subject: [PATCH 43/50] resource_group: skip
 syncBurstabilityWithServiceLimitLocked's group lock when no service limit is
 set

syncBurstabilityWithServiceLimitLocked unconditionally took the
group's write lock before checking whether the keyspace even has an
active service limit configured, regressing from the previous
short-circuit that used only read locks in that case. This function
runs on every group insert/update across the system, including the
async bulk merge's up-to-500k-group batches (see
BenchmarkAsyncLoadMergeReaderStall), so escalating to a write lock in
the common no-op case - most keyspaces have no service limit set -
added needless contention against a concurrent RequestRU call already
holding the same, already-live group's lock.

Fixed by checking isSet/serviceLimit (a cheap read of krgm's
already-locked state, no extra lock) before taking group's lock,
returning immediately when there is nothing to apply.

Regression test (TestSyncBurstabilityWithServiceLimitLockedSkipsGroupLockWhenNoServiceLimit)
holds the group's lock externally and confirms the call returns
without blocking when no service limit is set; verified it fails
(blocks until timeout) against the pre-fix code and passes against the
fix.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Signed-off-by: bufferflies <1045931706@qq.com>
---
 .../server/keyspace_manager.go                |  9 +++++
 .../server/keyspace_manager_test.go           | 35 +++++++++++++++++++
 2 files changed, 44 insertions(+)

diff --git a/pkg/mcs/resourcemanager/server/keyspace_manager.go b/pkg/mcs/resourcemanager/server/keyspace_manager.go
index 1da724e787..81d3e37980 100644
--- a/pkg/mcs/resourcemanager/server/keyspace_manager.go
+++ b/pkg/mcs/resourcemanager/server/keyspace_manager.go
@@ -1016,6 +1016,15 @@ func (krgm *keyspaceResourceGroupManager) syncBurstabilityWithServiceLimitLocked
 		return
 	}
 	serviceLimit, isSet := krgm.getServiceLimitLocked()
+	// Cheap short-circuit before taking group's write lock: most keyspaces
+	// have no active service limit at all, and this runs on every group
+	// insert/update across the whole system (including the async bulk
+	// merge's up-to-500k-group batches), so skipping the write lock entirely
+	// in the common no-op case avoids needless contention against concurrent
+	// RequestRU calls on the same, already-live group.
+	if !isSet || serviceLimit <= 0 {
+		return
+	}
 	group.Lock()
 	defer group.Unlock()
 	applyBurstabilitySyncLocked(group, serviceLimit, isSet)
diff --git a/pkg/mcs/resourcemanager/server/keyspace_manager_test.go b/pkg/mcs/resourcemanager/server/keyspace_manager_test.go
index 3f2d82ec80..e3978ed07b 100644
--- a/pkg/mcs/resourcemanager/server/keyspace_manager_test.go
+++ b/pkg/mcs/resourcemanager/server/keyspace_manager_test.go
@@ -107,6 +107,41 @@ func TestInitDefaultResourceGroup(t *testing.T) {
 	re.Equal(int64(UnlimitedBurstLimit), defaultGroup.getBurstLimit())
 }
 
+// TestSyncBurstabilityWithServiceLimitLockedSkipsGroupLockWhenNoServiceLimit
+// guards against syncBurstabilityWithServiceLimitLocked unconditionally
+// taking group's write lock before checking whether the keyspace even has an
+// active service limit. This runs on every group insert/update across the
+// system, including the async bulk merge's up-to-500k-group batches, so
+// escalating to a write lock in the common no-op case (most keyspaces have
+// no service limit set) is needless contention against a concurrent
+// RequestRU call already holding the same group's lock. The fix checks
+// isSet/serviceLimit first (cheap: it only reads krgm's already-locked
+// state) and returns before ever touching group's lock when there is
+// nothing to apply.
+func TestSyncBurstabilityWithServiceLimitLockedSkipsGroupLockWhenNoServiceLimit(t *testing.T) {
+	krgm := newKeyspaceResourceGroupManager(1, storage.NewStorageWithMemoryBackend())
+	group := newDefaultResourceGroup()
+
+	// No service limit has been configured on krgm, so
+	// syncBurstabilityWithServiceLimitLocked has nothing to do here. Hold
+	// group's lock externally to prove it's never contended.
+	group.Lock()
+	defer group.Unlock()
+
+	done := make(chan struct{})
+	go func() {
+		krgm.RLock()
+		krgm.syncBurstabilityWithServiceLimitLocked(group)
+		krgm.RUnlock()
+		close(done)
+	}()
+	select {
+	case <-done:
+	case <-time.After(200 * time.Millisecond):
+		t.Fatal("must not try to take group's lock when the keyspace has no active service limit")
+	}
+}
+
 func TestAddResourceGroup(t *testing.T) {
 	re := require.New(t)
 

From 984e64de5845450d4d3d2845964521b2e00ef120 Mon Sep 17 00:00:00 2001
From: bufferflies <1045931706@qq.com>
Date: Thu, 6 Aug 2026 03:09:00 +0200
Subject: [PATCH 44/50] resource_group: simplify locking helpers and dedupe
 error-wrapping

Quality-only cleanup, no behavior change. Ran a 4-angle simplify pass
(reuse/simplification/efficiency/altitude) over the full PR diff and
applied the findings that were safe, mechanical, and didn't touch
behavior:

- Moved WrapLoadingError (pkg/mcs/resourcemanager/server/grpc_service.go)
  into pkg/errs/errno.go as ErrResourceGroupsLoadingGRPC, matching the
  existing domain-error-to-gRPC-status pattern already used there
  (ErrGRPCRateLimitExceeded, ErrNotLeader) instead of a new,
  differently-located wrapper. Updated all 8 call sites across
  grpc_service.go and server/resource_group_proxy_service.go.

- persistResourceGroupRunningState re-derived the same "present and not
  a reserved placeholder" check hasConfirmedResourceGroup already
  encodes. Extracted confirmedResourceGroupLocked (returns the group
  too, so the persist loop doesn't need a second lock round trip) and
  had both hasConfirmedResourceGroup and the persist loop use it.

- Dropped RLock/RUnlock around reads of tempKrgms in the async merge's
  flatten step: tempKrgms is a map this goroutine alone constructs and
  holds - not yet reachable from m.krgms or any other goroutine at that
  point - so the locking was pure overhead on every keyspace loaded per
  cycle.

- The "look up or create in m.krgms" snippet was duplicated at 4 call
  sites (3 of which already hold m.Lock() and couldn't call the public
  getOrCreateKeyspaceResourceGroupManager without double-locking).
  Extracted getOrCreateKeyspaceResourceGroupManagerLocked and had all 4
  sites (SetKeyspaceServiceLimit's mirror step, the merge loop,
  loadResourceGroupIfNeeded's retry loop, publishResourceGroupMutation)
  use it.

- Same shape for the sync-loaded marker write: extracted
  markResourceGroupSyncLoadedLocked and had markResourceGroupSyncLoaded,
  loadResourceGroupIfNeeded, and publishResourceGroupMutation share it
  instead of each inlining the map write.

Findings considered and deliberately not applied:
- Deleting krgm.addResourceGroup/deleteResourceGroup as "dead code": both
  still have ~19 test call sites using them as setup helpers; not dead.
- Folding the defaultGroupMu guard duplicated in AddResourceGroup/
  ModifyResourceGroup into persistResourceGroup/modifyResourceGroup:
  would self-deadlock when called from initDefaultResourceGroup, which
  already holds the same (non-reentrant) defaultGroupMu before calling
  persistResourceGroup - the duplication at the two call sites is what
  lets each caller decide for itself whether it already holds the lock.
- Folding the duplicated "!enableMetadataWatcher && LoadingStateCompleted"
  check in getOrCreateKeyspaceResourceGroupManager into
  loadResourceGroupIfNeeded: the existing comment on that check
  explicitly documents this split exists to avoid a recursive call
  between the two functions.
- Returning the resolved krgm/group from loadResourceGroupIfNeeded so
  GetResourceGroup/GetMutableResourceGroup don't re-resolve it in
  metadata-watcher mode: real double-lookup, but changes a heavily-used
  function's return contract and its callers - left as a follow-up
  rather than folded into a no-behavior-change cleanup pass.

Full pkg/mcs/resourcemanager/... suite green under -race with
failpoints enabled, 2 iterations; go build ./... and go vet clean.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Signed-off-by: bufferflies <1045931706@qq.com>
---
 pkg/errs/errno.go                             | 10 +++
 .../resourcemanager/server/grpc_service.go    | 23 ++-----
 .../server/keyspace_manager.go                | 35 ++++++----
 pkg/mcs/resourcemanager/server/manager.go     | 69 +++++++++----------
 server/resource_group_proxy_service.go        |  7 +-
 5 files changed, 74 insertions(+), 70 deletions(-)

diff --git a/pkg/errs/errno.go b/pkg/errs/errno.go
index e840594dec..04243a584f 100644
--- a/pkg/errs/errno.go
+++ b/pkg/errs/errno.go
@@ -67,6 +67,16 @@ var (
 	ErrGRPCRateLimitExceeded            = func(err error) error {
 		return status.Error(codes.ResourceExhausted, err.Error())
 	}
+	// ErrResourceGroupsLoadingGRPC converts the retryable
+	// ErrResourceGroupsLoading into codes.Unavailable, so generic
+	// client-side retry logic can act on it instead of seeing an opaque
+	// codes.Unknown. Other errors pass through unchanged.
+	ErrResourceGroupsLoadingGRPC = func(err error) error {
+		if ErrResourceGroupsLoading.Equal(err) {
+			return status.Error(codes.Unavailable, err.Error())
+		}
+		return err
+	}
 
 	// FailedPrecondition indicates operation was rejected because the
 	// system is not in a state required for the operation's execution.
diff --git a/pkg/mcs/resourcemanager/server/grpc_service.go b/pkg/mcs/resourcemanager/server/grpc_service.go
index 36c6306c7f..4e0340b538 100644
--- a/pkg/mcs/resourcemanager/server/grpc_service.go
+++ b/pkg/mcs/resourcemanager/server/grpc_service.go
@@ -102,19 +102,6 @@ func (s *Service) checkServing() error {
 	return nil
 }
 
-// WrapLoadingError converts the retryable "resource groups are still loading"
-// error into codes.Unavailable, so generic client-side retry logic can act on
-// it instead of seeing an opaque codes.Unknown. Other errors pass through.
-// Exported because the standalone-mode metadata proxy (server.resourceGroup-
-// ProxyServer) calls the local Manager directly instead of going through
-// this gRPC service, and must apply the same mapping itself.
-func WrapLoadingError(err error) error {
-	if errs.ErrResourceGroupsLoading.Equal(err) {
-		return status.Error(codes.Unavailable, err.Error())
-	}
-	return err
-}
-
 // GetResourceGroup implements ResourceManagerServer.GetResourceGroup.
 func (s *Service) GetResourceGroup(_ context.Context, req *rmpb.GetResourceGroupRequest) (*rmpb.GetResourceGroupResponse, error) {
 	if err := s.checkServing(); err != nil {
@@ -123,7 +110,7 @@ func (s *Service) GetResourceGroup(_ context.Context, req *rmpb.GetResourceGroup
 	keyspaceID := ExtractKeyspaceID(req.GetKeyspaceId())
 	rg, err := s.manager.GetResourceGroup(keyspaceID, req.ResourceGroupName, req.WithRuStats)
 	if err != nil {
-		return nil, WrapLoadingError(err)
+		return nil, errs.ErrResourceGroupsLoadingGRPC(err)
 	}
 	if rg == nil {
 		return nil, errs.ErrResourceGroupNotExists.FastGenByArgs(req.ResourceGroupName)
@@ -142,7 +129,7 @@ func (s *Service) ListResourceGroups(_ context.Context, req *rmpb.ListResourceGr
 	keyspaceID := ExtractKeyspaceID(req.GetKeyspaceId())
 	groups, err := s.manager.GetResourceGroupList(keyspaceID, req.WithRuStats)
 	if err != nil {
-		return nil, WrapLoadingError(err)
+		return nil, errs.ErrResourceGroupsLoadingGRPC(err)
 	}
 	resps := &rmpb.ListResourceGroupsResponse{
 		Groups: make([]*rmpb.ResourceGroup, 0, len(groups)),
@@ -164,7 +151,7 @@ func (s *Service) AddResourceGroup(_ context.Context, req *rmpb.PutResourceGroup
 	}
 	err := s.manager.AddResourceGroup(req.GetGroup())
 	if err != nil {
-		return nil, WrapLoadingError(err)
+		return nil, errs.ErrResourceGroupsLoadingGRPC(err)
 	}
 	return &rmpb.PutResourceGroupResponse{Body: "Success!"}, nil
 }
@@ -179,7 +166,7 @@ func (s *Service) DeleteResourceGroup(_ context.Context, req *rmpb.DeleteResourc
 	}
 	err := s.manager.DeleteResourceGroup(ExtractKeyspaceID(req.GetKeyspaceId()), req.ResourceGroupName)
 	if err != nil {
-		return nil, WrapLoadingError(err)
+		return nil, errs.ErrResourceGroupsLoadingGRPC(err)
 	}
 	return &rmpb.DeleteResourceGroupResponse{Body: "Success!"}, nil
 }
@@ -194,7 +181,7 @@ func (s *Service) ModifyResourceGroup(_ context.Context, req *rmpb.PutResourceGr
 	}
 	err := s.manager.ModifyResourceGroup(req.GetGroup())
 	if err != nil {
-		return nil, WrapLoadingError(err)
+		return nil, errs.ErrResourceGroupsLoadingGRPC(err)
 	}
 	return &rmpb.PutResourceGroupResponse{Body: "Success!"}, nil
 }
diff --git a/pkg/mcs/resourcemanager/server/keyspace_manager.go b/pkg/mcs/resourcemanager/server/keyspace_manager.go
index 81d3e37980..0e1773f61d 100644
--- a/pkg/mcs/resourcemanager/server/keyspace_manager.go
+++ b/pkg/mcs/resourcemanager/server/keyspace_manager.go
@@ -417,11 +417,24 @@ func (krgm *keyspaceResourceGroupManager) isReserved(name string) bool {
 func (krgm *keyspaceResourceGroupManager) hasConfirmedResourceGroup(name string) bool {
 	krgm.RLock()
 	defer krgm.RUnlock()
-	if _, ok := krgm.groups[name]; !ok {
-		return false
+	_, ok := krgm.confirmedResourceGroupLocked(name)
+	return ok
+}
+
+// confirmedResourceGroupLocked is hasConfirmedResourceGroup for a caller that
+// already holds krgm's lock (read or write), returning the group itself too
+// so a caller that needs both the confirmed check and the group (e.g. the
+// state persist loop below) doesn't have to re-derive the same "present and
+// not a reserved placeholder" logic or take a second lock round trip.
+func (krgm *keyspaceResourceGroupManager) confirmedResourceGroupLocked(name string) (*ResourceGroup, bool) {
+	group, ok := krgm.groups[name]
+	if !ok {
+		return nil, false
+	}
+	if _, reserved := krgm.reservedGroups[name]; reserved {
+		return nil, false
 	}
-	_, reserved := krgm.reservedGroups[name]
-	return !reserved
+	return group, true
 }
 
 func (krgm *keyspaceResourceGroupManager) getResourceGroup(name string, withStats bool) *ResourceGroup {
@@ -477,15 +490,11 @@ func (krgm *keyspaceResourceGroupManager) persistResourceGroupRunningState() {
 	krgm.RUnlock()
 	for idx := range keys {
 		krgm.RLock()
-		group, ok := krgm.groups[keys[idx]]
-		_, reserved := krgm.reservedGroups[keys[idx]]
-		if ok && reserved {
-			// The entry is still just an unconfirmed placeholder (e.g. the
-			// synthetic default installed before async loading completes);
-			// persisting its fresh state would permanently overwrite any
-			// real persisted state still waiting to be loaded.
-			ok = false
-		}
+		// A reserved placeholder (e.g. the synthetic default installed before
+		// async loading completes) is skipped: persisting its fresh state
+		// would permanently overwrite any real persisted state still waiting
+		// to be loaded.
+		group, ok := krgm.confirmedResourceGroupLocked(keys[idx])
 		if ok {
 			if err := group.persistStates(krgm.keyspaceID, krgm.storage); err != nil {
 				log.Error("persist keyspace resource group state failed",
diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go
index eb514e08d5..5c1e38024e 100644
--- a/pkg/mcs/resourcemanager/server/manager.go
+++ b/pkg/mcs/resourcemanager/server/manager.go
@@ -318,11 +318,7 @@ func (m *Manager) SetKeyspaceServiceLimit(keyspaceID uint32, serviceLimit float6
 	// for this keyspace can run concurrently, so whatever's in storage now is
 	// still what this call itself just wrote - safe to mirror unconditionally.
 	m.Lock()
-	cur, ok := m.krgms[keyspaceID]
-	if !ok {
-		cur = newKeyspaceResourceGroupManager(keyspaceID, m.storage, m.writeRole)
-		m.krgms[keyspaceID] = cur
-	}
+	cur := m.getOrCreateKeyspaceResourceGroupManagerLocked(keyspaceID)
 	m.Unlock()
 	if cur != krgm {
 		cur.setServiceLimitFromStorage(serviceLimit)
@@ -380,11 +376,7 @@ func (m *Manager) GetRUVersionPolicy() *RUVersionPolicy {
 // from the cache truly doesn't exist anywhere, so it's synthesized directly.
 func (m *Manager) getOrCreateKeyspaceResourceGroupManager(keyspaceID uint32, initDefault bool) *keyspaceResourceGroupManager {
 	m.Lock()
-	krgm, ok := m.krgms[keyspaceID]
-	if !ok {
-		krgm = newKeyspaceResourceGroupManager(keyspaceID, m.storage, m.writeRole)
-		m.krgms[keyspaceID] = krgm
-	}
+	krgm := m.getOrCreateKeyspaceResourceGroupManagerLocked(keyspaceID)
 	m.Unlock()
 	if initDefault {
 		// In metadata-watcher mode, LoadingStateCompleted only means the
@@ -416,6 +408,18 @@ func (m *Manager) getOrCreateKeyspaceResourceGroupManager(keyspaceID uint32, ini
 	return krgm
 }
 
+// getOrCreateKeyspaceResourceGroupManagerLocked is the m.krgms lookup/create
+// step of getOrCreateKeyspaceResourceGroupManager for a caller that already
+// holds m.Lock().
+func (m *Manager) getOrCreateKeyspaceResourceGroupManagerLocked(keyspaceID uint32) *keyspaceResourceGroupManager {
+	krgm, ok := m.krgms[keyspaceID]
+	if !ok {
+		krgm = newKeyspaceResourceGroupManager(keyspaceID, m.storage, m.writeRole)
+		m.krgms[keyspaceID] = krgm
+	}
+	return krgm
+}
+
 func (m *Manager) getKeyspaceResourceGroupManager(keyspaceID uint32) *keyspaceResourceGroupManager {
 	m.RLock()
 	defer m.RUnlock()
@@ -663,20 +667,21 @@ func (m *Manager) asyncLoadResourceGroups(ctx context.Context, epoch uint64) {
 			group      *ResourceGroup
 		}
 		// Size the slice up front: it holds every loaded group, which is exactly
-		// the scale this loader exists to handle.
+		// the scale this loader exists to handle. tempKrgms is a local map this
+		// goroutine alone constructed and holds - not yet reachable from
+		// m.krgms or any other goroutine - so reading it here needs no locking;
+		// only the individual *ResourceGroup values get published later, into
+		// whichever keyspace manager is current at merge time, not tempKrgm
+		// itself.
 		totalGroups := 0
 		for _, tempKrgm := range tempKrgms {
-			tempKrgm.RLock()
 			totalGroups += len(tempKrgm.groups)
-			tempKrgm.RUnlock()
 		}
 		pending := make([]mergeItem, 0, totalGroups)
 		for keyspaceID, tempKrgm := range tempKrgms {
-			tempKrgm.RLock()
 			for name, group := range tempKrgm.groups {
 				pending = append(pending, mergeItem{keyspaceID: keyspaceID, name: name, group: group})
 			}
-			tempKrgm.RUnlock()
 		}
 
 		const mergeBatchSize = 1024
@@ -699,11 +704,7 @@ func (m *Manager) asyncLoadResourceGroups(ctx context.Context, epoch uint64) {
 				if m.syncLoadedGroups[key] {
 					continue
 				}
-				krgm := m.krgms[it.keyspaceID]
-				if krgm == nil {
-					krgm = newKeyspaceResourceGroupManager(it.keyspaceID, m.storage, m.writeRole)
-					m.krgms[it.keyspaceID] = krgm
-				}
+				krgm := m.getOrCreateKeyspaceResourceGroupManagerLocked(it.keyspaceID)
 				krgm.Lock()
 				krgm.groups[it.name] = it.group
 				// This group is now confirmed, fully-loaded data (settings
@@ -858,11 +859,7 @@ func (m *Manager) loadResourceGroupIfNeeded(keyspaceID uint32, name string) erro
 		// the new bulk merge skip a group its cache doesn't contain.
 		m.Lock()
 		epoch := m.loadEpoch
-		krgm = m.krgms[keyspaceID]
-		if krgm == nil {
-			krgm = newKeyspaceResourceGroupManager(keyspaceID, m.storage, m.writeRole)
-			m.krgms[keyspaceID] = krgm
-		}
+		krgm = m.getOrCreateKeyspaceResourceGroupManagerLocked(keyspaceID)
 		m.Unlock()
 		// Snapshot the delete generation before the lock-free storage read,
 		// so a Delete that lands after the read is detected under the insert
@@ -919,7 +916,6 @@ func (m *Manager) loadResourceGroupIfNeeded(keyspaceID uint32, name string) erro
 			return err
 		}
 		inserted := false
-		markKey := trackerKey{keyspaceID: keyspaceID, groupName: name}
 		m.Lock()
 		if m.loadEpoch != epoch || m.krgms[keyspaceID] != krgm {
 			// The manager was reinitialized for a new term while the storage
@@ -962,9 +958,7 @@ func (m *Manager) loadResourceGroupIfNeeded(keyspaceID uint32, name string) erro
 			krgm.syncBurstabilityWithServiceLimitLocked(group)
 		}
 		krgm.Unlock()
-		if m.syncLoadedGroups != nil {
-			m.syncLoadedGroups[markKey] = true
-		}
+		m.markResourceGroupSyncLoadedLocked(keyspaceID, name)
 		m.Unlock()
 		failpoint.Inject("lazyLoadAfterCachePublish", func() {})
 		syncLoadGroupCounter.Inc()
@@ -984,6 +978,13 @@ func (m *Manager) markResourceGroupSyncLoaded(keyspaceID uint32, krgm *keyspaceR
 	if m.krgms[keyspaceID] != krgm {
 		return
 	}
+	m.markResourceGroupSyncLoadedLocked(keyspaceID, name)
+}
+
+// markResourceGroupSyncLoadedLocked is markResourceGroupSyncLoaded's marker
+// write for a caller that already holds m.Lock() and has already verified
+// krgm is still the live entry for keyspaceID.
+func (m *Manager) markResourceGroupSyncLoadedLocked(keyspaceID uint32, name string) {
 	if m.syncLoadedGroups != nil {
 		m.syncLoadedGroups[trackerKey{keyspaceID: keyspaceID, groupName: name}] = true
 	}
@@ -1054,11 +1055,7 @@ func (m *Manager) publishResourceGroupMutation(
 ) {
 	m.Lock()
 	defer m.Unlock()
-	cur, ok := m.krgms[keyspaceID]
-	if !ok {
-		cur = newKeyspaceResourceGroupManager(keyspaceID, m.storage, m.writeRole)
-		m.krgms[keyspaceID] = cur
-	}
+	cur := m.getOrCreateKeyspaceResourceGroupManagerLocked(keyspaceID)
 	if cur != krgm && cur.hasConfirmedResourceGroup(name) {
 		log.Info("skip publishing resource group mutation: a newer confirmed write already exists",
 			zap.Uint32("keyspace-id", keyspaceID), zap.String("group-name", name))
@@ -1074,8 +1071,8 @@ func (m *Manager) publishResourceGroupMutation(
 	}
 	cur.Unlock()
 	failpoint.InjectCall("publishMutationBeforeMark")
-	if mark && m.syncLoadedGroups != nil {
-		m.syncLoadedGroups[trackerKey{keyspaceID: keyspaceID, groupName: name}] = true
+	if mark {
+		m.markResourceGroupSyncLoadedLocked(keyspaceID, name)
 	}
 }
 
diff --git a/server/resource_group_proxy_service.go b/server/resource_group_proxy_service.go
index b9ea083e89..977186aa96 100644
--- a/server/resource_group_proxy_service.go
+++ b/server/resource_group_proxy_service.go
@@ -25,6 +25,7 @@ import (
 	"github.com/pingcap/kvproto/pkg/resource_manager"
 	"github.com/pingcap/log"
 
+	"github.com/tikv/pd/pkg/errs"
 	rm_server "github.com/tikv/pd/pkg/mcs/resourcemanager/server"
 	"github.com/tikv/pd/pkg/mcs/utils/constant"
 	"github.com/tikv/pd/pkg/utils/grpcutil"
@@ -138,7 +139,7 @@ func (s *resourceGroupProxyServer) AddResourceGroup(ctx context.Context, req *re
 		return nil, status.Error(codes.Internal, "resource group metadata manager is not initialized")
 	}
 	if err := s.metadataManager.AddResourceGroup(req.GetGroup()); err != nil {
-		return nil, rm_server.WrapLoadingError(err)
+		return nil, errs.ErrResourceGroupsLoadingGRPC(err)
 	}
 	return &resource_manager.PutResourceGroupResponse{Body: "Success!"}, nil
 }
@@ -166,7 +167,7 @@ func (s *resourceGroupProxyServer) ModifyResourceGroup(ctx context.Context, req
 		return nil, status.Error(codes.Internal, "resource group metadata manager is not initialized")
 	}
 	if err := s.metadataManager.ModifyResourceGroup(req.GetGroup()); err != nil {
-		return nil, rm_server.WrapLoadingError(err)
+		return nil, errs.ErrResourceGroupsLoadingGRPC(err)
 	}
 	return &resource_manager.PutResourceGroupResponse{Body: "Success!"}, nil
 }
@@ -197,7 +198,7 @@ func (s *resourceGroupProxyServer) DeleteResourceGroup(ctx context.Context, req
 		rm_server.ExtractKeyspaceID(req.GetKeyspaceId()),
 		req.GetResourceGroupName(),
 	); err != nil {
-		return nil, rm_server.WrapLoadingError(err)
+		return nil, errs.ErrResourceGroupsLoadingGRPC(err)
 	}
 	return &resource_manager.DeleteResourceGroupResponse{Body: "Success!"}, nil
 }

From b35672d3e832dd7d6bc15152c75de4ae4448dac6 Mon Sep 17 00:00:00 2001
From: bufferflies <1045931706@qq.com>
Date: Thu, 6 Aug 2026 09:37:32 +0200
Subject: [PATCH 45/50] resource_group: retry and fall back to bulk value when
 loadServiceLimits' point re-read fails

loadServiceLimits re-reads each keyspace's service limit from storage
under serviceLimitLocks to avoid applying a bulk-scan snapshot that a
concurrent SetKeyspaceServiceLimit might have already made stale. But
that point re-read can itself fail transiently, and until now a single
failure just logged and dropped the update - leaving the keyspace with
no service limit cached at all until the next Init, silently letting
burstable groups bypass the configured cap indefinitely.

Retry the point read a few times, and on exhausting the budget, fall
back to the bulk-scanned value instead of dropping the update. This
re-admits the point re-read's own narrow staleness window only in this
rare failure case, which is strictly better than losing the limit
entirely.

Added TestLoadServiceLimitsRetriesPointReadBeforeFallingBack and
TestLoadServiceLimitsFallsBackToBulkValueOnPersistentFailure, both
verified to fail against the pre-fix code. Full
pkg/mcs/resourcemanager/... suite green under -race with failpoints
enabled; go build/vet and gofmt clean.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Signed-off-by: bufferflies <1045931706@qq.com>
---
 pkg/mcs/resourcemanager/server/manager.go     | 31 ++++++-
 .../server/manager_async_test.go              | 83 +++++++++++++++++++
 2 files changed, 110 insertions(+), 4 deletions(-)

diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go
index 5c1e38024e..0584e36bca 100644
--- a/pkg/mcs/resourcemanager/server/manager.go
+++ b/pkg/mcs/resourcemanager/server/manager.go
@@ -543,8 +543,12 @@ func (m *Manager) initMetadata(ctx context.Context) error {
 	return nil
 }
 
+// maxServiceLimitReloadAttempts bounds the point re-read retries in
+// loadServiceLimits below before it falls back to the bulk-scanned value.
+const maxServiceLimitReloadAttempts = 3
+
 func (m *Manager) loadServiceLimits() error {
-	return m.storage.LoadServiceLimits(func(keyspaceID uint32, _ float64) {
+	return m.storage.LoadServiceLimits(func(keyspaceID uint32, bulkScannedLimit float64) {
 		failpoint.InjectCall("loadServiceLimitsBeforeApply", keyspaceID)
 		// Serialize against SetKeyspaceServiceLimit for this keyspace, and
 		// re-read the value from storage instead of trusting the one the
@@ -559,10 +563,29 @@ func (m *Manager) loadServiceLimits() error {
 		// completes before SetKeyspaceServiceLimit releases this lock.
 		m.serviceLimitLocks.Lock(keyspaceID)
 		defer m.serviceLimitLocks.Unlock(keyspaceID)
-		serviceLimit, err := m.storage.LoadServiceLimit(keyspaceID)
+		var (
+			serviceLimit float64
+			err          error
+		)
+		for attempt := 1; attempt <= maxServiceLimitReloadAttempts; attempt++ {
+			serviceLimit, err = m.storage.LoadServiceLimit(keyspaceID)
+			if err == nil {
+				break
+			}
+			log.Warn("failed to reload service limit, retrying",
+				zap.Uint32("keyspace-id", keyspaceID), zap.Int("attempt", attempt), zap.Error(err))
+		}
 		if err != nil {
-			log.Warn("failed to reload service limit", zap.Uint32("keyspace-id", keyspaceID), zap.Error(err))
-			return
+			// Retries exhausted, e.g. a persistent storage failure: fall back
+			// to the bulk-scanned value instead of dropping the update
+			// entirely. This re-admits the narrow staleness window the point
+			// re-read above exists to close, but only in this rare failure
+			// case - strictly better than leaving the keyspace with no
+			// service limit cached at all (silently allowing burstable
+			// groups to bypass it) until the next Init.
+			log.Error("giving up reloading service limit, falling back to the bulk-scanned value",
+				zap.Uint32("keyspace-id", keyspaceID), zap.Error(err))
+			serviceLimit = bulkScannedLimit
 		}
 		m.getOrCreateKeyspaceResourceGroupManager(keyspaceID, false).setServiceLimitFromStorage(serviceLimit)
 	})
diff --git a/pkg/mcs/resourcemanager/server/manager_async_test.go b/pkg/mcs/resourcemanager/server/manager_async_test.go
index b942bc7161..7d11675762 100644
--- a/pkg/mcs/resourcemanager/server/manager_async_test.go
+++ b/pkg/mcs/resourcemanager/server/manager_async_test.go
@@ -19,6 +19,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"math"
 	"sync"
 	"sync/atomic"
 	"testing"
@@ -1217,6 +1218,88 @@ func TestLoadServiceLimitsDoesNotClobberConcurrentSet(t *testing.T) {
 		"the replay must not overwrite a concurrently-set newer value with its own stale snapshot")
 }
 
+// failingServiceLimitLoadStorage makes LoadServiceLimit (the point re-read
+// loadServiceLimits performs under serviceLimitLocks) fail a controlled
+// number of times for one keyspace, to exercise its retry-then-fallback
+// behavior. LoadServiceLimits (the bulk scan) is left untouched.
+type failingServiceLimitLoadStorage struct {
+	storage.Storage
+	keyspaceID   uint32
+	failuresLeft atomic.Int32
+	calls        atomic.Int32
+}
+
+func (s *failingServiceLimitLoadStorage) LoadServiceLimit(keyspaceID uint32) (float64, error) {
+	if keyspaceID == s.keyspaceID {
+		s.calls.Add(1)
+		if s.failuresLeft.Add(-1) >= 0 {
+			return 0, errors.New("injected service limit load failure")
+		}
+	}
+	return s.Storage.LoadServiceLimit(keyspaceID)
+}
+
+// TestLoadServiceLimitsRetriesPointReadBeforeFallingBack guards against
+// loadServiceLimits treating a single point-read failure as fatal. The point
+// re-read added to close the race in TestLoadServiceLimitsDoesNotClobberConcurrentSet
+// above can itself fail transiently (e.g. a storage blip), so it must retry a
+// few times and apply the value once a retry succeeds, instead of giving up
+// and falling back to the (potentially stale) bulk-scanned value on the very
+// first failure.
+func TestLoadServiceLimitsRetriesPointReadBeforeFallingBack(t *testing.T) {
+	re := require.New(t)
+	const keyspaceID = 1
+	base := storage.NewStorageWithMemoryBackend()
+	re.NoError(base.SaveServiceLimit(keyspaceID, 100))
+	store := &failingServiceLimitLoadStorage{Storage: base, keyspaceID: keyspaceID}
+	// Fail once, then succeed on the second attempt - strictly fewer calls
+	// than the retry budget, to prove it stops retrying once a read succeeds
+	// instead of always spending the full budget.
+	store.failuresLeft.Store(1)
+
+	m := NewManager[*mockConfigProvider](&mockConfigProvider{})
+	m.storage = store
+
+	re.NoError(m.loadServiceLimits())
+
+	limiter := m.GetKeyspaceServiceLimiter(keyspaceID)
+	re.NotNil(limiter)
+	re.Equal(float64(100), limiter.ServiceLimit,
+		"a point read that succeeds within the retry budget must be applied")
+	re.EqualValues(2, store.calls.Load(),
+		"must retry the point read instead of falling back after only one failure, and stop once it succeeds")
+}
+
+// TestLoadServiceLimitsFallsBackToBulkValueOnPersistentFailure guards against
+// loadServiceLimits silently dropping a keyspace's service limit entirely
+// when its point re-read keeps failing (e.g. a persistent storage issue).
+// Without a fallback, the keyspace would run with no service limit cached at
+// all - letting burstable groups bypass the configured cap indefinitely,
+// since nothing else retries this load until the next Init - which is worse
+// than falling back to the bulk-scanned value and re-admitting its narrow,
+// already-covered staleness window.
+func TestLoadServiceLimitsFallsBackToBulkValueOnPersistentFailure(t *testing.T) {
+	re := require.New(t)
+	const keyspaceID = 1
+	base := storage.NewStorageWithMemoryBackend()
+	re.NoError(base.SaveServiceLimit(keyspaceID, 100))
+	store := &failingServiceLimitLoadStorage{Storage: base, keyspaceID: keyspaceID}
+	// Always fail the point read, however many times it's retried.
+	store.failuresLeft.Store(math.MaxInt32)
+
+	m := NewManager[*mockConfigProvider](&mockConfigProvider{})
+	m.storage = store
+
+	re.NoError(m.loadServiceLimits())
+
+	limiter := m.GetKeyspaceServiceLimiter(keyspaceID)
+	re.NotNil(limiter)
+	re.Equal(float64(100), limiter.ServiceLimit,
+		"must fall back to the bulk-scanned value rather than leaving the service limit uncached")
+	re.EqualValues(maxServiceLimitReloadAttempts, store.calls.Load(),
+		"must exhaust the retry budget before falling back")
+}
+
 // TestAsyncLoadResourceGroupsMergeGroupNeverVisibleUnsynced guards against a
 // window in the bulk merge where a newly-loaded group became visible in
 // krgm.groups (readable by any concurrent GetResourceGroup/token request)

From 422a3cf8a0271be935269c30c846b19280d4ae21 Mon Sep 17 00:00:00 2001
From: bufferflies <1045931706@qq.com>
Date: Thu, 6 Aug 2026 11:29:19 +0200
Subject: [PATCH 46/50] resource_group: don't let loadServiceLimits' fallback
 clobber a concurrently-set newer value

loadServiceLimits' fallback to the bulk-scanned value on a persistent
point-read failure (b35672d3e) could overwrite a value a concurrent
SetKeyspaceServiceLimit had already fully committed - both storage and
cache - for the same keyspace before the callback ever acquired
serviceLimitLocks. SetKeyspaceServiceLimit holds that same per-keyspace
lock across its entire storage-write-then-cache-mirror sequence, so by
the time the callback holds the lock, a concurrent call has either
fully landed or hasn't started - there's no partial state to race
against. Check the cache for an already-newer value before applying
the fallback, and leave it untouched instead of overwriting it.

Known, narrower gap left undocumented as a TODO only, not fixed: since
a service limit has no flag separate from its value, a concurrent
SetKeyspaceServiceLimit(id, 0) landing in the same window is
indistinguishable here from "never configured" and would still get
overwritten by the stale fallback. Left as a documented residual gap
per user decision - it additionally requires the point read to keep
failing for the whole retry budget on top of that exact interleaving.

Added TestLoadServiceLimitsDoesNotClobberConcurrentSetOnPersistentPointReadFailure,
verified to fail against the pre-fix code. Full
pkg/mcs/resourcemanager/... suite green under -race with failpoints
enabled; go build/vet and gofmt clean.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Signed-off-by: bufferflies <1045931706@qq.com>
---
 pkg/mcs/resourcemanager/server/manager.go     | 42 ++++++++++---
 .../server/manager_async_test.go              | 61 +++++++++++++++++++
 2 files changed, 95 insertions(+), 8 deletions(-)

diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go
index 0584e36bca..f96c829904 100644
--- a/pkg/mcs/resourcemanager/server/manager.go
+++ b/pkg/mcs/resourcemanager/server/manager.go
@@ -563,6 +563,7 @@ func (m *Manager) loadServiceLimits() error {
 		// completes before SetKeyspaceServiceLimit releases this lock.
 		m.serviceLimitLocks.Lock(keyspaceID)
 		defer m.serviceLimitLocks.Unlock(keyspaceID)
+		krgm := m.getOrCreateKeyspaceResourceGroupManager(keyspaceID, false)
 		var (
 			serviceLimit float64
 			err          error
@@ -576,18 +577,43 @@ func (m *Manager) loadServiceLimits() error {
 				zap.Uint32("keyspace-id", keyspaceID), zap.Int("attempt", attempt), zap.Error(err))
 		}
 		if err != nil {
-			// Retries exhausted, e.g. a persistent storage failure: fall back
-			// to the bulk-scanned value instead of dropping the update
-			// entirely. This re-admits the narrow staleness window the point
-			// re-read above exists to close, but only in this rare failure
-			// case - strictly better than leaving the keyspace with no
-			// service limit cached at all (silently allowing burstable
-			// groups to bypass it) until the next Init.
+			// Retries exhausted, e.g. a persistent storage failure. A
+			// concurrent SetKeyspaceServiceLimit call for this keyspace holds
+			// this same serviceLimitLocks entry across both its storage write
+			// and its cache mirror step, so there is no partially-applied
+			// state it could leave behind: by the time we hold the lock, it
+			// has either fully landed (cache already reflects a write that is
+			// strictly newer than the bulk scan's pre-lock snapshot) or has
+			// not started yet (nothing to race with the fallback below). If
+			// the cache already carries such a value, leave it untouched
+			// instead of clobbering it with the stale bulk-scanned one.
+			//
+			// Known gap: a service limit has no separate "has this ever been
+			// explicitly configured" flag - getServiceLimit's isSet also
+			// reads as false for a limit explicitly set to 0 (unlimited), the
+			// same as a never-configured one. So a concurrent
+			// SetKeyspaceServiceLimit(keyspaceID, 0) that lands in this same
+			// window is indistinguishable here from "nothing set yet", and
+			// the fallback below would incorrectly reapply the stale
+			// bulk-scanned (pre-clear) value over the just-cleared limit.
+			// Narrower than the gap this whole retry/fallback exists to
+			// close - it additionally requires the point read to keep
+			// failing for the full retry budget - and not closed here.
+			if _, alreadySet := krgm.getServiceLimit(); alreadySet {
+				log.Warn("giving up reloading service limit; a newer value is already cached, leaving it untouched",
+					zap.Uint32("keyspace-id", keyspaceID), zap.Error(err))
+				return
+			}
+			// Nothing newer is cached yet: fall back to the bulk-scanned
+			// value instead of dropping the update entirely - strictly
+			// better than leaving the keyspace with no service limit cached
+			// at all (silently allowing burstable groups to bypass it) until
+			// the next Init.
 			log.Error("giving up reloading service limit, falling back to the bulk-scanned value",
 				zap.Uint32("keyspace-id", keyspaceID), zap.Error(err))
 			serviceLimit = bulkScannedLimit
 		}
-		m.getOrCreateKeyspaceResourceGroupManager(keyspaceID, false).setServiceLimitFromStorage(serviceLimit)
+		krgm.setServiceLimitFromStorage(serviceLimit)
 	})
 }
 
diff --git a/pkg/mcs/resourcemanager/server/manager_async_test.go b/pkg/mcs/resourcemanager/server/manager_async_test.go
index 7d11675762..4a32091e7a 100644
--- a/pkg/mcs/resourcemanager/server/manager_async_test.go
+++ b/pkg/mcs/resourcemanager/server/manager_async_test.go
@@ -1300,6 +1300,67 @@ func TestLoadServiceLimitsFallsBackToBulkValueOnPersistentFailure(t *testing.T)
 		"must exhaust the retry budget before falling back")
 }
 
+// TestLoadServiceLimitsDoesNotClobberConcurrentSetOnPersistentPointReadFailure
+// guards against the bulk-scanned fallback above overwriting a newer value
+// that a concurrent SetKeyspaceServiceLimit call already fully committed - to
+// storage and the live cache - before loadServiceLimits' callback for this
+// keyspace ever acquired serviceLimitLocks. SetKeyspaceServiceLimit holds
+// that same per-keyspace lock across both its storage write and its cache
+// mirror, so there is no partially-applied state the callback can observe:
+// by the time it holds the lock, a concurrent call has either fully landed
+// (cache already reflects a value strictly newer than the bulk scan's
+// pre-lock snapshot) or has not started at all. The fix checks the cache
+// before falling back and leaves an already-newer value untouched.
+func TestLoadServiceLimitsDoesNotClobberConcurrentSetOnPersistentPointReadFailure(t *testing.T) {
+	re := require.New(t)
+	const keyspaceID = 1
+	base := storage.NewStorageWithMemoryBackend()
+	re.NoError(base.SaveServiceLimit(keyspaceID, 100))
+	store := &failingServiceLimitLoadStorage{Storage: base, keyspaceID: keyspaceID}
+	// Always fail the point read, however many times it's retried.
+	store.failuresLeft.Store(math.MaxInt32)
+
+	m := NewManager[*mockConfigProvider](&mockConfigProvider{})
+	m.storage = store
+
+	reached := make(chan struct{})
+	release := make(chan struct{})
+	re.NoError(failpoint.EnableCall("github.com/tikv/pd/pkg/mcs/resourcemanager/server/loadServiceLimitsBeforeApply", func(gotKeyspaceID uint32) {
+		if gotKeyspaceID != keyspaceID {
+			return
+		}
+		close(reached)
+		<-release
+	}))
+	defer func() {
+		re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/mcs/resourcemanager/server/loadServiceLimitsBeforeApply"))
+	}()
+
+	loadDone := make(chan struct{})
+	go func() {
+		defer close(loadDone)
+		re.NoError(m.loadServiceLimits())
+	}()
+	select {
+	case <-reached:
+	case <-time.After(time.Second):
+		t.Fatal("timed out waiting for loadServiceLimits to reach its replay callback")
+	}
+
+	// While the replay is parked before ever acquiring serviceLimitLocks, a
+	// concurrent SetKeyspaceServiceLimit call fully completes - both its
+	// storage write and its cache mirror - for the same keyspace.
+	re.NoError(m.SetKeyspaceServiceLimit(keyspaceID, 200))
+
+	close(release)
+	<-loadDone
+
+	limiter := m.GetKeyspaceServiceLimiter(keyspaceID)
+	re.NotNil(limiter)
+	re.Equal(float64(200), limiter.ServiceLimit,
+		"a concurrently-completed newer write must not be overwritten by the stale bulk-scanned value even when the point read keeps failing")
+}
+
 // TestAsyncLoadResourceGroupsMergeGroupNeverVisibleUnsynced guards against a
 // window in the bulk merge where a newly-loaded group became visible in
 // krgm.groups (readable by any concurrent GetResourceGroup/token request)

From 6cc24c7ab71da20c3370befded040677696378be Mon Sep 17 00:00:00 2001
From: bufferflies <1045931706@qq.com>
Date: Fri, 7 Aug 2026 04:08:31 +0200
Subject: [PATCH 47/50] resource_group: fail fast on Add/Modify/Delete's
 parked-before-storage cross-term collision

publishResourceGroupMutation's confirmed-write guard silently drops a
mutation's cache effect when a newer term already has confirmed data
for the same group. If the mutation's storage write still lands after
that guard was checked, storage ends up mutated but the cache is
stuck on stale data indefinitely - the #11105 interleaving.

For the specific case where the mutation was parked *before* its
storage-write phase even started (not while a write was already in
flight - that still needs #11105's CAS work), this is preventable:
re-check whether the current term already has confirmed data for the
target group right before the write, and abort with a retryable error
instead of writing on behalf of a result that's going to be dropped
anyway.

Deliberately narrower than a plain "did the term change" check: a
term change alone with nothing yet confirmed for this group is
harmless and already handled gracefully by publishResourceGroupMutation
re-resolving the current manager - only abort when there's an actual
confirmed collision to race against (hasNewerConfirmedWrite). Verified
this distinction against TestAsyncLoadResourceGroupsCrossTermDeletePublishesToNewTerm
and TestAsyncLoadResourceGroupsCrossTermModifyDefaultStaysConfirmed,
whose "harmless term change, mutation should still succeed" scenarios
must keep passing unchanged.

Added new addResourceGroupBeforeStorage/modifyResourceGroupBeforeStorage
failpoints (Delete reuses the existing deleteResourceGroupBeforeStorage)
and three regression tests, one per mutation, each verified to fail
against the pre-fix code by reproducing the #11105 shape directly
(mutation returns success while its cache effect is silently dropped).

Also updated tikv/pd#11105 with the initDefaultResourceGroup in-flight
window as an explicit tracked item, per review discussion.

Full pkg/mcs/resourcemanager/... suite green under -race with
failpoints enabled, 2 iterations; go build/vet and gofmt clean.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Signed-off-by: bufferflies <1045931706@qq.com>
---
 pkg/mcs/resourcemanager/server/manager.go     |  67 +++++-
 .../server/manager_async_test.go              | 206 ++++++++++++++++++
 2 files changed, 270 insertions(+), 3 deletions(-)

diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go
index f96c829904..9fe40d710f 100644
--- a/pkg/mcs/resourcemanager/server/manager.go
+++ b/pkg/mcs/resourcemanager/server/manager.go
@@ -397,9 +397,7 @@ func (m *Manager) getOrCreateKeyspaceResourceGroupManager(keyspaceID uint32, ini
 			// request for the default group will retry through
 			// loadResourceGroupIfNeeded, which does surface such errors.
 			_, _ = m.initDefaultResourceGroup(keyspaceID, krgm, func() bool {
-				m.RLock()
-				defer m.RUnlock()
-				return m.krgms[keyspaceID] == krgm
+				return m.isKeyspaceManagerCurrent(keyspaceID, krgm)
 			})
 		} else if err := m.loadResourceGroupIfNeeded(keyspaceID, DefaultResourceGroupName); err != nil {
 			log.Debug("failed to load default resource group", zap.Uint32("keyspace-id", keyspaceID), zap.Error(err))
@@ -1201,6 +1199,43 @@ func (m *Manager) initDefaultResourceGroup(keyspaceID uint32, krgm *keyspaceReso
 	return true, nil
 }
 
+// isKeyspaceManagerCurrent reports whether krgm is still the live keyspace
+// manager for keyspaceID. Used where any term change - confirmed or not - is
+// reason enough to bail out, e.g. initDefaultResourceGroup's best-effort,
+// no-caller-waiting synthesis: publishResourceGroupMutation would apply it
+// into the new term anyway if nothing there has confirmed a default yet, so
+// aborting a harmless case here just costs a later retry, not correctness.
+func (m *Manager) isKeyspaceManagerCurrent(keyspaceID uint32, krgm *keyspaceResourceGroupManager) bool {
+	m.RLock()
+	defer m.RUnlock()
+	return m.krgms[keyspaceID] == krgm
+}
+
+// hasNewerConfirmedWrite reports whether the current keyspace manager for
+// keyspaceID is no longer krgm and already has confirmed data for name -
+// i.e. whether a mutation resolved against krgm is racing a leadership
+// change that already landed a newer, competing write for the same group.
+// Mirrors the guard publishResourceGroupMutation applies at publish time.
+// AddResourceGroup/ModifyResourceGroup/DeleteResourceGroup check this
+// immediately before their storage-write phase to fail fast with a
+// retryable error in that specific case, instead of writing to storage on
+// behalf of a term whose result is going to be silently dropped at publish
+// time anyway. Deliberately narrower than isKeyspaceManagerCurrent: a term
+// change with nothing yet confirmed for this group in the new term is
+// harmless and already handled gracefully - publishResourceGroupMutation
+// re-resolves the current manager and applies the result there - so it must
+// not raise the same reject here that only the confirmed-collision case
+// warrants.
+// This only catches a collision that completes before the storage write
+// starts; one that lands while the write is already in flight needs the
+// storage-side revision/CAS check tracked in #11105.
+func (m *Manager) hasNewerConfirmedWrite(keyspaceID uint32, name string, krgm *keyspaceResourceGroupManager) bool {
+	m.RLock()
+	defer m.RUnlock()
+	cur, ok := m.krgms[keyspaceID]
+	return ok && cur != krgm && cur.hasConfirmedResourceGroup(name)
+}
+
 func (m *Manager) isResourceGroupLoadingComplete() bool {
 	return m.getLoadingState() == LoadingStateCompleted
 }
@@ -1413,6 +1448,17 @@ func (m *Manager) AddResourceGroup(grouppb *rmpb.ResourceGroup) error {
 		krgm.defaultGroupMu.Lock()
 		defer krgm.defaultGroupMu.Unlock()
 	}
+	failpoint.InjectCall("addResourceGroupBeforeStorage")
+	// Fail fast only if a leadership change already replaced krgm AND the new
+	// term already has confirmed data for this group - i.e. persisting here
+	// would be racing a write that's already won and would just get silently
+	// dropped by publishResourceGroupMutation's own matching guard below. A
+	// harmless term change with nothing yet confirmed in the new term is not
+	// rejected: publishResourceGroupMutation already applies the result there
+	// gracefully in that case. See hasNewerConfirmedWrite.
+	if m.hasNewerConfirmedWrite(keyspaceID, grouppb.Name, krgm) {
+		return errs.ErrResourceGroupsLoading
+	}
 	// Storage phase: validate and persist. Publishing the cache effect is done
 	// separately below, against krgm if it's still the live manager for
 	// keyspaceID by then, or dropped otherwise - see publishResourceGroupMutation.
@@ -1449,6 +1495,12 @@ func (m *Manager) ModifyResourceGroup(grouppb *rmpb.ResourceGroup) error {
 		krgm.defaultGroupMu.Lock()
 		defer krgm.defaultGroupMu.Unlock()
 	}
+	failpoint.InjectCall("modifyResourceGroupBeforeStorage")
+	// Fail fast only on a newer confirmed collision; see the matching guard
+	// in AddResourceGroup.
+	if m.hasNewerConfirmedWrite(keyspaceID, grouppb.Name, krgm) {
+		return errs.ErrResourceGroupsLoading
+	}
 	patched, err := krgm.modifyResourceGroup(grouppb)
 	if err != nil {
 		return err
@@ -1498,6 +1550,15 @@ func (m *Manager) DeleteResourceGroup(keyspaceID uint32, name string) error {
 		return errs.ErrKeyspaceNotExists.FastGenByArgs(keyspaceID)
 	}
 	failpoint.InjectCall("deleteResourceGroupBeforeStorage")
+	// Fail fast only on a newer confirmed collision; see the matching guard
+	// in AddResourceGroup. A harmless term change alone (nothing yet
+	// confirmed in the new term) must still let the delete proceed, storage
+	// removal is independent of which term's krgm initiated it, and
+	// publishResourceGroupMutation republishes the result against whichever
+	// term is current at publish time.
+	if m.hasNewerConfirmedWrite(keyspaceID, name, krgm) {
+		return errs.ErrResourceGroupsLoading
+	}
 	// Storage phase: validate and remove from storage. Publishing the cache
 	// effect is done separately below, against krgm if it's still the live
 	// manager for keyspaceID by then, or dropped otherwise (a delete
diff --git a/pkg/mcs/resourcemanager/server/manager_async_test.go b/pkg/mcs/resourcemanager/server/manager_async_test.go
index 4a32091e7a..ff1ecaf3ff 100644
--- a/pkg/mcs/resourcemanager/server/manager_async_test.go
+++ b/pkg/mcs/resourcemanager/server/manager_async_test.go
@@ -766,6 +766,212 @@ func TestAsyncLoadResourceGroupsCrossTermDeletePublishesToNewTerm(t *testing.T)
 	re.Nil(g, "the deleted group must not be resurrected by the new term's merge")
 }
 
+// TestDeleteResourceGroupAbortsOnNewerConfirmedWriteAcrossTermChange guards
+// the other half of the #11105 "parked-before-storage" interleaving that
+// TestAsyncLoadResourceGroupsCrossTermDeletePublishesToNewTerm above does not
+// cover: unlike that test, here the new term's own reload confirms the group
+// (storage is unchanged - the Delete never reached its storage phase) before
+// the parked Delete resumes. Proceeding to remove it from storage now would
+// race a write that has already won and would just get silently dropped by
+// publishResourceGroupMutation's confirmed-write guard, leaving storage
+// empty but the live cache still serving the group forever - the exact
+// #11105 shape. hasNewerConfirmedWrite must catch this and abort before the
+// storage write, while still leaving the sibling "nothing confirmed yet"
+// case (the other test) to succeed normally.
+func TestDeleteResourceGroupAbortsOnNewerConfirmedWriteAcrossTermChange(t *testing.T) {
+	re := require.New(t)
+	store := newBlockingResourceGroupStorage()
+	re.NoError(store.SaveResourceGroupSetting(1, "ct-del-abort", newAsyncTestGroup("ct-del-abort")))
+
+	m := NewManager[*mockConfigProvider](&mockConfigProvider{})
+	m.storage = store
+	re.NoError(m.Init(context.Background()))
+	cancelTerm1 := m.cancel
+	defer stopAsyncTestManager(m)
+	defer store.unblock()
+
+	// Let term 1 load fully so the Delete starts against a settled term.
+	store.waitEntered(t)
+	store.unblock()
+	testutil.Eventually(re, func() bool {
+		groups, err := m.GetResourceGroupList(1, false)
+		return err == nil && len(groups) == 2
+	}, testutil.WithTickInterval(20*time.Millisecond))
+
+	// Park the Delete between resolving its keyspace manager and its storage
+	// phase.
+	reached := make(chan struct{})
+	release := make(chan struct{})
+	re.NoError(failpoint.EnableCall("github.com/tikv/pd/pkg/mcs/resourcemanager/server/deleteResourceGroupBeforeStorage", func() {
+		close(reached)
+		<-release
+	}))
+	defer func() {
+		re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/mcs/resourcemanager/server/deleteResourceGroupBeforeStorage"))
+	}()
+	var delErr error
+	delDone := make(chan struct{})
+	go func() {
+		defer close(delDone)
+		delErr = m.DeleteResourceGroup(1, "ct-del-abort")
+	}()
+	select {
+	case <-reached:
+	case <-time.After(time.Second):
+		t.Fatal("timed out waiting for the delete to reach its storage phase")
+	}
+
+	// Leadership changes and term 2 fully completes loading. Storage is
+	// unchanged (the Delete never reached its storage phase), so term 2's own
+	// reload confirms the group in the new term's cache.
+	cancelTerm1()
+	re.NoError(m.Init(context.Background()))
+	testutil.Eventually(re, func() bool {
+		groups, err := m.GetResourceGroupList(1, false)
+		return err == nil && len(groups) == 2
+	}, testutil.WithTickInterval(20*time.Millisecond))
+
+	// Resume the parked Delete: it must see term 2 already has confirmed data
+	// for this group and abort, instead of removing it from storage on
+	// behalf of the detached term-1 manager.
+	close(release)
+	<-delDone
+	re.ErrorIs(delErr, errs.ErrResourceGroupsLoading)
+
+	g, err := m.GetResourceGroup(1, "ct-del-abort", false)
+	re.NoError(err)
+	re.NotNil(g, "the aborted delete must not have removed the group from storage")
+}
+
+// TestAddResourceGroupAbortsOnNewerConfirmedWriteAcrossTermChange is
+// AddResourceGroup's counterpart to the Delete test above: an old-term Add
+// parked before its storage phase must abort, not overwrite storage, once it
+// wakes up to find the new term already has confirmed data for the same
+// group (here, from the new term's own reload of unchanged storage).
+func TestAddResourceGroupAbortsOnNewerConfirmedWriteAcrossTermChange(t *testing.T) {
+	re := require.New(t)
+	store := newBlockingResourceGroupStorage()
+	re.NoError(store.SaveResourceGroupSetting(1, "ct-add-abort", newAsyncTestGroup("ct-add-abort")))
+
+	m := NewManager[*mockConfigProvider](&mockConfigProvider{})
+	m.storage = store
+	re.NoError(m.Init(context.Background()))
+	cancelTerm1 := m.cancel
+	defer stopAsyncTestManager(m)
+	defer store.unblock()
+
+	store.waitEntered(t)
+	store.unblock()
+	testutil.Eventually(re, func() bool {
+		groups, err := m.GetResourceGroupList(1, false)
+		return err == nil && len(groups) == 2
+	}, testutil.WithTickInterval(20*time.Millisecond))
+
+	reached := make(chan struct{})
+	release := make(chan struct{})
+	re.NoError(failpoint.EnableCall("github.com/tikv/pd/pkg/mcs/resourcemanager/server/addResourceGroupBeforeStorage", func() {
+		close(reached)
+		<-release
+	}))
+	defer func() {
+		re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/mcs/resourcemanager/server/addResourceGroupBeforeStorage"))
+	}()
+	updated := newAsyncTestGroup("ct-add-abort")
+	updated.KeyspaceId = &resource_manager.KeyspaceIDValue{Keyspace: &resource_manager.KeyspaceIDValue_Value{Value: 1}}
+	updated.RUSettings.RU.Settings.FillRate = asyncTestGroupFillRate * 2
+	var addErr error
+	addDone := make(chan struct{})
+	go func() {
+		defer close(addDone)
+		addErr = m.AddResourceGroup(updated)
+	}()
+	select {
+	case <-reached:
+	case <-time.After(time.Second):
+		t.Fatal("timed out waiting for the add to reach its storage phase")
+	}
+
+	cancelTerm1()
+	re.NoError(m.Init(context.Background()))
+	testutil.Eventually(re, func() bool {
+		groups, err := m.GetResourceGroupList(1, false)
+		return err == nil && len(groups) == 2
+	}, testutil.WithTickInterval(20*time.Millisecond))
+
+	close(release)
+	<-addDone
+	re.ErrorIs(addErr, errs.ErrResourceGroupsLoading)
+
+	g, err := m.GetResourceGroup(1, "ct-add-abort", false)
+	re.NoError(err)
+	re.NotNil(g)
+	re.Equal(float64(asyncTestGroupFillRate), g.RUSettings.RU.getFillRate(),
+		"the aborted add must not have overwritten storage with the stale term-1 value")
+}
+
+// TestModifyResourceGroupAbortsOnNewerConfirmedWriteAcrossTermChange is
+// ModifyResourceGroup's counterpart to the Delete and Add tests above.
+func TestModifyResourceGroupAbortsOnNewerConfirmedWriteAcrossTermChange(t *testing.T) {
+	re := require.New(t)
+	store := newBlockingResourceGroupStorage()
+	re.NoError(store.SaveResourceGroupSetting(1, "ct-modify-abort", newAsyncTestGroup("ct-modify-abort")))
+
+	m := NewManager[*mockConfigProvider](&mockConfigProvider{})
+	m.storage = store
+	re.NoError(m.Init(context.Background()))
+	cancelTerm1 := m.cancel
+	defer stopAsyncTestManager(m)
+	defer store.unblock()
+
+	store.waitEntered(t)
+	store.unblock()
+	testutil.Eventually(re, func() bool {
+		groups, err := m.GetResourceGroupList(1, false)
+		return err == nil && len(groups) == 2
+	}, testutil.WithTickInterval(20*time.Millisecond))
+
+	reached := make(chan struct{})
+	release := make(chan struct{})
+	re.NoError(failpoint.EnableCall("github.com/tikv/pd/pkg/mcs/resourcemanager/server/modifyResourceGroupBeforeStorage", func() {
+		close(reached)
+		<-release
+	}))
+	defer func() {
+		re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/mcs/resourcemanager/server/modifyResourceGroupBeforeStorage"))
+	}()
+	modified := newAsyncTestGroup("ct-modify-abort")
+	modified.KeyspaceId = &resource_manager.KeyspaceIDValue{Keyspace: &resource_manager.KeyspaceIDValue_Value{Value: 1}}
+	modified.RUSettings.RU.Settings.FillRate = asyncTestGroupFillRate * 2
+	var modErr error
+	modDone := make(chan struct{})
+	go func() {
+		defer close(modDone)
+		modErr = m.ModifyResourceGroup(modified)
+	}()
+	select {
+	case <-reached:
+	case <-time.After(time.Second):
+		t.Fatal("timed out waiting for the modify to reach its storage phase")
+	}
+
+	cancelTerm1()
+	re.NoError(m.Init(context.Background()))
+	testutil.Eventually(re, func() bool {
+		groups, err := m.GetResourceGroupList(1, false)
+		return err == nil && len(groups) == 2
+	}, testutil.WithTickInterval(20*time.Millisecond))
+
+	close(release)
+	<-modDone
+	re.ErrorIs(modErr, errs.ErrResourceGroupsLoading)
+
+	g, err := m.GetResourceGroup(1, "ct-modify-abort", false)
+	re.NoError(err)
+	re.NotNil(g)
+	re.Equal(float64(asyncTestGroupFillRate), g.RUSettings.RU.getFillRate(),
+		"the aborted modify must not have overwritten storage with the stale term-1 value")
+}
+
 // TestAsyncLoadResourceGroupsExhaustedRetriesReturnLoadingError guards the
 // exhausted-retry path of the lazy load: when every attempt loses the
 // delete-generation race, the load must fail with a retryable loading error

From e26e741509a7a763a13cbece5eca1da6376f708c Mon Sep 17 00:00:00 2001
From: bufferflies <1045931706@qq.com>
Date: Tue, 11 Aug 2026 15:09:38 +0200
Subject: [PATCH 48/50] mcs, resourcemanager: hold the lock across
 SetKeyspaceRUVersion's save

SetKeyspaceRUVersion mutated the live m.controllerConfig.RUVersionPolicy
object in place and only called storage.SaveControllerConfig after
unlocking. That left RUVersionPolicy.Overrides - a plain map - both
mutable by a concurrent SetKeyspaceRUVersion call and readable by this
call's own unlocked marshal (an unsynchronized concurrent map
read/write), and let this call's save land out of order relative to
UpdateControllerConfigItem's, which clones, saves, and publishes fully
inside its own lock - a save delayed past unlock here could silently
overwrite a newer config UpdateControllerConfigItem had already
committed.

Fix it to match UpdateControllerConfigItem's existing pattern: clone
before mutating, and hold the lock across save + publish so the two
config mutators are fully serialized against each other, not just
consistent with the in-memory state at capture time.

Adds a setKeyspaceRUVersionBeforeSave failpoint and a regression test
proving UpdateControllerConfigItem blocks while SetKeyspaceRUVersion is
parked before its save, and that neither call's change is lost once
both complete. Verified against pre-fix code (temporarily reverted the
production change, kept the test) to confirm it actually catches the
bug.

Reported by rleungx: https://github.com/tikv/pd/pull/10873#discussion_r3757142028
---
 pkg/mcs/resourcemanager/server/manager.go     | 47 +++++++-----
 .../server/manager_async_test.go              | 74 +++++++++++++++++++
 2 files changed, 103 insertions(+), 18 deletions(-)

diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go
index 9fe40d710f..0f973e8c43 100644
--- a/pkg/mcs/resourcemanager/server/manager.go
+++ b/pkg/mcs/resourcemanager/server/manager.go
@@ -332,31 +332,42 @@ func (m *Manager) SetKeyspaceRUVersion(keyspaceID uint32, ruVersion int32) error
 		return errMetadataWriteDisabled
 	}
 	m.Lock()
-	if m.controllerConfig.RUVersionPolicy == nil {
+	defer m.Unlock()
+	// Mutate a clone, not m.controllerConfig directly, and save it before
+	// publishing it to the field - matching UpdateControllerConfigItem's
+	// pattern below. Mutating the live object in place and saving it after
+	// unlocking (the previous approach here) left RUVersionPolicy.Overrides,
+	// a plain map, both mutable by a concurrent call to this same function and
+	// readable by this call's own unlocked marshal - an unsynchronized
+	// concurrent map read/write. It also let this call's save land out of
+	// order relative to UpdateControllerConfigItem's, which clones, saves,
+	// and publishes fully inside its own lock: a save delayed past unlock
+	// here could persist a stale snapshot over a newer config
+	// UpdateControllerConfigItem had already committed. Holding the lock
+	// across the storage write closes both: no other config mutator can run
+	// until this call's save and publish are done.
+	controllerConfig := cloneControllerConfig(m.controllerConfig)
+	if controllerConfig.RUVersionPolicy == nil {
 		// DefaultRUVersion (v1) means no RU model change.
 		// There is currently no API to modify this global default; it is
 		// intentionally fixed so that only per-keyspace overrides drive version bumps.
-		m.controllerConfig.RUVersionPolicy = &RUVersionPolicy{Default: DefaultRUVersion}
+		controllerConfig.RUVersionPolicy = &RUVersionPolicy{Default: DefaultRUVersion}
 	}
-	if m.controllerConfig.RUVersionPolicy.Overrides == nil {
-		m.controllerConfig.RUVersionPolicy.Overrides = make(map[uint32]RUVersion)
+	if controllerConfig.RUVersionPolicy.Overrides == nil {
+		controllerConfig.RUVersionPolicy.Overrides = make(map[uint32]RUVersion)
 	}
-	defaultVersion := m.controllerConfig.RUVersionPolicy.Default
+	defaultVersion := controllerConfig.RUVersionPolicy.Default
 	if ruVersion == defaultVersion {
-		delete(m.controllerConfig.RUVersionPolicy.Overrides, keyspaceID)
+		delete(controllerConfig.RUVersionPolicy.Overrides, keyspaceID)
 	} else {
-		m.controllerConfig.RUVersionPolicy.Overrides[keyspaceID] = ruVersion
-	}
-	// Capture the config object to save while still holding the lock, instead
-	// of re-reading the m.controllerConfig field after unlocking below:
-	// initControllerConfig can reassign that field wholesale (under the same
-	// lock) on a leadership change, so an unlocked re-read races with it and
-	// can end up saving a different config object than the one just mutated
-	// above. Matches the pattern initControllerConfig itself already uses -
-	// save a locally captured reference, never the field.
-	controllerConfig := m.controllerConfig
-	m.Unlock()
-	return m.storage.SaveControllerConfig(controllerConfig)
+		controllerConfig.RUVersionPolicy.Overrides[keyspaceID] = ruVersion
+	}
+	failpoint.InjectCall("setKeyspaceRUVersionBeforeSave")
+	if err := m.storage.SaveControllerConfig(controllerConfig); err != nil {
+		return err
+	}
+	m.controllerConfig = controllerConfig
+	return nil
 }
 
 // GetRUVersionPolicy returns a deep copy of the current RU version policy from the controller config.
diff --git a/pkg/mcs/resourcemanager/server/manager_async_test.go b/pkg/mcs/resourcemanager/server/manager_async_test.go
index ff1ecaf3ff..33e3489c81 100644
--- a/pkg/mcs/resourcemanager/server/manager_async_test.go
+++ b/pkg/mcs/resourcemanager/server/manager_async_test.go
@@ -16,6 +16,7 @@ package server
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -1819,3 +1820,76 @@ func TestAcquireTokenBucketsSurvivesLazyLoadFailure(t *testing.T) {
 	re.Len(stream.sent[0].Responses, 1, "only the loadable group should be answered")
 	re.Equal("good-group", stream.sent[0].Responses[0].ResourceGroupName)
 }
+
+// TestSetKeyspaceRUVersionSerializesAgainstUpdateControllerConfigItem guards
+// against SetKeyspaceRUVersion mutating RUVersionPolicy.Overrides and saving
+// it to storage outside its lock. That used to capture a reference to the
+// live (not cloned) controller config, unlock, and only then call
+// storage.SaveControllerConfig - leaving Overrides, a plain map, both
+// mutable by a concurrent SetKeyspaceRUVersion call and readable by this
+// call's own unlocked marshal (an unsynchronized concurrent map read/write),
+// and letting this call's save land after UpdateControllerConfigItem's own
+// clone-save-publish (which runs fully inside its lock), silently
+// overwriting UpdateControllerConfigItem's newer write. The fix clones
+// before mutating and holds the lock across the whole
+// mutate-save-publish sequence, so a concurrent config mutator can't even
+// start until this call is done - proven here by asserting
+// UpdateControllerConfigItem blocks while SetKeyspaceRUVersion is parked
+// right before its save, and that neither call's change is lost once both
+// complete.
+func TestSetKeyspaceRUVersionSerializesAgainstUpdateControllerConfigItem(t *testing.T) {
+	re := require.New(t)
+	m := prepareManager()
+
+	reached := make(chan struct{})
+	release := make(chan struct{})
+	re.NoError(failpoint.EnableCall("github.com/tikv/pd/pkg/mcs/resourcemanager/server/setKeyspaceRUVersionBeforeSave", func() {
+		close(reached)
+		<-release
+	}))
+	defer func() {
+		re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/mcs/resourcemanager/server/setKeyspaceRUVersionBeforeSave"))
+	}()
+
+	setDone := make(chan error, 1)
+	go func() {
+		setDone <- m.SetKeyspaceRUVersion(1, 2)
+	}()
+	select {
+	case <-reached:
+	case <-time.After(time.Second):
+		t.Fatal("timed out waiting for SetKeyspaceRUVersion to reach its pre-save failpoint")
+	}
+
+	// While SetKeyspaceRUVersion is parked holding the lock, a concurrent
+	// UpdateControllerConfigItem call must block rather than run - proving
+	// the two are mutually exclusive across the whole save, not just the
+	// in-memory mutation step.
+	updateDone := make(chan error, 1)
+	go func() {
+		updateDone <- m.UpdateControllerConfigItem("request-unit.read-base-cost", 1.5)
+	}()
+	select {
+	case err := <-updateDone:
+		t.Fatalf("UpdateControllerConfigItem completed (err=%v) while SetKeyspaceRUVersion still held the lock", err)
+	case <-time.After(100 * time.Millisecond):
+	}
+
+	close(release)
+	re.NoError(<-setDone)
+	re.NoError(<-updateDone)
+
+	re.InDelta(1.5, m.controllerConfig.RequestUnit.ReadBaseCost, 0.00001,
+		"UpdateControllerConfigItem's change must survive")
+	re.Equal(RUVersion(2), m.controllerConfig.RUVersionPolicy.Overrides[1],
+		"SetKeyspaceRUVersion's change must not be lost to a later save landing out of order")
+
+	raw, err := m.storage.LoadControllerConfig()
+	re.NoError(err)
+	persisted := &ControllerConfig{}
+	re.NoError(json.Unmarshal([]byte(raw), persisted))
+	re.InDelta(1.5, persisted.RequestUnit.ReadBaseCost, 0.00001,
+		"UpdateControllerConfigItem's change must be persisted")
+	re.Equal(RUVersion(2), persisted.RUVersionPolicy.Overrides[1],
+		"SetKeyspaceRUVersion's change must be persisted, not clobbered by an out-of-order save")
+}

From 619761d105f093fa41834c8b6b626e2063577132 Mon Sep 17 00:00:00 2001
From: bufferflies <1045931706@qq.com>
Date: Wed, 12 Aug 2026 04:15:44 +0200
Subject: [PATCH 49/50] mcs, resourcemanager: hold the lock across
 initControllerConfig's save

initControllerConfig read the current config under RLock, unmarshaled
storage's value into a clone, saved that clone back to storage while
unlocked, and only then published it under Lock. Save-before-publish
ordering avoided one specific race (a concurrent mutator marshaling
the not-yet-cloned live object), but the unlocked save itself was
never mutually exclusive with SetKeyspaceRUVersion or
UpdateControllerConfigItem, which now both save+publish fully inside
their own lock: either could commit a newer config to storage and
m.controllerConfig in the window between this call's read and its own
save, and this call's now-stale snapshot would silently overwrite it -
the same class of lost update just fixed for SetKeyspaceRUVersion.

Fold the whole clone-merge-save-publish sequence into one m.Lock()
critical section, same as the other two config mutators. Requests are
gated behind IsServing(), which only flips true after Init (and thus
this call) returns, so this does not add contention with live request
traffic in the common case.
---
 pkg/mcs/resourcemanager/server/manager.go | 27 +++++++++++++----------
 1 file changed, 15 insertions(+), 12 deletions(-)

diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go
index 0f973e8c43..8b65f33e84 100644
--- a/pkg/mcs/resourcemanager/server/manager.go
+++ b/pkg/mcs/resourcemanager/server/manager.go
@@ -504,28 +504,31 @@ func (m *Manager) initControllerConfig() error {
 		log.Error("resource controller config load failed", zap.Error(err), zap.String("v", v))
 		return err
 	}
-	// Unmarshal into a clone and publish it under the lock: on a
-	// re-initialization after a leadership change, the previous term's
-	// background goroutines may still be reading the current config.
-	m.RLock()
+	// Clone, merge, save, and publish all under the same lock, matching
+	// SetKeyspaceRUVersion/UpdateControllerConfigItem's pattern: this makes
+	// initControllerConfig's own re-save fully serialized against every
+	// other config mutator too, instead of only against itself via ordering
+	// (save-before-publish). Without this, a concurrent SetKeyspaceRUVersion
+	// or UpdateControllerConfigItem call could commit its change to storage
+	// and m.controllerConfig in between this call's unlocked read/merge and
+	// its own unlocked save, and this call's now-stale snapshot would
+	// silently overwrite it - the same class of lost update fixed for
+	// SetKeyspaceRUVersion. Requests are gated behind IsServing(), which
+	// only flips true after Init (and thus this call) returns, so holding
+	// the lock across the save here does not add contention with live
+	// request traffic in the common case.
+	m.Lock()
+	defer m.Unlock()
 	controllerConfig := cloneControllerConfig(m.controllerConfig)
-	m.RUnlock()
 	if err = json.Unmarshal([]byte(v), &controllerConfig); err != nil {
 		log.Warn("un-marshall controller config failed, fallback to default", zap.Error(err), zap.String("v", v))
 	}
-	// re-save the config to make sure the config has been persisted. This
-	// must run before controllerConfig is published into m.controllerConfig
-	// below: once published, it's reachable (and mutable) by any concurrent
-	// caller holding m.Lock() - e.g. SetKeyspaceRUVersion - which would race
-	// with this unlocked marshal-and-save if it ran after instead.
 	if m.writeRole.AllowsMetadataWrite() {
 		if err := m.storage.SaveControllerConfig(controllerConfig); err != nil {
 			return err
 		}
 	}
-	m.Lock()
 	m.controllerConfig = controllerConfig
-	m.Unlock()
 	return nil
 }
 

From 716cbbe9021dda5c91b83652d15b28f952d85574 Mon Sep 17 00:00:00 2001
From: bufferflies <1045931706@qq.com>
Date: Wed, 12 Aug 2026 08:41:43 +0200
Subject: [PATCH 50/50] mcs, resourcemanager: table-drive the three
 abort-on-newer-confirmed-write tests

TestDeleteResourceGroupAbortsOnNewerConfirmedWriteAcrossTermChange,
TestAddResourceGroupAbortsOnNewerConfirmedWriteAcrossTermChange, and
TestModifyResourceGroupAbortsOnNewerConfirmedWriteAcrossTermChange were
near-identical: same setup, same park-before-storage/term-change/resume
choreography, same final assertions - differing only in which API is
called, which failpoint parks it, and the group name.

Consolidate into TestResourceGroupMutationAbortsOnNewerConfirmedWriteAcrossTermChange,
a single table-driven test with one shared harness and a small
per-mutation-kind table entry. Verified each subtest still independently
catches its corresponding regression: temporarily disabling
DeleteResourceGroup's hasNewerConfirmedWrite guard fails only the
"delete" subtest, leaving "add" and "modify" green.

No production code change.
---
 .../server/manager_async_test.go              | 320 +++++++-----------
 1 file changed, 120 insertions(+), 200 deletions(-)

diff --git a/pkg/mcs/resourcemanager/server/manager_async_test.go b/pkg/mcs/resourcemanager/server/manager_async_test.go
index 33e3489c81..cf6cd13195 100644
--- a/pkg/mcs/resourcemanager/server/manager_async_test.go
+++ b/pkg/mcs/resourcemanager/server/manager_async_test.go
@@ -767,210 +767,130 @@ func TestAsyncLoadResourceGroupsCrossTermDeletePublishesToNewTerm(t *testing.T)
 	re.Nil(g, "the deleted group must not be resurrected by the new term's merge")
 }
 
-// TestDeleteResourceGroupAbortsOnNewerConfirmedWriteAcrossTermChange guards
-// the other half of the #11105 "parked-before-storage" interleaving that
-// TestAsyncLoadResourceGroupsCrossTermDeletePublishesToNewTerm above does not
-// cover: unlike that test, here the new term's own reload confirms the group
-// (storage is unchanged - the Delete never reached its storage phase) before
-// the parked Delete resumes. Proceeding to remove it from storage now would
-// race a write that has already won and would just get silently dropped by
-// publishResourceGroupMutation's confirmed-write guard, leaving storage
-// empty but the live cache still serving the group forever - the exact
-// #11105 shape. hasNewerConfirmedWrite must catch this and abort before the
-// storage write, while still leaving the sibling "nothing confirmed yet"
-// case (the other test) to succeed normally.
-func TestDeleteResourceGroupAbortsOnNewerConfirmedWriteAcrossTermChange(t *testing.T) {
-	re := require.New(t)
-	store := newBlockingResourceGroupStorage()
-	re.NoError(store.SaveResourceGroupSetting(1, "ct-del-abort", newAsyncTestGroup("ct-del-abort")))
-
-	m := NewManager[*mockConfigProvider](&mockConfigProvider{})
-	m.storage = store
-	re.NoError(m.Init(context.Background()))
-	cancelTerm1 := m.cancel
-	defer stopAsyncTestManager(m)
-	defer store.unblock()
-
-	// Let term 1 load fully so the Delete starts against a settled term.
-	store.waitEntered(t)
-	store.unblock()
-	testutil.Eventually(re, func() bool {
-		groups, err := m.GetResourceGroupList(1, false)
-		return err == nil && len(groups) == 2
-	}, testutil.WithTickInterval(20*time.Millisecond))
-
-	// Park the Delete between resolving its keyspace manager and its storage
-	// phase.
-	reached := make(chan struct{})
-	release := make(chan struct{})
-	re.NoError(failpoint.EnableCall("github.com/tikv/pd/pkg/mcs/resourcemanager/server/deleteResourceGroupBeforeStorage", func() {
-		close(reached)
-		<-release
-	}))
-	defer func() {
-		re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/mcs/resourcemanager/server/deleteResourceGroupBeforeStorage"))
-	}()
-	var delErr error
-	delDone := make(chan struct{})
-	go func() {
-		defer close(delDone)
-		delErr = m.DeleteResourceGroup(1, "ct-del-abort")
-	}()
-	select {
-	case <-reached:
-	case <-time.After(time.Second):
-		t.Fatal("timed out waiting for the delete to reach its storage phase")
-	}
-
-	// Leadership changes and term 2 fully completes loading. Storage is
-	// unchanged (the Delete never reached its storage phase), so term 2's own
-	// reload confirms the group in the new term's cache.
-	cancelTerm1()
-	re.NoError(m.Init(context.Background()))
-	testutil.Eventually(re, func() bool {
-		groups, err := m.GetResourceGroupList(1, false)
-		return err == nil && len(groups) == 2
-	}, testutil.WithTickInterval(20*time.Millisecond))
-
-	// Resume the parked Delete: it must see term 2 already has confirmed data
-	// for this group and abort, instead of removing it from storage on
-	// behalf of the detached term-1 manager.
-	close(release)
-	<-delDone
-	re.ErrorIs(delErr, errs.ErrResourceGroupsLoading)
-
-	g, err := m.GetResourceGroup(1, "ct-del-abort", false)
-	re.NoError(err)
-	re.NotNil(g, "the aborted delete must not have removed the group from storage")
-}
-
-// TestAddResourceGroupAbortsOnNewerConfirmedWriteAcrossTermChange is
-// AddResourceGroup's counterpart to the Delete test above: an old-term Add
-// parked before its storage phase must abort, not overwrite storage, once it
-// wakes up to find the new term already has confirmed data for the same
-// group (here, from the new term's own reload of unchanged storage).
-func TestAddResourceGroupAbortsOnNewerConfirmedWriteAcrossTermChange(t *testing.T) {
-	re := require.New(t)
-	store := newBlockingResourceGroupStorage()
-	re.NoError(store.SaveResourceGroupSetting(1, "ct-add-abort", newAsyncTestGroup("ct-add-abort")))
-
-	m := NewManager[*mockConfigProvider](&mockConfigProvider{})
-	m.storage = store
-	re.NoError(m.Init(context.Background()))
-	cancelTerm1 := m.cancel
-	defer stopAsyncTestManager(m)
-	defer store.unblock()
-
-	store.waitEntered(t)
-	store.unblock()
-	testutil.Eventually(re, func() bool {
-		groups, err := m.GetResourceGroupList(1, false)
-		return err == nil && len(groups) == 2
-	}, testutil.WithTickInterval(20*time.Millisecond))
-
-	reached := make(chan struct{})
-	release := make(chan struct{})
-	re.NoError(failpoint.EnableCall("github.com/tikv/pd/pkg/mcs/resourcemanager/server/addResourceGroupBeforeStorage", func() {
-		close(reached)
-		<-release
-	}))
-	defer func() {
-		re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/mcs/resourcemanager/server/addResourceGroupBeforeStorage"))
-	}()
-	updated := newAsyncTestGroup("ct-add-abort")
-	updated.KeyspaceId = &resource_manager.KeyspaceIDValue{Keyspace: &resource_manager.KeyspaceIDValue_Value{Value: 1}}
-	updated.RUSettings.RU.Settings.FillRate = asyncTestGroupFillRate * 2
-	var addErr error
-	addDone := make(chan struct{})
-	go func() {
-		defer close(addDone)
-		addErr = m.AddResourceGroup(updated)
-	}()
-	select {
-	case <-reached:
-	case <-time.After(time.Second):
-		t.Fatal("timed out waiting for the add to reach its storage phase")
+// TestResourceGroupMutationAbortsOnNewerConfirmedWriteAcrossTermChange
+// guards the "parked-before-storage" side of the #11105 interleaving for all
+// three mutation kinds: an old-term Add/Modify/Delete parked before its
+// storage phase must abort, not touch storage, once it wakes up to find the
+// new term already has confirmed data for the same group (here, from the
+// new term's own reload of unchanged storage, since the mutation never
+// reached its own storage phase). Proceeding would race a write that has
+// already won and would just get silently dropped by
+// publishResourceGroupMutation's confirmed-write guard - for Delete that
+// would leave storage empty but the cache still serving the group forever,
+// the exact #11105 shape; for Add/Modify it would overwrite storage with a
+// stale value nobody asked for. hasNewerConfirmedWrite must catch this and
+// abort before the storage write in every case, while still leaving the
+// sibling "nothing confirmed yet" case
+// (TestAsyncLoadResourceGroupsCrossTermDeletePublishesToNewTerm) to succeed
+// normally.
+func TestResourceGroupMutationAbortsOnNewerConfirmedWriteAcrossTermChange(t *testing.T) {
+	cases := []struct {
+		name      string
+		group     string
+		failpoint string
+		mutate    func(m *Manager, group string) error
+	}{
+		{
+			name:      "delete",
+			group:     "ct-del-abort",
+			failpoint: "deleteResourceGroupBeforeStorage",
+			mutate: func(m *Manager, group string) error {
+				return m.DeleteResourceGroup(1, group)
+			},
+		},
+		{
+			name:      "add",
+			group:     "ct-add-abort",
+			failpoint: "addResourceGroupBeforeStorage",
+			mutate: func(m *Manager, group string) error {
+				updated := newAsyncTestGroup(group)
+				updated.KeyspaceId = &resource_manager.KeyspaceIDValue{Keyspace: &resource_manager.KeyspaceIDValue_Value{Value: 1}}
+				updated.RUSettings.RU.Settings.FillRate = asyncTestGroupFillRate * 2
+				return m.AddResourceGroup(updated)
+			},
+		},
+		{
+			name:      "modify",
+			group:     "ct-modify-abort",
+			failpoint: "modifyResourceGroupBeforeStorage",
+			mutate: func(m *Manager, group string) error {
+				modified := newAsyncTestGroup(group)
+				modified.KeyspaceId = &resource_manager.KeyspaceIDValue{Keyspace: &resource_manager.KeyspaceIDValue_Value{Value: 1}}
+				modified.RUSettings.RU.Settings.FillRate = asyncTestGroupFillRate * 2
+				return m.ModifyResourceGroup(modified)
+			},
+		},
 	}
 
-	cancelTerm1()
-	re.NoError(m.Init(context.Background()))
-	testutil.Eventually(re, func() bool {
-		groups, err := m.GetResourceGroupList(1, false)
-		return err == nil && len(groups) == 2
-	}, testutil.WithTickInterval(20*time.Millisecond))
-
-	close(release)
-	<-addDone
-	re.ErrorIs(addErr, errs.ErrResourceGroupsLoading)
-
-	g, err := m.GetResourceGroup(1, "ct-add-abort", false)
-	re.NoError(err)
-	re.NotNil(g)
-	re.Equal(float64(asyncTestGroupFillRate), g.RUSettings.RU.getFillRate(),
-		"the aborted add must not have overwritten storage with the stale term-1 value")
-}
-
-// TestModifyResourceGroupAbortsOnNewerConfirmedWriteAcrossTermChange is
-// ModifyResourceGroup's counterpart to the Delete and Add tests above.
-func TestModifyResourceGroupAbortsOnNewerConfirmedWriteAcrossTermChange(t *testing.T) {
-	re := require.New(t)
-	store := newBlockingResourceGroupStorage()
-	re.NoError(store.SaveResourceGroupSetting(1, "ct-modify-abort", newAsyncTestGroup("ct-modify-abort")))
-
-	m := NewManager[*mockConfigProvider](&mockConfigProvider{})
-	m.storage = store
-	re.NoError(m.Init(context.Background()))
-	cancelTerm1 := m.cancel
-	defer stopAsyncTestManager(m)
-	defer store.unblock()
-
-	store.waitEntered(t)
-	store.unblock()
-	testutil.Eventually(re, func() bool {
-		groups, err := m.GetResourceGroupList(1, false)
-		return err == nil && len(groups) == 2
-	}, testutil.WithTickInterval(20*time.Millisecond))
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			re := require.New(t)
+			store := newBlockingResourceGroupStorage()
+			re.NoError(store.SaveResourceGroupSetting(1, tc.group, newAsyncTestGroup(tc.group)))
+
+			m := NewManager[*mockConfigProvider](&mockConfigProvider{})
+			m.storage = store
+			re.NoError(m.Init(context.Background()))
+			cancelTerm1 := m.cancel
+			defer stopAsyncTestManager(m)
+			defer store.unblock()
+
+			// Let term 1 load fully so the mutation starts against a settled term.
+			store.waitEntered(t)
+			store.unblock()
+			testutil.Eventually(re, func() bool {
+				groups, err := m.GetResourceGroupList(1, false)
+				return err == nil && len(groups) == 2
+			}, testutil.WithTickInterval(20*time.Millisecond))
+
+			// Park the mutation between resolving its keyspace manager and its
+			// storage phase.
+			reached := make(chan struct{})
+			release := make(chan struct{})
+			fpPath := "github.com/tikv/pd/pkg/mcs/resourcemanager/server/" + tc.failpoint
+			re.NoError(failpoint.EnableCall(fpPath, func() {
+				close(reached)
+				<-release
+			}))
+			defer func() {
+				re.NoError(failpoint.Disable(fpPath))
+			}()
+			var mutErr error
+			mutDone := make(chan struct{})
+			go func() {
+				defer close(mutDone)
+				mutErr = tc.mutate(m, tc.group)
+			}()
+			select {
+			case <-reached:
+			case <-time.After(time.Second):
+				t.Fatal("timed out waiting for the mutation to reach its storage phase")
+			}
 
-	reached := make(chan struct{})
-	release := make(chan struct{})
-	re.NoError(failpoint.EnableCall("github.com/tikv/pd/pkg/mcs/resourcemanager/server/modifyResourceGroupBeforeStorage", func() {
-		close(reached)
-		<-release
-	}))
-	defer func() {
-		re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/mcs/resourcemanager/server/modifyResourceGroupBeforeStorage"))
-	}()
-	modified := newAsyncTestGroup("ct-modify-abort")
-	modified.KeyspaceId = &resource_manager.KeyspaceIDValue{Keyspace: &resource_manager.KeyspaceIDValue_Value{Value: 1}}
-	modified.RUSettings.RU.Settings.FillRate = asyncTestGroupFillRate * 2
-	var modErr error
-	modDone := make(chan struct{})
-	go func() {
-		defer close(modDone)
-		modErr = m.ModifyResourceGroup(modified)
-	}()
-	select {
-	case <-reached:
-	case <-time.After(time.Second):
-		t.Fatal("timed out waiting for the modify to reach its storage phase")
+			// Leadership changes and term 2 fully completes loading. Storage is
+			// unchanged (the mutation never reached its storage phase), so term
+			// 2's own reload confirms the group in the new term's cache.
+			cancelTerm1()
+			re.NoError(m.Init(context.Background()))
+			testutil.Eventually(re, func() bool {
+				groups, err := m.GetResourceGroupList(1, false)
+				return err == nil && len(groups) == 2
+			}, testutil.WithTickInterval(20*time.Millisecond))
+
+			// Resume the parked mutation: it must see term 2 already has
+			// confirmed data for this group and abort, instead of touching
+			// storage on behalf of the detached term-1 manager.
+			close(release)
+			<-mutDone
+			re.ErrorIs(mutErr, errs.ErrResourceGroupsLoading)
+
+			g, err := m.GetResourceGroup(1, tc.group, false)
+			re.NoError(err)
+			re.NotNil(g, "the aborted mutation must not have removed the group from storage")
+			re.Equal(float64(asyncTestGroupFillRate), g.RUSettings.RU.getFillRate(),
+				"the aborted mutation must not have overwritten storage with a stale value")
+		})
 	}
-
-	cancelTerm1()
-	re.NoError(m.Init(context.Background()))
-	testutil.Eventually(re, func() bool {
-		groups, err := m.GetResourceGroupList(1, false)
-		return err == nil && len(groups) == 2
-	}, testutil.WithTickInterval(20*time.Millisecond))
-
-	close(release)
-	<-modDone
-	re.ErrorIs(modErr, errs.ErrResourceGroupsLoading)
-
-	g, err := m.GetResourceGroup(1, "ct-modify-abort", false)
-	re.NoError(err)
-	re.NotNil(g)
-	re.Equal(float64(asyncTestGroupFillRate), g.RUSettings.RU.getFillRate(),
-		"the aborted modify must not have overwritten storage with the stale term-1 value")
 }
 
 // TestAsyncLoadResourceGroupsExhaustedRetriesReturnLoadingError guards the