From 256f11d3b7f6d6fb1585337d28c25d4654121ce5 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Mon, 13 Jul 2026 11:33:43 +0800 Subject: [PATCH 01/15] server: clean up microservice metadata in PD mode Signed-off-by: Ryan Leung --- server/microservice_cleanup.go | 256 ++++++++++++++++++++++ server/microservice_cleanup_test.go | 220 +++++++++++++++++++ server/server.go | 1 + tests/integrations/mcs/tso/server_test.go | 133 +++++++++++ 4 files changed, 610 insertions(+) create mode 100644 server/microservice_cleanup.go create mode 100644 server/microservice_cleanup_test.go diff --git a/server/microservice_cleanup.go b/server/microservice_cleanup.go new file mode 100644 index 0000000000..3df4838ae2 --- /dev/null +++ b/server/microservice_cleanup.go @@ -0,0 +1,256 @@ +// Copyright 2026 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "context" + "fmt" + "strconv" + "time" + + clientv3 "go.etcd.io/etcd/client/v3" + "go.uber.org/zap" + + "github.com/pingcap/errors" + "github.com/pingcap/failpoint" + "github.com/pingcap/log" + + "github.com/tikv/pd/pkg/errs" + "github.com/tikv/pd/pkg/keyspace" + "github.com/tikv/pd/pkg/keyspace/constant" + mcs "github.com/tikv/pd/pkg/mcs/utils/constant" + "github.com/tikv/pd/pkg/storage/kv" + "github.com/tikv/pd/pkg/utils/etcdutil" + "github.com/tikv/pd/pkg/utils/keypath" + "github.com/tikv/pd/pkg/utils/logutil" +) + +// microserviceMetadataCleanupDelay lets clients observe the service mode change +// before stale microservice metadata disappears from etcd. +const ( + microserviceMetadataCleanupDelay = 10 * time.Second + microserviceMetadataCleanupRetryInterval = 5 * time.Second +) + +func (s *Server) scheduleMicroserviceMetadataCleanup(ctx context.Context) { + if s.IsKeyspaceGroupEnabled() { + return + } + s.serverLoopWg.Add(1) + go func() { + defer logutil.LogPanic() + defer s.serverLoopWg.Done() + if !waitMicroserviceMetadataCleanupDelay(ctx) { + return + } + for { + if s.member != nil && !s.member.IsServing() { + return + } + if err := s.cleanupMicroserviceMetadataInPDMode(ctx); err == nil { + return + } else { + log.Warn("failed to clean up microservice metadata in PD mode, retry later", + errs.ZapError(err)) + } + timer := time.NewTimer(microserviceMetadataCleanupRetryInterval) + select { + case <-ctx.Done(): + timer.Stop() + return + case <-timer.C: + } + } + }() +} + +func waitMicroserviceMetadataCleanupDelay(ctx context.Context) bool { + delay := microserviceMetadataCleanupDelay + failpoint.Inject("skipMicroserviceMetadataCleanupDelay", func() { + delay = 0 + }) + if delay == 0 { + return true + } + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-ctx.Done(): + return false + case <-timer.C: + return true + } +} + +func (s *Server) cleanupMicroserviceMetadataInPDMode(ctx context.Context) error { + if s.IsKeyspaceGroupEnabled() { + return nil + } + start := time.Now() + + defaultGroupExists, err := s.checkDefaultOnlyTSOKeyspaceGroup() + if err != nil { + return err + } + cleanedKeyspaces, err := s.cleanupDefaultTSOKeyspaceGroupConfig(ctx) + if err != nil { + return err + } + deletedDefaultGroup, err := s.deleteDefaultTSOKeyspaceGroup(ctx) + if err != nil { + return err + } + deletedMicroserviceKeys, err := s.deleteMicroserviceEtcdKeys(ctx) + if err != nil { + return err + } + + if defaultGroupExists || deletedDefaultGroup || cleanedKeyspaces > 0 || deletedMicroserviceKeys > 0 { + log.Info("cleaned up microservice metadata in PD mode", + zap.Bool("default-keyspace-group-exists", defaultGroupExists), + zap.Bool("deleted-default-keyspace-group", deletedDefaultGroup), + zap.Int("cleaned-keyspace-configs", cleanedKeyspaces), + zap.Int64("deleted-microservice-keys", deletedMicroserviceKeys), + zap.Duration("cost", time.Since(start))) + } + return nil +} + +func (s *Server) checkDefaultOnlyTSOKeyspaceGroup() (bool, error) { + groups, err := s.storage.LoadKeyspaceGroups(constant.DefaultKeyspaceGroupID, 0) + if err != nil { + return false, err + } + defaultGroupExists := false + for _, group := range groups { + if group == nil { + continue + } + if group.ID != constant.DefaultKeyspaceGroupID { + return false, errors.Errorf("found non-default TSO keyspace group %d when cleaning up PD mode microservice metadata", group.ID) + } + if group.IsSplitting() { + return false, errors.Errorf("default TSO keyspace group is splitting when cleaning up PD mode microservice metadata") + } + if group.IsMerging() { + return false, errors.Errorf("default TSO keyspace group is merging when cleaning up PD mode microservice metadata") + } + defaultGroupExists = true + } + return defaultGroupExists, nil +} + +func (s *Server) cleanupDefaultTSOKeyspaceGroupConfig(ctx context.Context) (int, error) { + var ( + cleaned int + startID uint32 = constant.StartKeyspaceID + ) + for { + var ( + batchCleaned int + keyspaceIDs []uint32 + loaded int + ) + err := s.storage.RunInTxn(ctx, func(txn kv.Txn) error { + metas, err := s.storage.LoadRangeKeyspace(txn, startID, etcdutil.MaxEtcdTxnOps) + if err != nil { + return err + } + loaded = len(metas) + keyspaceIDs = make([]uint32, 0, len(metas)) + for _, meta := range metas { + if meta == nil { + continue + } + keyspaceIDs = append(keyspaceIDs, meta.GetId()) + if meta.Config == nil { + continue + } + groupIDText, ok := meta.Config[keyspace.TSOKeyspaceGroupIDKey] + if !ok { + continue + } + groupID, err := strconv.ParseUint(groupIDText, 10, 32) + if err != nil { + return errors.Errorf("keyspace %d has invalid TSO keyspace group ID %q", meta.GetId(), groupIDText) + } + if uint32(groupID) != constant.DefaultKeyspaceGroupID { + return errors.Errorf("keyspace %d is assigned to non-default TSO keyspace group %d", meta.GetId(), groupID) + } + delete(meta.Config, keyspace.TSOKeyspaceGroupIDKey) + if err := s.storage.SaveKeyspaceMeta(txn, meta); err != nil { + return err + } + batchCleaned++ + } + return nil + }) + if err != nil { + return cleaned, err + } + cleaned += batchCleaned + if loaded < etcdutil.MaxEtcdTxnOps { + break + } + if len(keyspaceIDs) == 0 { + break + } + lastID := keyspaceIDs[len(keyspaceIDs)-1] + if lastID == ^uint32(0) { + break + } + startID = lastID + 1 + } + return cleaned, nil +} + +func (s *Server) deleteDefaultTSOKeyspaceGroup(ctx context.Context) (bool, error) { + deleted := false + err := s.storage.RunInTxn(ctx, func(txn kv.Txn) error { + group, err := s.storage.LoadKeyspaceGroup(txn, constant.DefaultKeyspaceGroupID) + if err != nil || group == nil { + return err + } + if group.IsSplitting() { + return errors.Errorf("default TSO keyspace group is splitting when deleting PD mode microservice metadata") + } + if group.IsMerging() { + return errors.Errorf("default TSO keyspace group is merging when deleting PD mode microservice metadata") + } + if err := s.storage.DeleteKeyspaceGroup(txn, constant.DefaultKeyspaceGroupID); err != nil { + return err + } + deleted = true + return nil + }) + return deleted, err +} + +func (s *Server) deleteMicroserviceEtcdKeys(ctx context.Context) (int64, error) { + if s.client == nil { + return 0, nil + } + ctx, cancel := context.WithTimeout(ctx, etcdutil.DefaultRequestTimeout) + defer cancel() + resp, err := s.client.Delete(ctx, microserviceEtcdPrefix(), clientv3.WithPrefix()) + if err != nil { + return 0, errs.ErrEtcdKVDelete.Wrap(err).GenWithStackByCause() + } + return resp.Deleted, nil +} + +func microserviceEtcdPrefix() string { + return fmt.Sprintf("%s/%d/", mcs.MicroserviceRootPath, keypath.ClusterID()) +} diff --git a/server/microservice_cleanup_test.go b/server/microservice_cleanup_test.go new file mode 100644 index 0000000000..e22a548fca --- /dev/null +++ b/server/microservice_cleanup_test.go @@ -0,0 +1,220 @@ +// Copyright 2026 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "context" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/require" + clientv3 "go.etcd.io/etcd/client/v3" + + "github.com/pingcap/failpoint" + "github.com/pingcap/kvproto/pkg/keyspacepb" + + "github.com/tikv/pd/pkg/keyspace" + "github.com/tikv/pd/pkg/keyspace/constant" + mcs "github.com/tikv/pd/pkg/mcs/utils/constant" + "github.com/tikv/pd/pkg/storage" + "github.com/tikv/pd/pkg/storage/endpoint" + "github.com/tikv/pd/pkg/storage/kv" + "github.com/tikv/pd/pkg/utils/etcdutil" + "github.com/tikv/pd/pkg/utils/keypath" +) + +func TestCleanupMicroserviceMetadataInPDMode(t *testing.T) { + re := require.New(t) + ctx := context.Background() + _, client, clean := etcdutil.NewTestEtcdCluster(t, 1, nil) + defer clean() + keypath.SetClusterID(12345) + defer keypath.ResetClusterID() + + store := storage.NewStorageWithEtcdBackend(client) + svr := &Server{storage: store, client: client} + re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { + if err := store.SaveKeyspaceGroup(txn, &endpoint.KeyspaceGroup{ + ID: constant.DefaultKeyspaceGroupID, + UserKind: endpoint.Basic.String(), + Keyspaces: []uint32{1}, + }); err != nil { + return err + } + if err := store.SaveKeyspaceMeta(txn, &keyspacepb.KeyspaceMeta{ + Id: 1, + Name: "keyspace-1", + Config: map[string]string{ + keyspace.TSOKeyspaceGroupIDKey: strconv.FormatUint(uint64(constant.DefaultKeyspaceGroupID), 10), + "gc_life_time": "10m", + }, + }); err != nil { + return err + } + return store.SaveKeyspaceMeta(txn, &keyspacepb.KeyspaceMeta{ + Id: 2, + Name: "keyspace-2", + Config: map[string]string{ + "custom": "value", + }, + }) + })) + _, err := client.Put(ctx, keypath.RegistryPath(mcs.TSOServiceName, "127.0.0.1:3379"), "tso") + re.NoError(err) + _, err = client.Put(ctx, keypath.RegistryPath(mcs.SchedulingServiceName, "127.0.0.1:3379"), "scheduling") + re.NoError(err) + _, err = client.Put(ctx, keypath.ElectionPath(&keypath.MsParam{ + ServiceName: mcs.TSOServiceName, + GroupID: constant.DefaultKeyspaceGroupID, + }), "primary") + re.NoError(err) + _, err = client.Put(ctx, keypath.TimestampPath(constant.DefaultKeyspaceGroupID), "timestamp") + re.NoError(err) + + re.NoError(svr.cleanupMicroserviceMetadataInPDMode(ctx)) + re.NoError(svr.cleanupMicroserviceMetadataInPDMode(ctx)) + + groups, err := store.LoadKeyspaceGroups(constant.DefaultKeyspaceGroupID, 0) + re.NoError(err) + re.Empty(groups) + re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { + meta, err := store.LoadKeyspaceMeta(txn, 1) + re.NoError(err) + re.NotNil(meta) + re.NotContains(meta.GetConfig(), keyspace.TSOKeyspaceGroupIDKey) + re.Equal("10m", meta.GetConfig()["gc_life_time"]) + + meta, err = store.LoadKeyspaceMeta(txn, 2) + re.NoError(err) + re.NotNil(meta) + re.Equal("value", meta.GetConfig()["custom"]) + return nil + })) + resp, err := etcdutil.EtcdKVGet(client, microserviceEtcdPrefix(), clientv3.WithPrefix()) + re.NoError(err) + re.Empty(resp.Kvs) + resp, err = etcdutil.EtcdKVGet(client, keypath.TimestampPath(constant.DefaultKeyspaceGroupID)) + re.NoError(err) + re.Len(resp.Kvs, 1) + re.Equal("timestamp", string(resp.Kvs[0].Value)) +} + +func TestScheduleMicroserviceMetadataCleanupReturnsImmediately(t *testing.T) { + re := require.New(t) + ctx := context.Background() + _, client, clean := etcdutil.NewTestEtcdCluster(t, 1, nil) + defer clean() + keypath.SetClusterID(12348) + defer keypath.ResetClusterID() + + store := storage.NewStorageWithEtcdBackend(client) + svr := &Server{storage: store, client: client} + re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { + return store.SaveKeyspaceGroup(txn, &endpoint.KeyspaceGroup{ + ID: constant.DefaultKeyspaceGroupID, + UserKind: endpoint.Basic.String(), + }) + })) + re.NoError(failpoint.Enable("github.com/tikv/pd/server/skipMicroserviceMetadataCleanupDelay", "return(true)")) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/utils/etcdutil/SlowEtcdKVGet", "return(1)")) + t.Cleanup(func() { + _ = failpoint.Disable("github.com/tikv/pd/server/skipMicroserviceMetadataCleanupDelay") + _ = failpoint.Disable("github.com/tikv/pd/pkg/utils/etcdutil/SlowEtcdKVGet") + }) + + start := time.Now() + svr.scheduleMicroserviceMetadataCleanup(ctx) + re.Less(time.Since(start), 200*time.Millisecond) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/utils/etcdutil/SlowEtcdKVGet")) + svr.serverLoopWg.Wait() +} + +func TestCleanupMicroserviceMetadataInPDModeRejectsNonDefaultGroup(t *testing.T) { + re := require.New(t) + ctx := context.Background() + _, client, clean := etcdutil.NewTestEtcdCluster(t, 1, nil) + defer clean() + keypath.SetClusterID(12346) + defer keypath.ResetClusterID() + + store := storage.NewStorageWithEtcdBackend(client) + svr := &Server{storage: store, client: client} + re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { + if err := store.SaveKeyspaceGroup(txn, &endpoint.KeyspaceGroup{ + ID: constant.DefaultKeyspaceGroupID, + UserKind: endpoint.Basic.String(), + }); err != nil { + return err + } + return store.SaveKeyspaceGroup(txn, &endpoint.KeyspaceGroup{ + ID: 1, + UserKind: endpoint.Standard.String(), + }) + })) + _, err := client.Put(ctx, keypath.RegistryPath(mcs.TSOServiceName, "127.0.0.1:3379"), "tso") + re.NoError(err) + + err = svr.cleanupMicroserviceMetadataInPDMode(ctx) + re.ErrorContains(err, "non-default TSO keyspace group 1") + groups, err := store.LoadKeyspaceGroups(constant.DefaultKeyspaceGroupID, 0) + re.NoError(err) + re.Len(groups, 2) + resp, err := etcdutil.EtcdKVGet(client, microserviceEtcdPrefix(), clientv3.WithPrefix()) + re.NoError(err) + re.Len(resp.Kvs, 1) +} + +func TestCleanupMicroserviceMetadataInPDModeRejectsNonDefaultKeyspaceAssignment(t *testing.T) { + re := require.New(t) + ctx := context.Background() + _, client, clean := etcdutil.NewTestEtcdCluster(t, 1, nil) + defer clean() + keypath.SetClusterID(12347) + defer keypath.ResetClusterID() + + store := storage.NewStorageWithEtcdBackend(client) + svr := &Server{storage: store, client: client} + re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { + if err := store.SaveKeyspaceGroup(txn, &endpoint.KeyspaceGroup{ + ID: constant.DefaultKeyspaceGroupID, + UserKind: endpoint.Basic.String(), + }); err != nil { + return err + } + return store.SaveKeyspaceMeta(txn, &keyspacepb.KeyspaceMeta{ + Id: 1, + Name: "keyspace-1", + Config: map[string]string{ + keyspace.TSOKeyspaceGroupIDKey: "1", + }, + }) + })) + _, err := client.Put(ctx, keypath.RegistryPath(mcs.TSOServiceName, "127.0.0.1:3379"), "tso") + re.NoError(err) + + err = svr.cleanupMicroserviceMetadataInPDMode(ctx) + re.ErrorContains(err, "keyspace 1 is assigned to non-default TSO keyspace group 1") + var group *endpoint.KeyspaceGroup + re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { + var err error + group, err = store.LoadKeyspaceGroup(txn, constant.DefaultKeyspaceGroupID) + return err + })) + re.NotNil(group) + resp, err := etcdutil.EtcdKVGet(client, microserviceEtcdPrefix(), clientv3.WithPrefix()) + re.NoError(err) + re.Len(resp.Kvs, 1) +} diff --git a/server/server.go b/server/server.go index fe0a2a37d1..a3d9dd71ad 100644 --- a/server/server.go +++ b/server/server.go @@ -2049,6 +2049,7 @@ func (s *Server) campaignLeader() { zap.String("leader-name", s.Name()), zap.Duration("total-cost", totalDuration), zap.Duration("cost", enableLeaderDuration)) + s.scheduleMicroserviceMetadataCleanup(ctx) leaderTicker := time.NewTicker(mcs.LeaderTickInterval) defer leaderTicker.Stop() diff --git a/tests/integrations/mcs/tso/server_test.go b/tests/integrations/mcs/tso/server_test.go index d20fd7642a..d569ea41bf 100644 --- a/tests/integrations/mcs/tso/server_test.go +++ b/tests/integrations/mcs/tso/server_test.go @@ -34,11 +34,13 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/metapb" + "github.com/pingcap/kvproto/pkg/pdpb" pd "github.com/tikv/pd/client" "github.com/tikv/pd/client/opt" "github.com/tikv/pd/client/pkg/caller" "github.com/tikv/pd/pkg/core" + "github.com/tikv/pd/pkg/keyspace" "github.com/tikv/pd/pkg/keyspace/constant" "github.com/tikv/pd/pkg/mcs/discovery" tso "github.com/tikv/pd/pkg/mcs/tso/server" @@ -46,6 +48,7 @@ import ( mcs "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/etcdutil" + "github.com/tikv/pd/pkg/utils/keypath" "github.com/tikv/pd/pkg/utils/tempurl" "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/pkg/utils/tsoutil" @@ -769,6 +772,136 @@ func TestTSOServiceSwitch(t *testing.T) { re.NoError(failpoint.Disable("github.com/tikv/pd/client/servicediscovery/fastUpdateServiceMode")) } +func TestPDModeSwitchBetweenMicroserviceAndMonolithMultipleTimes(t *testing.T) { + re := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + re.NoError(failpoint.Enable("github.com/tikv/pd/server/skipMicroserviceMetadataCleanupDelay", "return(true)")) + defer func() { + re.NoError(failpoint.Disable("github.com/tikv/pd/server/skipMicroserviceMetadataCleanupDelay")) + }() + + tc, err := tests.NewTestClusterWithKeyspaceGroup(ctx, 1, func(conf *config.Config, _ string) { + conf.Microservice.EnableTSODynamicSwitching = false + }) + re.NoError(err) + defer tc.Destroy() + + re.NoError(tc.RunInitialServers()) + pdServer := tc.GetServer(tc.WaitLeader()) + re.NotNil(pdServer) + re.NoError(pdServer.BootstrapCluster()) + + const switchRounds = 2 + for range switchRounds { + waitDefaultKeyspaceGroupReady(re, pdServer) + tsoCluster, err := tests.NewTestTSOCluster(ctx, 1, pdServer.GetAddr()) + re.NoError(err) + waitTSOServiceReady(re, tsoCluster) + checkMicroserviceTSOAvailable(ctx, re, pdServer) + tsoCluster.Destroy() + + pdServer = restartPDForServiceMode(ctx, re, tc, pdServer, nil) + checkPDTSOAvailable(ctx, re, pdServer) + waitMicroserviceMetadataCleaned(re, pdServer) + + pdServer = restartPDForServiceMode(ctx, re, tc, pdServer, []string{mcs.PDServiceName}) + } + + waitDefaultKeyspaceGroupReady(re, pdServer) + tsoCluster, err := tests.NewTestTSOCluster(ctx, 1, pdServer.GetAddr()) + re.NoError(err) + defer tsoCluster.Destroy() + waitTSOServiceReady(re, tsoCluster) + checkMicroserviceTSOAvailable(ctx, re, pdServer) +} + +func restartPDForServiceMode( + ctx context.Context, + re *require.Assertions, + tc *tests.TestCluster, + oldServer *tests.TestServer, + services []string, +) *tests.TestServer { + cfg := oldServer.GetConfig() + serverName := cfg.Name + re.NoError(oldServer.Stop()) + + newServer, err := tests.NewTestServer(ctx, cfg, services) + re.NoError(err) + tc.GetServers()[serverName] = newServer + re.NoError(newServer.Run()) + re.True(newServer.WaitLeader()) + return newServer +} + +func waitDefaultKeyspaceGroupReady(re *require.Assertions, pdServer *tests.TestServer) { + testutil.Eventually(re, func() bool { + resp, err := etcdutil.EtcdKVGet(pdServer.GetEtcdClient(), keypath.KeyspaceGroupIDPath(constant.DefaultKeyspaceGroupID)) + return err == nil && len(resp.Kvs) == 1 + }, testutil.WithWaitFor(10*time.Second), testutil.WithTickInterval(100*time.Millisecond)) +} + +func waitTSOServiceReady(re *require.Assertions, tsoCluster *tests.TestTSOCluster) { + primary := tsoCluster.WaitForDefaultPrimaryServing(re) + testutil.Eventually(re, func() bool { + resp, err := tests.TestDialClient.Get(primary.GetAddr() + tsoapi.APIPathPrefix + "/health") + if err != nil { + return false + } + defer resp.Body.Close() + return resp.StatusCode == http.StatusOK + }, testutil.WithWaitFor(10*time.Second), testutil.WithTickInterval(100*time.Millisecond)) +} + +func checkPDTSOAvailable(ctx context.Context, re *require.Assertions, pdServer *tests.TestServer) { + cli := utils.SetupClientWithAPIContext(ctx, re, pd.NewAPIContextV1(), []string{pdServer.GetAddr()}) + defer cli.Close() + physical, logical, err := cli.GetTS(ctx) + re.NoError(err) + re.NotZero(tsoutil.ComposeTS(physical, logical)) +} + +func checkMicroserviceTSOAvailable(ctx context.Context, re *require.Assertions, pdServer *tests.TestServer) { + addr := strings.TrimPrefix(pdServer.GetAddr(), "http://") + conn, err := grpc.Dial(addr, grpc.WithTransportCredentials(insecure.NewCredentials())) //nolint:staticcheck + re.NoError(err) + defer conn.Close() + + resp, err := pdpb.NewPDClient(conn).GetClusterInfo(ctx, &pdpb.GetClusterInfoRequest{}) + re.NoError(err) + re.Equal([]pdpb.ServiceMode{pdpb.ServiceMode_API_SVC_MODE}, resp.GetServiceModes()) + re.NotEmpty(resp.GetTsoUrls()) + + checkPDTSOAvailable(ctx, re, pdServer) +} + +func waitMicroserviceMetadataCleaned(re *require.Assertions, pdServer *tests.TestServer) { + testutil.Eventually(re, func() bool { + keyspaceGroupResp, err := etcdutil.EtcdKVGet(pdServer.GetEtcdClient(), keypath.KeyspaceGroupIDPrefix(), clientv3.WithPrefix()) + if err != nil || len(keyspaceGroupResp.Kvs) != 0 { + return false + } + microserviceResp, err := etcdutil.EtcdKVGet( + pdServer.GetEtcdClient(), + fmt.Sprintf("%s/%d/", mcs.MicroserviceRootPath, keypath.ClusterID()), + clientv3.WithPrefix()) + if err != nil || len(microserviceResp.Kvs) != 0 { + return false + } + keyspaces, err := pdServer.GetKeyspaceManager().LoadRangeKeyspace(constant.StartKeyspaceID, 0) + if err != nil { + return false + } + for _, meta := range keyspaces { + if _, ok := meta.GetConfig()[keyspace.TSOKeyspaceGroupIDKey]; ok { + return false + } + } + return true + }, testutil.WithWaitFor(30*time.Second), testutil.WithTickInterval(100*time.Millisecond)) +} + func checkTSOMonotonic(ctx context.Context, pdClient pd.Client, globalLastTS *uint64, count int) error { for range count { physical, logical, err := pdClient.GetTS(ctx) From e3fe646c7a27b8ee8c395810fa6a00472fdb25c6 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Mon, 13 Jul 2026 12:12:24 +0800 Subject: [PATCH 02/15] server: make mode cleanup safe and non-blocking Signed-off-by: Ryan Leung --- server/microservice_cleanup.go | 110 ++++++++++++++-------- server/microservice_cleanup_test.go | 95 ++++++++++++++++--- tests/integrations/mcs/tso/server_test.go | 4 - 3 files changed, 152 insertions(+), 57 deletions(-) diff --git a/server/microservice_cleanup.go b/server/microservice_cleanup.go index 3df4838ae2..a98274c9d5 100644 --- a/server/microservice_cleanup.go +++ b/server/microservice_cleanup.go @@ -24,7 +24,6 @@ import ( "go.uber.org/zap" "github.com/pingcap/errors" - "github.com/pingcap/failpoint" "github.com/pingcap/log" "github.com/tikv/pd/pkg/errs" @@ -37,11 +36,12 @@ import ( "github.com/tikv/pd/pkg/utils/logutil" ) -// microserviceMetadataCleanupDelay lets clients observe the service mode change -// before stale microservice metadata disappears from etcd. const ( - microserviceMetadataCleanupDelay = 10 * time.Second microserviceMetadataCleanupRetryInterval = 5 * time.Second + // Each loaded key adds one comparison and each updated key adds one write to + // the etcd transaction. Use half of the operation limit so a full batch stays + // within the transaction limit. + microserviceMetadataCleanupBatchSize = etcdutil.MaxEtcdTxnOps / 2 ) func (s *Server) scheduleMicroserviceMetadataCleanup(ctx context.Context) { @@ -52,19 +52,16 @@ func (s *Server) scheduleMicroserviceMetadataCleanup(ctx context.Context) { go func() { defer logutil.LogPanic() defer s.serverLoopWg.Done() - if !waitMicroserviceMetadataCleanupDelay(ctx) { - return - } for { if s.member != nil && !s.member.IsServing() { return } - if err := s.cleanupMicroserviceMetadataInPDMode(ctx); err == nil { + err := s.cleanupMicroserviceMetadataInPDMode(ctx) + if err == nil { return - } else { - log.Warn("failed to clean up microservice metadata in PD mode, retry later", - errs.ZapError(err)) } + log.Warn("failed to clean up microservice metadata in PD mode, retry later", + errs.ZapError(err)) timer := time.NewTimer(microserviceMetadataCleanupRetryInterval) select { case <-ctx.Done(): @@ -76,24 +73,6 @@ func (s *Server) scheduleMicroserviceMetadataCleanup(ctx context.Context) { }() } -func waitMicroserviceMetadataCleanupDelay(ctx context.Context) bool { - delay := microserviceMetadataCleanupDelay - failpoint.Inject("skipMicroserviceMetadataCleanupDelay", func() { - delay = 0 - }) - if delay == 0 { - return true - } - timer := time.NewTimer(delay) - defer timer.Stop() - select { - case <-ctx.Done(): - return false - case <-timer.C: - return true - } -} - func (s *Server) cleanupMicroserviceMetadataInPDMode(ctx context.Context) error { if s.IsKeyspaceGroupEnabled() { return nil @@ -104,6 +83,9 @@ func (s *Server) cleanupMicroserviceMetadataInPDMode(ctx context.Context) error if err != nil { return err } + if err := s.validateDefaultTSOKeyspaceGroupConfig(ctx); err != nil { + return err + } cleanedKeyspaces, err := s.cleanupDefaultTSOKeyspaceGroupConfig(ctx) if err != nil { return err @@ -128,6 +110,60 @@ func (s *Server) cleanupMicroserviceMetadataInPDMode(ctx context.Context) error return nil } +func (s *Server) validateDefaultTSOKeyspaceGroupConfig(ctx context.Context) error { + startID := constant.StartKeyspaceID + for { + var ( + keyspaceIDs []uint32 + loaded int + ) + err := s.storage.RunInTxn(ctx, func(txn kv.Txn) error { + metas, err := s.storage.LoadRangeKeyspace(txn, startID, etcdutil.MaxEtcdTxnOps) + if err != nil { + return err + } + loaded = len(metas) + keyspaceIDs = make([]uint32, 0, len(metas)) + for _, meta := range metas { + if meta == nil { + continue + } + keyspaceIDs = append(keyspaceIDs, meta.GetId()) + if err := validateDefaultTSOKeyspaceGroupAssignment(meta.GetId(), meta.GetConfig()); err != nil { + return err + } + } + return nil + }) + if err != nil { + return err + } + if loaded < etcdutil.MaxEtcdTxnOps || len(keyspaceIDs) == 0 { + return nil + } + lastID := keyspaceIDs[len(keyspaceIDs)-1] + if lastID == ^uint32(0) { + return nil + } + startID = lastID + 1 + } +} + +func validateDefaultTSOKeyspaceGroupAssignment(keyspaceID uint32, config map[string]string) error { + groupIDText, ok := config[keyspace.TSOKeyspaceGroupIDKey] + if !ok { + return nil + } + groupID, err := strconv.ParseUint(groupIDText, 10, 32) + if err != nil { + return errors.Errorf("keyspace %d has invalid TSO keyspace group ID %q", keyspaceID, groupIDText) + } + if groupID != uint64(constant.DefaultKeyspaceGroupID) { + return errors.Errorf("keyspace %d is assigned to non-default TSO keyspace group %d", keyspaceID, groupID) + } + return nil +} + func (s *Server) checkDefaultOnlyTSOKeyspaceGroup() (bool, error) { groups, err := s.storage.LoadKeyspaceGroups(constant.DefaultKeyspaceGroupID, 0) if err != nil { @@ -155,7 +191,7 @@ func (s *Server) checkDefaultOnlyTSOKeyspaceGroup() (bool, error) { func (s *Server) cleanupDefaultTSOKeyspaceGroupConfig(ctx context.Context) (int, error) { var ( cleaned int - startID uint32 = constant.StartKeyspaceID + startID = constant.StartKeyspaceID ) for { var ( @@ -164,7 +200,7 @@ func (s *Server) cleanupDefaultTSOKeyspaceGroupConfig(ctx context.Context) (int, loaded int ) err := s.storage.RunInTxn(ctx, func(txn kv.Txn) error { - metas, err := s.storage.LoadRangeKeyspace(txn, startID, etcdutil.MaxEtcdTxnOps) + metas, err := s.storage.LoadRangeKeyspace(txn, startID, microserviceMetadataCleanupBatchSize) if err != nil { return err } @@ -178,16 +214,12 @@ func (s *Server) cleanupDefaultTSOKeyspaceGroupConfig(ctx context.Context) (int, if meta.Config == nil { continue } - groupIDText, ok := meta.Config[keyspace.TSOKeyspaceGroupIDKey] + _, ok := meta.Config[keyspace.TSOKeyspaceGroupIDKey] if !ok { continue } - groupID, err := strconv.ParseUint(groupIDText, 10, 32) - if err != nil { - return errors.Errorf("keyspace %d has invalid TSO keyspace group ID %q", meta.GetId(), groupIDText) - } - if uint32(groupID) != constant.DefaultKeyspaceGroupID { - return errors.Errorf("keyspace %d is assigned to non-default TSO keyspace group %d", meta.GetId(), groupID) + if err := validateDefaultTSOKeyspaceGroupAssignment(meta.GetId(), meta.GetConfig()); err != nil { + return err } delete(meta.Config, keyspace.TSOKeyspaceGroupIDKey) if err := s.storage.SaveKeyspaceMeta(txn, meta); err != nil { @@ -201,7 +233,7 @@ func (s *Server) cleanupDefaultTSOKeyspaceGroupConfig(ctx context.Context) (int, return cleaned, err } cleaned += batchCleaned - if loaded < etcdutil.MaxEtcdTxnOps { + if loaded < microserviceMetadataCleanupBatchSize { break } if len(keyspaceIDs) == 0 { diff --git a/server/microservice_cleanup_test.go b/server/microservice_cleanup_test.go index e22a548fca..1f01d752cb 100644 --- a/server/microservice_cleanup_test.go +++ b/server/microservice_cleanup_test.go @@ -128,10 +128,8 @@ func TestScheduleMicroserviceMetadataCleanupReturnsImmediately(t *testing.T) { UserKind: endpoint.Basic.String(), }) })) - re.NoError(failpoint.Enable("github.com/tikv/pd/server/skipMicroserviceMetadataCleanupDelay", "return(true)")) re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/utils/etcdutil/SlowEtcdKVGet", "return(1)")) t.Cleanup(func() { - _ = failpoint.Disable("github.com/tikv/pd/server/skipMicroserviceMetadataCleanupDelay") _ = failpoint.Disable("github.com/tikv/pd/pkg/utils/etcdutil/SlowEtcdKVGet") }) @@ -188,33 +186,102 @@ func TestCleanupMicroserviceMetadataInPDModeRejectsNonDefaultKeyspaceAssignment( store := storage.NewStorageWithEtcdBackend(client) svr := &Server{storage: store, client: client} re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { - if err := store.SaveKeyspaceGroup(txn, &endpoint.KeyspaceGroup{ + return store.SaveKeyspaceGroup(txn, &endpoint.KeyspaceGroup{ ID: constant.DefaultKeyspaceGroupID, UserKind: endpoint.Basic.String(), - }); err != nil { - return err - } - return store.SaveKeyspaceMeta(txn, &keyspacepb.KeyspaceMeta{ - Id: 1, - Name: "keyspace-1", - Config: map[string]string{ - keyspace.TSOKeyspaceGroupIDKey: "1", - }, }) })) + metas := make([]*keyspacepb.KeyspaceMeta, 0, etcdutil.MaxEtcdTxnOps+1) + for id := uint32(1); id <= etcdutil.MaxEtcdTxnOps; id++ { + metas = append(metas, newKeyspaceMetaWithTSOGroup(id, "0")) + } + invalidID := uint32(etcdutil.MaxEtcdTxnOps + 1) + metas = append(metas, newKeyspaceMetaWithTSOGroup(invalidID, "1")) + saveKeyspaceMetas(ctx, re, store, metas) _, err := client.Put(ctx, keypath.RegistryPath(mcs.TSOServiceName, "127.0.0.1:3379"), "tso") re.NoError(err) err = svr.cleanupMicroserviceMetadataInPDMode(ctx) - re.ErrorContains(err, "keyspace 1 is assigned to non-default TSO keyspace group 1") + re.ErrorContains(err, "keyspace "+strconv.FormatUint(uint64(invalidID), 10)+" is assigned to non-default TSO keyspace group 1") var group *endpoint.KeyspaceGroup re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { var err error group, err = store.LoadKeyspaceGroup(txn, constant.DefaultKeyspaceGroupID) - return err + if err != nil { + return err + } + meta, err := store.LoadKeyspaceMeta(txn, 1) + re.NoError(err) + re.Equal("0", meta.GetConfig()[keyspace.TSOKeyspaceGroupIDKey]) + return nil })) re.NotNil(group) resp, err := etcdutil.EtcdKVGet(client, microserviceEtcdPrefix(), clientv3.WithPrefix()) re.NoError(err) re.Len(resp.Kvs, 1) } + +func TestCleanupMicroserviceMetadataInPDModeWithLargeKeyspaceBatch(t *testing.T) { + re := require.New(t) + ctx := context.Background() + _, client, clean := etcdutil.NewTestEtcdCluster(t, 1, nil) + defer clean() + keypath.SetClusterID(12349) + defer keypath.ResetClusterID() + + store := storage.NewStorageWithEtcdBackend(client) + svr := &Server{storage: store, client: client} + re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { + return store.SaveKeyspaceGroup(txn, &endpoint.KeyspaceGroup{ + ID: constant.DefaultKeyspaceGroupID, + UserKind: endpoint.Basic.String(), + }) + })) + metas := make([]*keyspacepb.KeyspaceMeta, 0, etcdutil.MaxEtcdTxnOps) + for id := uint32(1); id <= etcdutil.MaxEtcdTxnOps; id++ { + metas = append(metas, newKeyspaceMetaWithTSOGroup(id, "0")) + } + saveKeyspaceMetas(ctx, re, store, metas) + + re.NoError(svr.cleanupMicroserviceMetadataInPDMode(ctx)) + re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { + loaded, err := store.LoadRangeKeyspace(txn, constant.StartKeyspaceID, 0) + if err != nil { + return err + } + re.Len(loaded, etcdutil.MaxEtcdTxnOps) + for _, meta := range loaded { + re.NotContains(meta.GetConfig(), keyspace.TSOKeyspaceGroupIDKey) + } + return nil + })) +} + +func newKeyspaceMetaWithTSOGroup(id uint32, groupID string) *keyspacepb.KeyspaceMeta { + return &keyspacepb.KeyspaceMeta{ + Id: id, + Name: "keyspace-" + strconv.FormatUint(uint64(id), 10), + Config: map[string]string{ + keyspace.TSOKeyspaceGroupIDKey: groupID, + }, + } +} + +func saveKeyspaceMetas( + ctx context.Context, + re *require.Assertions, + store storage.Storage, + metas []*keyspacepb.KeyspaceMeta, +) { + for start := 0; start < len(metas); start += microserviceMetadataCleanupBatchSize { + end := min(start+microserviceMetadataCleanupBatchSize, len(metas)) + re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { + for _, meta := range metas[start:end] { + if err := store.SaveKeyspaceMeta(txn, meta); err != nil { + return err + } + } + return nil + })) + } +} diff --git a/tests/integrations/mcs/tso/server_test.go b/tests/integrations/mcs/tso/server_test.go index d569ea41bf..eeec9f8ff2 100644 --- a/tests/integrations/mcs/tso/server_test.go +++ b/tests/integrations/mcs/tso/server_test.go @@ -776,10 +776,6 @@ func TestPDModeSwitchBetweenMicroserviceAndMonolithMultipleTimes(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - re.NoError(failpoint.Enable("github.com/tikv/pd/server/skipMicroserviceMetadataCleanupDelay", "return(true)")) - defer func() { - re.NoError(failpoint.Disable("github.com/tikv/pd/server/skipMicroserviceMetadataCleanupDelay")) - }() tc, err := tests.NewTestClusterWithKeyspaceGroup(ctx, 1, func(conf *config.Config, _ string) { conf.Microservice.EnableTSODynamicSwitching = false From 3509b605d38cb3753184a938d17e17d3cc3e62ca Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Mon, 13 Jul 2026 12:21:50 +0800 Subject: [PATCH 03/15] server: simplify mode cleanup flow Signed-off-by: Ryan Leung --- server/microservice_cleanup.go | 185 +++++++++++++-------------------- 1 file changed, 74 insertions(+), 111 deletions(-) diff --git a/server/microservice_cleanup.go b/server/microservice_cleanup.go index a98274c9d5..4045cb1cc7 100644 --- a/server/microservice_cleanup.go +++ b/server/microservice_cleanup.go @@ -24,6 +24,7 @@ import ( "go.uber.org/zap" "github.com/pingcap/errors" + "github.com/pingcap/kvproto/pkg/keyspacepb" "github.com/pingcap/log" "github.com/tikv/pd/pkg/errs" @@ -79,11 +80,7 @@ func (s *Server) cleanupMicroserviceMetadataInPDMode(ctx context.Context) error } start := time.Now() - defaultGroupExists, err := s.checkDefaultOnlyTSOKeyspaceGroup() - if err != nil { - return err - } - if err := s.validateDefaultTSOKeyspaceGroupConfig(ctx); err != nil { + if err := s.validateMicroserviceMetadataCleanup(ctx); err != nil { return err } cleanedKeyspaces, err := s.cleanupDefaultTSOKeyspaceGroupConfig(ctx) @@ -99,9 +96,8 @@ func (s *Server) cleanupMicroserviceMetadataInPDMode(ctx context.Context) error return err } - if defaultGroupExists || deletedDefaultGroup || cleanedKeyspaces > 0 || deletedMicroserviceKeys > 0 { + if deletedDefaultGroup || cleanedKeyspaces > 0 || deletedMicroserviceKeys > 0 { log.Info("cleaned up microservice metadata in PD mode", - zap.Bool("default-keyspace-group-exists", defaultGroupExists), zap.Bool("deleted-default-keyspace-group", deletedDefaultGroup), zap.Int("cleaned-keyspace-configs", cleanedKeyspaces), zap.Int64("deleted-microservice-keys", deletedMicroserviceKeys), @@ -110,38 +106,89 @@ func (s *Server) cleanupMicroserviceMetadataInPDMode(ctx context.Context) error return nil } -func (s *Server) validateDefaultTSOKeyspaceGroupConfig(ctx context.Context) error { +func (s *Server) validateMicroserviceMetadataCleanup(ctx context.Context) error { + groups, err := s.storage.LoadKeyspaceGroups(constant.DefaultKeyspaceGroupID, 0) + if err != nil { + return err + } + for _, group := range groups { + if group == nil { + continue + } + if group.ID != constant.DefaultKeyspaceGroupID { + return errors.Errorf("found non-default TSO keyspace group %d when cleaning up PD mode microservice metadata", group.ID) + } + if group.IsSplitting() { + return errors.Errorf("default TSO keyspace group is splitting when cleaning up PD mode microservice metadata") + } + if group.IsMerging() { + return errors.Errorf("default TSO keyspace group is merging when cleaning up PD mode microservice metadata") + } + } + return s.scanKeyspaceMetadata(ctx, etcdutil.MaxEtcdTxnOps, func(_ kv.Txn, metas []*keyspacepb.KeyspaceMeta) error { + for _, meta := range metas { + if meta == nil { + continue + } + if err := validateDefaultTSOKeyspaceGroupAssignment(meta.GetId(), meta.GetConfig()); err != nil { + return err + } + } + return nil + }) +} + +func (s *Server) cleanupDefaultTSOKeyspaceGroupConfig(ctx context.Context) (int, error) { + cleaned := 0 + err := s.scanKeyspaceMetadata(ctx, microserviceMetadataCleanupBatchSize, func(txn kv.Txn, metas []*keyspacepb.KeyspaceMeta) error { + for _, meta := range metas { + if meta == nil || meta.Config == nil { + continue + } + if _, ok := meta.Config[keyspace.TSOKeyspaceGroupIDKey]; !ok { + continue + } + if err := validateDefaultTSOKeyspaceGroupAssignment(meta.GetId(), meta.GetConfig()); err != nil { + return err + } + delete(meta.Config, keyspace.TSOKeyspaceGroupIDKey) + if err := s.storage.SaveKeyspaceMeta(txn, meta); err != nil { + return err + } + cleaned++ + } + return nil + }) + return cleaned, err +} + +func (s *Server) scanKeyspaceMetadata( + ctx context.Context, + batchSize int, + process func(kv.Txn, []*keyspacepb.KeyspaceMeta) error, +) error { startID := constant.StartKeyspaceID for { - var ( - keyspaceIDs []uint32 - loaded int - ) + var metas []*keyspacepb.KeyspaceMeta err := s.storage.RunInTxn(ctx, func(txn kv.Txn) error { - metas, err := s.storage.LoadRangeKeyspace(txn, startID, etcdutil.MaxEtcdTxnOps) + var err error + metas, err = s.storage.LoadRangeKeyspace(txn, startID, batchSize) if err != nil { return err } - loaded = len(metas) - keyspaceIDs = make([]uint32, 0, len(metas)) - for _, meta := range metas { - if meta == nil { - continue - } - keyspaceIDs = append(keyspaceIDs, meta.GetId()) - if err := validateDefaultTSOKeyspaceGroupAssignment(meta.GetId(), meta.GetConfig()); err != nil { - return err - } - } - return nil + return process(txn, metas) }) if err != nil { return err } - if loaded < etcdutil.MaxEtcdTxnOps || len(keyspaceIDs) == 0 { + if len(metas) < batchSize { + return nil + } + lastMeta := metas[len(metas)-1] + if lastMeta == nil { return nil } - lastID := keyspaceIDs[len(keyspaceIDs)-1] + lastID := lastMeta.GetId() if lastID == ^uint32(0) { return nil } @@ -164,90 +211,6 @@ func validateDefaultTSOKeyspaceGroupAssignment(keyspaceID uint32, config map[str return nil } -func (s *Server) checkDefaultOnlyTSOKeyspaceGroup() (bool, error) { - groups, err := s.storage.LoadKeyspaceGroups(constant.DefaultKeyspaceGroupID, 0) - if err != nil { - return false, err - } - defaultGroupExists := false - for _, group := range groups { - if group == nil { - continue - } - if group.ID != constant.DefaultKeyspaceGroupID { - return false, errors.Errorf("found non-default TSO keyspace group %d when cleaning up PD mode microservice metadata", group.ID) - } - if group.IsSplitting() { - return false, errors.Errorf("default TSO keyspace group is splitting when cleaning up PD mode microservice metadata") - } - if group.IsMerging() { - return false, errors.Errorf("default TSO keyspace group is merging when cleaning up PD mode microservice metadata") - } - defaultGroupExists = true - } - return defaultGroupExists, nil -} - -func (s *Server) cleanupDefaultTSOKeyspaceGroupConfig(ctx context.Context) (int, error) { - var ( - cleaned int - startID = constant.StartKeyspaceID - ) - for { - var ( - batchCleaned int - keyspaceIDs []uint32 - loaded int - ) - err := s.storage.RunInTxn(ctx, func(txn kv.Txn) error { - metas, err := s.storage.LoadRangeKeyspace(txn, startID, microserviceMetadataCleanupBatchSize) - if err != nil { - return err - } - loaded = len(metas) - keyspaceIDs = make([]uint32, 0, len(metas)) - for _, meta := range metas { - if meta == nil { - continue - } - keyspaceIDs = append(keyspaceIDs, meta.GetId()) - if meta.Config == nil { - continue - } - _, ok := meta.Config[keyspace.TSOKeyspaceGroupIDKey] - if !ok { - continue - } - if err := validateDefaultTSOKeyspaceGroupAssignment(meta.GetId(), meta.GetConfig()); err != nil { - return err - } - delete(meta.Config, keyspace.TSOKeyspaceGroupIDKey) - if err := s.storage.SaveKeyspaceMeta(txn, meta); err != nil { - return err - } - batchCleaned++ - } - return nil - }) - if err != nil { - return cleaned, err - } - cleaned += batchCleaned - if loaded < microserviceMetadataCleanupBatchSize { - break - } - if len(keyspaceIDs) == 0 { - break - } - lastID := keyspaceIDs[len(keyspaceIDs)-1] - if lastID == ^uint32(0) { - break - } - startID = lastID + 1 - } - return cleaned, nil -} - func (s *Server) deleteDefaultTSOKeyspaceGroup(ctx context.Context) (bool, error) { deleted := false err := s.storage.RunInTxn(ctx, func(txn kv.Txn) error { From ac16bb41d102b7679b6620e7c47fab153d77c623 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Mon, 13 Jul 2026 12:32:11 +0800 Subject: [PATCH 04/15] server: skip cleanup outside mode transitions Signed-off-by: Ryan Leung --- server/microservice_cleanup.go | 33 +++++++++++++++++++++++------ server/microservice_cleanup_test.go | 21 ++++++++++++++++++ 2 files changed, 47 insertions(+), 7 deletions(-) diff --git a/server/microservice_cleanup.go b/server/microservice_cleanup.go index 4045cb1cc7..073261dcb3 100644 --- a/server/microservice_cleanup.go +++ b/server/microservice_cleanup.go @@ -80,9 +80,13 @@ func (s *Server) cleanupMicroserviceMetadataInPDMode(ctx context.Context) error } start := time.Now() - if err := s.validateMicroserviceMetadataCleanup(ctx); err != nil { + needsCleanup, err := s.validateMicroserviceMetadataCleanup(ctx) + if err != nil { return err } + if !needsCleanup { + return nil + } cleanedKeyspaces, err := s.cleanupDefaultTSOKeyspaceGroupConfig(ctx) if err != nil { return err @@ -106,26 +110,40 @@ func (s *Server) cleanupMicroserviceMetadataInPDMode(ctx context.Context) error return nil } -func (s *Server) validateMicroserviceMetadataCleanup(ctx context.Context) error { +func (s *Server) validateMicroserviceMetadataCleanup(ctx context.Context) (bool, error) { groups, err := s.storage.LoadKeyspaceGroups(constant.DefaultKeyspaceGroupID, 0) if err != nil { - return err + return false, err } + needsCleanup := false for _, group := range groups { if group == nil { continue } + needsCleanup = true if group.ID != constant.DefaultKeyspaceGroupID { - return errors.Errorf("found non-default TSO keyspace group %d when cleaning up PD mode microservice metadata", group.ID) + return false, errors.Errorf("found non-default TSO keyspace group %d when cleaning up PD mode microservice metadata", group.ID) } if group.IsSplitting() { - return errors.Errorf("default TSO keyspace group is splitting when cleaning up PD mode microservice metadata") + return false, errors.Errorf("default TSO keyspace group is splitting when cleaning up PD mode microservice metadata") } if group.IsMerging() { - return errors.Errorf("default TSO keyspace group is merging when cleaning up PD mode microservice metadata") + return false, errors.Errorf("default TSO keyspace group is merging when cleaning up PD mode microservice metadata") } } - return s.scanKeyspaceMetadata(ctx, etcdutil.MaxEtcdTxnOps, func(_ kv.Txn, metas []*keyspacepb.KeyspaceMeta) error { + if !needsCleanup && s.client != nil { + getCtx, cancel := context.WithTimeout(ctx, etcdutil.DefaultRequestTimeout) + resp, err := s.client.Get(getCtx, microserviceEtcdPrefix(), clientv3.WithPrefix(), clientv3.WithLimit(1)) + cancel() + if err != nil { + return false, errs.ErrEtcdKVGet.Wrap(err).GenWithStackByCause() + } + needsCleanup = len(resp.Kvs) > 0 + } + if !needsCleanup { + return false, nil + } + err = s.scanKeyspaceMetadata(ctx, etcdutil.MaxEtcdTxnOps, func(_ kv.Txn, metas []*keyspacepb.KeyspaceMeta) error { for _, meta := range metas { if meta == nil { continue @@ -136,6 +154,7 @@ func (s *Server) validateMicroserviceMetadataCleanup(ctx context.Context) error } return nil }) + return true, err } func (s *Server) cleanupDefaultTSOKeyspaceGroupConfig(ctx context.Context) (int, error) { diff --git a/server/microservice_cleanup_test.go b/server/microservice_cleanup_test.go index 1f01d752cb..2e9599621f 100644 --- a/server/microservice_cleanup_test.go +++ b/server/microservice_cleanup_test.go @@ -140,6 +140,27 @@ func TestScheduleMicroserviceMetadataCleanupReturnsImmediately(t *testing.T) { svr.serverLoopWg.Wait() } +func TestMicroserviceMetadataCleanupTransitionDetection(t *testing.T) { + re := require.New(t) + ctx := context.Background() + _, client, clean := etcdutil.NewTestEtcdCluster(t, 1, nil) + defer clean() + keypath.SetClusterID(12350) + defer keypath.ResetClusterID() + + store := storage.NewStorageWithEtcdBackend(client) + svr := &Server{storage: store, client: client} + needsCleanup, err := svr.validateMicroserviceMetadataCleanup(ctx) + re.NoError(err) + re.False(needsCleanup) + + _, err = client.Put(ctx, keypath.RegistryPath(mcs.TSOServiceName, "127.0.0.1:3379"), "tso") + re.NoError(err) + needsCleanup, err = svr.validateMicroserviceMetadataCleanup(ctx) + re.NoError(err) + re.True(needsCleanup) +} + func TestCleanupMicroserviceMetadataInPDModeRejectsNonDefaultGroup(t *testing.T) { re := require.New(t) ctx := context.Background() From ee7064875ae49dab2d6d2442fecb8a847a5a0509 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Wed, 15 Jul 2026 14:50:25 +0800 Subject: [PATCH 05/15] server: protect microservice metadata cleanup Signed-off-by: Ryan Leung --- server/microservice_cleanup.go | 64 ++++++++---- server/microservice_cleanup_test.go | 145 ++++++++++++++++++++++++++++ 2 files changed, 192 insertions(+), 17 deletions(-) diff --git a/server/microservice_cleanup.go b/server/microservice_cleanup.go index 073261dcb3..5c675f47e1 100644 --- a/server/microservice_cleanup.go +++ b/server/microservice_cleanup.go @@ -16,6 +16,7 @@ package server import ( "context" + goerrors "errors" "fmt" "strconv" "time" @@ -24,6 +25,7 @@ import ( "go.uber.org/zap" "github.com/pingcap/errors" + "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/keyspacepb" "github.com/pingcap/log" @@ -45,6 +47,12 @@ const ( microserviceMetadataCleanupBatchSize = etcdutil.MaxEtcdTxnOps / 2 ) +var errMicroserviceMetadataCleanupRejected = errors.New("microservice metadata cleanup rejected") + +func rejectMicroserviceMetadataCleanup(format string, args ...any) error { + return errors.Wrapf(errMicroserviceMetadataCleanupRejected, format, args...) +} + func (s *Server) scheduleMicroserviceMetadataCleanup(ctx context.Context) { if s.IsKeyspaceGroupEnabled() { return @@ -61,6 +69,11 @@ func (s *Server) scheduleMicroserviceMetadataCleanup(ctx context.Context) { if err == nil { return } + if goerrors.Is(err, errMicroserviceMetadataCleanupRejected) { + log.Warn("cannot safely clean up microservice metadata in PD mode", + errs.ZapError(err)) + return + } log.Warn("failed to clean up microservice metadata in PD mode, retry later", errs.ZapError(err)) timer := time.NewTimer(microserviceMetadataCleanupRetryInterval) @@ -95,6 +108,10 @@ func (s *Server) cleanupMicroserviceMetadataInPDMode(ctx context.Context) error if err != nil { return err } + failpoint.InjectCall("beforeDeleteMicroserviceEtcdKeys") + if _, err := s.checkMicroserviceEtcdKeysNotLeased(ctx); err != nil { + return err + } deletedMicroserviceKeys, err := s.deleteMicroserviceEtcdKeys(ctx) if err != nil { return err @@ -110,35 +127,48 @@ func (s *Server) cleanupMicroserviceMetadataInPDMode(ctx context.Context) error return nil } +func (s *Server) checkMicroserviceEtcdKeysNotLeased(ctx context.Context) (bool, error) { + if s.client == nil { + return false, nil + } + getCtx, cancel := context.WithTimeout(ctx, etcdutil.DefaultRequestTimeout) + defer cancel() + resp, err := s.client.Get(getCtx, microserviceEtcdPrefix(), clientv3.WithPrefix(), clientv3.WithKeysOnly()) + if err != nil { + return false, errs.ErrEtcdKVGet.Wrap(err).GenWithStackByCause() + } + for _, item := range resp.Kvs { + if item.Lease != 0 { + return true, errors.Errorf("microservice key %q is still leased", string(item.Key)) + } + } + return len(resp.Kvs) > 0, nil +} + func (s *Server) validateMicroserviceMetadataCleanup(ctx context.Context) (bool, error) { + microserviceKeysExist, err := s.checkMicroserviceEtcdKeysNotLeased(ctx) + if err != nil { + return false, err + } groups, err := s.storage.LoadKeyspaceGroups(constant.DefaultKeyspaceGroupID, 0) if err != nil { return false, err } - needsCleanup := false + needsCleanup := microserviceKeysExist for _, group := range groups { if group == nil { continue } needsCleanup = true if group.ID != constant.DefaultKeyspaceGroupID { - return false, errors.Errorf("found non-default TSO keyspace group %d when cleaning up PD mode microservice metadata", group.ID) + return false, rejectMicroserviceMetadataCleanup("found non-default TSO keyspace group %d when cleaning up PD mode microservice metadata", group.ID) } if group.IsSplitting() { - return false, errors.Errorf("default TSO keyspace group is splitting when cleaning up PD mode microservice metadata") + return false, rejectMicroserviceMetadataCleanup("default TSO keyspace group is splitting when cleaning up PD mode microservice metadata") } if group.IsMerging() { - return false, errors.Errorf("default TSO keyspace group is merging when cleaning up PD mode microservice metadata") - } - } - if !needsCleanup && s.client != nil { - getCtx, cancel := context.WithTimeout(ctx, etcdutil.DefaultRequestTimeout) - resp, err := s.client.Get(getCtx, microserviceEtcdPrefix(), clientv3.WithPrefix(), clientv3.WithLimit(1)) - cancel() - if err != nil { - return false, errs.ErrEtcdKVGet.Wrap(err).GenWithStackByCause() + return false, rejectMicroserviceMetadataCleanup("default TSO keyspace group is merging when cleaning up PD mode microservice metadata") } - needsCleanup = len(resp.Kvs) > 0 } if !needsCleanup { return false, nil @@ -222,10 +252,10 @@ func validateDefaultTSOKeyspaceGroupAssignment(keyspaceID uint32, config map[str } groupID, err := strconv.ParseUint(groupIDText, 10, 32) if err != nil { - return errors.Errorf("keyspace %d has invalid TSO keyspace group ID %q", keyspaceID, groupIDText) + return rejectMicroserviceMetadataCleanup("keyspace %d has invalid TSO keyspace group ID %q", keyspaceID, groupIDText) } if groupID != uint64(constant.DefaultKeyspaceGroupID) { - return errors.Errorf("keyspace %d is assigned to non-default TSO keyspace group %d", keyspaceID, groupID) + return rejectMicroserviceMetadataCleanup("keyspace %d is assigned to non-default TSO keyspace group %d", keyspaceID, groupID) } return nil } @@ -238,10 +268,10 @@ func (s *Server) deleteDefaultTSOKeyspaceGroup(ctx context.Context) (bool, error return err } if group.IsSplitting() { - return errors.Errorf("default TSO keyspace group is splitting when deleting PD mode microservice metadata") + return rejectMicroserviceMetadataCleanup("default TSO keyspace group is splitting when deleting PD mode microservice metadata") } if group.IsMerging() { - return errors.Errorf("default TSO keyspace group is merging when deleting PD mode microservice metadata") + return rejectMicroserviceMetadataCleanup("default TSO keyspace group is merging when deleting PD mode microservice metadata") } if err := s.storage.DeleteKeyspaceGroup(txn, constant.DefaultKeyspaceGroupID); err != nil { return err diff --git a/server/microservice_cleanup_test.go b/server/microservice_cleanup_test.go index 2e9599621f..559458dada 100644 --- a/server/microservice_cleanup_test.go +++ b/server/microservice_cleanup_test.go @@ -112,6 +112,112 @@ func TestCleanupMicroserviceMetadataInPDMode(t *testing.T) { re.Equal("timestamp", string(resp.Kvs[0].Value)) } +func TestCleanupMicroserviceMetadataWaitsForLeasedKeys(t *testing.T) { + re := require.New(t) + ctx := context.Background() + _, client, clean := etcdutil.NewTestEtcdCluster(t, 1, nil) + defer clean() + keypath.SetClusterID(12351) + defer keypath.ResetClusterID() + + store := storage.NewStorageWithEtcdBackend(client) + svr := &Server{storage: store, client: client} + re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { + if err := store.SaveKeyspaceGroup(txn, &endpoint.KeyspaceGroup{ + ID: constant.DefaultKeyspaceGroupID, + UserKind: endpoint.Basic.String(), + Keyspaces: []uint32{1}, + }); err != nil { + return err + } + return store.SaveKeyspaceMeta(txn, newKeyspaceMetaWithTSOGroup(1, "0")) + })) + registryPath := keypath.RegistryPath(mcs.TSOServiceName, "127.0.0.1:3379") + leaseID, err := etcdutil.EtcdKVPutWithTTL(ctx, client, registryPath, "tso", 60) + re.NoError(err) + re.NotZero(leaseID) + + err = svr.cleanupMicroserviceMetadataInPDMode(ctx) + re.ErrorContains(err, "is still leased") + + groups, err := store.LoadKeyspaceGroups(constant.DefaultKeyspaceGroupID, 0) + re.NoError(err) + re.Len(groups, 1) + re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { + meta, err := store.LoadKeyspaceMeta(txn, 1) + if err != nil { + return err + } + re.Contains(meta.GetConfig(), keyspace.TSOKeyspaceGroupIDKey) + return nil + })) + resp, err := etcdutil.EtcdKVGet(client, registryPath) + re.NoError(err) + re.Len(resp.Kvs, 1) + re.NotZero(resp.Kvs[0].Lease) + + _, err = client.Revoke(ctx, leaseID) + re.NoError(err) + re.NoError(svr.cleanupMicroserviceMetadataInPDMode(ctx)) + groups, err = store.LoadKeyspaceGroups(constant.DefaultKeyspaceGroupID, 0) + re.NoError(err) + re.Empty(groups) + re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { + meta, err := store.LoadKeyspaceMeta(txn, 1) + if err != nil { + return err + } + re.NotContains(meta.GetConfig(), keyspace.TSOKeyspaceGroupIDKey) + return nil + })) +} + +func TestCleanupMicroserviceMetadataRechecksLeasesBeforeDelete(t *testing.T) { + re := require.New(t) + ctx := context.Background() + _, client, clean := etcdutil.NewTestEtcdCluster(t, 1, nil) + defer clean() + keypath.SetClusterID(12352) + defer keypath.ResetClusterID() + + store := storage.NewStorageWithEtcdBackend(client) + svr := &Server{storage: store, client: client} + re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { + if err := store.SaveKeyspaceGroup(txn, &endpoint.KeyspaceGroup{ + ID: constant.DefaultKeyspaceGroupID, + UserKind: endpoint.Basic.String(), + Keyspaces: []uint32{1}, + }); err != nil { + return err + } + return store.SaveKeyspaceMeta(txn, newKeyspaceMetaWithTSOGroup(1, "0")) + })) + registryPath := keypath.RegistryPath(mcs.TSOServiceName, "127.0.0.1:3379") + var leaseID clientv3.LeaseID + re.NoError(failpoint.EnableCall("github.com/tikv/pd/server/beforeDeleteMicroserviceEtcdKeys", func() { + if leaseID != clientv3.NoLease { + return + } + var err error + leaseID, err = etcdutil.EtcdKVPutWithTTL(ctx, client, registryPath, "tso", 60) + re.NoError(err) + })) + t.Cleanup(func() { + _ = failpoint.Disable("github.com/tikv/pd/server/beforeDeleteMicroserviceEtcdKeys") + }) + + err := svr.cleanupMicroserviceMetadataInPDMode(ctx) + re.ErrorContains(err, "is still leased") + re.NotZero(leaseID) + resp, err := etcdutil.EtcdKVGet(client, registryPath) + re.NoError(err) + re.Len(resp.Kvs, 1) + re.Equal(leaseID, clientv3.LeaseID(resp.Kvs[0].Lease)) + + _, err = client.Revoke(ctx, leaseID) + re.NoError(err) +} + func TestScheduleMicroserviceMetadataCleanupReturnsImmediately(t *testing.T) { re := require.New(t) ctx := context.Background() @@ -140,6 +246,45 @@ func TestScheduleMicroserviceMetadataCleanupReturnsImmediately(t *testing.T) { svr.serverLoopWg.Wait() } +func TestScheduleMicroserviceMetadataCleanupStopsOnRejectedState(t *testing.T) { + re := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + _, client, clean := etcdutil.NewTestEtcdCluster(t, 1, nil) + defer clean() + keypath.SetClusterID(12353) + defer keypath.ResetClusterID() + + store := storage.NewStorageWithEtcdBackend(client) + svr := &Server{storage: store, client: client} + re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { + if err := store.SaveKeyspaceGroup(txn, &endpoint.KeyspaceGroup{ + ID: constant.DefaultKeyspaceGroupID, + UserKind: endpoint.Basic.String(), + }); err != nil { + return err + } + return store.SaveKeyspaceGroup(txn, &endpoint.KeyspaceGroup{ + ID: 1, + UserKind: endpoint.Standard.String(), + }) + })) + + svr.scheduleMicroserviceMetadataCleanup(ctx) + done := make(chan struct{}) + go func() { + svr.serverLoopWg.Wait() + close(done) + }() + select { + case <-done: + case <-time.After(time.Second): + cancel() + <-done + t.Fatal("cleanup task kept retrying a rejected metadata state") + } +} + func TestMicroserviceMetadataCleanupTransitionDetection(t *testing.T) { re := require.New(t) ctx := context.Background() From 87e17fb7d79b90f1f21dab4f62d7b5c9640f7f5c Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Mon, 27 Jul 2026 15:12:39 +0800 Subject: [PATCH 06/15] server: narrow PD mode metadata cleanup Signed-off-by: Ryan Leung --- server/microservice_cleanup.go | 182 +------------- server/microservice_cleanup_test.go | 293 ++++++++-------------- tests/integrations/mcs/tso/server_test.go | 58 +++-- 3 files changed, 148 insertions(+), 385 deletions(-) diff --git a/server/microservice_cleanup.go b/server/microservice_cleanup.go index 5c675f47e1..9b9aca92e8 100644 --- a/server/microservice_cleanup.go +++ b/server/microservice_cleanup.go @@ -17,35 +17,20 @@ package server import ( "context" goerrors "errors" - "fmt" - "strconv" "time" - clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" "github.com/pingcap/errors" - "github.com/pingcap/failpoint" - "github.com/pingcap/kvproto/pkg/keyspacepb" "github.com/pingcap/log" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/keyspace" "github.com/tikv/pd/pkg/keyspace/constant" - mcs "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/storage/kv" - "github.com/tikv/pd/pkg/utils/etcdutil" - "github.com/tikv/pd/pkg/utils/keypath" "github.com/tikv/pd/pkg/utils/logutil" ) -const ( - microserviceMetadataCleanupRetryInterval = 5 * time.Second - // Each loaded key adds one comparison and each updated key adds one write to - // the etcd transaction. Use half of the operation limit so a full batch stays - // within the transaction limit. - microserviceMetadataCleanupBatchSize = etcdutil.MaxEtcdTxnOps / 2 -) +const microserviceMetadataCleanupRetryInterval = 5 * time.Second var errMicroserviceMetadataCleanupRejected = errors.New("microservice metadata cleanup rejected") @@ -91,70 +76,31 @@ func (s *Server) cleanupMicroserviceMetadataInPDMode(ctx context.Context) error if s.IsKeyspaceGroupEnabled() { return nil } - start := time.Now() - needsCleanup, err := s.validateMicroserviceMetadataCleanup(ctx) - if err != nil { - return err - } - if !needsCleanup { - return nil - } - cleanedKeyspaces, err := s.cleanupDefaultTSOKeyspaceGroupConfig(ctx) - if err != nil { - return err - } - deletedDefaultGroup, err := s.deleteDefaultTSOKeyspaceGroup(ctx) - if err != nil { - return err - } - failpoint.InjectCall("beforeDeleteMicroserviceEtcdKeys") - if _, err := s.checkMicroserviceEtcdKeysNotLeased(ctx); err != nil { + // The persisted keyspace group contains the stale TSO member addresses that + // block a later switch back to API service mode. Keyspace assignment markers + // and lease-owned microservice keys are intentionally left untouched. + needsCleanup, err := s.validateMicroserviceMetadataCleanup() + if err != nil || !needsCleanup { return err } - deletedMicroserviceKeys, err := s.deleteMicroserviceEtcdKeys(ctx) + deleted, err := s.deleteDefaultTSOKeyspaceGroup(ctx) if err != nil { return err } - - if deletedDefaultGroup || cleanedKeyspaces > 0 || deletedMicroserviceKeys > 0 { + if deleted { log.Info("cleaned up microservice metadata in PD mode", - zap.Bool("deleted-default-keyspace-group", deletedDefaultGroup), - zap.Int("cleaned-keyspace-configs", cleanedKeyspaces), - zap.Int64("deleted-microservice-keys", deletedMicroserviceKeys), - zap.Duration("cost", time.Since(start))) + zap.Bool("deleted-default-keyspace-group", true)) } return nil } -func (s *Server) checkMicroserviceEtcdKeysNotLeased(ctx context.Context) (bool, error) { - if s.client == nil { - return false, nil - } - getCtx, cancel := context.WithTimeout(ctx, etcdutil.DefaultRequestTimeout) - defer cancel() - resp, err := s.client.Get(getCtx, microserviceEtcdPrefix(), clientv3.WithPrefix(), clientv3.WithKeysOnly()) - if err != nil { - return false, errs.ErrEtcdKVGet.Wrap(err).GenWithStackByCause() - } - for _, item := range resp.Kvs { - if item.Lease != 0 { - return true, errors.Errorf("microservice key %q is still leased", string(item.Key)) - } - } - return len(resp.Kvs) > 0, nil -} - -func (s *Server) validateMicroserviceMetadataCleanup(ctx context.Context) (bool, error) { - microserviceKeysExist, err := s.checkMicroserviceEtcdKeysNotLeased(ctx) - if err != nil { - return false, err - } +func (s *Server) validateMicroserviceMetadataCleanup() (bool, error) { groups, err := s.storage.LoadKeyspaceGroups(constant.DefaultKeyspaceGroupID, 0) if err != nil { return false, err } - needsCleanup := microserviceKeysExist + needsCleanup := false for _, group := range groups { if group == nil { continue @@ -170,94 +116,7 @@ func (s *Server) validateMicroserviceMetadataCleanup(ctx context.Context) (bool, return false, rejectMicroserviceMetadataCleanup("default TSO keyspace group is merging when cleaning up PD mode microservice metadata") } } - if !needsCleanup { - return false, nil - } - err = s.scanKeyspaceMetadata(ctx, etcdutil.MaxEtcdTxnOps, func(_ kv.Txn, metas []*keyspacepb.KeyspaceMeta) error { - for _, meta := range metas { - if meta == nil { - continue - } - if err := validateDefaultTSOKeyspaceGroupAssignment(meta.GetId(), meta.GetConfig()); err != nil { - return err - } - } - return nil - }) - return true, err -} - -func (s *Server) cleanupDefaultTSOKeyspaceGroupConfig(ctx context.Context) (int, error) { - cleaned := 0 - err := s.scanKeyspaceMetadata(ctx, microserviceMetadataCleanupBatchSize, func(txn kv.Txn, metas []*keyspacepb.KeyspaceMeta) error { - for _, meta := range metas { - if meta == nil || meta.Config == nil { - continue - } - if _, ok := meta.Config[keyspace.TSOKeyspaceGroupIDKey]; !ok { - continue - } - if err := validateDefaultTSOKeyspaceGroupAssignment(meta.GetId(), meta.GetConfig()); err != nil { - return err - } - delete(meta.Config, keyspace.TSOKeyspaceGroupIDKey) - if err := s.storage.SaveKeyspaceMeta(txn, meta); err != nil { - return err - } - cleaned++ - } - return nil - }) - return cleaned, err -} - -func (s *Server) scanKeyspaceMetadata( - ctx context.Context, - batchSize int, - process func(kv.Txn, []*keyspacepb.KeyspaceMeta) error, -) error { - startID := constant.StartKeyspaceID - for { - var metas []*keyspacepb.KeyspaceMeta - err := s.storage.RunInTxn(ctx, func(txn kv.Txn) error { - var err error - metas, err = s.storage.LoadRangeKeyspace(txn, startID, batchSize) - if err != nil { - return err - } - return process(txn, metas) - }) - if err != nil { - return err - } - if len(metas) < batchSize { - return nil - } - lastMeta := metas[len(metas)-1] - if lastMeta == nil { - return nil - } - lastID := lastMeta.GetId() - if lastID == ^uint32(0) { - return nil - } - startID = lastID + 1 - } -} - -func validateDefaultTSOKeyspaceGroupAssignment(keyspaceID uint32, config map[string]string) error { - groupIDText, ok := config[keyspace.TSOKeyspaceGroupIDKey] - if !ok { - return nil - } - groupID, err := strconv.ParseUint(groupIDText, 10, 32) - if err != nil { - return rejectMicroserviceMetadataCleanup("keyspace %d has invalid TSO keyspace group ID %q", keyspaceID, groupIDText) - } - if groupID != uint64(constant.DefaultKeyspaceGroupID) { - return rejectMicroserviceMetadataCleanup("keyspace %d is assigned to non-default TSO keyspace group %d", keyspaceID, groupID) - } - return nil + return needsCleanup, nil } func (s *Server) deleteDefaultTSOKeyspaceGroup(ctx context.Context) (bool, error) { @@ -281,20 +140,3 @@ func (s *Server) deleteDefaultTSOKeyspaceGroup(ctx context.Context) (bool, error }) return deleted, err } - -func (s *Server) deleteMicroserviceEtcdKeys(ctx context.Context) (int64, error) { - if s.client == nil { - return 0, nil - } - ctx, cancel := context.WithTimeout(ctx, etcdutil.DefaultRequestTimeout) - defer cancel() - resp, err := s.client.Delete(ctx, microserviceEtcdPrefix(), clientv3.WithPrefix()) - if err != nil { - return 0, errs.ErrEtcdKVDelete.Wrap(err).GenWithStackByCause() - } - return resp.Deleted, nil -} - -func microserviceEtcdPrefix() string { - return fmt.Sprintf("%s/%d/", mcs.MicroserviceRootPath, keypath.ClusterID()) -} diff --git a/server/microservice_cleanup_test.go b/server/microservice_cleanup_test.go index 559458dada..c8ae24308b 100644 --- a/server/microservice_cleanup_test.go +++ b/server/microservice_cleanup_test.go @@ -54,35 +54,30 @@ func TestCleanupMicroserviceMetadataInPDMode(t *testing.T) { }); err != nil { return err } - if err := store.SaveKeyspaceMeta(txn, &keyspacepb.KeyspaceMeta{ + return store.SaveKeyspaceMeta(txn, &keyspacepb.KeyspaceMeta{ Id: 1, Name: "keyspace-1", Config: map[string]string{ keyspace.TSOKeyspaceGroupIDKey: strconv.FormatUint(uint64(constant.DefaultKeyspaceGroupID), 10), "gc_life_time": "10m", }, - }); err != nil { - return err - } - return store.SaveKeyspaceMeta(txn, &keyspacepb.KeyspaceMeta{ - Id: 2, - Name: "keyspace-2", - Config: map[string]string{ - "custom": "value", - }, }) })) - _, err := client.Put(ctx, keypath.RegistryPath(mcs.TSOServiceName, "127.0.0.1:3379"), "tso") - re.NoError(err) - _, err = client.Put(ctx, keypath.RegistryPath(mcs.SchedulingServiceName, "127.0.0.1:3379"), "scheduling") - re.NoError(err) - _, err = client.Put(ctx, keypath.ElectionPath(&keypath.MsParam{ + + registryPath := keypath.RegistryPath(mcs.TSOServiceName, "127.0.0.1:3379") + electionPath := keypath.ElectionPath(&keypath.MsParam{ ServiceName: mcs.TSOServiceName, GroupID: constant.DefaultKeyspaceGroupID, - }), "primary") - re.NoError(err) - _, err = client.Put(ctx, keypath.TimestampPath(constant.DefaultKeyspaceGroupID), "timestamp") - re.NoError(err) + }) + timestampPath := keypath.TimestampPath(constant.DefaultKeyspaceGroupID) + for path, value := range map[string]string{ + registryPath: "tso", + electionPath: "primary", + timestampPath: "timestamp", + } { + _, err := client.Put(ctx, path, value) + re.NoError(err) + } re.NoError(svr.cleanupMicroserviceMetadataInPDMode(ctx)) re.NoError(svr.cleanupMicroserviceMetadataInPDMode(ctx)) @@ -94,25 +89,23 @@ func TestCleanupMicroserviceMetadataInPDMode(t *testing.T) { meta, err := store.LoadKeyspaceMeta(txn, 1) re.NoError(err) re.NotNil(meta) - re.NotContains(meta.GetConfig(), keyspace.TSOKeyspaceGroupIDKey) + re.Equal("0", meta.GetConfig()[keyspace.TSOKeyspaceGroupIDKey]) re.Equal("10m", meta.GetConfig()["gc_life_time"]) - - meta, err = store.LoadKeyspaceMeta(txn, 2) - re.NoError(err) - re.NotNil(meta) - re.Equal("value", meta.GetConfig()["custom"]) return nil })) - resp, err := etcdutil.EtcdKVGet(client, microserviceEtcdPrefix(), clientv3.WithPrefix()) - re.NoError(err) - re.Empty(resp.Kvs) - resp, err = etcdutil.EtcdKVGet(client, keypath.TimestampPath(constant.DefaultKeyspaceGroupID)) - re.NoError(err) - re.Len(resp.Kvs, 1) - re.Equal("timestamp", string(resp.Kvs[0].Value)) + for path, value := range map[string]string{ + registryPath: "tso", + electionPath: "primary", + timestampPath: "timestamp", + } { + resp, err := etcdutil.EtcdKVGet(client, path) + re.NoError(err) + re.Len(resp.Kvs, 1) + re.Equal(value, string(resp.Kvs[0].Value)) + } } -func TestCleanupMicroserviceMetadataWaitsForLeasedKeys(t *testing.T) { +func TestCleanupMicroserviceMetadataIgnoresLeasedRegistryKeys(t *testing.T) { re := require.New(t) ctx := context.Background() _, client, clean := etcdutil.NewTestEtcdCluster(t, 1, nil) @@ -123,99 +116,29 @@ func TestCleanupMicroserviceMetadataWaitsForLeasedKeys(t *testing.T) { store := storage.NewStorageWithEtcdBackend(client) svr := &Server{storage: store, client: client} re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { - if err := store.SaveKeyspaceGroup(txn, &endpoint.KeyspaceGroup{ + return store.SaveKeyspaceGroup(txn, &endpoint.KeyspaceGroup{ ID: constant.DefaultKeyspaceGroupID, UserKind: endpoint.Basic.String(), Keyspaces: []uint32{1}, - }); err != nil { - return err - } - return store.SaveKeyspaceMeta(txn, newKeyspaceMetaWithTSOGroup(1, "0")) + }) })) + registryPath := keypath.RegistryPath(mcs.TSOServiceName, "127.0.0.1:3379") leaseID, err := etcdutil.EtcdKVPutWithTTL(ctx, client, registryPath, "tso", 60) re.NoError(err) re.NotZero(leaseID) + defer func() { + _, _ = client.Revoke(ctx, leaseID) + }() - err = svr.cleanupMicroserviceMetadataInPDMode(ctx) - re.ErrorContains(err, "is still leased") - - groups, err := store.LoadKeyspaceGroups(constant.DefaultKeyspaceGroupID, 0) - re.NoError(err) - re.Len(groups, 1) - re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { - meta, err := store.LoadKeyspaceMeta(txn, 1) - if err != nil { - return err - } - re.Contains(meta.GetConfig(), keyspace.TSOKeyspaceGroupIDKey) - return nil - })) - resp, err := etcdutil.EtcdKVGet(client, registryPath) - re.NoError(err) - re.Len(resp.Kvs, 1) - re.NotZero(resp.Kvs[0].Lease) - - _, err = client.Revoke(ctx, leaseID) - re.NoError(err) re.NoError(svr.cleanupMicroserviceMetadataInPDMode(ctx)) - groups, err = store.LoadKeyspaceGroups(constant.DefaultKeyspaceGroupID, 0) + groups, err := store.LoadKeyspaceGroups(constant.DefaultKeyspaceGroupID, 0) re.NoError(err) re.Empty(groups) - re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { - meta, err := store.LoadKeyspaceMeta(txn, 1) - if err != nil { - return err - } - re.NotContains(meta.GetConfig(), keyspace.TSOKeyspaceGroupIDKey) - return nil - })) -} - -func TestCleanupMicroserviceMetadataRechecksLeasesBeforeDelete(t *testing.T) { - re := require.New(t) - ctx := context.Background() - _, client, clean := etcdutil.NewTestEtcdCluster(t, 1, nil) - defer clean() - keypath.SetClusterID(12352) - defer keypath.ResetClusterID() - - store := storage.NewStorageWithEtcdBackend(client) - svr := &Server{storage: store, client: client} - re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { - if err := store.SaveKeyspaceGroup(txn, &endpoint.KeyspaceGroup{ - ID: constant.DefaultKeyspaceGroupID, - UserKind: endpoint.Basic.String(), - Keyspaces: []uint32{1}, - }); err != nil { - return err - } - return store.SaveKeyspaceMeta(txn, newKeyspaceMetaWithTSOGroup(1, "0")) - })) - registryPath := keypath.RegistryPath(mcs.TSOServiceName, "127.0.0.1:3379") - var leaseID clientv3.LeaseID - re.NoError(failpoint.EnableCall("github.com/tikv/pd/server/beforeDeleteMicroserviceEtcdKeys", func() { - if leaseID != clientv3.NoLease { - return - } - var err error - leaseID, err = etcdutil.EtcdKVPutWithTTL(ctx, client, registryPath, "tso", 60) - re.NoError(err) - })) - t.Cleanup(func() { - _ = failpoint.Disable("github.com/tikv/pd/server/beforeDeleteMicroserviceEtcdKeys") - }) - - err := svr.cleanupMicroserviceMetadataInPDMode(ctx) - re.ErrorContains(err, "is still leased") - re.NotZero(leaseID) resp, err := etcdutil.EtcdKVGet(client, registryPath) re.NoError(err) re.Len(resp.Kvs, 1) re.Equal(leaseID, clientv3.LeaseID(resp.Kvs[0].Lease)) - - _, err = client.Revoke(ctx, leaseID) - re.NoError(err) } func TestScheduleMicroserviceMetadataCleanupReturnsImmediately(t *testing.T) { @@ -295,13 +218,23 @@ func TestMicroserviceMetadataCleanupTransitionDetection(t *testing.T) { store := storage.NewStorageWithEtcdBackend(client) svr := &Server{storage: store, client: client} - needsCleanup, err := svr.validateMicroserviceMetadataCleanup(ctx) + needsCleanup, err := svr.validateMicroserviceMetadataCleanup() re.NoError(err) re.False(needsCleanup) _, err = client.Put(ctx, keypath.RegistryPath(mcs.TSOServiceName, "127.0.0.1:3379"), "tso") re.NoError(err) - needsCleanup, err = svr.validateMicroserviceMetadataCleanup(ctx) + needsCleanup, err = svr.validateMicroserviceMetadataCleanup() + re.NoError(err) + re.False(needsCleanup) + + re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { + return store.SaveKeyspaceGroup(txn, &endpoint.KeyspaceGroup{ + ID: constant.DefaultKeyspaceGroupID, + UserKind: endpoint.Basic.String(), + }) + })) + needsCleanup, err = svr.validateMicroserviceMetadataCleanup() re.NoError(err) re.True(needsCleanup) } @@ -328,20 +261,15 @@ func TestCleanupMicroserviceMetadataInPDModeRejectsNonDefaultGroup(t *testing.T) UserKind: endpoint.Standard.String(), }) })) - _, err := client.Put(ctx, keypath.RegistryPath(mcs.TSOServiceName, "127.0.0.1:3379"), "tso") - re.NoError(err) - err = svr.cleanupMicroserviceMetadataInPDMode(ctx) + err := svr.cleanupMicroserviceMetadataInPDMode(ctx) re.ErrorContains(err, "non-default TSO keyspace group 1") groups, err := store.LoadKeyspaceGroups(constant.DefaultKeyspaceGroupID, 0) re.NoError(err) re.Len(groups, 2) - resp, err := etcdutil.EtcdKVGet(client, microserviceEtcdPrefix(), clientv3.WithPrefix()) - re.NoError(err) - re.Len(resp.Kvs, 1) } -func TestCleanupMicroserviceMetadataInPDModeRejectsNonDefaultKeyspaceAssignment(t *testing.T) { +func TestCleanupMicroserviceMetadataInPDModePreservesAssignmentMarkers(t *testing.T) { re := require.New(t) ctx := context.Background() _, client, clean := etcdutil.NewTestEtcdCluster(t, 1, nil) @@ -352,75 +280,73 @@ func TestCleanupMicroserviceMetadataInPDModeRejectsNonDefaultKeyspaceAssignment( store := storage.NewStorageWithEtcdBackend(client) svr := &Server{storage: store, client: client} re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { - return store.SaveKeyspaceGroup(txn, &endpoint.KeyspaceGroup{ + if err := store.SaveKeyspaceGroup(txn, &endpoint.KeyspaceGroup{ ID: constant.DefaultKeyspaceGroupID, UserKind: endpoint.Basic.String(), - }) + }); err != nil { + return err + } + return store.SaveKeyspaceMeta(txn, newKeyspaceMetaWithTSOGroup(1, "1")) })) - metas := make([]*keyspacepb.KeyspaceMeta, 0, etcdutil.MaxEtcdTxnOps+1) - for id := uint32(1); id <= etcdutil.MaxEtcdTxnOps; id++ { - metas = append(metas, newKeyspaceMetaWithTSOGroup(id, "0")) - } - invalidID := uint32(etcdutil.MaxEtcdTxnOps + 1) - metas = append(metas, newKeyspaceMetaWithTSOGroup(invalidID, "1")) - saveKeyspaceMetas(ctx, re, store, metas) - _, err := client.Put(ctx, keypath.RegistryPath(mcs.TSOServiceName, "127.0.0.1:3379"), "tso") - re.NoError(err) - err = svr.cleanupMicroserviceMetadataInPDMode(ctx) - re.ErrorContains(err, "keyspace "+strconv.FormatUint(uint64(invalidID), 10)+" is assigned to non-default TSO keyspace group 1") - var group *endpoint.KeyspaceGroup + re.NoError(svr.cleanupMicroserviceMetadataInPDMode(ctx)) re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { - var err error - group, err = store.LoadKeyspaceGroup(txn, constant.DefaultKeyspaceGroupID) - if err != nil { - return err - } meta, err := store.LoadKeyspaceMeta(txn, 1) re.NoError(err) - re.Equal("0", meta.GetConfig()[keyspace.TSOKeyspaceGroupIDKey]) + re.Equal("1", meta.GetConfig()[keyspace.TSOKeyspaceGroupIDKey]) return nil })) - re.NotNil(group) - resp, err := etcdutil.EtcdKVGet(client, microserviceEtcdPrefix(), clientv3.WithPrefix()) - re.NoError(err) - re.Len(resp.Kvs, 1) } -func TestCleanupMicroserviceMetadataInPDModeWithLargeKeyspaceBatch(t *testing.T) { - re := require.New(t) - ctx := context.Background() - _, client, clean := etcdutil.NewTestEtcdCluster(t, 1, nil) - defer clean() - keypath.SetClusterID(12349) - defer keypath.ResetClusterID() +func TestCleanupMicroserviceMetadataInPDModeRejectsGroupTransition(t *testing.T) { + testCases := []struct { + name string + transition func(*endpoint.KeyspaceGroup) + errText string + }{ + { + name: "splitting", + transition: func(group *endpoint.KeyspaceGroup) { + group.SplitState = &endpoint.SplitState{SplitSource: group.ID} + }, + errText: "splitting", + }, + { + name: "merging", + transition: func(group *endpoint.KeyspaceGroup) { + group.MergeState = &endpoint.MergeState{MergeList: []uint32{1}} + }, + errText: "merging", + }, + } - store := storage.NewStorageWithEtcdBackend(client) - svr := &Server{storage: store, client: client} - re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { - return store.SaveKeyspaceGroup(txn, &endpoint.KeyspaceGroup{ - ID: constant.DefaultKeyspaceGroupID, - UserKind: endpoint.Basic.String(), + for i, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + re := require.New(t) + ctx := context.Background() + _, client, clean := etcdutil.NewTestEtcdCluster(t, 1, nil) + defer clean() + keypath.SetClusterID(uint64(12400 + i)) + defer keypath.ResetClusterID() + + store := storage.NewStorageWithEtcdBackend(client) + svr := &Server{storage: store, client: client} + group := &endpoint.KeyspaceGroup{ + ID: constant.DefaultKeyspaceGroupID, + UserKind: endpoint.Basic.String(), + } + testCase.transition(group) + re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { + return store.SaveKeyspaceGroup(txn, group) + })) + + err := svr.cleanupMicroserviceMetadataInPDMode(ctx) + re.ErrorContains(err, testCase.errText) + groups, err := store.LoadKeyspaceGroups(constant.DefaultKeyspaceGroupID, 0) + re.NoError(err) + re.Len(groups, 1) }) - })) - metas := make([]*keyspacepb.KeyspaceMeta, 0, etcdutil.MaxEtcdTxnOps) - for id := uint32(1); id <= etcdutil.MaxEtcdTxnOps; id++ { - metas = append(metas, newKeyspaceMetaWithTSOGroup(id, "0")) } - saveKeyspaceMetas(ctx, re, store, metas) - - re.NoError(svr.cleanupMicroserviceMetadataInPDMode(ctx)) - re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { - loaded, err := store.LoadRangeKeyspace(txn, constant.StartKeyspaceID, 0) - if err != nil { - return err - } - re.Len(loaded, etcdutil.MaxEtcdTxnOps) - for _, meta := range loaded { - re.NotContains(meta.GetConfig(), keyspace.TSOKeyspaceGroupIDKey) - } - return nil - })) } func newKeyspaceMetaWithTSOGroup(id uint32, groupID string) *keyspacepb.KeyspaceMeta { @@ -432,22 +358,3 @@ func newKeyspaceMetaWithTSOGroup(id uint32, groupID string) *keyspacepb.Keyspace }, } } - -func saveKeyspaceMetas( - ctx context.Context, - re *require.Assertions, - store storage.Storage, - metas []*keyspacepb.KeyspaceMeta, -) { - for start := 0; start < len(metas); start += microserviceMetadataCleanupBatchSize { - end := min(start+microserviceMetadataCleanupBatchSize, len(metas)) - re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { - for _, meta := range metas[start:end] { - if err := store.SaveKeyspaceMeta(txn, meta); err != nil { - return err - } - } - return nil - })) - } -} diff --git a/tests/integrations/mcs/tso/server_test.go b/tests/integrations/mcs/tso/server_test.go index eeec9f8ff2..98d028e723 100644 --- a/tests/integrations/mcs/tso/server_test.go +++ b/tests/integrations/mcs/tso/server_test.go @@ -776,9 +776,14 @@ func TestPDModeSwitchBetweenMicroserviceAndMonolithMultipleTimes(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() + re.NoError(failpoint.Enable("github.com/tikv/pd/server/skipKeyspaceRegionCheck", "return")) + t.Cleanup(func() { + re.NoError(failpoint.Disable("github.com/tikv/pd/server/skipKeyspaceRegionCheck")) + }) tc, err := tests.NewTestClusterWithKeyspaceGroup(ctx, 1, func(conf *config.Config, _ string) { conf.Microservice.EnableTSODynamicSwitching = false + conf.Keyspace.WaitRegionSplit = false }) re.NoError(err) defer tc.Destroy() @@ -787,6 +792,11 @@ func TestPDModeSwitchBetweenMicroserviceAndMonolithMultipleTimes(t *testing.T) { pdServer := tc.GetServer(tc.WaitLeader()) re.NotNil(pdServer) re.NoError(pdServer.BootstrapCluster()) + const userKeyspaceName = "mode_switch_user_ks" + _, err = pdServer.GetServer().GetKeyspaceManager().CreateKeyspace(&keyspace.CreateKeyspaceRequest{ + Name: userKeyspaceName, + }) + re.NoError(err) const switchRounds = 2 for range switchRounds { @@ -795,11 +805,13 @@ func TestPDModeSwitchBetweenMicroserviceAndMonolithMultipleTimes(t *testing.T) { re.NoError(err) waitTSOServiceReady(re, tsoCluster) checkMicroserviceTSOAvailable(ctx, re, pdServer) + checkKeyspaceTSOAvailable(ctx, re, pdServer, userKeyspaceName) tsoCluster.Destroy() pdServer = restartPDForServiceMode(ctx, re, tc, pdServer, nil) checkPDTSOAvailable(ctx, re, pdServer) - waitMicroserviceMetadataCleaned(re, pdServer) + checkKeyspaceTSOAvailable(ctx, re, pdServer, userKeyspaceName) + waitDefaultKeyspaceGroupCleaned(re, pdServer) pdServer = restartPDForServiceMode(ctx, re, tc, pdServer, []string{mcs.PDServiceName}) } @@ -810,6 +822,7 @@ func TestPDModeSwitchBetweenMicroserviceAndMonolithMultipleTimes(t *testing.T) { defer tsoCluster.Destroy() waitTSOServiceReady(re, tsoCluster) checkMicroserviceTSOAvailable(ctx, re, pdServer) + checkKeyspaceTSOAvailable(ctx, re, pdServer, userKeyspaceName) } func restartPDForServiceMode( @@ -872,29 +885,30 @@ func checkMicroserviceTSOAvailable(ctx context.Context, re *require.Assertions, checkPDTSOAvailable(ctx, re, pdServer) } -func waitMicroserviceMetadataCleaned(re *require.Assertions, pdServer *tests.TestServer) { +func checkKeyspaceTSOAvailable( + ctx context.Context, + re *require.Assertions, + pdServer *tests.TestServer, + keyspaceName string, +) { + cli := utils.SetupClientWithAPIContext( + ctx, + re, + pd.NewAPIContextV2(keyspaceName), + []string{pdServer.GetAddr()}, + ) + defer cli.Close() + physical, logical, err := cli.GetTS(ctx) + re.NoError(err) + re.NotZero(tsoutil.ComposeTS(physical, logical)) +} + +func waitDefaultKeyspaceGroupCleaned(re *require.Assertions, pdServer *tests.TestServer) { testutil.Eventually(re, func() bool { - keyspaceGroupResp, err := etcdutil.EtcdKVGet(pdServer.GetEtcdClient(), keypath.KeyspaceGroupIDPrefix(), clientv3.WithPrefix()) - if err != nil || len(keyspaceGroupResp.Kvs) != 0 { - return false - } - microserviceResp, err := etcdutil.EtcdKVGet( + resp, err := etcdutil.EtcdKVGet( pdServer.GetEtcdClient(), - fmt.Sprintf("%s/%d/", mcs.MicroserviceRootPath, keypath.ClusterID()), - clientv3.WithPrefix()) - if err != nil || len(microserviceResp.Kvs) != 0 { - return false - } - keyspaces, err := pdServer.GetKeyspaceManager().LoadRangeKeyspace(constant.StartKeyspaceID, 0) - if err != nil { - return false - } - for _, meta := range keyspaces { - if _, ok := meta.GetConfig()[keyspace.TSOKeyspaceGroupIDKey]; ok { - return false - } - } - return true + keypath.KeyspaceGroupIDPath(constant.DefaultKeyspaceGroupID)) + return err == nil && len(resp.Kvs) == 0 }, testutil.WithWaitFor(30*time.Second), testutil.WithTickInterval(100*time.Millisecond)) } From cf726762a7dd70ea3420eaa3d4e0fdd0db898b30 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Tue, 11 Aug 2026 17:27:22 +0800 Subject: [PATCH 07/15] server: safely clear stale default group members Signed-off-by: Ryan Leung --- server/microservice_cleanup.go | 134 +++++++-- server/microservice_cleanup_test.go | 326 +++++++++++++++++++--- tests/integrations/mcs/tso/server_test.go | 77 +++-- 3 files changed, 460 insertions(+), 77 deletions(-) diff --git a/server/microservice_cleanup.go b/server/microservice_cleanup.go index 9b9aca92e8..7f00f06d59 100644 --- a/server/microservice_cleanup.go +++ b/server/microservice_cleanup.go @@ -16,17 +16,23 @@ package server import ( "context" + "encoding/json" goerrors "errors" "time" + clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" "github.com/pingcap/errors" + "github.com/pingcap/failpoint" "github.com/pingcap/log" "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/keyspace/constant" + "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/storage/kv" + "github.com/tikv/pd/pkg/utils/etcdutil" + "github.com/tikv/pd/pkg/utils/keypath" "github.com/tikv/pd/pkg/utils/logutil" ) @@ -34,6 +40,12 @@ const microserviceMetadataCleanupRetryInterval = 5 * time.Second var errMicroserviceMetadataCleanupRejected = errors.New("microservice metadata cleanup rejected") +type microserviceMetadataCleanupTerm struct { + leaderKey string + leaderValue string + leaseID clientv3.LeaseID +} + func rejectMicroserviceMetadataCleanup(format string, args ...any) error { return errors.Wrapf(errMicroserviceMetadataCleanupRejected, format, args...) } @@ -42,6 +54,10 @@ func (s *Server) scheduleMicroserviceMetadataCleanup(ctx context.Context) { if s.IsKeyspaceGroupEnabled() { return } + term, ok := s.captureMicroserviceMetadataCleanupTerm() + if !ok { + return + } s.serverLoopWg.Add(1) go func() { defer logutil.LogPanic() @@ -50,7 +66,7 @@ func (s *Server) scheduleMicroserviceMetadataCleanup(ctx context.Context) { if s.member != nil && !s.member.IsServing() { return } - err := s.cleanupMicroserviceMetadataInPDMode(ctx) + err := s.cleanupMicroserviceMetadataInPDMode(ctx, term) if err == nil { return } @@ -72,31 +88,56 @@ func (s *Server) scheduleMicroserviceMetadataCleanup(ctx context.Context) { }() } -func (s *Server) cleanupMicroserviceMetadataInPDMode(ctx context.Context) error { +func (s *Server) captureMicroserviceMetadataCleanupTerm() (microserviceMetadataCleanupTerm, bool) { + if s.member == nil { + return microserviceMetadataCleanupTerm{}, false + } + leadership := s.member.GetLeadership() + if leadership == nil { + return microserviceMetadataCleanupTerm{}, false + } + lease := leadership.GetLease() + if lease == nil { + return microserviceMetadataCleanupTerm{}, false + } + term := microserviceMetadataCleanupTerm{ + leaderKey: leadership.GetLeaderKey(), + leaderValue: leadership.GetLeaderValue(), + leaseID: lease.GetID(), + } + return term, term.leaderKey != "" && term.leaderValue != "" && term.leaseID != 0 +} + +func (s *Server) cleanupMicroserviceMetadataInPDMode( + ctx context.Context, + term microserviceMetadataCleanupTerm, +) error { if s.IsKeyspaceGroupEnabled() { return nil } // The persisted keyspace group contains the stale TSO member addresses that // block a later switch back to API service mode. Keyspace assignment markers - // and lease-owned microservice keys are intentionally left untouched. + // and the rest of the keyspace group metadata are intentionally left untouched. needsCleanup, err := s.validateMicroserviceMetadataCleanup() if err != nil || !needsCleanup { return err } - deleted, err := s.deleteDefaultTSOKeyspaceGroup(ctx) + cleared, err := s.clearDefaultTSOKeyspaceGroupMembers(ctx, term) if err != nil { return err } - if deleted { + if cleared { log.Info("cleaned up microservice metadata in PD mode", - zap.Bool("deleted-default-keyspace-group", true)) + zap.Bool("cleared-default-keyspace-group-members", true)) } return nil } func (s *Server) validateMicroserviceMetadataCleanup() (bool, error) { - groups, err := s.storage.LoadKeyspaceGroups(constant.DefaultKeyspaceGroupID, 0) + // At most two records are needed: the default group and the first unsupported + // non-default group, if one exists. + groups, err := s.storage.LoadKeyspaceGroups(constant.DefaultKeyspaceGroupID, 2) if err != nil { return false, err } @@ -105,7 +146,6 @@ func (s *Server) validateMicroserviceMetadataCleanup() (bool, error) { if group == nil { continue } - needsCleanup = true if group.ID != constant.DefaultKeyspaceGroupID { return false, rejectMicroserviceMetadataCleanup("found non-default TSO keyspace group %d when cleaning up PD mode microservice metadata", group.ID) } @@ -115,28 +155,66 @@ func (s *Server) validateMicroserviceMetadataCleanup() (bool, error) { if group.IsMerging() { return false, rejectMicroserviceMetadataCleanup("default TSO keyspace group is merging when cleaning up PD mode microservice metadata") } + needsCleanup = len(group.Members) > 0 } return needsCleanup, nil } -func (s *Server) deleteDefaultTSOKeyspaceGroup(ctx context.Context) (bool, error) { - deleted := false - err := s.storage.RunInTxn(ctx, func(txn kv.Txn) error { - group, err := s.storage.LoadKeyspaceGroup(txn, constant.DefaultKeyspaceGroupID) - if err != nil || group == nil { - return err - } - if group.IsSplitting() { - return rejectMicroserviceMetadataCleanup("default TSO keyspace group is splitting when deleting PD mode microservice metadata") - } - if group.IsMerging() { - return rejectMicroserviceMetadataCleanup("default TSO keyspace group is merging when deleting PD mode microservice metadata") - } - if err := s.storage.DeleteKeyspaceGroup(txn, constant.DefaultKeyspaceGroupID); err != nil { - return err - } - deleted = true - return nil - }) - return deleted, err +func (s *Server) clearDefaultTSOKeyspaceGroupMembers( + ctx context.Context, + term microserviceMetadataCleanupTerm, +) (bool, error) { + groupKey := keypath.KeyspaceGroupIDPath(constant.DefaultKeyspaceGroupID) + resp, err := etcdutil.EtcdKVGet(s.client, groupKey) + if err != nil { + return false, err + } + if len(resp.Kvs) == 0 { + return false, nil + } + if len(resp.Kvs) != 1 { + return false, errs.ErrEtcdKVGetResponse.FastGenByArgs(resp.Kvs) + } + + groupKV := resp.Kvs[0] + group := &endpoint.KeyspaceGroup{} + if err := json.Unmarshal(groupKV.Value, group); err != nil { + return false, errs.ErrJSONUnmarshal.Wrap(err).GenWithStackByCause() + } + if group.ID != constant.DefaultKeyspaceGroupID { + return false, rejectMicroserviceMetadataCleanup( + "found TSO keyspace group %d at the default group path when clearing PD mode microservice metadata", group.ID) + } + if group.IsSplitting() { + return false, rejectMicroserviceMetadataCleanup("default TSO keyspace group is splitting when clearing PD mode microservice metadata") + } + if group.IsMerging() { + return false, rejectMicroserviceMetadataCleanup("default TSO keyspace group is merging when clearing PD mode microservice metadata") + } + if len(group.Members) == 0 { + return false, nil + } + + group.Members = nil + value, err := json.Marshal(group) + if err != nil { + return false, errs.ErrJSONMarshal.Wrap(err).GenWithStackByCause() + } + + failpoint.InjectCall("beforeClearDefaultTSOKeyspaceGroupMembersCommit") + txnResp, err := kv.NewSlowLogTxnWithContext(ctx, s.client). + If( + clientv3.Compare(clientv3.Value(term.leaderKey), "=", term.leaderValue), + clientv3.Compare(clientv3.LeaseValue(term.leaderKey), "=", term.leaseID), + clientv3.Compare(clientv3.ModRevision(groupKey), "=", groupKV.ModRevision), + ). + Then(clientv3.OpPut(groupKey, string(value))). + Commit() + if err != nil { + return false, errs.ErrEtcdTxnInternal.Wrap(err).GenWithStackByCause() + } + if !txnResp.Succeeded { + return false, errs.ErrEtcdTxnConflict.FastGenByArgs() + } + return true, nil } diff --git a/server/microservice_cleanup_test.go b/server/microservice_cleanup_test.go index c8ae24308b..96bc7d6649 100644 --- a/server/microservice_cleanup_test.go +++ b/server/microservice_cleanup_test.go @@ -17,6 +17,7 @@ package server import ( "context" "strconv" + "sync" "testing" "time" @@ -26,9 +27,11 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/keyspacepb" + "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/keyspace" "github.com/tikv/pd/pkg/keyspace/constant" mcs "github.com/tikv/pd/pkg/mcs/utils/constant" + "github.com/tikv/pd/pkg/member" "github.com/tikv/pd/pkg/storage" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/storage/kv" @@ -40,16 +43,21 @@ func TestCleanupMicroserviceMetadataInPDMode(t *testing.T) { re := require.New(t) ctx := context.Background() _, client, clean := etcdutil.NewTestEtcdCluster(t, 1, nil) - defer clean() + t.Cleanup(clean) keypath.SetClusterID(12345) defer keypath.ResetClusterID() store := storage.NewStorageWithEtcdBackend(client) - svr := &Server{storage: store, client: client} + svr, term := newMicroserviceMetadataCleanupTestServer(t, client, store) + staleMembers := []endpoint.KeyspaceGroupMember{{ + Address: "http://127.0.0.1:3379", + Priority: mcs.DefaultKeyspaceGroupReplicaPriority, + }} re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { if err := store.SaveKeyspaceGroup(txn, &endpoint.KeyspaceGroup{ ID: constant.DefaultKeyspaceGroupID, UserKind: endpoint.Basic.String(), + Members: staleMembers, Keyspaces: []uint32{1}, }); err != nil { return err @@ -79,12 +87,16 @@ func TestCleanupMicroserviceMetadataInPDMode(t *testing.T) { re.NoError(err) } - re.NoError(svr.cleanupMicroserviceMetadataInPDMode(ctx)) - re.NoError(svr.cleanupMicroserviceMetadataInPDMode(ctx)) + re.NoError(svr.cleanupMicroserviceMetadataInPDMode(ctx, term)) + re.NoError(svr.cleanupMicroserviceMetadataInPDMode(ctx, term)) groups, err := store.LoadKeyspaceGroups(constant.DefaultKeyspaceGroupID, 0) re.NoError(err) - re.Empty(groups) + re.Len(groups, 1) + re.Equal(constant.DefaultKeyspaceGroupID, groups[0].ID) + re.Equal(endpoint.Basic.String(), groups[0].UserKind) + re.Empty(groups[0].Members) + re.Equal([]uint32{1}, groups[0].Keyspaces) re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { meta, err := store.LoadKeyspaceMeta(txn, 1) re.NoError(err) @@ -109,16 +121,20 @@ func TestCleanupMicroserviceMetadataIgnoresLeasedRegistryKeys(t *testing.T) { re := require.New(t) ctx := context.Background() _, client, clean := etcdutil.NewTestEtcdCluster(t, 1, nil) - defer clean() + t.Cleanup(clean) keypath.SetClusterID(12351) defer keypath.ResetClusterID() store := storage.NewStorageWithEtcdBackend(client) - svr := &Server{storage: store, client: client} + svr, term := newMicroserviceMetadataCleanupTestServer(t, client, store) re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { return store.SaveKeyspaceGroup(txn, &endpoint.KeyspaceGroup{ - ID: constant.DefaultKeyspaceGroupID, - UserKind: endpoint.Basic.String(), + ID: constant.DefaultKeyspaceGroupID, + UserKind: endpoint.Basic.String(), + Members: []endpoint.KeyspaceGroupMember{{ + Address: "http://127.0.0.1:3379", + Priority: mcs.DefaultKeyspaceGroupReplicaPriority, + }}, Keyspaces: []uint32{1}, }) })) @@ -131,10 +147,11 @@ func TestCleanupMicroserviceMetadataIgnoresLeasedRegistryKeys(t *testing.T) { _, _ = client.Revoke(ctx, leaseID) }() - re.NoError(svr.cleanupMicroserviceMetadataInPDMode(ctx)) + re.NoError(svr.cleanupMicroserviceMetadataInPDMode(ctx, term)) groups, err := store.LoadKeyspaceGroups(constant.DefaultKeyspaceGroupID, 0) re.NoError(err) - re.Empty(groups) + re.Len(groups, 1) + re.Empty(groups[0].Members) resp, err := etcdutil.EtcdKVGet(client, registryPath) re.NoError(err) re.Len(resp.Kvs, 1) @@ -145,27 +162,30 @@ func TestScheduleMicroserviceMetadataCleanupReturnsImmediately(t *testing.T) { re := require.New(t) ctx := context.Background() _, client, clean := etcdutil.NewTestEtcdCluster(t, 1, nil) - defer clean() + t.Cleanup(clean) keypath.SetClusterID(12348) defer keypath.ResetClusterID() store := storage.NewStorageWithEtcdBackend(client) - svr := &Server{storage: store, client: client} + svr, _ := newMicroserviceMetadataCleanupTestServer(t, client, store) re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { return store.SaveKeyspaceGroup(txn, &endpoint.KeyspaceGroup{ ID: constant.DefaultKeyspaceGroupID, UserKind: endpoint.Basic.String(), + Members: []endpoint.KeyspaceGroupMember{{ + Address: "http://127.0.0.1:3379", + Priority: mcs.DefaultKeyspaceGroupReplicaPriority, + }}, }) })) - re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/utils/etcdutil/SlowEtcdKVGet", "return(1)")) - t.Cleanup(func() { - _ = failpoint.Disable("github.com/tikv/pd/pkg/utils/etcdutil/SlowEtcdKVGet") - }) + blocker := enableMicroserviceMetadataCleanupCommitBlocker(t) start := time.Now() svr.scheduleMicroserviceMetadataCleanup(ctx) re.Less(time.Since(start), 200*time.Millisecond) - re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/utils/etcdutil/SlowEtcdKVGet")) + blocker.wait(t) + blocker.releaseCleanup() + blocker.disable(t) svr.serverLoopWg.Wait() } @@ -174,16 +194,20 @@ func TestScheduleMicroserviceMetadataCleanupStopsOnRejectedState(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() _, client, clean := etcdutil.NewTestEtcdCluster(t, 1, nil) - defer clean() + t.Cleanup(clean) keypath.SetClusterID(12353) defer keypath.ResetClusterID() store := storage.NewStorageWithEtcdBackend(client) - svr := &Server{storage: store, client: client} + svr, _ := newMicroserviceMetadataCleanupTestServer(t, client, store) re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { if err := store.SaveKeyspaceGroup(txn, &endpoint.KeyspaceGroup{ ID: constant.DefaultKeyspaceGroupID, UserKind: endpoint.Basic.String(), + Members: []endpoint.KeyspaceGroupMember{{ + Address: "http://127.0.0.1:3379", + Priority: mcs.DefaultKeyspaceGroupReplicaPriority, + }}, }); err != nil { return err } @@ -212,7 +236,7 @@ func TestMicroserviceMetadataCleanupTransitionDetection(t *testing.T) { re := require.New(t) ctx := context.Background() _, client, clean := etcdutil.NewTestEtcdCluster(t, 1, nil) - defer clean() + t.Cleanup(clean) keypath.SetClusterID(12350) defer keypath.ResetClusterID() @@ -236,6 +260,21 @@ func TestMicroserviceMetadataCleanupTransitionDetection(t *testing.T) { })) needsCleanup, err = svr.validateMicroserviceMetadataCleanup() re.NoError(err) + re.False(needsCleanup) + + re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { + group, err := store.LoadKeyspaceGroup(txn, constant.DefaultKeyspaceGroupID) + if err != nil { + return err + } + group.Members = []endpoint.KeyspaceGroupMember{{ + Address: "http://127.0.0.1:3379", + Priority: mcs.DefaultKeyspaceGroupReplicaPriority, + }} + return store.SaveKeyspaceGroup(txn, group) + })) + needsCleanup, err = svr.validateMicroserviceMetadataCleanup() + re.NoError(err) re.True(needsCleanup) } @@ -243,16 +282,21 @@ func TestCleanupMicroserviceMetadataInPDModeRejectsNonDefaultGroup(t *testing.T) re := require.New(t) ctx := context.Background() _, client, clean := etcdutil.NewTestEtcdCluster(t, 1, nil) - defer clean() + t.Cleanup(clean) keypath.SetClusterID(12346) defer keypath.ResetClusterID() store := storage.NewStorageWithEtcdBackend(client) - svr := &Server{storage: store, client: client} + svr, term := newMicroserviceMetadataCleanupTestServer(t, client, store) + staleMembers := []endpoint.KeyspaceGroupMember{{ + Address: "http://127.0.0.1:3379", + Priority: mcs.DefaultKeyspaceGroupReplicaPriority, + }} re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { if err := store.SaveKeyspaceGroup(txn, &endpoint.KeyspaceGroup{ ID: constant.DefaultKeyspaceGroupID, UserKind: endpoint.Basic.String(), + Members: staleMembers, }); err != nil { return err } @@ -262,40 +306,49 @@ func TestCleanupMicroserviceMetadataInPDModeRejectsNonDefaultGroup(t *testing.T) }) })) - err := svr.cleanupMicroserviceMetadataInPDMode(ctx) + err := svr.cleanupMicroserviceMetadataInPDMode(ctx, term) re.ErrorContains(err, "non-default TSO keyspace group 1") groups, err := store.LoadKeyspaceGroups(constant.DefaultKeyspaceGroupID, 0) re.NoError(err) re.Len(groups, 2) + re.Equal(staleMembers, groups[0].Members) } func TestCleanupMicroserviceMetadataInPDModePreservesAssignmentMarkers(t *testing.T) { re := require.New(t) ctx := context.Background() _, client, clean := etcdutil.NewTestEtcdCluster(t, 1, nil) - defer clean() + t.Cleanup(clean) keypath.SetClusterID(12347) defer keypath.ResetClusterID() store := storage.NewStorageWithEtcdBackend(client) - svr := &Server{storage: store, client: client} + svr, term := newMicroserviceMetadataCleanupTestServer(t, client, store) re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { if err := store.SaveKeyspaceGroup(txn, &endpoint.KeyspaceGroup{ ID: constant.DefaultKeyspaceGroupID, UserKind: endpoint.Basic.String(), + Members: []endpoint.KeyspaceGroupMember{{ + Address: "http://127.0.0.1:3379", + Priority: mcs.DefaultKeyspaceGroupReplicaPriority, + }}, }); err != nil { return err } - return store.SaveKeyspaceMeta(txn, newKeyspaceMetaWithTSOGroup(1, "1")) + return store.SaveKeyspaceMeta(txn, newKeyspaceMetaWithTSOGroup(1, "0")) })) - re.NoError(svr.cleanupMicroserviceMetadataInPDMode(ctx)) + re.NoError(svr.cleanupMicroserviceMetadataInPDMode(ctx, term)) re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { meta, err := store.LoadKeyspaceMeta(txn, 1) re.NoError(err) - re.Equal("1", meta.GetConfig()[keyspace.TSOKeyspaceGroupIDKey]) + re.Equal("0", meta.GetConfig()[keyspace.TSOKeyspaceGroupIDKey]) return nil })) + groups, err := store.LoadKeyspaceGroups(constant.DefaultKeyspaceGroupID, 0) + re.NoError(err) + re.Len(groups, 1) + re.Empty(groups[0].Members) } func TestCleanupMicroserviceMetadataInPDModeRejectsGroupTransition(t *testing.T) { @@ -325,27 +378,236 @@ func TestCleanupMicroserviceMetadataInPDModeRejectsGroupTransition(t *testing.T) re := require.New(t) ctx := context.Background() _, client, clean := etcdutil.NewTestEtcdCluster(t, 1, nil) - defer clean() + t.Cleanup(clean) keypath.SetClusterID(uint64(12400 + i)) defer keypath.ResetClusterID() store := storage.NewStorageWithEtcdBackend(client) - svr := &Server{storage: store, client: client} + svr, term := newMicroserviceMetadataCleanupTestServer(t, client, store) group := &endpoint.KeyspaceGroup{ ID: constant.DefaultKeyspaceGroupID, UserKind: endpoint.Basic.String(), + Members: []endpoint.KeyspaceGroupMember{{ + Address: "http://127.0.0.1:3379", + Priority: mcs.DefaultKeyspaceGroupReplicaPriority, + }}, } testCase.transition(group) re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { return store.SaveKeyspaceGroup(txn, group) })) - err := svr.cleanupMicroserviceMetadataInPDMode(ctx) + err := svr.cleanupMicroserviceMetadataInPDMode(ctx, term) re.ErrorContains(err, testCase.errText) groups, err := store.LoadKeyspaceGroups(constant.DefaultKeyspaceGroupID, 0) re.NoError(err) re.Len(groups, 1) + re.Equal(group, groups[0]) + }) + } +} + +func TestClearDefaultTSOKeyspaceGroupMembersIsFencedByLeadershipTerm(t *testing.T) { + re := require.New(t) + ctx := context.Background() + _, client, clean := etcdutil.NewTestEtcdCluster(t, 1, nil) + t.Cleanup(clean) + keypath.SetClusterID(12354) + t.Cleanup(keypath.ResetClusterID) + + store := storage.NewStorageWithEtcdBackend(client) + svr, oldTerm := newMicroserviceMetadataCleanupTestServer(t, client, store) + staleMembers := []endpoint.KeyspaceGroupMember{{ + Address: "http://127.0.0.1:3379", + Priority: mcs.DefaultKeyspaceGroupReplicaPriority, + }} + re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { + return store.SaveKeyspaceGroup(txn, &endpoint.KeyspaceGroup{ + ID: constant.DefaultKeyspaceGroupID, + UserKind: endpoint.Basic.String(), + Members: staleMembers, + Keyspaces: []uint32{1}, + }) + })) + + blocker := enableMicroserviceMetadataCleanupCommitBlocker(t) + resultCh := make(chan clearDefaultTSOKeyspaceGroupMembersResult, 1) + go func() { + cleared, err := svr.clearDefaultTSOKeyspaceGroupMembers(ctx, oldTerm) + resultCh <- clearDefaultTSOKeyspaceGroupMembersResult{cleared: cleared, err: err} + }() + blocker.wait(t) + + // Campaign again with the same member value to prove that comparing only the + // leader value would allow an old-term cleanup to pass. + svr.member.Resign() + re.NoError(svr.member.GetLeadership().Campaign( + testMicroserviceMetadataCleanupLeaseTimeout, + svr.member.MemberValue(), + )) + svr.member.PromoteSelf() + newTerm, ok := svr.captureMicroserviceMetadataCleanupTerm() + re.True(ok) + re.Equal(oldTerm.leaderValue, newTerm.leaderValue) + re.NotEqual(oldTerm.leaseID, newTerm.leaseID) + + blocker.releaseCleanup() + blocker.disable(t) + result := waitClearDefaultTSOKeyspaceGroupMembersResult(t, resultCh) + re.False(result.cleared) + re.ErrorIs(result.err, errs.ErrEtcdTxnConflict) + + groups, err := store.LoadKeyspaceGroups(constant.DefaultKeyspaceGroupID, 0) + re.NoError(err) + re.Len(groups, 1) + re.Equal(staleMembers, groups[0].Members) +} + +func TestClearDefaultTSOKeyspaceGroupMembersPreservesConcurrentUpdate(t *testing.T) { + re := require.New(t) + ctx := context.Background() + _, client, clean := etcdutil.NewTestEtcdCluster(t, 1, nil) + t.Cleanup(clean) + keypath.SetClusterID(12355) + t.Cleanup(keypath.ResetClusterID) + + store := storage.NewStorageWithEtcdBackend(client) + svr, term := newMicroserviceMetadataCleanupTestServer(t, client, store) + re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { + return store.SaveKeyspaceGroup(txn, &endpoint.KeyspaceGroup{ + ID: constant.DefaultKeyspaceGroupID, + UserKind: endpoint.Basic.String(), + Members: []endpoint.KeyspaceGroupMember{{ + Address: "http://127.0.0.1:3379", + Priority: mcs.DefaultKeyspaceGroupReplicaPriority, + }}, + Keyspaces: []uint32{1}, }) + })) + + blocker := enableMicroserviceMetadataCleanupCommitBlocker(t) + resultCh := make(chan clearDefaultTSOKeyspaceGroupMembersResult, 1) + go func() { + cleared, err := svr.clearDefaultTSOKeyspaceGroupMembers(ctx, term) + resultCh <- clearDefaultTSOKeyspaceGroupMembersResult{cleared: cleared, err: err} + }() + blocker.wait(t) + + newMembers := []endpoint.KeyspaceGroupMember{{ + Address: "http://127.0.0.1:3380", + Priority: mcs.DefaultKeyspaceGroupReplicaPriority, + }} + re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { + group, err := store.LoadKeyspaceGroup(txn, constant.DefaultKeyspaceGroupID) + if err != nil { + return err + } + group.Members = newMembers + group.Keyspaces = append(group.Keyspaces, 2) + return store.SaveKeyspaceGroup(txn, group) + })) + + blocker.releaseCleanup() + blocker.disable(t) + result := waitClearDefaultTSOKeyspaceGroupMembersResult(t, resultCh) + re.False(result.cleared) + re.ErrorIs(result.err, errs.ErrEtcdTxnConflict) + + groups, err := store.LoadKeyspaceGroups(constant.DefaultKeyspaceGroupID, 0) + re.NoError(err) + re.Len(groups, 1) + re.Equal(newMembers, groups[0].Members) + re.Equal([]uint32{1, 2}, groups[0].Keyspaces) +} + +const testMicroserviceMetadataCleanupLeaseTimeout = 60 + +type clearDefaultTSOKeyspaceGroupMembersResult struct { + cleared bool + err error +} + +type microserviceMetadataCleanupCommitBlocker struct { + name string + reached chan struct{} + release chan struct{} + reachedOnce sync.Once + releaseOnce sync.Once + disableOnce sync.Once +} + +func newMicroserviceMetadataCleanupTestServer( + t *testing.T, + client *clientv3.Client, + store storage.Storage, +) (*Server, microserviceMetadataCleanupTerm) { + t.Helper() + pdMember := member.NewMember(nil, client, 1) + pdMember.InitMemberInfo("http://127.0.0.1:2379", "http://127.0.0.1:2380", "pd-test") + re := require.New(t) + re.NoError(pdMember.GetLeadership().Campaign(testMicroserviceMetadataCleanupLeaseTimeout, pdMember.MemberValue())) + pdMember.PromoteSelf() + t.Cleanup(pdMember.Resign) + + svr := &Server{storage: store, client: client, member: pdMember} + term, ok := svr.captureMicroserviceMetadataCleanupTerm() + re.True(ok) + return svr, term +} + +func enableMicroserviceMetadataCleanupCommitBlocker(t *testing.T) *microserviceMetadataCleanupCommitBlocker { + t.Helper() + blocker := µserviceMetadataCleanupCommitBlocker{ + name: "github.com/tikv/pd/server/beforeClearDefaultTSOKeyspaceGroupMembersCommit", + reached: make(chan struct{}), + release: make(chan struct{}), + } + require.NoError(t, failpoint.EnableCall(blocker.name, func() { + blocker.reachedOnce.Do(func() { + close(blocker.reached) + }) + <-blocker.release + })) + t.Cleanup(func() { + blocker.releaseCleanup() + blocker.disable(t) + }) + return blocker +} + +func (b *microserviceMetadataCleanupCommitBlocker) wait(t *testing.T) { + t.Helper() + select { + case <-b.reached: + case <-time.After(10 * time.Second): + t.Fatal("microservice metadata cleanup did not reach the commit hook") + } +} + +func (b *microserviceMetadataCleanupCommitBlocker) releaseCleanup() { + b.releaseOnce.Do(func() { + close(b.release) + }) +} + +func (b *microserviceMetadataCleanupCommitBlocker) disable(t *testing.T) { + t.Helper() + b.disableOnce.Do(func() { + require.NoError(t, failpoint.Disable(b.name)) + }) +} + +func waitClearDefaultTSOKeyspaceGroupMembersResult( + t *testing.T, + resultCh <-chan clearDefaultTSOKeyspaceGroupMembersResult, +) clearDefaultTSOKeyspaceGroupMembersResult { + t.Helper() + select { + case result := <-resultCh: + return result + case <-time.After(10 * time.Second): + t.Fatal("microservice metadata cleanup did not return") + return clearDefaultTSOKeyspaceGroupMembersResult{} } } diff --git a/tests/integrations/mcs/tso/server_test.go b/tests/integrations/mcs/tso/server_test.go index 98d028e723..7a1bd67e37 100644 --- a/tests/integrations/mcs/tso/server_test.go +++ b/tests/integrations/mcs/tso/server_test.go @@ -20,6 +20,7 @@ import ( "fmt" "io" "net/http" + "slices" "strings" "sync" "testing" @@ -48,7 +49,6 @@ import ( mcs "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/etcdutil" - "github.com/tikv/pd/pkg/utils/keypath" "github.com/tikv/pd/pkg/utils/tempurl" "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/pkg/utils/tsoutil" @@ -793,36 +793,39 @@ func TestPDModeSwitchBetweenMicroserviceAndMonolithMultipleTimes(t *testing.T) { re.NotNil(pdServer) re.NoError(pdServer.BootstrapCluster()) const userKeyspaceName = "mode_switch_user_ks" - _, err = pdServer.GetServer().GetKeyspaceManager().CreateKeyspace(&keyspace.CreateKeyspaceRequest{ + userKeyspace, err := pdServer.GetServer().GetKeyspaceManager().CreateKeyspace(&keyspace.CreateKeyspaceRequest{ Name: userKeyspaceName, }) re.NoError(err) + userKeyspaceID := userKeyspace.GetId() const switchRounds = 2 for range switchRounds { - waitDefaultKeyspaceGroupReady(re, pdServer) + waitDefaultKeyspaceGroupReady(re, pdServer, userKeyspaceID) tsoCluster, err := tests.NewTestTSOCluster(ctx, 1, pdServer.GetAddr()) re.NoError(err) waitTSOServiceReady(re, tsoCluster) checkMicroserviceTSOAvailable(ctx, re, pdServer) checkKeyspaceTSOAvailable(ctx, re, pdServer, userKeyspaceName) + waitDefaultKeyspaceGroupMembers(re, pdServer, tsoCluster.GetKeyspaceGroupMember()) tsoCluster.Destroy() pdServer = restartPDForServiceMode(ctx, re, tc, pdServer, nil) checkPDTSOAvailable(ctx, re, pdServer) checkKeyspaceTSOAvailable(ctx, re, pdServer, userKeyspaceName) - waitDefaultKeyspaceGroupCleaned(re, pdServer) + waitDefaultKeyspaceGroupCleaned(re, pdServer, userKeyspaceID) pdServer = restartPDForServiceMode(ctx, re, tc, pdServer, []string{mcs.PDServiceName}) } - waitDefaultKeyspaceGroupReady(re, pdServer) + waitDefaultKeyspaceGroupReady(re, pdServer, userKeyspaceID) tsoCluster, err := tests.NewTestTSOCluster(ctx, 1, pdServer.GetAddr()) re.NoError(err) defer tsoCluster.Destroy() waitTSOServiceReady(re, tsoCluster) checkMicroserviceTSOAvailable(ctx, re, pdServer) checkKeyspaceTSOAvailable(ctx, re, pdServer, userKeyspaceName) + waitDefaultKeyspaceGroupMembers(re, pdServer, tsoCluster.GetKeyspaceGroupMember()) } func restartPDForServiceMode( @@ -844,13 +847,62 @@ func restartPDForServiceMode( return newServer } -func waitDefaultKeyspaceGroupReady(re *require.Assertions, pdServer *tests.TestServer) { +func waitDefaultKeyspaceGroupReady( + re *require.Assertions, + pdServer *tests.TestServer, + userKeyspaceID uint32, +) { testutil.Eventually(re, func() bool { - resp, err := etcdutil.EtcdKVGet(pdServer.GetEtcdClient(), keypath.KeyspaceGroupIDPath(constant.DefaultKeyspaceGroupID)) - return err == nil && len(resp.Kvs) == 1 + group, err := loadDefaultKeyspaceGroup(pdServer) + return err == nil && slices.Contains(group.Keyspaces, userKeyspaceID) }, testutil.WithWaitFor(10*time.Second), testutil.WithTickInterval(100*time.Millisecond)) } +func waitDefaultKeyspaceGroupCleaned( + re *require.Assertions, + pdServer *tests.TestServer, + userKeyspaceID uint32, +) { + testutil.Eventually(re, func() bool { + group, err := loadDefaultKeyspaceGroup(pdServer) + return err == nil && + len(group.Members) == 0 && + slices.Contains(group.Keyspaces, userKeyspaceID) + }, testutil.WithWaitFor(30*time.Second), testutil.WithTickInterval(100*time.Millisecond)) +} + +func waitDefaultKeyspaceGroupMembers( + re *require.Assertions, + pdServer *tests.TestServer, + expected []endpoint.KeyspaceGroupMember, +) { + testutil.Eventually(re, func() bool { + group, err := loadDefaultKeyspaceGroup(pdServer) + if err != nil || len(group.Members) != len(expected) { + return false + } + for _, expectedMember := range expected { + if !slices.ContainsFunc(group.Members, func(member endpoint.KeyspaceGroupMember) bool { + return member.IsAddressEquivalent(expectedMember.Address) + }) { + return false + } + } + return true + }, testutil.WithWaitFor(10*time.Second), testutil.WithTickInterval(100*time.Millisecond)) +} + +func loadDefaultKeyspaceGroup(pdServer *tests.TestServer) (*endpoint.KeyspaceGroup, error) { + groups, err := pdServer.GetServer().GetStorage().LoadKeyspaceGroups(constant.DefaultKeyspaceGroupID, 1) + if err != nil { + return nil, err + } + if len(groups) != 1 || groups[0] == nil || groups[0].ID != constant.DefaultKeyspaceGroupID { + return nil, fmt.Errorf("default keyspace group %d does not exist", constant.DefaultKeyspaceGroupID) + } + return groups[0], nil +} + func waitTSOServiceReady(re *require.Assertions, tsoCluster *tests.TestTSOCluster) { primary := tsoCluster.WaitForDefaultPrimaryServing(re) testutil.Eventually(re, func() bool { @@ -903,15 +955,6 @@ func checkKeyspaceTSOAvailable( re.NotZero(tsoutil.ComposeTS(physical, logical)) } -func waitDefaultKeyspaceGroupCleaned(re *require.Assertions, pdServer *tests.TestServer) { - testutil.Eventually(re, func() bool { - resp, err := etcdutil.EtcdKVGet( - pdServer.GetEtcdClient(), - keypath.KeyspaceGroupIDPath(constant.DefaultKeyspaceGroupID)) - return err == nil && len(resp.Kvs) == 0 - }, testutil.WithWaitFor(30*time.Second), testutil.WithTickInterval(100*time.Millisecond)) -} - func checkTSOMonotonic(ctx context.Context, pdClient pd.Client, globalLastTS *uint64, count int) error { for range count { physical, logical, err := pdClient.GetTS(ctx) From a261200c821d2d30b506034375ad3f4be9c5d46c Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Tue, 11 Aug 2026 21:11:41 +0800 Subject: [PATCH 08/15] server: gate PD readiness on mode cleanup Signed-off-by: Ryan Leung --- pkg/utils/etcdutil/etcdutil.go | 14 +- server/metrics.go | 8 ++ server/microservice_cleanup.go | 164 ++++++++++++++-------- server/microservice_cleanup_test.go | 107 ++++++++++---- server/server.go | 10 +- tests/integrations/mcs/tso/server_test.go | 101 ++++++++++--- 6 files changed, 299 insertions(+), 105 deletions(-) diff --git a/pkg/utils/etcdutil/etcdutil.go b/pkg/utils/etcdutil/etcdutil.go index 34a54738f5..5592cc5cc2 100644 --- a/pkg/utils/etcdutil/etcdutil.go +++ b/pkg/utils/etcdutil/etcdutil.go @@ -161,7 +161,19 @@ func RemoveEtcdMember(client *clientv3.Client, id uint64) (*clientv3.MemberRemov // EtcdKVGet returns the etcd GetResponse by given key or key prefix func EtcdKVGet(c *clientv3.Client, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) { - ctx, cancel := context.WithTimeout(c.Ctx(), DefaultRequestTimeout) + return EtcdKVGetWithContext(c.Ctx(), c, key, opts...) +} + +// EtcdKVGetWithContext returns the etcd GetResponse by the given key or key +// prefix. The request stops when either ctx is canceled or the default request +// timeout is reached. +func EtcdKVGetWithContext( + ctx context.Context, + c *clientv3.Client, + key string, + opts ...clientv3.OpOption, +) (*clientv3.GetResponse, error) { + ctx, cancel := context.WithTimeout(ctx, DefaultRequestTimeout) defer cancel() start := time.Now() diff --git a/server/metrics.go b/server/metrics.go index f995cf5efa..faa09aebf3 100644 --- a/server/metrics.go +++ b/server/metrics.go @@ -211,6 +211,13 @@ var ( Name: "region_request_cnt", Help: "Counter of region request.", }, []string{"request", "caller_id", "caller_component", "event"}) + microserviceMetadataCleanupCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pd", + Subsystem: "server", + Name: "microservice_metadata_cleanup_total", + Help: "Counter of microservice metadata cleanup events during PD-mode leader preparation.", + }, []string{"outcome"}) ) func init() { @@ -237,4 +244,5 @@ func init() { prometheus.MustRegister(forwardFailCounter) prometheus.MustRegister(forwardTsoDuration) prometheus.MustRegister(regionRequestCounter) + prometheus.MustRegister(microserviceMetadataCleanupCounter) } diff --git a/server/microservice_cleanup.go b/server/microservice_cleanup.go index 7f00f06d59..58a989b06a 100644 --- a/server/microservice_cleanup.go +++ b/server/microservice_cleanup.go @@ -20,6 +20,7 @@ import ( goerrors "errors" "time" + "go.etcd.io/etcd/api/v3/mvccpb" clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" @@ -33,10 +34,12 @@ import ( "github.com/tikv/pd/pkg/storage/kv" "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/keypath" - "github.com/tikv/pd/pkg/utils/logutil" ) -const microserviceMetadataCleanupRetryInterval = 5 * time.Second +const ( + microserviceMetadataCleanupInitialRetryInterval = time.Second + microserviceMetadataCleanupMaxRetryInterval = 30 * time.Second +) var errMicroserviceMetadataCleanupRejected = errors.New("microservice metadata cleanup rejected") @@ -50,42 +53,69 @@ func rejectMicroserviceMetadataCleanup(format string, args ...any) error { return errors.Wrapf(errMicroserviceMetadataCleanupRejected, format, args...) } -func (s *Server) scheduleMicroserviceMetadataCleanup(ctx context.Context) { +// prepareMicroserviceMetadataCleanup makes the supported API-service-to-PD-mode +// transition durable before the PD leader becomes ready. Unsafe metadata states +// are left untouched and do not prevent PD mode from serving. +func (s *Server) prepareMicroserviceMetadataCleanup(ctx context.Context) error { if s.IsKeyspaceGroupEnabled() { - return + return nil } term, ok := s.captureMicroserviceMetadataCleanupTerm() if !ok { - return - } - s.serverLoopWg.Add(1) - go func() { - defer logutil.LogPanic() - defer s.serverLoopWg.Done() - for { - if s.member != nil && !s.member.IsServing() { - return - } - err := s.cleanupMicroserviceMetadataInPDMode(ctx, term) - if err == nil { - return - } - if goerrors.Is(err, errMicroserviceMetadataCleanupRejected) { - log.Warn("cannot safely clean up microservice metadata in PD mode", - errs.ZapError(err)) - return - } - log.Warn("failed to clean up microservice metadata in PD mode, retry later", + return errors.New("cannot capture the PD leadership term for microservice metadata cleanup") + } + + retryInterval := microserviceMetadataCleanupInitialRetryInterval + for { + if !s.isMicroserviceMetadataCleanupTermCurrent(term) { + return errs.ErrEtcdTxnConflict.FastGenByArgs() + } + err := s.cleanupMicroserviceMetadataInPDMode(ctx, term) + if err == nil { + return nil + } + if goerrors.Is(err, errMicroserviceMetadataCleanupRejected) { + microserviceMetadataCleanupCounter.WithLabelValues("rejected").Inc() + log.Warn("cannot safely clean up microservice metadata in PD mode", errs.ZapError(err)) - timer := time.NewTimer(microserviceMetadataCleanupRetryInterval) - select { - case <-ctx.Done(): - timer.Stop() - return - case <-timer.C: + return nil + } + if ctx.Err() != nil { + return ctx.Err() + } + + microserviceMetadataCleanupCounter.WithLabelValues("retry").Inc() + log.Warn("failed to clean up microservice metadata in PD mode, retry later", + zap.Duration("retry-interval", retryInterval), + errs.ZapError(err)) + if err := s.waitMicroserviceMetadataCleanupRetry(ctx, term, retryInterval); err != nil { + return err + } + retryInterval = min(retryInterval*2, microserviceMetadataCleanupMaxRetryInterval) + } +} + +func (s *Server) waitMicroserviceMetadataCleanupRetry( + ctx context.Context, + term microserviceMetadataCleanupTerm, + retryInterval time.Duration, +) error { + retryTimer := time.NewTimer(retryInterval) + defer retryTimer.Stop() + leadershipTicker := time.NewTicker(time.Second) + defer leadershipTicker.Stop() + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-retryTimer.C: + return nil + case <-leadershipTicker.C: + if !s.isMicroserviceMetadataCleanupTermCurrent(term) { + return errs.ErrEtcdTxnConflict.FastGenByArgs() } } - }() + } } func (s *Server) captureMicroserviceMetadataCleanupTerm() (microserviceMetadataCleanupTerm, bool) { @@ -108,6 +138,14 @@ func (s *Server) captureMicroserviceMetadataCleanupTerm() (microserviceMetadataC return term, term.leaderKey != "" && term.leaderValue != "" && term.leaseID != 0 } +func (s *Server) isMicroserviceMetadataCleanupTermCurrent(term microserviceMetadataCleanupTerm) bool { + if s.member == nil || s.member.GetLeadership() == nil || !s.member.GetLeadership().Check() { + return false + } + current, ok := s.captureMicroserviceMetadataCleanupTerm() + return ok && current == term +} + func (s *Server) cleanupMicroserviceMetadataInPDMode( ctx context.Context, term microserviceMetadataCleanupTerm, @@ -119,64 +157,78 @@ func (s *Server) cleanupMicroserviceMetadataInPDMode( // The persisted keyspace group contains the stale TSO member addresses that // block a later switch back to API service mode. Keyspace assignment markers // and the rest of the keyspace group metadata are intentionally left untouched. - needsCleanup, err := s.validateMicroserviceMetadataCleanup() - if err != nil || !needsCleanup { + groupKV, err := s.loadMicroserviceMetadataCleanupCandidate(ctx) + if err != nil { return err } - cleared, err := s.clearDefaultTSOKeyspaceGroupMembers(ctx, term) + if groupKV == nil { + microserviceMetadataCleanupCounter.WithLabelValues("skipped").Inc() + return nil + } + cleared, err := s.clearDefaultTSOKeyspaceGroupMembers(ctx, term, groupKV) if err != nil { return err } if cleared { + microserviceMetadataCleanupCounter.WithLabelValues("success").Inc() log.Info("cleaned up microservice metadata in PD mode", zap.Bool("cleared-default-keyspace-group-members", true)) } return nil } -func (s *Server) validateMicroserviceMetadataCleanup() (bool, error) { +func (s *Server) loadMicroserviceMetadataCleanupCandidate(ctx context.Context) (*mvccpb.KeyValue, error) { // At most two records are needed: the default group and the first unsupported // non-default group, if one exists. - groups, err := s.storage.LoadKeyspaceGroups(constant.DefaultKeyspaceGroupID, 2) + groupKey := keypath.KeyspaceGroupIDPath(constant.DefaultKeyspaceGroupID) + resp, err := etcdutil.EtcdKVGetWithContext( + ctx, + s.client, + groupKey, + clientv3.WithRange(clientv3.GetPrefixRangeEnd(keypath.KeyspaceGroupIDPrefix())), + clientv3.WithLimit(2), + ) if err != nil { - return false, err + return nil, err } - needsCleanup := false - for _, group := range groups { - if group == nil { - continue + var candidate *mvccpb.KeyValue + for _, groupKV := range resp.Kvs { + group := &endpoint.KeyspaceGroup{} + if err := json.Unmarshal(groupKV.Value, group); err != nil { + return nil, rejectMicroserviceMetadataCleanup( + "cannot decode TSO keyspace group metadata when cleaning up PD mode microservice metadata: %v", err) + } + if string(groupKV.Key) != groupKey { + return nil, rejectMicroserviceMetadataCleanup( + "found non-default TSO keyspace group %d when cleaning up PD mode microservice metadata", group.ID) } if group.ID != constant.DefaultKeyspaceGroupID { - return false, rejectMicroserviceMetadataCleanup("found non-default TSO keyspace group %d when cleaning up PD mode microservice metadata", group.ID) + return nil, rejectMicroserviceMetadataCleanup("found TSO keyspace group %d at the default group path when cleaning up PD mode microservice metadata", group.ID) } if group.IsSplitting() { - return false, rejectMicroserviceMetadataCleanup("default TSO keyspace group is splitting when cleaning up PD mode microservice metadata") + return nil, rejectMicroserviceMetadataCleanup("default TSO keyspace group is splitting when cleaning up PD mode microservice metadata") } if group.IsMerging() { - return false, rejectMicroserviceMetadataCleanup("default TSO keyspace group is merging when cleaning up PD mode microservice metadata") + return nil, rejectMicroserviceMetadataCleanup("default TSO keyspace group is merging when cleaning up PD mode microservice metadata") + } + if len(group.Members) > 0 { + candidate = groupKV } - needsCleanup = len(group.Members) > 0 } - return needsCleanup, nil + return candidate, nil } func (s *Server) clearDefaultTSOKeyspaceGroupMembers( ctx context.Context, term microserviceMetadataCleanupTerm, + groupKV *mvccpb.KeyValue, ) (bool, error) { groupKey := keypath.KeyspaceGroupIDPath(constant.DefaultKeyspaceGroupID) - resp, err := etcdutil.EtcdKVGet(s.client, groupKey) - if err != nil { - return false, err - } - if len(resp.Kvs) == 0 { - return false, nil - } - if len(resp.Kvs) != 1 { - return false, errs.ErrEtcdKVGetResponse.FastGenByArgs(resp.Kvs) + if groupKV == nil || string(groupKV.Key) != groupKey { + return false, rejectMicroserviceMetadataCleanup( + "default TSO keyspace group cleanup candidate does not match its persisted path") } - groupKV := resp.Kvs[0] group := &endpoint.KeyspaceGroup{} if err := json.Unmarshal(groupKV.Value, group); err != nil { return false, errs.ErrJSONUnmarshal.Wrap(err).GenWithStackByCause() diff --git a/server/microservice_cleanup_test.go b/server/microservice_cleanup_test.go index 96bc7d6649..2cd6153691 100644 --- a/server/microservice_cleanup_test.go +++ b/server/microservice_cleanup_test.go @@ -63,8 +63,8 @@ func TestCleanupMicroserviceMetadataInPDMode(t *testing.T) { return err } return store.SaveKeyspaceMeta(txn, &keyspacepb.KeyspaceMeta{ - Id: 1, - Name: "keyspace-1", + Keyspace: &keyspacepb.KeyspaceMeta_Id{Id: 1}, + Name: "keyspace-1", Config: map[string]string{ keyspace.TSOKeyspaceGroupIDKey: strconv.FormatUint(uint64(constant.DefaultKeyspaceGroupID), 10), "gc_life_time": "10m", @@ -158,7 +158,7 @@ func TestCleanupMicroserviceMetadataIgnoresLeasedRegistryKeys(t *testing.T) { re.Equal(leaseID, clientv3.LeaseID(resp.Kvs[0].Lease)) } -func TestScheduleMicroserviceMetadataCleanupReturnsImmediately(t *testing.T) { +func TestPrepareMicroserviceMetadataCleanupBlocksUntilCommitted(t *testing.T) { re := require.New(t) ctx := context.Background() _, client, clean := etcdutil.NewTestEtcdCluster(t, 1, nil) @@ -180,16 +180,28 @@ func TestScheduleMicroserviceMetadataCleanupReturnsImmediately(t *testing.T) { })) blocker := enableMicroserviceMetadataCleanupCommitBlocker(t) - start := time.Now() - svr.scheduleMicroserviceMetadataCleanup(ctx) - re.Less(time.Since(start), 200*time.Millisecond) + resultCh := make(chan error, 1) + go func() { + resultCh <- svr.prepareMicroserviceMetadataCleanup(ctx) + }() blocker.wait(t) + select { + case err := <-resultCh: + re.NoError(err) + t.Fatal("PD leader preparation completed before microservice metadata cleanup committed") + default: + } blocker.releaseCleanup() blocker.disable(t) - svr.serverLoopWg.Wait() + select { + case err := <-resultCh: + re.NoError(err) + case <-time.After(10 * time.Second): + t.Fatal("microservice metadata cleanup did not return") + } } -func TestScheduleMicroserviceMetadataCleanupStopsOnRejectedState(t *testing.T) { +func TestPrepareMicroserviceMetadataCleanupStopsOnRejectedState(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -217,18 +229,17 @@ func TestScheduleMicroserviceMetadataCleanupStopsOnRejectedState(t *testing.T) { }) })) - svr.scheduleMicroserviceMetadataCleanup(ctx) - done := make(chan struct{}) + resultCh := make(chan error, 1) go func() { - svr.serverLoopWg.Wait() - close(done) + resultCh <- svr.prepareMicroserviceMetadataCleanup(ctx) }() select { - case <-done: - case <-time.After(time.Second): + case err := <-resultCh: + re.NoError(err) + case <-time.After(10 * time.Second): cancel() - <-done - t.Fatal("cleanup task kept retrying a rejected metadata state") + <-resultCh + t.Fatal("cleanup kept retrying a rejected metadata state") } } @@ -242,15 +253,15 @@ func TestMicroserviceMetadataCleanupTransitionDetection(t *testing.T) { store := storage.NewStorageWithEtcdBackend(client) svr := &Server{storage: store, client: client} - needsCleanup, err := svr.validateMicroserviceMetadataCleanup() + candidate, err := svr.loadMicroserviceMetadataCleanupCandidate(ctx) re.NoError(err) - re.False(needsCleanup) + re.Nil(candidate) _, err = client.Put(ctx, keypath.RegistryPath(mcs.TSOServiceName, "127.0.0.1:3379"), "tso") re.NoError(err) - needsCleanup, err = svr.validateMicroserviceMetadataCleanup() + candidate, err = svr.loadMicroserviceMetadataCleanupCandidate(ctx) re.NoError(err) - re.False(needsCleanup) + re.Nil(candidate) re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { return store.SaveKeyspaceGroup(txn, &endpoint.KeyspaceGroup{ @@ -258,9 +269,9 @@ func TestMicroserviceMetadataCleanupTransitionDetection(t *testing.T) { UserKind: endpoint.Basic.String(), }) })) - needsCleanup, err = svr.validateMicroserviceMetadataCleanup() + candidate, err = svr.loadMicroserviceMetadataCleanupCandidate(ctx) re.NoError(err) - re.False(needsCleanup) + re.Nil(candidate) re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { group, err := store.LoadKeyspaceGroup(txn, constant.DefaultKeyspaceGroupID) @@ -273,9 +284,43 @@ func TestMicroserviceMetadataCleanupTransitionDetection(t *testing.T) { }} return store.SaveKeyspaceGroup(txn, group) })) - needsCleanup, err = svr.validateMicroserviceMetadataCleanup() + candidate, err = svr.loadMicroserviceMetadataCleanupCandidate(ctx) re.NoError(err) - re.True(needsCleanup) + re.NotNil(candidate) +} + +func TestPrepareMicroserviceMetadataCleanupStopsOnContextCancellation(t *testing.T) { + re := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + _, client, clean := etcdutil.NewTestEtcdCluster(t, 1, nil) + t.Cleanup(clean) + keypath.SetClusterID(12356) + t.Cleanup(keypath.ResetClusterID) + + store := storage.NewStorageWithEtcdBackend(client) + svr, _ := newMicroserviceMetadataCleanupTestServer(t, client, store) + re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { + return store.SaveKeyspaceGroup(txn, &endpoint.KeyspaceGroup{ + ID: constant.DefaultKeyspaceGroupID, + UserKind: endpoint.Basic.String(), + Members: []endpoint.KeyspaceGroupMember{{ + Address: "http://127.0.0.1:3379", + Priority: mcs.DefaultKeyspaceGroupReplicaPriority, + }}, + }) + })) + cancel() + + resultCh := make(chan error, 1) + go func() { + resultCh <- svr.prepareMicroserviceMetadataCleanup(ctx) + }() + select { + case err := <-resultCh: + re.ErrorIs(err, context.Canceled) + case <-time.After(10 * time.Second): + t.Fatal("cleanup did not stop after its context was canceled") + } } func TestCleanupMicroserviceMetadataInPDModeRejectsNonDefaultGroup(t *testing.T) { @@ -430,10 +475,13 @@ func TestClearDefaultTSOKeyspaceGroupMembersIsFencedByLeadershipTerm(t *testing. }) })) + candidate, err := svr.loadMicroserviceMetadataCleanupCandidate(ctx) + re.NoError(err) + re.NotNil(candidate) blocker := enableMicroserviceMetadataCleanupCommitBlocker(t) resultCh := make(chan clearDefaultTSOKeyspaceGroupMembersResult, 1) go func() { - cleared, err := svr.clearDefaultTSOKeyspaceGroupMembers(ctx, oldTerm) + cleared, err := svr.clearDefaultTSOKeyspaceGroupMembers(ctx, oldTerm, candidate) resultCh <- clearDefaultTSOKeyspaceGroupMembersResult{cleared: cleared, err: err} }() blocker.wait(t) @@ -485,10 +533,13 @@ func TestClearDefaultTSOKeyspaceGroupMembersPreservesConcurrentUpdate(t *testing }) })) + candidate, err := svr.loadMicroserviceMetadataCleanupCandidate(ctx) + re.NoError(err) + re.NotNil(candidate) blocker := enableMicroserviceMetadataCleanupCommitBlocker(t) resultCh := make(chan clearDefaultTSOKeyspaceGroupMembersResult, 1) go func() { - cleared, err := svr.clearDefaultTSOKeyspaceGroupMembers(ctx, term) + cleared, err := svr.clearDefaultTSOKeyspaceGroupMembers(ctx, term, candidate) resultCh <- clearDefaultTSOKeyspaceGroupMembersResult{cleared: cleared, err: err} }() blocker.wait(t) @@ -613,8 +664,8 @@ func waitClearDefaultTSOKeyspaceGroupMembersResult( func newKeyspaceMetaWithTSOGroup(id uint32, groupID string) *keyspacepb.KeyspaceMeta { return &keyspacepb.KeyspaceMeta{ - Id: id, - Name: "keyspace-" + strconv.FormatUint(uint64(id), 10), + Keyspace: &keyspacepb.KeyspaceMeta_Id{Id: id}, + Name: "keyspace-" + strconv.FormatUint(uint64(id), 10), Config: map[string]string{ keyspace.TSOKeyspaceGroupIDKey: groupID, }, diff --git a/server/server.go b/server/server.go index de4a1a7f57..ea5697e46a 100644 --- a/server/server.go +++ b/server/server.go @@ -2049,6 +2049,15 @@ func (s *Server) campaignLeader() { } rebaseDuration := time.Since(rebaseStart) log.Info("sync id from etcd completed", zap.Duration("cost", rebaseDuration)) + cleanupStart := time.Now() + if err := s.prepareMicroserviceMetadataCleanup(ctx); err != nil { + log.Warn("failed to prepare microservice metadata before serving as PD leader", + errs.ZapError(err), + zap.Duration("cost", time.Since(cleanupStart))) + return + } + cleanupDuration := time.Since(cleanupStart) + log.Info("prepare microservice metadata completed", zap.Duration("cost", cleanupDuration)) // PromoteSelf to accept the remaining service, such as GetStore, GetRegion. enableLeaderStart := time.Now() s.member.PromoteSelf() @@ -2068,7 +2077,6 @@ func (s *Server) campaignLeader() { zap.String("leader-name", s.Name()), zap.Duration("total-cost", totalDuration), zap.Duration("cost", enableLeaderDuration)) - s.scheduleMicroserviceMetadataCleanup(ctx) leaderTicker := time.NewTicker(mcs.LeaderTickInterval) defer leaderTicker.Stop() diff --git a/tests/integrations/mcs/tso/server_test.go b/tests/integrations/mcs/tso/server_test.go index 7a1bd67e37..31f3e129e8 100644 --- a/tests/integrations/mcs/tso/server_test.go +++ b/tests/integrations/mcs/tso/server_test.go @@ -48,6 +48,7 @@ import ( tsoapi "github.com/tikv/pd/pkg/mcs/tso/server/apis/v1" mcs "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/storage/endpoint" + "github.com/tikv/pd/pkg/storage/kv" "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/tempurl" "github.com/tikv/pd/pkg/utils/testutil" @@ -798,9 +799,25 @@ func TestPDModeSwitchBetweenMicroserviceAndMonolithMultipleTimes(t *testing.T) { }) re.NoError(err) userKeyspaceID := userKeyspace.GetId() + cleanupReached := make(chan struct{}) + cleanupRelease := make(chan struct{}) + var cleanupReachedOnce, cleanupReleaseOnce sync.Once + const cleanupBlockerName = "github.com/tikv/pd/server/beforeClearDefaultTSOKeyspaceGroupMembersCommit" + re.NoError(failpoint.EnableCall(cleanupBlockerName, func() { + cleanupReachedOnce.Do(func() { + close(cleanupReached) + }) + <-cleanupRelease + })) + t.Cleanup(func() { + cleanupReleaseOnce.Do(func() { + close(cleanupRelease) + }) + re.NoError(failpoint.Disable(cleanupBlockerName)) + }) const switchRounds = 2 - for range switchRounds { + for round := range switchRounds { waitDefaultKeyspaceGroupReady(re, pdServer, userKeyspaceID) tsoCluster, err := tests.NewTestTSOCluster(ctx, 1, pdServer.GetAddr()) re.NoError(err) @@ -809,13 +826,35 @@ func TestPDModeSwitchBetweenMicroserviceAndMonolithMultipleTimes(t *testing.T) { checkKeyspaceTSOAvailable(ctx, re, pdServer, userKeyspaceName) waitDefaultKeyspaceGroupMembers(re, pdServer, tsoCluster.GetKeyspaceGroupMember()) tsoCluster.Destroy() + seedDefaultKeyspaceGroupMembers(ctx, re, pdServer, []endpoint.KeyspaceGroupMember{{ + Address: "http://127.0.0.1:1", + Priority: mcs.DefaultKeyspaceGroupReplicaPriority, + }}) - pdServer = restartPDForServiceMode(ctx, re, tc, pdServer, nil) + if round == 0 { + pdServer, err = startPDForServiceMode(ctx, tc, pdServer, nil) + re.NoError(err) + select { + case <-cleanupReached: + case <-time.After(10 * time.Second): + t.Fatal("normal-mode PD did not reach the cleanup commit hook") + } + + readyBeforeCleanup := pdServer.GetServer().GetMember().IsServing() + cleanupReleaseOnce.Do(func() { + close(cleanupRelease) + }) + re.False(readyBeforeCleanup, "normal-mode PD became ready before stale metadata was cleaned") + re.True(pdServer.WaitLeader()) + } else { + pdServer, err = restartPDForServiceMode(ctx, tc, pdServer, nil) + re.NoError(err) + } checkPDTSOAvailable(ctx, re, pdServer) checkKeyspaceTSOAvailable(ctx, re, pdServer, userKeyspaceName) - waitDefaultKeyspaceGroupCleaned(re, pdServer, userKeyspaceID) - pdServer = restartPDForServiceMode(ctx, re, tc, pdServer, []string{mcs.PDServiceName}) + pdServer, err = restartPDForServiceMode(ctx, tc, pdServer, []string{mcs.PDServiceName}) + re.NoError(err) } waitDefaultKeyspaceGroupReady(re, pdServer, userKeyspaceID) @@ -830,21 +869,41 @@ func TestPDModeSwitchBetweenMicroserviceAndMonolithMultipleTimes(t *testing.T) { func restartPDForServiceMode( ctx context.Context, - re *require.Assertions, tc *tests.TestCluster, oldServer *tests.TestServer, services []string, -) *tests.TestServer { +) (*tests.TestServer, error) { + newServer, err := startPDForServiceMode(ctx, tc, oldServer, services) + if err != nil { + return nil, err + } + if !newServer.WaitLeader() { + return nil, fmt.Errorf("PD server %s did not become ready", newServer.GetConfig().Name) + } + return newServer, nil +} + +func startPDForServiceMode( + ctx context.Context, + tc *tests.TestCluster, + oldServer *tests.TestServer, + services []string, +) (*tests.TestServer, error) { cfg := oldServer.GetConfig() serverName := cfg.Name - re.NoError(oldServer.Stop()) + if err := oldServer.Stop(); err != nil { + return nil, err + } newServer, err := tests.NewTestServer(ctx, cfg, services) - re.NoError(err) + if err != nil { + return nil, err + } tc.GetServers()[serverName] = newServer - re.NoError(newServer.Run()) - re.True(newServer.WaitLeader()) - return newServer + if err := newServer.Run(); err != nil { + return nil, err + } + return newServer, nil } func waitDefaultKeyspaceGroupReady( @@ -858,17 +917,21 @@ func waitDefaultKeyspaceGroupReady( }, testutil.WithWaitFor(10*time.Second), testutil.WithTickInterval(100*time.Millisecond)) } -func waitDefaultKeyspaceGroupCleaned( +func seedDefaultKeyspaceGroupMembers( + ctx context.Context, re *require.Assertions, pdServer *tests.TestServer, - userKeyspaceID uint32, + members []endpoint.KeyspaceGroupMember, ) { - testutil.Eventually(re, func() bool { - group, err := loadDefaultKeyspaceGroup(pdServer) - return err == nil && - len(group.Members) == 0 && - slices.Contains(group.Keyspaces, userKeyspaceID) - }, testutil.WithWaitFor(30*time.Second), testutil.WithTickInterval(100*time.Millisecond)) + store := pdServer.GetServer().GetStorage() + re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { + group, err := store.LoadKeyspaceGroup(txn, constant.DefaultKeyspaceGroupID) + if err != nil { + return err + } + group.Members = members + return store.SaveKeyspaceGroup(txn, group) + })) } func waitDefaultKeyspaceGroupMembers( From 0f9a0b89e940682fb6dc38e7ede96439aa98bfeb Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Wed, 12 Aug 2026 15:23:31 +0800 Subject: [PATCH 09/15] server: avoid group scan when cleanup is unnecessary Signed-off-by: Ryan Leung --- server/microservice_cleanup.go | 65 +++++++++++++++++------------ server/microservice_cleanup_test.go | 32 ++++++++++++++ 2 files changed, 71 insertions(+), 26 deletions(-) diff --git a/server/microservice_cleanup.go b/server/microservice_cleanup.go index 58a989b06a..9482b882c0 100644 --- a/server/microservice_cleanup.go +++ b/server/microservice_cleanup.go @@ -178,44 +178,57 @@ func (s *Server) cleanupMicroserviceMetadataInPDMode( } func (s *Server) loadMicroserviceMetadataCleanupCandidate(ctx context.Context) (*mvccpb.KeyValue, error) { - // At most two records are needed: the default group and the first unsupported - // non-default group, if one exists. + // Read the default group first so normal PD-mode leader campaigns do not scan + // all keyspace groups. Non-default groups only matter when cleanup would mutate + // the default group. groupKey := keypath.KeyspaceGroupIDPath(constant.DefaultKeyspaceGroupID) - resp, err := etcdutil.EtcdKVGetWithContext( + resp, err := etcdutil.EtcdKVGetWithContext(ctx, s.client, groupKey) + if err != nil { + return nil, err + } + if len(resp.Kvs) == 0 { + return nil, nil + } + + groupKV := resp.Kvs[0] + group := &endpoint.KeyspaceGroup{} + if err := json.Unmarshal(groupKV.Value, group); err != nil { + return nil, rejectMicroserviceMetadataCleanup( + "cannot decode TSO keyspace group metadata when cleaning up PD mode microservice metadata: %v", err) + } + if group.ID != constant.DefaultKeyspaceGroupID { + return nil, rejectMicroserviceMetadataCleanup("found TSO keyspace group %d at the default group path when cleaning up PD mode microservice metadata", group.ID) + } + if group.IsSplitting() { + return nil, rejectMicroserviceMetadataCleanup("default TSO keyspace group is splitting when cleaning up PD mode microservice metadata") + } + if group.IsMerging() { + return nil, rejectMicroserviceMetadataCleanup("default TSO keyspace group is merging when cleaning up PD mode microservice metadata") + } + if len(group.Members) == 0 { + return nil, nil + } + + resp, err = etcdutil.EtcdKVGetWithContext( ctx, s.client, - groupKey, + keypath.KeyspaceGroupIDPath(constant.DefaultKeyspaceGroupID+1), clientv3.WithRange(clientv3.GetPrefixRangeEnd(keypath.KeyspaceGroupIDPrefix())), - clientv3.WithLimit(2), + clientv3.WithLimit(1), ) if err != nil { return nil, err } - var candidate *mvccpb.KeyValue - for _, groupKV := range resp.Kvs { - group := &endpoint.KeyspaceGroup{} - if err := json.Unmarshal(groupKV.Value, group); err != nil { + if len(resp.Kvs) > 0 { + nonDefaultGroup := &endpoint.KeyspaceGroup{} + if err := json.Unmarshal(resp.Kvs[0].Value, nonDefaultGroup); err != nil { return nil, rejectMicroserviceMetadataCleanup( "cannot decode TSO keyspace group metadata when cleaning up PD mode microservice metadata: %v", err) } - if string(groupKV.Key) != groupKey { - return nil, rejectMicroserviceMetadataCleanup( - "found non-default TSO keyspace group %d when cleaning up PD mode microservice metadata", group.ID) - } - if group.ID != constant.DefaultKeyspaceGroupID { - return nil, rejectMicroserviceMetadataCleanup("found TSO keyspace group %d at the default group path when cleaning up PD mode microservice metadata", group.ID) - } - if group.IsSplitting() { - return nil, rejectMicroserviceMetadataCleanup("default TSO keyspace group is splitting when cleaning up PD mode microservice metadata") - } - if group.IsMerging() { - return nil, rejectMicroserviceMetadataCleanup("default TSO keyspace group is merging when cleaning up PD mode microservice metadata") - } - if len(group.Members) > 0 { - candidate = groupKV - } + return nil, rejectMicroserviceMetadataCleanup( + "found non-default TSO keyspace group %d when cleaning up PD mode microservice metadata", nonDefaultGroup.ID) } - return candidate, nil + return groupKV, nil } func (s *Server) clearDefaultTSOKeyspaceGroupMembers( diff --git a/server/microservice_cleanup_test.go b/server/microservice_cleanup_test.go index 2cd6153691..fe6f5b3e11 100644 --- a/server/microservice_cleanup_test.go +++ b/server/microservice_cleanup_test.go @@ -289,6 +289,38 @@ func TestMicroserviceMetadataCleanupTransitionDetection(t *testing.T) { re.NotNil(candidate) } +func TestMicroserviceMetadataCleanupSkipsNonDefaultGroupsWithoutStaleMembers(t *testing.T) { + re := require.New(t) + ctx := context.Background() + _, client, clean := etcdutil.NewTestEtcdCluster(t, 1, nil) + t.Cleanup(clean) + keypath.SetClusterID(12357) + t.Cleanup(keypath.ResetClusterID) + + store := storage.NewStorageWithEtcdBackend(client) + svr := &Server{storage: store, client: client} + re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { + return store.SaveKeyspaceGroup(txn, &endpoint.KeyspaceGroup{ + ID: 1, + UserKind: endpoint.Standard.String(), + }) + })) + + candidate, err := svr.loadMicroserviceMetadataCleanupCandidate(ctx) + re.NoError(err) + re.Nil(candidate) + + re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { + return store.SaveKeyspaceGroup(txn, &endpoint.KeyspaceGroup{ + ID: constant.DefaultKeyspaceGroupID, + UserKind: endpoint.Basic.String(), + }) + })) + candidate, err = svr.loadMicroserviceMetadataCleanupCandidate(ctx) + re.NoError(err) + re.Nil(candidate) +} + func TestPrepareMicroserviceMetadataCleanupStopsOnContextCancellation(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) From 44b056eed5070d15f180796a157254921574a216 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Wed, 12 Aug 2026 16:28:10 +0800 Subject: [PATCH 10/15] server: run mode cleanup asynchronously Signed-off-by: Ryan Leung --- server/microservice_cleanup.go | 28 +++++++++++++----- server/microservice_cleanup_test.go | 36 ++++++++--------------- server/server.go | 10 +------ tests/integrations/mcs/tso/server_test.go | 29 +++++++++++------- 4 files changed, 54 insertions(+), 49 deletions(-) diff --git a/server/microservice_cleanup.go b/server/microservice_cleanup.go index 9482b882c0..de1ded0851 100644 --- a/server/microservice_cleanup.go +++ b/server/microservice_cleanup.go @@ -34,6 +34,7 @@ import ( "github.com/tikv/pd/pkg/storage/kv" "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/keypath" + "github.com/tikv/pd/pkg/utils/logutil" ) const ( @@ -53,18 +54,31 @@ func rejectMicroserviceMetadataCleanup(format string, args ...any) error { return errors.Wrapf(errMicroserviceMetadataCleanupRejected, format, args...) } -// prepareMicroserviceMetadataCleanup makes the supported API-service-to-PD-mode -// transition durable before the PD leader becomes ready. Unsafe metadata states -// are left untouched and do not prevent PD mode from serving. -func (s *Server) prepareMicroserviceMetadataCleanup(ctx context.Context) error { +// scheduleMicroserviceMetadataCleanup cleans up supported API-service metadata +// in the background after the PD leader starts serving. +func (s *Server) scheduleMicroserviceMetadataCleanup(ctx context.Context) { if s.IsKeyspaceGroupEnabled() { - return nil + return } term, ok := s.captureMicroserviceMetadataCleanupTerm() if !ok { - return errors.New("cannot capture the PD leadership term for microservice metadata cleanup") - } + log.Warn("cannot capture the PD leadership term for microservice metadata cleanup") + return + } + s.serverLoopWg.Add(1) + go func() { + defer logutil.LogPanic() + defer s.serverLoopWg.Done() + if err := s.runMicroserviceMetadataCleanup(ctx, term); err != nil && ctx.Err() == nil { + log.Warn("microservice metadata cleanup stopped before completion", errs.ZapError(err)) + } + }() +} +func (s *Server) runMicroserviceMetadataCleanup( + ctx context.Context, + term microserviceMetadataCleanupTerm, +) error { retryInterval := microserviceMetadataCleanupInitialRetryInterval for { if !s.isMicroserviceMetadataCleanupTermCurrent(term) { diff --git a/server/microservice_cleanup_test.go b/server/microservice_cleanup_test.go index fe6f5b3e11..56ebc719bc 100644 --- a/server/microservice_cleanup_test.go +++ b/server/microservice_cleanup_test.go @@ -158,7 +158,7 @@ func TestCleanupMicroserviceMetadataIgnoresLeasedRegistryKeys(t *testing.T) { re.Equal(leaseID, clientv3.LeaseID(resp.Kvs[0].Lease)) } -func TestPrepareMicroserviceMetadataCleanupBlocksUntilCommitted(t *testing.T) { +func TestScheduleMicroserviceMetadataCleanupDoesNotBlockServing(t *testing.T) { re := require.New(t) ctx := context.Background() _, client, clean := etcdutil.NewTestEtcdCluster(t, 1, nil) @@ -180,28 +180,18 @@ func TestPrepareMicroserviceMetadataCleanupBlocksUntilCommitted(t *testing.T) { })) blocker := enableMicroserviceMetadataCleanupCommitBlocker(t) - resultCh := make(chan error, 1) - go func() { - resultCh <- svr.prepareMicroserviceMetadataCleanup(ctx) - }() + svr.scheduleMicroserviceMetadataCleanup(ctx) blocker.wait(t) - select { - case err := <-resultCh: - re.NoError(err) - t.Fatal("PD leader preparation completed before microservice metadata cleanup committed") - default: - } + re.True(svr.member.IsServing()) blocker.releaseCleanup() blocker.disable(t) - select { - case err := <-resultCh: - re.NoError(err) - case <-time.After(10 * time.Second): - t.Fatal("microservice metadata cleanup did not return") - } + re.Eventually(func() bool { + groups, err := store.LoadKeyspaceGroups(constant.DefaultKeyspaceGroupID, 1) + return err == nil && len(groups) == 1 && len(groups[0].Members) == 0 + }, 10*time.Second, 100*time.Millisecond) } -func TestPrepareMicroserviceMetadataCleanupStopsOnRejectedState(t *testing.T) { +func TestRunMicroserviceMetadataCleanupStopsOnRejectedState(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -211,7 +201,7 @@ func TestPrepareMicroserviceMetadataCleanupStopsOnRejectedState(t *testing.T) { defer keypath.ResetClusterID() store := storage.NewStorageWithEtcdBackend(client) - svr, _ := newMicroserviceMetadataCleanupTestServer(t, client, store) + svr, term := newMicroserviceMetadataCleanupTestServer(t, client, store) re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { if err := store.SaveKeyspaceGroup(txn, &endpoint.KeyspaceGroup{ ID: constant.DefaultKeyspaceGroupID, @@ -231,7 +221,7 @@ func TestPrepareMicroserviceMetadataCleanupStopsOnRejectedState(t *testing.T) { resultCh := make(chan error, 1) go func() { - resultCh <- svr.prepareMicroserviceMetadataCleanup(ctx) + resultCh <- svr.runMicroserviceMetadataCleanup(ctx, term) }() select { case err := <-resultCh: @@ -321,7 +311,7 @@ func TestMicroserviceMetadataCleanupSkipsNonDefaultGroupsWithoutStaleMembers(t * re.Nil(candidate) } -func TestPrepareMicroserviceMetadataCleanupStopsOnContextCancellation(t *testing.T) { +func TestRunMicroserviceMetadataCleanupStopsOnContextCancellation(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) _, client, clean := etcdutil.NewTestEtcdCluster(t, 1, nil) @@ -330,7 +320,7 @@ func TestPrepareMicroserviceMetadataCleanupStopsOnContextCancellation(t *testing t.Cleanup(keypath.ResetClusterID) store := storage.NewStorageWithEtcdBackend(client) - svr, _ := newMicroserviceMetadataCleanupTestServer(t, client, store) + svr, term := newMicroserviceMetadataCleanupTestServer(t, client, store) re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { return store.SaveKeyspaceGroup(txn, &endpoint.KeyspaceGroup{ ID: constant.DefaultKeyspaceGroupID, @@ -345,7 +335,7 @@ func TestPrepareMicroserviceMetadataCleanupStopsOnContextCancellation(t *testing resultCh := make(chan error, 1) go func() { - resultCh <- svr.prepareMicroserviceMetadataCleanup(ctx) + resultCh <- svr.runMicroserviceMetadataCleanup(ctx, term) }() select { case err := <-resultCh: diff --git a/server/server.go b/server/server.go index ea5697e46a..de4a1a7f57 100644 --- a/server/server.go +++ b/server/server.go @@ -2049,15 +2049,6 @@ func (s *Server) campaignLeader() { } rebaseDuration := time.Since(rebaseStart) log.Info("sync id from etcd completed", zap.Duration("cost", rebaseDuration)) - cleanupStart := time.Now() - if err := s.prepareMicroserviceMetadataCleanup(ctx); err != nil { - log.Warn("failed to prepare microservice metadata before serving as PD leader", - errs.ZapError(err), - zap.Duration("cost", time.Since(cleanupStart))) - return - } - cleanupDuration := time.Since(cleanupStart) - log.Info("prepare microservice metadata completed", zap.Duration("cost", cleanupDuration)) // PromoteSelf to accept the remaining service, such as GetStore, GetRegion. enableLeaderStart := time.Now() s.member.PromoteSelf() @@ -2077,6 +2068,7 @@ func (s *Server) campaignLeader() { zap.String("leader-name", s.Name()), zap.Duration("total-cost", totalDuration), zap.Duration("cost", enableLeaderDuration)) + s.scheduleMicroserviceMetadataCleanup(ctx) leaderTicker := time.NewTicker(mcs.LeaderTickInterval) defer leaderTicker.Stop() diff --git a/tests/integrations/mcs/tso/server_test.go b/tests/integrations/mcs/tso/server_test.go index 31f3e129e8..6734dc4d58 100644 --- a/tests/integrations/mcs/tso/server_test.go +++ b/tests/integrations/mcs/tso/server_test.go @@ -773,6 +773,8 @@ func TestTSOServiceSwitch(t *testing.T) { re.NoError(failpoint.Disable("github.com/tikv/pd/client/servicediscovery/fastUpdateServiceMode")) } +const modeSwitchUserKeyspaceName = "mode_switch_user_ks" + func TestPDModeSwitchBetweenMicroserviceAndMonolithMultipleTimes(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) @@ -793,9 +795,8 @@ func TestPDModeSwitchBetweenMicroserviceAndMonolithMultipleTimes(t *testing.T) { pdServer := tc.GetServer(tc.WaitLeader()) re.NotNil(pdServer) re.NoError(pdServer.BootstrapCluster()) - const userKeyspaceName = "mode_switch_user_ks" userKeyspace, err := pdServer.GetServer().GetKeyspaceManager().CreateKeyspace(&keyspace.CreateKeyspaceRequest{ - Name: userKeyspaceName, + Name: modeSwitchUserKeyspaceName, }) re.NoError(err) userKeyspaceID := userKeyspace.GetId() @@ -823,7 +824,7 @@ func TestPDModeSwitchBetweenMicroserviceAndMonolithMultipleTimes(t *testing.T) { re.NoError(err) waitTSOServiceReady(re, tsoCluster) checkMicroserviceTSOAvailable(ctx, re, pdServer) - checkKeyspaceTSOAvailable(ctx, re, pdServer, userKeyspaceName) + checkKeyspaceTSOAvailable(ctx, re, pdServer) waitDefaultKeyspaceGroupMembers(re, pdServer, tsoCluster.GetKeyspaceGroupMember()) tsoCluster.Destroy() seedDefaultKeyspaceGroupMembers(ctx, re, pdServer, []endpoint.KeyspaceGroupMember{{ @@ -840,18 +841,20 @@ func TestPDModeSwitchBetweenMicroserviceAndMonolithMultipleTimes(t *testing.T) { t.Fatal("normal-mode PD did not reach the cleanup commit hook") } - readyBeforeCleanup := pdServer.GetServer().GetMember().IsServing() + re.True(pdServer.WaitLeader(), "normal-mode PD did not become ready while mode cleanup was pending") + checkPDLeaderAPIAvailable(ctx, re, pdServer) + checkPDTSOAvailable(ctx, re, pdServer) + checkKeyspaceTSOAvailable(ctx, re, pdServer) cleanupReleaseOnce.Do(func() { close(cleanupRelease) }) - re.False(readyBeforeCleanup, "normal-mode PD became ready before stale metadata was cleaned") - re.True(pdServer.WaitLeader()) } else { pdServer, err = restartPDForServiceMode(ctx, tc, pdServer, nil) re.NoError(err) } checkPDTSOAvailable(ctx, re, pdServer) - checkKeyspaceTSOAvailable(ctx, re, pdServer, userKeyspaceName) + checkKeyspaceTSOAvailable(ctx, re, pdServer) + waitDefaultKeyspaceGroupMembers(re, pdServer, nil) pdServer, err = restartPDForServiceMode(ctx, tc, pdServer, []string{mcs.PDServiceName}) re.NoError(err) @@ -863,7 +866,7 @@ func TestPDModeSwitchBetweenMicroserviceAndMonolithMultipleTimes(t *testing.T) { defer tsoCluster.Destroy() waitTSOServiceReady(re, tsoCluster) checkMicroserviceTSOAvailable(ctx, re, pdServer) - checkKeyspaceTSOAvailable(ctx, re, pdServer, userKeyspaceName) + checkKeyspaceTSOAvailable(ctx, re, pdServer) waitDefaultKeyspaceGroupMembers(re, pdServer, tsoCluster.GetKeyspaceGroupMember()) } @@ -986,6 +989,13 @@ func checkPDTSOAvailable(ctx context.Context, re *require.Assertions, pdServer * re.NotZero(tsoutil.ComposeTS(physical, logical)) } +func checkPDLeaderAPIAvailable(ctx context.Context, re *require.Assertions, pdServer *tests.TestServer) { + cli := utils.SetupClientWithAPIContext(ctx, re, pd.NewAPIContextV1(), []string{pdServer.GetAddr()}) + defer cli.Close() + _, err := cli.GetAllStores(ctx) + re.NoError(err) +} + func checkMicroserviceTSOAvailable(ctx context.Context, re *require.Assertions, pdServer *tests.TestServer) { addr := strings.TrimPrefix(pdServer.GetAddr(), "http://") conn, err := grpc.Dial(addr, grpc.WithTransportCredentials(insecure.NewCredentials())) //nolint:staticcheck @@ -1004,12 +1014,11 @@ func checkKeyspaceTSOAvailable( ctx context.Context, re *require.Assertions, pdServer *tests.TestServer, - keyspaceName string, ) { cli := utils.SetupClientWithAPIContext( ctx, re, - pd.NewAPIContextV2(keyspaceName), + pd.NewAPIContextV2(modeSwitchUserKeyspaceName), []string{pdServer.GetAddr()}, ) defer cli.Close() From 25b33c3d6ef48991eaeab60e8796fecb02df01a3 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Fri, 14 Aug 2026 14:13:09 +0800 Subject: [PATCH 11/15] server: document best-effort mode cleanup Signed-off-by: Ryan Leung --- server/metrics.go | 2 +- server/microservice_cleanup.go | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/server/metrics.go b/server/metrics.go index faa09aebf3..08b3dfeda1 100644 --- a/server/metrics.go +++ b/server/metrics.go @@ -216,7 +216,7 @@ var ( Namespace: "pd", Subsystem: "server", Name: "microservice_metadata_cleanup_total", - Help: "Counter of microservice metadata cleanup events during PD-mode leader preparation.", + Help: "Counter of best-effort microservice metadata cleanup events in PD mode.", }, []string{"outcome"}) ) diff --git a/server/microservice_cleanup.go b/server/microservice_cleanup.go index de1ded0851..170cef21ed 100644 --- a/server/microservice_cleanup.go +++ b/server/microservice_cleanup.go @@ -54,8 +54,9 @@ func rejectMicroserviceMetadataCleanup(format string, args ...any) error { return errors.Wrapf(errMicroserviceMetadataCleanupRejected, format, args...) } -// scheduleMicroserviceMetadataCleanup cleans up supported API-service metadata -// in the background after the PD leader starts serving. +// scheduleMicroserviceMetadataCleanup starts best-effort cleanup of supported +// API-service metadata after the PD leader starts serving. It does not gate +// leader readiness and is scoped to the current leadership term. func (s *Server) scheduleMicroserviceMetadataCleanup(ctx context.Context) { if s.IsKeyspaceGroupEnabled() { return From 4a5a05905bb44ab4f59d7ea8f2dc24942a0e6ddc Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Fri, 14 Aug 2026 14:18:25 +0800 Subject: [PATCH 12/15] server: remove mode cleanup metric Signed-off-by: Ryan Leung --- server/metrics.go | 8 -------- server/microservice_cleanup.go | 4 ---- 2 files changed, 12 deletions(-) diff --git a/server/metrics.go b/server/metrics.go index 08b3dfeda1..f995cf5efa 100644 --- a/server/metrics.go +++ b/server/metrics.go @@ -211,13 +211,6 @@ var ( Name: "region_request_cnt", Help: "Counter of region request.", }, []string{"request", "caller_id", "caller_component", "event"}) - microserviceMetadataCleanupCounter = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: "pd", - Subsystem: "server", - Name: "microservice_metadata_cleanup_total", - Help: "Counter of best-effort microservice metadata cleanup events in PD mode.", - }, []string{"outcome"}) ) func init() { @@ -244,5 +237,4 @@ func init() { prometheus.MustRegister(forwardFailCounter) prometheus.MustRegister(forwardTsoDuration) prometheus.MustRegister(regionRequestCounter) - prometheus.MustRegister(microserviceMetadataCleanupCounter) } diff --git a/server/microservice_cleanup.go b/server/microservice_cleanup.go index 170cef21ed..5deae19b5b 100644 --- a/server/microservice_cleanup.go +++ b/server/microservice_cleanup.go @@ -90,7 +90,6 @@ func (s *Server) runMicroserviceMetadataCleanup( return nil } if goerrors.Is(err, errMicroserviceMetadataCleanupRejected) { - microserviceMetadataCleanupCounter.WithLabelValues("rejected").Inc() log.Warn("cannot safely clean up microservice metadata in PD mode", errs.ZapError(err)) return nil @@ -99,7 +98,6 @@ func (s *Server) runMicroserviceMetadataCleanup( return ctx.Err() } - microserviceMetadataCleanupCounter.WithLabelValues("retry").Inc() log.Warn("failed to clean up microservice metadata in PD mode, retry later", zap.Duration("retry-interval", retryInterval), errs.ZapError(err)) @@ -177,7 +175,6 @@ func (s *Server) cleanupMicroserviceMetadataInPDMode( return err } if groupKV == nil { - microserviceMetadataCleanupCounter.WithLabelValues("skipped").Inc() return nil } cleared, err := s.clearDefaultTSOKeyspaceGroupMembers(ctx, term, groupKV) @@ -185,7 +182,6 @@ func (s *Server) cleanupMicroserviceMetadataInPDMode( return err } if cleared { - microserviceMetadataCleanupCounter.WithLabelValues("success").Inc() log.Info("cleaned up microservice metadata in PD mode", zap.Bool("cleared-default-keyspace-group-members", true)) } From 1c6599677978b4280a80bd8549ad3678d94acd24 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Fri, 14 Aug 2026 14:37:19 +0800 Subject: [PATCH 13/15] server: simplify mode cleanup Signed-off-by: Ryan Leung --- server/microservice_cleanup.go | 119 ++-------- server/microservice_cleanup_test.go | 269 ++++------------------ tests/integrations/mcs/tso/server_test.go | 2 +- 3 files changed, 68 insertions(+), 322 deletions(-) diff --git a/server/microservice_cleanup.go b/server/microservice_cleanup.go index 5deae19b5b..fb28f3a7ae 100644 --- a/server/microservice_cleanup.go +++ b/server/microservice_cleanup.go @@ -20,7 +20,6 @@ import ( goerrors "errors" "time" - "go.etcd.io/etcd/api/v3/mvccpb" clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" @@ -82,9 +81,6 @@ func (s *Server) runMicroserviceMetadataCleanup( ) error { retryInterval := microserviceMetadataCleanupInitialRetryInterval for { - if !s.isMicroserviceMetadataCleanupTermCurrent(term) { - return errs.ErrEtcdTxnConflict.FastGenByArgs() - } err := s.cleanupMicroserviceMetadataInPDMode(ctx, term) if err == nil { return nil @@ -101,33 +97,14 @@ func (s *Server) runMicroserviceMetadataCleanup( log.Warn("failed to clean up microservice metadata in PD mode, retry later", zap.Duration("retry-interval", retryInterval), errs.ZapError(err)) - if err := s.waitMicroserviceMetadataCleanupRetry(ctx, term, retryInterval); err != nil { - return err - } - retryInterval = min(retryInterval*2, microserviceMetadataCleanupMaxRetryInterval) - } -} - -func (s *Server) waitMicroserviceMetadataCleanupRetry( - ctx context.Context, - term microserviceMetadataCleanupTerm, - retryInterval time.Duration, -) error { - retryTimer := time.NewTimer(retryInterval) - defer retryTimer.Stop() - leadershipTicker := time.NewTicker(time.Second) - defer leadershipTicker.Stop() - for { + retryTimer := time.NewTimer(retryInterval) select { case <-ctx.Done(): + retryTimer.Stop() return ctx.Err() case <-retryTimer.C: - return nil - case <-leadershipTicker.C: - if !s.isMicroserviceMetadataCleanupTermCurrent(term) { - return errs.ErrEtcdTxnConflict.FastGenByArgs() - } } + retryInterval = min(retryInterval*2, microserviceMetadataCleanupMaxRetryInterval) } } @@ -151,14 +128,6 @@ func (s *Server) captureMicroserviceMetadataCleanupTerm() (microserviceMetadataC return term, term.leaderKey != "" && term.leaderValue != "" && term.leaseID != 0 } -func (s *Server) isMicroserviceMetadataCleanupTermCurrent(term microserviceMetadataCleanupTerm) bool { - if s.member == nil || s.member.GetLeadership() == nil || !s.member.GetLeadership().Check() { - return false - } - current, ok := s.captureMicroserviceMetadataCleanupTerm() - return ok && current == term -} - func (s *Server) cleanupMicroserviceMetadataInPDMode( ctx context.Context, term microserviceMetadataCleanupTerm, @@ -170,54 +139,35 @@ func (s *Server) cleanupMicroserviceMetadataInPDMode( // The persisted keyspace group contains the stale TSO member addresses that // block a later switch back to API service mode. Keyspace assignment markers // and the rest of the keyspace group metadata are intentionally left untouched. - groupKV, err := s.loadMicroserviceMetadataCleanupCandidate(ctx) - if err != nil { - return err - } - if groupKV == nil { - return nil - } - cleared, err := s.clearDefaultTSOKeyspaceGroupMembers(ctx, term, groupKV) - if err != nil { - return err - } - if cleared { - log.Info("cleaned up microservice metadata in PD mode", - zap.Bool("cleared-default-keyspace-group-members", true)) - } - return nil -} - -func (s *Server) loadMicroserviceMetadataCleanupCandidate(ctx context.Context) (*mvccpb.KeyValue, error) { // Read the default group first so normal PD-mode leader campaigns do not scan // all keyspace groups. Non-default groups only matter when cleanup would mutate // the default group. groupKey := keypath.KeyspaceGroupIDPath(constant.DefaultKeyspaceGroupID) resp, err := etcdutil.EtcdKVGetWithContext(ctx, s.client, groupKey) if err != nil { - return nil, err + return err } if len(resp.Kvs) == 0 { - return nil, nil + return nil } groupKV := resp.Kvs[0] group := &endpoint.KeyspaceGroup{} if err := json.Unmarshal(groupKV.Value, group); err != nil { - return nil, rejectMicroserviceMetadataCleanup( + return rejectMicroserviceMetadataCleanup( "cannot decode TSO keyspace group metadata when cleaning up PD mode microservice metadata: %v", err) } if group.ID != constant.DefaultKeyspaceGroupID { - return nil, rejectMicroserviceMetadataCleanup("found TSO keyspace group %d at the default group path when cleaning up PD mode microservice metadata", group.ID) + return rejectMicroserviceMetadataCleanup("found TSO keyspace group %d at the default group path when cleaning up PD mode microservice metadata", group.ID) } if group.IsSplitting() { - return nil, rejectMicroserviceMetadataCleanup("default TSO keyspace group is splitting when cleaning up PD mode microservice metadata") + return rejectMicroserviceMetadataCleanup("default TSO keyspace group is splitting when cleaning up PD mode microservice metadata") } if group.IsMerging() { - return nil, rejectMicroserviceMetadataCleanup("default TSO keyspace group is merging when cleaning up PD mode microservice metadata") + return rejectMicroserviceMetadataCleanup("default TSO keyspace group is merging when cleaning up PD mode microservice metadata") } if len(group.Members) == 0 { - return nil, nil + return nil } resp, err = etcdutil.EtcdKVGetWithContext( @@ -228,56 +178,25 @@ func (s *Server) loadMicroserviceMetadataCleanupCandidate(ctx context.Context) ( clientv3.WithLimit(1), ) if err != nil { - return nil, err + return err } if len(resp.Kvs) > 0 { nonDefaultGroup := &endpoint.KeyspaceGroup{} if err := json.Unmarshal(resp.Kvs[0].Value, nonDefaultGroup); err != nil { - return nil, rejectMicroserviceMetadataCleanup( + return rejectMicroserviceMetadataCleanup( "cannot decode TSO keyspace group metadata when cleaning up PD mode microservice metadata: %v", err) } - return nil, rejectMicroserviceMetadataCleanup( + return rejectMicroserviceMetadataCleanup( "found non-default TSO keyspace group %d when cleaning up PD mode microservice metadata", nonDefaultGroup.ID) } - return groupKV, nil -} - -func (s *Server) clearDefaultTSOKeyspaceGroupMembers( - ctx context.Context, - term microserviceMetadataCleanupTerm, - groupKV *mvccpb.KeyValue, -) (bool, error) { - groupKey := keypath.KeyspaceGroupIDPath(constant.DefaultKeyspaceGroupID) - if groupKV == nil || string(groupKV.Key) != groupKey { - return false, rejectMicroserviceMetadataCleanup( - "default TSO keyspace group cleanup candidate does not match its persisted path") - } - - group := &endpoint.KeyspaceGroup{} - if err := json.Unmarshal(groupKV.Value, group); err != nil { - return false, errs.ErrJSONUnmarshal.Wrap(err).GenWithStackByCause() - } - if group.ID != constant.DefaultKeyspaceGroupID { - return false, rejectMicroserviceMetadataCleanup( - "found TSO keyspace group %d at the default group path when clearing PD mode microservice metadata", group.ID) - } - if group.IsSplitting() { - return false, rejectMicroserviceMetadataCleanup("default TSO keyspace group is splitting when clearing PD mode microservice metadata") - } - if group.IsMerging() { - return false, rejectMicroserviceMetadataCleanup("default TSO keyspace group is merging when clearing PD mode microservice metadata") - } - if len(group.Members) == 0 { - return false, nil - } group.Members = nil value, err := json.Marshal(group) if err != nil { - return false, errs.ErrJSONMarshal.Wrap(err).GenWithStackByCause() + return errs.ErrJSONMarshal.Wrap(err).GenWithStackByCause() } - failpoint.InjectCall("beforeClearDefaultTSOKeyspaceGroupMembersCommit") + failpoint.InjectCall("beforeMicroserviceMetadataCleanupCommit") txnResp, err := kv.NewSlowLogTxnWithContext(ctx, s.client). If( clientv3.Compare(clientv3.Value(term.leaderKey), "=", term.leaderValue), @@ -287,10 +206,12 @@ func (s *Server) clearDefaultTSOKeyspaceGroupMembers( Then(clientv3.OpPut(groupKey, string(value))). Commit() if err != nil { - return false, errs.ErrEtcdTxnInternal.Wrap(err).GenWithStackByCause() + return errs.ErrEtcdTxnInternal.Wrap(err).GenWithStackByCause() } if !txnResp.Succeeded { - return false, errs.ErrEtcdTxnConflict.FastGenByArgs() + return errs.ErrEtcdTxnConflict.FastGenByArgs() } - return true, nil + log.Info("cleaned up microservice metadata in PD mode", + zap.Bool("cleared-default-keyspace-group-members", true)) + return nil } diff --git a/server/microservice_cleanup_test.go b/server/microservice_cleanup_test.go index 56ebc719bc..6b064d3ab5 100644 --- a/server/microservice_cleanup_test.go +++ b/server/microservice_cleanup_test.go @@ -17,7 +17,6 @@ package server import ( "context" "strconv" - "sync" "testing" "time" @@ -117,47 +116,6 @@ func TestCleanupMicroserviceMetadataInPDMode(t *testing.T) { } } -func TestCleanupMicroserviceMetadataIgnoresLeasedRegistryKeys(t *testing.T) { - re := require.New(t) - ctx := context.Background() - _, client, clean := etcdutil.NewTestEtcdCluster(t, 1, nil) - t.Cleanup(clean) - keypath.SetClusterID(12351) - defer keypath.ResetClusterID() - - store := storage.NewStorageWithEtcdBackend(client) - svr, term := newMicroserviceMetadataCleanupTestServer(t, client, store) - re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { - return store.SaveKeyspaceGroup(txn, &endpoint.KeyspaceGroup{ - ID: constant.DefaultKeyspaceGroupID, - UserKind: endpoint.Basic.String(), - Members: []endpoint.KeyspaceGroupMember{{ - Address: "http://127.0.0.1:3379", - Priority: mcs.DefaultKeyspaceGroupReplicaPriority, - }}, - Keyspaces: []uint32{1}, - }) - })) - - registryPath := keypath.RegistryPath(mcs.TSOServiceName, "127.0.0.1:3379") - leaseID, err := etcdutil.EtcdKVPutWithTTL(ctx, client, registryPath, "tso", 60) - re.NoError(err) - re.NotZero(leaseID) - defer func() { - _, _ = client.Revoke(ctx, leaseID) - }() - - re.NoError(svr.cleanupMicroserviceMetadataInPDMode(ctx, term)) - groups, err := store.LoadKeyspaceGroups(constant.DefaultKeyspaceGroupID, 0) - re.NoError(err) - re.Len(groups, 1) - re.Empty(groups[0].Members) - resp, err := etcdutil.EtcdKVGet(client, registryPath) - re.NoError(err) - re.Len(resp.Kvs, 1) - re.Equal(leaseID, clientv3.LeaseID(resp.Kvs[0].Lease)) -} - func TestScheduleMicroserviceMetadataCleanupDoesNotBlockServing(t *testing.T) { re := require.New(t) ctx := context.Background() @@ -178,13 +136,12 @@ func TestScheduleMicroserviceMetadataCleanupDoesNotBlockServing(t *testing.T) { }}, }) })) - blocker := enableMicroserviceMetadataCleanupCommitBlocker(t) + cleanupReached, releaseCleanup := enableMicroserviceMetadataCleanupCommitBlocker(t) svr.scheduleMicroserviceMetadataCleanup(ctx) - blocker.wait(t) + waitMicroserviceMetadataCleanupCommit(t, cleanupReached) re.True(svr.member.IsServing()) - blocker.releaseCleanup() - blocker.disable(t) + releaseCleanup() re.Eventually(func() bool { groups, err := store.LoadKeyspaceGroups(constant.DefaultKeyspaceGroupID, 1) return err == nil && len(groups) == 1 && len(groups[0].Members) == 0 @@ -233,52 +190,6 @@ func TestRunMicroserviceMetadataCleanupStopsOnRejectedState(t *testing.T) { } } -func TestMicroserviceMetadataCleanupTransitionDetection(t *testing.T) { - re := require.New(t) - ctx := context.Background() - _, client, clean := etcdutil.NewTestEtcdCluster(t, 1, nil) - t.Cleanup(clean) - keypath.SetClusterID(12350) - defer keypath.ResetClusterID() - - store := storage.NewStorageWithEtcdBackend(client) - svr := &Server{storage: store, client: client} - candidate, err := svr.loadMicroserviceMetadataCleanupCandidate(ctx) - re.NoError(err) - re.Nil(candidate) - - _, err = client.Put(ctx, keypath.RegistryPath(mcs.TSOServiceName, "127.0.0.1:3379"), "tso") - re.NoError(err) - candidate, err = svr.loadMicroserviceMetadataCleanupCandidate(ctx) - re.NoError(err) - re.Nil(candidate) - - re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { - return store.SaveKeyspaceGroup(txn, &endpoint.KeyspaceGroup{ - ID: constant.DefaultKeyspaceGroupID, - UserKind: endpoint.Basic.String(), - }) - })) - candidate, err = svr.loadMicroserviceMetadataCleanupCandidate(ctx) - re.NoError(err) - re.Nil(candidate) - - re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { - group, err := store.LoadKeyspaceGroup(txn, constant.DefaultKeyspaceGroupID) - if err != nil { - return err - } - group.Members = []endpoint.KeyspaceGroupMember{{ - Address: "http://127.0.0.1:3379", - Priority: mcs.DefaultKeyspaceGroupReplicaPriority, - }} - return store.SaveKeyspaceGroup(txn, group) - })) - candidate, err = svr.loadMicroserviceMetadataCleanupCandidate(ctx) - re.NoError(err) - re.NotNil(candidate) -} - func TestMicroserviceMetadataCleanupSkipsNonDefaultGroupsWithoutStaleMembers(t *testing.T) { re := require.New(t) ctx := context.Background() @@ -288,7 +199,7 @@ func TestMicroserviceMetadataCleanupSkipsNonDefaultGroupsWithoutStaleMembers(t * t.Cleanup(keypath.ResetClusterID) store := storage.NewStorageWithEtcdBackend(client) - svr := &Server{storage: store, client: client} + svr, term := newMicroserviceMetadataCleanupTestServer(t, client, store) re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { return store.SaveKeyspaceGroup(txn, &endpoint.KeyspaceGroup{ ID: 1, @@ -296,9 +207,7 @@ func TestMicroserviceMetadataCleanupSkipsNonDefaultGroupsWithoutStaleMembers(t * }) })) - candidate, err := svr.loadMicroserviceMetadataCleanupCandidate(ctx) - re.NoError(err) - re.Nil(candidate) + re.NoError(svr.cleanupMicroserviceMetadataInPDMode(ctx, term)) re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { return store.SaveKeyspaceGroup(txn, &endpoint.KeyspaceGroup{ @@ -306,9 +215,7 @@ func TestMicroserviceMetadataCleanupSkipsNonDefaultGroupsWithoutStaleMembers(t * UserKind: endpoint.Basic.String(), }) })) - candidate, err = svr.loadMicroserviceMetadataCleanupCandidate(ctx) - re.NoError(err) - re.Nil(candidate) + re.NoError(svr.cleanupMicroserviceMetadataInPDMode(ctx, term)) } func TestRunMicroserviceMetadataCleanupStopsOnContextCancellation(t *testing.T) { @@ -381,43 +288,6 @@ func TestCleanupMicroserviceMetadataInPDModeRejectsNonDefaultGroup(t *testing.T) re.Equal(staleMembers, groups[0].Members) } -func TestCleanupMicroserviceMetadataInPDModePreservesAssignmentMarkers(t *testing.T) { - re := require.New(t) - ctx := context.Background() - _, client, clean := etcdutil.NewTestEtcdCluster(t, 1, nil) - t.Cleanup(clean) - keypath.SetClusterID(12347) - defer keypath.ResetClusterID() - - store := storage.NewStorageWithEtcdBackend(client) - svr, term := newMicroserviceMetadataCleanupTestServer(t, client, store) - re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { - if err := store.SaveKeyspaceGroup(txn, &endpoint.KeyspaceGroup{ - ID: constant.DefaultKeyspaceGroupID, - UserKind: endpoint.Basic.String(), - Members: []endpoint.KeyspaceGroupMember{{ - Address: "http://127.0.0.1:3379", - Priority: mcs.DefaultKeyspaceGroupReplicaPriority, - }}, - }); err != nil { - return err - } - return store.SaveKeyspaceMeta(txn, newKeyspaceMetaWithTSOGroup(1, "0")) - })) - - re.NoError(svr.cleanupMicroserviceMetadataInPDMode(ctx, term)) - re.NoError(store.RunInTxn(ctx, func(txn kv.Txn) error { - meta, err := store.LoadKeyspaceMeta(txn, 1) - re.NoError(err) - re.Equal("0", meta.GetConfig()[keyspace.TSOKeyspaceGroupIDKey]) - return nil - })) - groups, err := store.LoadKeyspaceGroups(constant.DefaultKeyspaceGroupID, 0) - re.NoError(err) - re.Len(groups, 1) - re.Empty(groups[0].Members) -} - func TestCleanupMicroserviceMetadataInPDModeRejectsGroupTransition(t *testing.T) { testCases := []struct { name string @@ -474,7 +344,7 @@ func TestCleanupMicroserviceMetadataInPDModeRejectsGroupTransition(t *testing.T) } } -func TestClearDefaultTSOKeyspaceGroupMembersIsFencedByLeadershipTerm(t *testing.T) { +func TestCleanupMicroserviceMetadataIsFencedByLeadershipTerm(t *testing.T) { re := require.New(t) ctx := context.Background() _, client, clean := etcdutil.NewTestEtcdCluster(t, 1, nil) @@ -497,16 +367,12 @@ func TestClearDefaultTSOKeyspaceGroupMembersIsFencedByLeadershipTerm(t *testing. }) })) - candidate, err := svr.loadMicroserviceMetadataCleanupCandidate(ctx) - re.NoError(err) - re.NotNil(candidate) - blocker := enableMicroserviceMetadataCleanupCommitBlocker(t) - resultCh := make(chan clearDefaultTSOKeyspaceGroupMembersResult, 1) + cleanupReached, releaseCleanup := enableMicroserviceMetadataCleanupCommitBlocker(t) + resultCh := make(chan error, 1) go func() { - cleared, err := svr.clearDefaultTSOKeyspaceGroupMembers(ctx, oldTerm, candidate) - resultCh <- clearDefaultTSOKeyspaceGroupMembersResult{cleared: cleared, err: err} + resultCh <- svr.cleanupMicroserviceMetadataInPDMode(ctx, oldTerm) }() - blocker.wait(t) + waitMicroserviceMetadataCleanupCommit(t, cleanupReached) // Campaign again with the same member value to prove that comparing only the // leader value would allow an old-term cleanup to pass. @@ -521,11 +387,8 @@ func TestClearDefaultTSOKeyspaceGroupMembersIsFencedByLeadershipTerm(t *testing. re.Equal(oldTerm.leaderValue, newTerm.leaderValue) re.NotEqual(oldTerm.leaseID, newTerm.leaseID) - blocker.releaseCleanup() - blocker.disable(t) - result := waitClearDefaultTSOKeyspaceGroupMembersResult(t, resultCh) - re.False(result.cleared) - re.ErrorIs(result.err, errs.ErrEtcdTxnConflict) + releaseCleanup() + re.ErrorIs(waitMicroserviceMetadataCleanupResult(t, resultCh), errs.ErrEtcdTxnConflict) groups, err := store.LoadKeyspaceGroups(constant.DefaultKeyspaceGroupID, 0) re.NoError(err) @@ -533,7 +396,7 @@ func TestClearDefaultTSOKeyspaceGroupMembersIsFencedByLeadershipTerm(t *testing. re.Equal(staleMembers, groups[0].Members) } -func TestClearDefaultTSOKeyspaceGroupMembersPreservesConcurrentUpdate(t *testing.T) { +func TestCleanupMicroserviceMetadataPreservesConcurrentUpdate(t *testing.T) { re := require.New(t) ctx := context.Background() _, client, clean := etcdutil.NewTestEtcdCluster(t, 1, nil) @@ -555,16 +418,12 @@ func TestClearDefaultTSOKeyspaceGroupMembersPreservesConcurrentUpdate(t *testing }) })) - candidate, err := svr.loadMicroserviceMetadataCleanupCandidate(ctx) - re.NoError(err) - re.NotNil(candidate) - blocker := enableMicroserviceMetadataCleanupCommitBlocker(t) - resultCh := make(chan clearDefaultTSOKeyspaceGroupMembersResult, 1) + cleanupReached, releaseCleanup := enableMicroserviceMetadataCleanupCommitBlocker(t) + resultCh := make(chan error, 1) go func() { - cleared, err := svr.clearDefaultTSOKeyspaceGroupMembers(ctx, term, candidate) - resultCh <- clearDefaultTSOKeyspaceGroupMembersResult{cleared: cleared, err: err} + resultCh <- svr.cleanupMicroserviceMetadataInPDMode(ctx, term) }() - blocker.wait(t) + waitMicroserviceMetadataCleanupCommit(t, cleanupReached) newMembers := []endpoint.KeyspaceGroupMember{{ Address: "http://127.0.0.1:3380", @@ -580,11 +439,8 @@ func TestClearDefaultTSOKeyspaceGroupMembersPreservesConcurrentUpdate(t *testing return store.SaveKeyspaceGroup(txn, group) })) - blocker.releaseCleanup() - blocker.disable(t) - result := waitClearDefaultTSOKeyspaceGroupMembersResult(t, resultCh) - re.False(result.cleared) - re.ErrorIs(result.err, errs.ErrEtcdTxnConflict) + releaseCleanup() + re.ErrorIs(waitMicroserviceMetadataCleanupResult(t, resultCh), errs.ErrEtcdTxnConflict) groups, err := store.LoadKeyspaceGroups(constant.DefaultKeyspaceGroupID, 0) re.NoError(err) @@ -595,20 +451,6 @@ func TestClearDefaultTSOKeyspaceGroupMembersPreservesConcurrentUpdate(t *testing const testMicroserviceMetadataCleanupLeaseTimeout = 60 -type clearDefaultTSOKeyspaceGroupMembersResult struct { - cleared bool - err error -} - -type microserviceMetadataCleanupCommitBlocker struct { - name string - reached chan struct{} - release chan struct{} - reachedOnce sync.Once - releaseOnce sync.Once - disableOnce sync.Once -} - func newMicroserviceMetadataCleanupTestServer( t *testing.T, client *clientv3.Client, @@ -628,68 +470,51 @@ func newMicroserviceMetadataCleanupTestServer( return svr, term } -func enableMicroserviceMetadataCleanupCommitBlocker(t *testing.T) *microserviceMetadataCleanupCommitBlocker { +func enableMicroserviceMetadataCleanupCommitBlocker(t *testing.T) (<-chan struct{}, func()) { t.Helper() - blocker := µserviceMetadataCleanupCommitBlocker{ - name: "github.com/tikv/pd/server/beforeClearDefaultTSOKeyspaceGroupMembersCommit", - reached: make(chan struct{}), - release: make(chan struct{}), + const name = "github.com/tikv/pd/server/beforeMicroserviceMetadataCleanupCommit" + reached := make(chan struct{}, 1) + release := make(chan struct{}) + unblock := func() { + select { + case <-release: + default: + close(release) + } } - require.NoError(t, failpoint.EnableCall(blocker.name, func() { - blocker.reachedOnce.Do(func() { - close(blocker.reached) - }) - <-blocker.release + require.NoError(t, failpoint.EnableCall(name, func() { + select { + case reached <- struct{}{}: + default: + } + <-release })) t.Cleanup(func() { - blocker.releaseCleanup() - blocker.disable(t) + unblock() + require.NoError(t, failpoint.Disable(name)) }) - return blocker + return reached, unblock } -func (b *microserviceMetadataCleanupCommitBlocker) wait(t *testing.T) { +func waitMicroserviceMetadataCleanupCommit(t *testing.T, reached <-chan struct{}) { t.Helper() select { - case <-b.reached: + case <-reached: case <-time.After(10 * time.Second): t.Fatal("microservice metadata cleanup did not reach the commit hook") } } -func (b *microserviceMetadataCleanupCommitBlocker) releaseCleanup() { - b.releaseOnce.Do(func() { - close(b.release) - }) -} - -func (b *microserviceMetadataCleanupCommitBlocker) disable(t *testing.T) { - t.Helper() - b.disableOnce.Do(func() { - require.NoError(t, failpoint.Disable(b.name)) - }) -} - -func waitClearDefaultTSOKeyspaceGroupMembersResult( +func waitMicroserviceMetadataCleanupResult( t *testing.T, - resultCh <-chan clearDefaultTSOKeyspaceGroupMembersResult, -) clearDefaultTSOKeyspaceGroupMembersResult { + resultCh <-chan error, +) error { t.Helper() select { - case result := <-resultCh: - return result + case err := <-resultCh: + return err case <-time.After(10 * time.Second): t.Fatal("microservice metadata cleanup did not return") - return clearDefaultTSOKeyspaceGroupMembersResult{} - } -} - -func newKeyspaceMetaWithTSOGroup(id uint32, groupID string) *keyspacepb.KeyspaceMeta { - return &keyspacepb.KeyspaceMeta{ - Keyspace: &keyspacepb.KeyspaceMeta_Id{Id: id}, - Name: "keyspace-" + strconv.FormatUint(uint64(id), 10), - Config: map[string]string{ - keyspace.TSOKeyspaceGroupIDKey: groupID, - }, + return nil } } diff --git a/tests/integrations/mcs/tso/server_test.go b/tests/integrations/mcs/tso/server_test.go index 6734dc4d58..c2019cc914 100644 --- a/tests/integrations/mcs/tso/server_test.go +++ b/tests/integrations/mcs/tso/server_test.go @@ -803,7 +803,7 @@ func TestPDModeSwitchBetweenMicroserviceAndMonolithMultipleTimes(t *testing.T) { cleanupReached := make(chan struct{}) cleanupRelease := make(chan struct{}) var cleanupReachedOnce, cleanupReleaseOnce sync.Once - const cleanupBlockerName = "github.com/tikv/pd/server/beforeClearDefaultTSOKeyspaceGroupMembersCommit" + const cleanupBlockerName = "github.com/tikv/pd/server/beforeMicroserviceMetadataCleanupCommit" re.NoError(failpoint.EnableCall(cleanupBlockerName, func() { cleanupReachedOnce.Do(func() { close(cleanupReached) From e4aa020dbc358564136b6c46a5e901cea8171903 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Fri, 14 Aug 2026 15:15:20 +0800 Subject: [PATCH 14/15] tests: cover both mode cleanup startup orders Signed-off-by: Ryan Leung --- tests/integrations/mcs/tso/server_test.go | 48 ++++++++++++++++------- 1 file changed, 33 insertions(+), 15 deletions(-) diff --git a/tests/integrations/mcs/tso/server_test.go b/tests/integrations/mcs/tso/server_test.go index c2019cc914..69fed6427d 100644 --- a/tests/integrations/mcs/tso/server_test.go +++ b/tests/integrations/mcs/tso/server_test.go @@ -810,29 +810,36 @@ func TestPDModeSwitchBetweenMicroserviceAndMonolithMultipleTimes(t *testing.T) { }) <-cleanupRelease })) - t.Cleanup(func() { + defer func() { cleanupReleaseOnce.Do(func() { close(cleanupRelease) }) re.NoError(failpoint.Disable(cleanupBlockerName)) - }) + }() - const switchRounds = 2 - for round := range switchRounds { + waitDefaultKeyspaceGroupReady(re, pdServer, userKeyspaceID) + tsoCluster, err := tests.NewTestTSOCluster(ctx, 1, pdServer.GetAddr()) + re.NoError(err) + defer func() { + if tsoCluster != nil { + tsoCluster.Destroy() + } + }() + + for _, tsoStartsBeforeCleanup := range []bool{true, false} { waitDefaultKeyspaceGroupReady(re, pdServer, userKeyspaceID) - tsoCluster, err := tests.NewTestTSOCluster(ctx, 1, pdServer.GetAddr()) - re.NoError(err) waitTSOServiceReady(re, tsoCluster) + waitDefaultKeyspaceGroupMembers(re, pdServer, tsoCluster.GetKeyspaceGroupMember()) checkMicroserviceTSOAvailable(ctx, re, pdServer) checkKeyspaceTSOAvailable(ctx, re, pdServer) - waitDefaultKeyspaceGroupMembers(re, pdServer, tsoCluster.GetKeyspaceGroupMember()) tsoCluster.Destroy() + tsoCluster = nil seedDefaultKeyspaceGroupMembers(ctx, re, pdServer, []endpoint.KeyspaceGroupMember{{ Address: "http://127.0.0.1:1", Priority: mcs.DefaultKeyspaceGroupReplicaPriority, }}) - if round == 0 { + if tsoStartsBeforeCleanup { pdServer, err = startPDForServiceMode(ctx, tc, pdServer, nil) re.NoError(err) select { @@ -845,13 +852,27 @@ func TestPDModeSwitchBetweenMicroserviceAndMonolithMultipleTimes(t *testing.T) { checkPDLeaderAPIAvailable(ctx, re, pdServer) checkPDTSOAvailable(ctx, re, pdServer) checkKeyspaceTSOAvailable(ctx, re, pdServer) - cleanupReleaseOnce.Do(func() { - close(cleanupRelease) - }) } else { pdServer, err = restartPDForServiceMode(ctx, tc, pdServer, nil) re.NoError(err) + waitDefaultKeyspaceGroupMembers(re, pdServer, nil) + } + + tsoCluster, err = tests.NewTestTSOCluster(ctx, 1, pdServer.GetAddr()) + re.NoError(err) + if tsoStartsBeforeCleanup { + for _, server := range tsoCluster.GetServers() { + resp, err := tests.TestDialClient.Get(server.GetAddr() + tsoapi.APIPathPrefix + "/health") + re.NoError(err) + statusCode := resp.StatusCode + re.NoError(resp.Body.Close()) + re.Equal(http.StatusNotFound, statusCode) + } + cleanupReleaseOnce.Do(func() { + close(cleanupRelease) + }) } + waitTSOServiceReady(re, tsoCluster) checkPDTSOAvailable(ctx, re, pdServer) checkKeyspaceTSOAvailable(ctx, re, pdServer) waitDefaultKeyspaceGroupMembers(re, pdServer, nil) @@ -861,13 +882,10 @@ func TestPDModeSwitchBetweenMicroserviceAndMonolithMultipleTimes(t *testing.T) { } waitDefaultKeyspaceGroupReady(re, pdServer, userKeyspaceID) - tsoCluster, err := tests.NewTestTSOCluster(ctx, 1, pdServer.GetAddr()) - re.NoError(err) - defer tsoCluster.Destroy() waitTSOServiceReady(re, tsoCluster) + waitDefaultKeyspaceGroupMembers(re, pdServer, tsoCluster.GetKeyspaceGroupMember()) checkMicroserviceTSOAvailable(ctx, re, pdServer) checkKeyspaceTSOAvailable(ctx, re, pdServer) - waitDefaultKeyspaceGroupMembers(re, pdServer, tsoCluster.GetKeyspaceGroupMember()) } func restartPDForServiceMode( From 3ba37d84b172707f54449db577bd15be702319df Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Fri, 14 Aug 2026 18:02:21 +0800 Subject: [PATCH 15/15] tests: cover keyspace created in PD mode Signed-off-by: Ryan Leung --- tests/integrations/mcs/tso/server_test.go | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/tests/integrations/mcs/tso/server_test.go b/tests/integrations/mcs/tso/server_test.go index 69fed6427d..a8ff55ee3d 100644 --- a/tests/integrations/mcs/tso/server_test.go +++ b/tests/integrations/mcs/tso/server_test.go @@ -774,6 +774,7 @@ func TestTSOServiceSwitch(t *testing.T) { } const modeSwitchUserKeyspaceName = "mode_switch_user_ks" +const modeSwitchPDModeKeyspaceName = "mode_switch_pd_mode_ks" func TestPDModeSwitchBetweenMicroserviceAndMonolithMultipleTimes(t *testing.T) { re := require.New(t) @@ -831,7 +832,7 @@ func TestPDModeSwitchBetweenMicroserviceAndMonolithMultipleTimes(t *testing.T) { waitTSOServiceReady(re, tsoCluster) waitDefaultKeyspaceGroupMembers(re, pdServer, tsoCluster.GetKeyspaceGroupMember()) checkMicroserviceTSOAvailable(ctx, re, pdServer) - checkKeyspaceTSOAvailable(ctx, re, pdServer) + checkKeyspaceTSOAvailable(ctx, re, pdServer, modeSwitchUserKeyspaceName) tsoCluster.Destroy() tsoCluster = nil seedDefaultKeyspaceGroupMembers(ctx, re, pdServer, []endpoint.KeyspaceGroupMember{{ @@ -851,12 +852,19 @@ func TestPDModeSwitchBetweenMicroserviceAndMonolithMultipleTimes(t *testing.T) { re.True(pdServer.WaitLeader(), "normal-mode PD did not become ready while mode cleanup was pending") checkPDLeaderAPIAvailable(ctx, re, pdServer) checkPDTSOAvailable(ctx, re, pdServer) - checkKeyspaceTSOAvailable(ctx, re, pdServer) + checkKeyspaceTSOAvailable(ctx, re, pdServer, modeSwitchUserKeyspaceName) } else { pdServer, err = restartPDForServiceMode(ctx, tc, pdServer, nil) re.NoError(err) waitDefaultKeyspaceGroupMembers(re, pdServer, nil) } + if tsoStartsBeforeCleanup { + // Create a keyspace while PD is running in normal mode. + _, err = pdServer.GetServer().GetKeyspaceManager().CreateKeyspace(&keyspace.CreateKeyspaceRequest{ + Name: modeSwitchPDModeKeyspaceName, + }) + re.NoError(err) + } tsoCluster, err = tests.NewTestTSOCluster(ctx, 1, pdServer.GetAddr()) re.NoError(err) @@ -874,18 +882,21 @@ func TestPDModeSwitchBetweenMicroserviceAndMonolithMultipleTimes(t *testing.T) { } waitTSOServiceReady(re, tsoCluster) checkPDTSOAvailable(ctx, re, pdServer) - checkKeyspaceTSOAvailable(ctx, re, pdServer) + checkKeyspaceTSOAvailable(ctx, re, pdServer, modeSwitchUserKeyspaceName) waitDefaultKeyspaceGroupMembers(re, pdServer, nil) pdServer, err = restartPDForServiceMode(ctx, tc, pdServer, []string{mcs.PDServiceName}) re.NoError(err) + if tsoStartsBeforeCleanup { + checkKeyspaceTSOAvailable(ctx, re, pdServer, modeSwitchPDModeKeyspaceName) + } } waitDefaultKeyspaceGroupReady(re, pdServer, userKeyspaceID) waitTSOServiceReady(re, tsoCluster) waitDefaultKeyspaceGroupMembers(re, pdServer, tsoCluster.GetKeyspaceGroupMember()) checkMicroserviceTSOAvailable(ctx, re, pdServer) - checkKeyspaceTSOAvailable(ctx, re, pdServer) + checkKeyspaceTSOAvailable(ctx, re, pdServer, modeSwitchUserKeyspaceName) } func restartPDForServiceMode( @@ -1032,11 +1043,12 @@ func checkKeyspaceTSOAvailable( ctx context.Context, re *require.Assertions, pdServer *tests.TestServer, + keyspaceName string, ) { cli := utils.SetupClientWithAPIContext( ctx, re, - pd.NewAPIContextV2(modeSwitchUserKeyspaceName), + pd.NewAPIContextV2(keyspaceName), []string{pdServer.GetAddr()}, ) defer cli.Close()