From 519ee198a19febbc29652e587db848ff54fa0352 Mon Sep 17 00:00:00 2001 From: Wenxuan Zhang Date: Mon, 27 Jul 2026 11:16:04 +0800 Subject: [PATCH 01/31] client: expose keyspace-level GC mode Signed-off-by: Wenxuan Zhang --- client/clients/gc/client.go | 13 ++++--- client/gc_client.go | 17 +++++---- client/gc_client_test.go | 70 +++++++++++++++++++++++++++++++++++++ 3 files changed, 88 insertions(+), 12 deletions(-) create mode 100644 client/gc_client_test.go diff --git a/client/clients/gc/client.go b/client/clients/gc/client.go index 9149def7e5..c1ed1f7984 100644 --- a/client/clients/gc/client.go +++ b/client/clients/gc/client.go @@ -286,11 +286,14 @@ func (b *GlobalGCBarrierInfo) isExpiredImpl(now time.Time) bool { //nolint:revive type GCState struct { // The ID of the keyspace this GC state belongs to. - KeyspaceID uint32 - TxnSafePoint uint64 - GCSafePoint uint64 - hasGCBarriers bool - gcBarriers []*GCBarrierInfo + KeyspaceID uint32 + // IsKeyspaceLevelGC reports whether this state belongs to an independent + // keyspace-level GC scope. + IsKeyspaceLevelGC bool + TxnSafePoint uint64 + GCSafePoint uint64 + hasGCBarriers bool + gcBarriers []*GCBarrierInfo hasGlobalGCBarriers bool globalGCBarriers []*GlobalGCBarrierInfo diff --git a/client/gc_client.go b/client/gc_client.go index ac8154e1c8..f89b1c98be 100644 --- a/client/gc_client.go +++ b/client/gc_client.go @@ -339,15 +339,18 @@ func pbToGCState(pb *pdpb.GCState, reqStartTime time.Time, excludeGCBarriers boo if pb.KeyspaceScope != nil { keyspaceID = pb.KeyspaceScope.GetKeyspaceId() } + var state gc.GCState if excludeGCBarriers { - return gc.NewGCStateWithoutGCBarriers(keyspaceID, pb.GetTxnSafePoint(), pb.GetGcSafePoint()) - } - - gcBarriers := make([]*gc.GCBarrierInfo, 0, len(pb.GetGcBarriers())) - for _, b := range pb.GetGcBarriers() { - gcBarriers = append(gcBarriers, pbToGCBarrierInfo(b, reqStartTime)) + state = gc.NewGCStateWithoutGCBarriers(keyspaceID, pb.GetTxnSafePoint(), pb.GetGcSafePoint()) + } else { + gcBarriers := make([]*gc.GCBarrierInfo, 0, len(pb.GetGcBarriers())) + for _, b := range pb.GetGcBarriers() { + gcBarriers = append(gcBarriers, pbToGCBarrierInfo(b, reqStartTime)) + } + state = gc.NewGCStateWithGCBarriers(keyspaceID, pb.GetTxnSafePoint(), pb.GetGcSafePoint(), gcBarriers) } - return gc.NewGCStateWithGCBarriers(keyspaceID, pb.GetTxnSafePoint(), pb.GetGcSafePoint(), gcBarriers) + state.IsKeyspaceLevelGC = pb.GetIsKeyspaceLevelGc() + return state } func pbToGCStateWithGlobalGCBarriers( diff --git a/client/gc_client_test.go b/client/gc_client_test.go new file mode 100644 index 0000000000..8bee74d5b1 --- /dev/null +++ b/client/gc_client_test.go @@ -0,0 +1,70 @@ +// Copyright 2026 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package pd + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/pingcap/kvproto/pkg/pdpb" +) + +func TestPBToGCStatePreservesKeyspaceLevelGC(t *testing.T) { + requestStart := time.Unix(100, 0) + for _, testCase := range []struct { + name string + isKeyspaceLevelGC bool + excludeBarriers bool + }{ + {name: "keyspace-level-with-barriers", isKeyspaceLevelGC: true}, + { + name: "keyspace-level-without-barriers", + isKeyspaceLevelGC: true, + excludeBarriers: true, + }, + {name: "unified-with-barriers", isKeyspaceLevelGC: false}, + { + name: "unified-without-barriers", + isKeyspaceLevelGC: false, + excludeBarriers: true, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + pbState := &pdpb.GCState{ + KeyspaceScope: &pdpb.KeyspaceScope{KeyspaceId: 42}, + IsKeyspaceLevelGc: testCase.isKeyspaceLevelGC, + TxnSafePoint: 100, + GcSafePoint: 90, + GcBarriers: []*pdpb.GCBarrierInfo{ + {BarrierId: "backup", BarrierTs: 95, TtlSeconds: 60}, + }, + } + + state := pbToGCState( + pbState, + requestStart, + testCase.excludeBarriers, + ) + require.Equal(t, testCase.isKeyspaceLevelGC, + state.IsKeyspaceLevelGC) + require.Equal(t, uint32(42), state.KeyspaceID) + require.Equal(t, uint64(100), state.TxnSafePoint) + require.Equal(t, uint64(90), state.GCSafePoint) + require.Equal(t, !testCase.excludeBarriers, + state.HasGCBarriers()) + }) + } +} From b4e660669d1525a11c2fae98bf545c3e0f26f333 Mon Sep 17 00:00:00 2001 From: Wenxuan Zhang Date: Mon, 27 Jul 2026 11:25:17 +0800 Subject: [PATCH 02/31] pd-ctl: add GC state JSON projections Signed-off-by: Wenxuan Zhang --- .../pd-ctl/pdctl/command/gc_state_command.go | 177 ++++++++++++++++++ .../pdctl/command/gc_state_command_test.go | 138 ++++++++++++++ 2 files changed, 315 insertions(+) create mode 100644 tools/pd-ctl/pdctl/command/gc_state_command.go create mode 100644 tools/pd-ctl/pdctl/command/gc_state_command_test.go diff --git a/tools/pd-ctl/pdctl/command/gc_state_command.go b/tools/pd-ctl/pdctl/command/gc_state_command.go new file mode 100644 index 0000000000..4b88786c32 --- /dev/null +++ b/tools/pd-ctl/pdctl/command/gc_state_command.go @@ -0,0 +1,177 @@ +// Copyright 2026 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package command + +import ( + "math" + "sort" + "strconv" + "time" + + "github.com/pingcap/errors" + + "github.com/tikv/pd/client/clients/gc" + "github.com/tikv/pd/pkg/keyspace/constant" +) + +type gcBarrierOutput struct { + BarrierID string `json:"barrier_id"` + BarrierTS uint64 `json:"barrier_ts"` + TTLSeconds int64 `json:"ttl_seconds"` +} + +type gcStateOutput struct { + KeyspaceID uint32 `json:"keyspace_id"` + IsKeyspaceLevelGC bool `json:"is_keyspace_level_gc"` + TxnSafePoint uint64 `json:"txn_safe_point"` + GCSafePoint uint64 `json:"gc_safe_point"` + GCBarriers []gcBarrierOutput `json:"gc_barriers"` +} + +type keyspaceGCStateOutput struct { + RequestedKeyspaceID uint32 `json:"requested_keyspace_id"` + EffectiveKeyspaceID uint32 `json:"effective_keyspace_id"` + IsKeyspaceLevelGC bool `json:"is_keyspace_level_gc"` + TxnSafePoint uint64 `json:"txn_safe_point"` + GCSafePoint uint64 `json:"gc_safe_point"` + GCBarriers []gcBarrierOutput `json:"gc_barriers"` +} + +type allGCStatesOutput struct { + GCStates []gcStateOutput `json:"gc_states"` + GlobalGCBarriers []gcBarrierOutput `json:"global_gc_barriers"` +} + +func parseGCStateKeyspaceID(value string) (uint32, error) { + parsed, err := strconv.ParseUint(value, 10, 32) + if err != nil { + return 0, errors.Annotatef(err, "invalid keyspace ID %q", value) + } + keyspaceID := uint32(parsed) + if keyspaceID > constant.MaxValidKeyspaceID && keyspaceID != constant.NullKeyspaceID { + return 0, errors.Errorf( + "invalid keyspace ID %q: expected 0 through %d or %d", + value, + constant.MaxValidKeyspaceID, + constant.NullKeyspaceID, + ) + } + return keyspaceID, nil +} + +func gcStateTTLSeconds(ttl time.Duration) int64 { + if ttl == gc.TTLNeverExpire { + return math.MaxInt64 + } + return int64(ttl / time.Second) +} + +func sortGCBarrierOutputs(barriers []gcBarrierOutput) { + sort.Slice(barriers, func(i, j int) bool { + if barriers[i].BarrierTS != barriers[j].BarrierTS { + return barriers[i].BarrierTS < barriers[j].BarrierTS + } + return barriers[i].BarrierID < barriers[j].BarrierID + }) +} + +func newLocalGCBarrierOutputs(barriers []*gc.GCBarrierInfo) []gcBarrierOutput { + result := make([]gcBarrierOutput, 0, len(barriers)) + for _, barrier := range barriers { + result = append(result, gcBarrierOutput{ + BarrierID: barrier.BarrierID, + BarrierTS: barrier.BarrierTS, + TTLSeconds: gcStateTTLSeconds(barrier.TTL), + }) + } + sortGCBarrierOutputs(result) + return result +} + +func newGlobalGCBarrierOutputs(barriers []*gc.GlobalGCBarrierInfo) []gcBarrierOutput { + result := make([]gcBarrierOutput, 0, len(barriers)) + for _, barrier := range barriers { + result = append(result, gcBarrierOutput{ + BarrierID: barrier.BarrierID, + BarrierTS: barrier.BarrierTS, + TTLSeconds: gcStateTTLSeconds(barrier.TTL), + }) + } + sortGCBarrierOutputs(result) + return result +} + +func newKeyspaceGCStateOutput( + requestedKeyspaceID uint32, + state gc.GCState, +) (keyspaceGCStateOutput, error) { + barriers, err := state.GetGCBarriers() + if err != nil { + return keyspaceGCStateOutput{}, errors.Annotatef( + err, + "failed to read GC barriers for keyspace %d", + requestedKeyspaceID, + ) + } + return keyspaceGCStateOutput{ + RequestedKeyspaceID: requestedKeyspaceID, + EffectiveKeyspaceID: state.KeyspaceID, + IsKeyspaceLevelGC: state.IsKeyspaceLevelGC, + TxnSafePoint: state.TxnSafePoint, + GCSafePoint: state.GCSafePoint, + GCBarriers: newLocalGCBarrierOutputs(barriers), + }, nil +} + +func newGCStateOutput(state gc.GCState) (gcStateOutput, error) { + barriers, err := state.GetGCBarriers() + if err != nil { + return gcStateOutput{}, errors.Annotatef( + err, + "failed to read GC barriers for keyspace %d", + state.KeyspaceID, + ) + } + return gcStateOutput{ + KeyspaceID: state.KeyspaceID, + IsKeyspaceLevelGC: state.IsKeyspaceLevelGC, + TxnSafePoint: state.TxnSafePoint, + GCSafePoint: state.GCSafePoint, + GCBarriers: newLocalGCBarrierOutputs(barriers), + }, nil +} + +func newAllGCStatesOutput(clusterState gc.ClusterGCStates) (allGCStatesOutput, error) { + states := make([]gcStateOutput, 0, len(clusterState.GCStates)) + for _, state := range clusterState.GCStates { + converted, err := newGCStateOutput(state) + if err != nil { + return allGCStatesOutput{}, err + } + states = append(states, converted) + } + sort.Slice(states, func(i, j int) bool { + return states[i].KeyspaceID < states[j].KeyspaceID + }) + + globalBarriers, err := clusterState.GetGlobalGCBarriers() + if err != nil { + return allGCStatesOutput{}, errors.Annotate(err, "failed to read global GC barriers") + } + return allGCStatesOutput{ + GCStates: states, + GlobalGCBarriers: newGlobalGCBarrierOutputs(globalBarriers), + }, nil +} diff --git a/tools/pd-ctl/pdctl/command/gc_state_command_test.go b/tools/pd-ctl/pdctl/command/gc_state_command_test.go new file mode 100644 index 0000000000..8fbc889c5d --- /dev/null +++ b/tools/pd-ctl/pdctl/command/gc_state_command_test.go @@ -0,0 +1,138 @@ +// Copyright 2026 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package command + +import ( + "encoding/json" + "math" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/tikv/pd/client/clients/gc" + "github.com/tikv/pd/pkg/keyspace/constant" +) + +func TestParseGCStateKeyspaceID(t *testing.T) { + for _, testCase := range []struct { + name string + input string + want uint32 + wantErr bool + }{ + {name: "zero", input: "0", want: 0}, + {name: "maximum-normal", input: "16777215", want: 16777215}, + {name: "null-keyspace", input: "4294967295", want: 4294967295}, + {name: "empty", input: "", wantErr: true}, + {name: "negative", input: "-1", wantErr: true}, + {name: "text", input: "tenant-a", wantErr: true}, + {name: "hexadecimal", input: "0xffffff", wantErr: true}, + {name: "first-invalid", input: "16777216", wantErr: true}, + {name: "null-synonym", input: "4294967294", wantErr: true}, + {name: "uint32-overflow", input: "4294967296", wantErr: true}, + } { + t.Run(testCase.name, func(t *testing.T) { + got, err := parseGCStateKeyspaceID(testCase.input) + if testCase.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + require.Equal(t, testCase.want, got) + }) + } +} + +func TestNewKeyspaceGCStateOutput(t *testing.T) { + state := gc.NewGCStateWithGCBarriers( + constant.NullKeyspaceID, + 100, + 90, + []*gc.GCBarrierInfo{ + gc.NewGCBarrierInfo("z-backup", 110, time.Hour, time.Time{}), + gc.NewGCBarrierInfo("b-backup", 105, gc.TTLNeverExpire, time.Time{}), + gc.NewGCBarrierInfo("a-backup", 110, 30*time.Second, time.Time{}), + }, + ) + state.IsKeyspaceLevelGC = false + + got, err := newKeyspaceGCStateOutput(42, state) + require.NoError(t, err) + require.Equal(t, uint32(42), got.RequestedKeyspaceID) + require.Equal(t, constant.NullKeyspaceID, got.EffectiveKeyspaceID) + require.False(t, got.IsKeyspaceLevelGC) + require.Equal(t, uint64(100), got.TxnSafePoint) + require.Equal(t, uint64(90), got.GCSafePoint) + require.Equal(t, []gcBarrierOutput{ + {BarrierID: "b-backup", BarrierTS: 105, TTLSeconds: math.MaxInt64}, + {BarrierID: "a-backup", BarrierTS: 110, TTLSeconds: 30}, + {BarrierID: "z-backup", BarrierTS: 110, TTLSeconds: 3600}, + }, got.GCBarriers) +} + +func TestNewAllGCStatesOutputSortsAndKeepsEmptyArrays(t *testing.T) { + empty := gc.NewGCStateWithGCBarriers(1, 20, 10, nil) + empty.IsKeyspaceLevelGC = true + nullState := gc.NewGCStateWithGCBarriers( + constant.NullKeyspaceID, + 40, + 30, + []*gc.GCBarrierInfo{ + gc.NewGCBarrierInfo("local", 45, gc.TTLNeverExpire, time.Time{}), + }, + ) + + clusterState := gc.NewClusterGCStatesWithGlobalGCBarriers( + map[uint32]gc.GCState{ + constant.NullKeyspaceID: nullState, + 1: empty, + }, + []*gc.GlobalGCBarrierInfo{ + gc.NewGlobalGCBarrierInfo("z-global", 60, time.Minute, time.Time{}), + gc.NewGlobalGCBarrierInfo("a-global", 60, gc.TTLNeverExpire, time.Time{}), + gc.NewGlobalGCBarrierInfo("first-global", 50, time.Second, time.Time{}), + }, + ) + + got, err := newAllGCStatesOutput(clusterState) + require.NoError(t, err) + require.Equal(t, []uint32{1, constant.NullKeyspaceID}, []uint32{ + got.GCStates[0].KeyspaceID, + got.GCStates[1].KeyspaceID, + }) + require.NotNil(t, got.GCStates[0].GCBarriers) + require.Empty(t, got.GCStates[0].GCBarriers) + require.Equal(t, []string{"first-global", "a-global", "z-global"}, []string{ + got.GlobalGCBarriers[0].BarrierID, + got.GlobalGCBarriers[1].BarrierID, + got.GlobalGCBarriers[2].BarrierID, + }) + + encoded, err := json.Marshal(got) + require.NoError(t, err) + require.Contains(t, string(encoded), `"gc_barriers":[]`) + require.Contains(t, string(encoded), `"global_gc_barriers":[`) +} + +func TestGCStateOutputRejectsExcludedBarriers(t *testing.T) { + state := gc.NewGCStateWithoutGCBarriers(42, 100, 90) + _, err := newKeyspaceGCStateOutput(42, state) + require.ErrorContains(t, err, "failed to read GC barriers for keyspace 42") + + clusterState := gc.NewClusterGCStatesWithoutGlobalGCBarriers(map[uint32]gc.GCState{}) + _, err = newAllGCStatesOutput(clusterState) + require.ErrorContains(t, err, "failed to read global GC barriers") +} From 6066b9dbf89c6c4c764ba674c649f6a38165b80e Mon Sep 17 00:00:00 2001 From: Wenxuan Zhang Date: Mon, 27 Jul 2026 11:28:58 +0800 Subject: [PATCH 03/31] pd-ctl: cover empty global GC barriers Signed-off-by: Wenxuan Zhang --- .../pdctl/command/gc_state_command_test.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tools/pd-ctl/pdctl/command/gc_state_command_test.go b/tools/pd-ctl/pdctl/command/gc_state_command_test.go index 8fbc889c5d..3cdc8fa12d 100644 --- a/tools/pd-ctl/pdctl/command/gc_state_command_test.go +++ b/tools/pd-ctl/pdctl/command/gc_state_command_test.go @@ -127,6 +127,22 @@ func TestNewAllGCStatesOutputSortsAndKeepsEmptyArrays(t *testing.T) { require.Contains(t, string(encoded), `"global_gc_barriers":[`) } +func TestNewAllGCStatesOutputKeepsEmptyGlobalBarrierArray(t *testing.T) { + clusterState := gc.NewClusterGCStatesWithGlobalGCBarriers( + map[uint32]gc.GCState{}, + nil, + ) + + got, err := newAllGCStatesOutput(clusterState) + require.NoError(t, err) + require.NotNil(t, got.GlobalGCBarriers) + require.Empty(t, got.GlobalGCBarriers) + + encoded, err := json.Marshal(got) + require.NoError(t, err) + require.Contains(t, string(encoded), `"global_gc_barriers":[]`) +} + func TestGCStateOutputRejectsExcludedBarriers(t *testing.T) { state := gc.NewGCStateWithoutGCBarriers(42, 100, 90) _, err := newKeyspaceGCStateOutput(42, state) From 045d188fbcabf8d06efa00f0dd89c18e63c5f064 Mon Sep 17 00:00:00 2001 From: Wenxuan Zhang Date: Mon, 27 Jul 2026 11:38:30 +0800 Subject: [PATCH 04/31] pd-ctl: add gc-state commands Signed-off-by: Wenxuan Zhang --- .../pd-ctl/pdctl/command/gc_state_command.go | 181 ++++++++++++++ .../pdctl/command/gc_state_command_test.go | 231 ++++++++++++++++++ 2 files changed, 412 insertions(+) diff --git a/tools/pd-ctl/pdctl/command/gc_state_command.go b/tools/pd-ctl/pdctl/command/gc_state_command.go index 4b88786c32..f44ed1488d 100644 --- a/tools/pd-ctl/pdctl/command/gc_state_command.go +++ b/tools/pd-ctl/pdctl/command/gc_state_command.go @@ -15,17 +15,91 @@ package command import ( + "context" + "encoding/json" "math" "sort" "strconv" "time" "github.com/pingcap/errors" + "github.com/spf13/cobra" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "github.com/tikv/pd/client" "github.com/tikv/pd/client/clients/gc" + "github.com/tikv/pd/client/pkg/caller" "github.com/tikv/pd/pkg/keyspace/constant" ) +type gcStateReader interface { + GetGCState(context.Context, uint32) (gc.GCState, error) + GetAllKeyspacesGCStates(context.Context) (gc.ClusterGCStates, error) + Close() +} + +type gcStateReaderFactory func(*cobra.Command) (gcStateReader, error) + +type pdGCStateReader struct { + client pd.Client +} + +func (r *pdGCStateReader) GetGCState( + ctx context.Context, + keyspaceID uint32, +) (gc.GCState, error) { + return r.client.GetGCStatesClient(keyspaceID).GetGCState( + ctx, + gc.ExcludeGCBarriers(false), + ) +} + +func (r *pdGCStateReader) GetAllKeyspacesGCStates( + ctx context.Context, +) (gc.ClusterGCStates, error) { + return r.client.GetGCStatesClient( + constant.NullKeyspaceID, + ).GetAllKeyspacesGCStates( + ctx, + gc.ExcludeGCBarriers(false), + gc.ExcludeGlobalGCBarriers(false), + ) +} + +func (r *pdGCStateReader) Close() { + r.client.Close() +} + +func newPDGCStateReader(cmd *cobra.Command) (gcStateReader, error) { + caPath, err := cmd.Flags().GetString("cacert") + if err != nil { + return nil, errors.WithStack(err) + } + certPath, err := cmd.Flags().GetString("cert") + if err != nil { + return nil, errors.WithStack(err) + } + keyPath, err := cmd.Flags().GetString("key") + if err != nil { + return nil, errors.WithStack(err) + } + client, err := pd.NewClientWithContext( + cmd.Context(), + caller.Component(PDControlCallerID), + getEndpoints(cmd), + pd.SecurityOption{ + CAPath: caPath, + CertPath: certPath, + KeyPath: keyPath, + }, + ) + if err != nil { + return nil, err + } + return &pdGCStateReader{client: client}, nil +} + type gcBarrierOutput struct { BarrierID string `json:"barrier_id"` BarrierTS uint64 `json:"barrier_ts"` @@ -175,3 +249,110 @@ func newAllGCStatesOutput(clusterState gc.ClusterGCStates) (allGCStatesOutput, e GlobalGCBarriers: newGlobalGCBarrierOutputs(globalBarriers), }, nil } + +// NewGCStateCommand returns the read-only GC state command. +func NewGCStateCommand() *cobra.Command { + return newGCStateCommand(newPDGCStateReader) +} + +func newGCStateCommand(factory gcStateReaderFactory) *cobra.Command { + command := &cobra.Command{ + Use: "gc-state", + Short: "show keyspace GC state and barriers", + Long: "Show effective per-keyspace GC safe points and barriers. " + + "Use the all subcommand to include cluster-wide global barriers.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + return cmd.Help() + }, + } + command.AddCommand( + newGCStateKeyspaceCommand(factory), + newGCStateAllCommand(factory), + ) + return command +} + +func newGCStateKeyspaceCommand(factory gcStateReaderFactory) *cobra.Command { + return &cobra.Command{ + Use: "keyspace ", + Short: "show one keyspace's effective GC state", + Long: "Show one keyspace's effective GC safe points and local " + + "barriers. Use gc-state all to inspect global barriers. " + + "The decimal NullKeyspace ID is 4294967295.", + Example: " pd-ctl gc-state keyspace 42\n" + + " pd-ctl gc-state keyspace 4294967295", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + keyspaceID, err := parseGCStateKeyspaceID(args[0]) + if err != nil { + return err + } + reader, err := factory(cmd) + if err != nil { + return errors.Annotate(err, "failed to create PD RPC client") + } + defer reader.Close() + + state, err := reader.GetGCState(cmd.Context(), keyspaceID) + if err != nil { + if status.Code(errors.Cause(err)) == codes.Unimplemented { + return errors.Annotate(err, + "gc-state requires a PD server that supports GetGCState") + } + return errors.Annotatef(err, + "failed to get GC state for keyspace %d", keyspaceID) + } + output, err := newKeyspaceGCStateOutput(keyspaceID, state) + if err != nil { + return err + } + return writeGCStateJSON(cmd, output) + }, + } +} + +func newGCStateAllCommand(factory gcStateReaderFactory) *cobra.Command { + return &cobra.Command{ + Use: "all", + Short: "show all keyspace GC states and global barriers", + Long: "Show all active keyspace GC states and local barriers. " + + "Cluster-wide global barriers appear once at the top level.", + Example: " pd-ctl gc-state all", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + reader, err := factory(cmd) + if err != nil { + return errors.Annotate(err, "failed to create PD RPC client") + } + defer reader.Close() + + clusterState, err := reader.GetAllKeyspacesGCStates(cmd.Context()) + if err != nil { + if status.Code(errors.Cause(err)) == codes.Unimplemented { + return errors.Annotate(err, + "gc-state all requires a PD server that supports "+ + "GetAllKeyspacesGCStates") + } + return errors.Annotate(err, "failed to get all keyspaces GC states") + } + output, err := newAllGCStatesOutput(clusterState) + if err != nil { + return err + } + return writeGCStateJSON(cmd, output) + }, + } +} + +func writeGCStateJSON(cmd *cobra.Command, value any) error { + data, err := json.MarshalIndent(value, "", " ") + if err != nil { + return errors.Annotate(err, "failed to marshal GC state JSON") + } + data = append(data, '\n') + if _, err := cmd.OutOrStdout().Write(data); err != nil { + return errors.Annotate(err, "failed to write GC state JSON") + } + return nil +} diff --git a/tools/pd-ctl/pdctl/command/gc_state_command_test.go b/tools/pd-ctl/pdctl/command/gc_state_command_test.go index 3cdc8fa12d..faf107cd60 100644 --- a/tools/pd-ctl/pdctl/command/gc_state_command_test.go +++ b/tools/pd-ctl/pdctl/command/gc_state_command_test.go @@ -15,17 +15,55 @@ package command import ( + "bytes" + "context" "encoding/json" + "errors" + "io" "math" + "strings" "testing" "time" + "github.com/spf13/cobra" "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "github.com/tikv/pd/client/clients/gc" "github.com/tikv/pd/pkg/keyspace/constant" ) +type fakeGCStateReader struct { + state gc.GCState + clusterState gc.ClusterGCStates + err error + requestedID uint32 + getStateCalls int + getAllCalls int + closed bool +} + +func (r *fakeGCStateReader) GetGCState( + _ context.Context, + keyspaceID uint32, +) (gc.GCState, error) { + r.requestedID = keyspaceID + r.getStateCalls++ + return r.state, r.err +} + +func (r *fakeGCStateReader) GetAllKeyspacesGCStates( + _ context.Context, +) (gc.ClusterGCStates, error) { + r.getAllCalls++ + return r.clusterState, r.err +} + +func (r *fakeGCStateReader) Close() { + r.closed = true +} + func TestParseGCStateKeyspaceID(t *testing.T) { for _, testCase := range []struct { name string @@ -152,3 +190,196 @@ func TestGCStateOutputRejectsExcludedBarriers(t *testing.T) { _, err = newAllGCStatesOutput(clusterState) require.ErrorContains(t, err, "failed to read global GC barriers") } + +func TestGCStateKeyspaceCommand(t *testing.T) { + state := gc.NewGCStateWithGCBarriers(42, 100, 90, nil) + state.IsKeyspaceLevelGC = true + reader := &fakeGCStateReader{state: state} + factoryCalls := 0 + cmd := newGCStateCommand(func(*cobra.Command) (gcStateReader, error) { + factoryCalls++ + return reader, nil + }) + output := new(bytes.Buffer) + cmd.SetOut(output) + cmd.SetErr(output) + cmd.SetArgs([]string{"keyspace", "42"}) + + require.NoError(t, cmd.Execute()) + require.Equal(t, 1, factoryCalls) + require.Equal(t, 1, reader.getStateCalls) + require.Equal(t, uint32(42), reader.requestedID) + require.True(t, reader.closed) + + var decoded map[string]json.RawMessage + require.NoError(t, json.Unmarshal(output.Bytes(), &decoded)) + require.Contains(t, decoded, "requested_keyspace_id") + require.Contains(t, decoded, "effective_keyspace_id") + require.Contains(t, decoded, "gc_barriers") + require.NotContains(t, decoded, "global_gc_barriers") +} + +func TestGCStateAllCommand(t *testing.T) { + reader := &fakeGCStateReader{ + clusterState: gc.NewClusterGCStatesWithGlobalGCBarriers( + map[uint32]gc.GCState{}, + nil, + ), + } + cmd := newGCStateCommand(func(*cobra.Command) (gcStateReader, error) { + return reader, nil + }) + output := new(bytes.Buffer) + cmd.SetOut(output) + cmd.SetErr(output) + cmd.SetArgs([]string{"all"}) + + require.NoError(t, cmd.Execute()) + require.Equal(t, 1, reader.getAllCalls) + require.True(t, reader.closed) + require.Contains(t, output.String(), `"gc_states": []`) + require.Contains(t, output.String(), `"global_gc_barriers": []`) +} + +func TestGCStateCommandValidatesBeforeCreatingClient(t *testing.T) { + for _, args := range [][]string{ + {}, + {"keyspace"}, + {"keyspace", "42", "extra"}, + {"keyspace", ""}, + {"keyspace", "-1"}, + {"keyspace", "tenant-a"}, + {"keyspace", "0xffffff"}, + {"keyspace", "16777216"}, + {"keyspace", "4294967294"}, + {"keyspace", "4294967296"}, + {"all", "extra"}, + } { + t.Run(strings.Join(args, "-"), func(t *testing.T) { + factoryCalled := false + cmd := newGCStateCommand( + func(*cobra.Command) (gcStateReader, error) { + factoryCalled = true + return nil, errors.New("factory must not run") + }, + ) + cmd.SetOut(io.Discard) + cmd.SetErr(io.Discard) + cmd.SetArgs(args) + err := cmd.Execute() + if len(args) == 0 { + require.NoError(t, err) + } else { + require.Error(t, err) + } + require.False(t, factoryCalled) + }) + } +} + +func TestGCStateCommandErrors(t *testing.T) { + for _, testCase := range []struct { + name string + args []string + factory gcStateReaderFactory + wantMessage string + }{ + { + name: "client-creation", + args: []string{"keyspace", "42"}, + factory: func(*cobra.Command) (gcStateReader, error) { + return nil, errors.New("dial rejected") + }, + wantMessage: "failed to create PD RPC client", + }, + { + name: "single-rpc-error", + args: []string{"keyspace", "42"}, + factory: func(*cobra.Command) (gcStateReader, error) { + return &fakeGCStateReader{err: errors.New("rpc rejected")}, nil + }, + wantMessage: "failed to get GC state for keyspace 42", + }, + { + name: "single-unimplemented", + args: []string{"keyspace", "42"}, + factory: func(*cobra.Command) (gcStateReader, error) { + return &fakeGCStateReader{ + err: status.Error(codes.Unimplemented, "method unavailable"), + }, nil + }, + wantMessage: "gc-state requires a PD server that supports GetGCState", + }, + { + name: "single-missing-barriers", + args: []string{"keyspace", "42"}, + factory: func(*cobra.Command) (gcStateReader, error) { + return &fakeGCStateReader{ + state: gc.NewGCStateWithoutGCBarriers(42, 100, 90), + }, nil + }, + wantMessage: "failed to read GC barriers for keyspace 42", + }, + { + name: "all-rpc-error", + args: []string{"all"}, + factory: func(*cobra.Command) (gcStateReader, error) { + return &fakeGCStateReader{err: errors.New("rpc rejected")}, nil + }, + wantMessage: "failed to get all keyspaces GC states", + }, + { + name: "all-unimplemented", + args: []string{"all"}, + factory: func(*cobra.Command) (gcStateReader, error) { + return &fakeGCStateReader{ + err: status.Error(codes.Unimplemented, "method unavailable"), + }, nil + }, + wantMessage: "gc-state all requires a PD server that supports " + + "GetAllKeyspacesGCStates", + }, + { + name: "all-missing-global-barriers", + args: []string{"all"}, + factory: func(*cobra.Command) (gcStateReader, error) { + return &fakeGCStateReader{ + clusterState: gc.NewClusterGCStatesWithoutGlobalGCBarriers( + map[uint32]gc.GCState{}, + ), + }, nil + }, + wantMessage: "failed to read global GC barriers", + }, + } { + t.Run(testCase.name, func(t *testing.T) { + cmd := newGCStateCommand(testCase.factory) + cmd.SetOut(io.Discard) + cmd.SetErr(io.Discard) + cmd.SetArgs(testCase.args) + err := cmd.Execute() + require.ErrorContains(t, err, testCase.wantMessage) + }) + } +} + +type failingWriter struct{} + +func (failingWriter) Write([]byte) (int, error) { + return 0, errors.New("output rejected") +} + +func TestGCStateCommandReturnsOutputError(t *testing.T) { + state := gc.NewGCStateWithGCBarriers(42, 100, 90, nil) + reader := &fakeGCStateReader{state: state} + cmd := newGCStateCommand(func(*cobra.Command) (gcStateReader, error) { + return reader, nil + }) + cmd.SetOut(failingWriter{}) + cmd.SetErr(io.Discard) + cmd.SetArgs([]string{"keyspace", "42"}) + + err := cmd.Execute() + require.ErrorContains(t, err, "failed to write GC state JSON") + require.True(t, reader.closed) +} From 6242d9e72b3ad8226aace8f22f6a11bbe57fdf30 Mon Sep 17 00:00:00 2001 From: Wenxuan Zhang Date: Mon, 27 Jul 2026 11:51:07 +0800 Subject: [PATCH 05/31] pd-ctl: test gc-state against PD Signed-off-by: Wenxuan Zhang --- tests/integrations/client/client_test.go | 3 + tools/pd-ctl/pdctl/ctl.go | 1 + tools/pd-ctl/tests/safepoint/gc_state_test.go | 302 ++++++++++++++++++ 3 files changed, 306 insertions(+) create mode 100644 tools/pd-ctl/tests/safepoint/gc_state_test.go diff --git a/tests/integrations/client/client_test.go b/tests/integrations/client/client_test.go index 97e5f3aeeb..6abb343c3d 100644 --- a/tests/integrations/client/client_test.go +++ b/tests/integrations/client/client_test.go @@ -2872,6 +2872,9 @@ func (s *clientStatefulTestSuite) TestGetAllKeyspaceGCStates() { re.NoError(err) res, err = cli.GetAllKeyspacesGCStates(ctx, gc.ExcludeGCBarriers(false), gc.ExcludeGlobalGCBarriers(false)) re.NoError(err) + re.True(res.GCStates[1].IsKeyspaceLevelGC) + re.True(res.GCStates[2].IsKeyspaceLevelGC) + re.False(res.GCStates[3].IsKeyspaceLevelGC) state, ok = res.GCStates[2] re.True(ok) gcBarriers, err = state.GetGCBarriers() diff --git a/tools/pd-ctl/pdctl/ctl.go b/tools/pd-ctl/pdctl/ctl.go index be4be12ff9..393fc82f67 100644 --- a/tools/pd-ctl/pdctl/ctl.go +++ b/tools/pd-ctl/pdctl/ctl.go @@ -68,6 +68,7 @@ func GetRootCmd() *cobra.Command { command.NewLogCommand(), command.NewPluginCommand(), command.NewServiceGCSafepointCommand(), + command.NewGCStateCommand(), command.NewMinResolvedTSCommand(), command.NewCompletionCommand(), command.NewUnsafeCommand(), diff --git a/tools/pd-ctl/tests/safepoint/gc_state_test.go b/tools/pd-ctl/tests/safepoint/gc_state_test.go new file mode 100644 index 0000000000..291d98c1eb --- /dev/null +++ b/tools/pd-ctl/tests/safepoint/gc_state_test.go @@ -0,0 +1,302 @@ +// Copyright 2026 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package safepoint_test + +import ( + "bytes" + "encoding/json" + "math" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/tikv/pd/pkg/keyspace" + "github.com/tikv/pd/pkg/keyspace/constant" + "github.com/tikv/pd/pkg/versioninfo/kerneltype" + "github.com/tikv/pd/server/config" + pdTests "github.com/tikv/pd/tests" + ctl "github.com/tikv/pd/tools/pd-ctl/pdctl" + "github.com/tikv/pd/tools/pd-ctl/tests" +) + +type gcStateCommandBarrier struct { + BarrierID string `json:"barrier_id"` + BarrierTS uint64 `json:"barrier_ts"` + TTLSeconds int64 `json:"ttl_seconds"` +} + +type gcStateCommandSingle struct { + RequestedKeyspaceID uint32 `json:"requested_keyspace_id"` + EffectiveKeyspaceID uint32 `json:"effective_keyspace_id"` + IsKeyspaceLevelGC bool `json:"is_keyspace_level_gc"` + TxnSafePoint uint64 `json:"txn_safe_point"` + GCSafePoint uint64 `json:"gc_safe_point"` + GCBarriers []gcStateCommandBarrier `json:"gc_barriers"` +} + +type gcStateCommandState struct { + KeyspaceID uint32 `json:"keyspace_id"` + IsKeyspaceLevelGC bool `json:"is_keyspace_level_gc"` + TxnSafePoint uint64 `json:"txn_safe_point"` + GCSafePoint uint64 `json:"gc_safe_point"` + GCBarriers []gcStateCommandBarrier `json:"gc_barriers"` +} + +type gcStateCommandAll struct { + GCStates []gcStateCommandState `json:"gc_states"` + GlobalGCBarriers []gcStateCommandBarrier `json:"global_gc_barriers"` +} + +type expectedGCStateCommandBarrier struct { + barrierID string + barrierTS uint64 + expires bool +} + +func requireGCStateCommandBarriers( + re *require.Assertions, + actual []gcStateCommandBarrier, + expected []expectedGCStateCommandBarrier, +) { + re.Len(actual, len(expected)) + for i, want := range expected { + re.Equal(want.barrierID, actual[i].BarrierID) + re.Equal(want.barrierTS, actual[i].BarrierTS) + if want.expires { + re.GreaterOrEqual(actual[i].TTLSeconds, int64(3595)) + re.LessOrEqual(actual[i].TTLSeconds, int64(3600)) + } else { + re.Equal(int64(math.MaxInt64), actual[i].TTLSeconds) + } + } +} + +func TestGCState(t *testing.T) { + re := require.New(t) + ctx := t.Context() + cluster, err := pdTests.NewTestCluster(ctx, 1, + func(conf *config.Config, _ string) { + conf.Keyspace.WaitRegionSplit = false + }, + ) + re.NoError(err) + defer cluster.Destroy() + re.NoError(cluster.RunInitialServers()) + re.NotEmpty(cluster.WaitLeader()) + leaderServer := cluster.GetLeaderServer() + re.NoError(leaderServer.BootstrapCluster()) + + keyspaceLevel, err := leaderServer.GetKeyspaceManager().CreateKeyspace( + &keyspace.CreateKeyspaceRequest{ + Name: "gc_state_ks_level", + Config: map[string]string{ + keyspace.GCManagementType: keyspace.KeyspaceLevelGC, + }, + CreateTime: time.Now().Unix(), + }, + ) + re.NoError(err) + + var unifiedKeyspaceID uint32 + if !kerneltype.IsNextGen() { + unified, err := leaderServer.GetKeyspaceManager().CreateKeyspace( + &keyspace.CreateKeyspaceRequest{ + Name: "gc_state_unified", + Config: map[string]string{ + keyspace.GCManagementType: keyspace.UnifiedGC, + }, + CreateTime: time.Now().Unix(), + }, + ) + re.NoError(err) + unifiedKeyspaceID = unified.Id + } + + manager := leaderServer.GetServer().GetGCStateManager() + now := time.Now() + _, err = manager.AdvanceTxnSafePoint(constant.NullKeyspaceID, 100, now) + re.NoError(err) + _, _, err = manager.AdvanceGCSafePoint(constant.NullKeyspaceID, 90) + re.NoError(err) + _, err = manager.SetGCBarrier( + constant.NullKeyspaceID, + "z-null", + 120, + time.Hour, + now, + ) + re.NoError(err) + _, err = manager.SetGCBarrier( + constant.NullKeyspaceID, + "a-null", + 110, + time.Duration(math.MaxInt64), + now, + ) + re.NoError(err) + + _, err = manager.AdvanceTxnSafePoint(keyspaceLevel.Id, 200, now) + re.NoError(err) + _, _, err = manager.AdvanceGCSafePoint(keyspaceLevel.Id, 190) + re.NoError(err) + _, err = manager.SetGCBarrier( + keyspaceLevel.Id, + "z-local", + 220, + time.Hour, + now, + ) + re.NoError(err) + _, err = manager.SetGCBarrier( + keyspaceLevel.Id, + "a-local", + 210, + time.Duration(math.MaxInt64), + now, + ) + re.NoError(err) + + _, err = manager.SetGlobalGCBarrier( + ctx, + "z-global", + 320, + time.Hour, + now, + ) + re.NoError(err) + _, err = manager.SetGlobalGCBarrier( + ctx, + "a-global", + 310, + time.Duration(math.MaxInt64), + now, + ) + re.NoError(err) + + pdAddr := cluster.GetConfig().GetClientURL() + keyspaceLevelID := strconv.FormatUint(uint64(keyspaceLevel.Id), 10) + output, err := tests.ExecuteCommand( + ctl.GetRootCmd(), "-u", pdAddr, "gc-state", "keyspace", keyspaceLevelID, + ) + re.NoError(err) + var singleProperties map[string]json.RawMessage + re.NoError(json.Unmarshal(output, &singleProperties), string(output)) + re.NotContains(singleProperties, "global_gc_barriers") + var single gcStateCommandSingle + re.NoError(json.Unmarshal(output, &single), string(output)) + re.Equal(keyspaceLevel.Id, single.RequestedKeyspaceID) + re.Equal(keyspaceLevel.Id, single.EffectiveKeyspaceID) + re.True(single.IsKeyspaceLevelGC) + re.Equal(uint64(200), single.TxnSafePoint) + re.Equal(uint64(190), single.GCSafePoint) + requireGCStateCommandBarriers(re, single.GCBarriers, []expectedGCStateCommandBarrier{ + {barrierID: "a-local", barrierTS: 210}, + {barrierID: "z-local", barrierTS: 220, expires: true}, + }) + + output, err = tests.ExecuteCommand( + ctl.GetRootCmd(), "-u", pdAddr, "gc-state", "keyspace", "4294967295", + ) + re.NoError(err) + re.NoError(json.Unmarshal(output, &single), string(output)) + re.Equal(constant.NullKeyspaceID, single.RequestedKeyspaceID) + re.Equal(constant.NullKeyspaceID, single.EffectiveKeyspaceID) + re.False(single.IsKeyspaceLevelGC) + re.Equal(uint64(100), single.TxnSafePoint) + re.Equal(uint64(90), single.GCSafePoint) + requireGCStateCommandBarriers(re, single.GCBarriers, []expectedGCStateCommandBarrier{ + {barrierID: "a-null", barrierTS: 110}, + {barrierID: "z-null", barrierTS: 120, expires: true}, + }) + + _, err = tests.ExecuteCommand( + ctl.GetRootCmd(), "-u", pdAddr, "gc-state", "keyspace", "16770000", + ) + re.ErrorContains(err, "failed to get GC state for keyspace 16770000") + + output, err = tests.ExecuteCommand(ctl.GetRootCmd(), "-u", pdAddr, "gc-state", "all") + re.NoError(err) + re.Equal(1, bytes.Count(output, []byte(`"global_gc_barriers"`)), string(output)) + var all gcStateCommandAll + re.NoError(json.Unmarshal(output, &all), string(output)) + for i := 1; i < len(all.GCStates); i++ { + re.Less(all.GCStates[i-1].KeyspaceID, all.GCStates[i].KeyspaceID) + } + statesByID := make(map[uint32]gcStateCommandState, len(all.GCStates)) + for _, state := range all.GCStates { + re.NotNil(state.GCBarriers) + statesByID[state.KeyspaceID] = state + } + + nullState, ok := statesByID[constant.NullKeyspaceID] + re.True(ok) + re.False(nullState.IsKeyspaceLevelGC) + re.Equal(uint64(100), nullState.TxnSafePoint) + re.Equal(uint64(90), nullState.GCSafePoint) + requireGCStateCommandBarriers(re, nullState.GCBarriers, []expectedGCStateCommandBarrier{ + {barrierID: "a-null", barrierTS: 110}, + {barrierID: "z-null", barrierTS: 120, expires: true}, + }) + + keyspaceLevelState, ok := statesByID[keyspaceLevel.Id] + re.True(ok) + re.True(keyspaceLevelState.IsKeyspaceLevelGC) + re.Equal(uint64(200), keyspaceLevelState.TxnSafePoint) + re.Equal(uint64(190), keyspaceLevelState.GCSafePoint) + requireGCStateCommandBarriers(re, keyspaceLevelState.GCBarriers, []expectedGCStateCommandBarrier{ + {barrierID: "a-local", barrierTS: 210}, + {barrierID: "z-local", barrierTS: 220, expires: true}, + }) + requireGCStateCommandBarriers(re, all.GlobalGCBarriers, []expectedGCStateCommandBarrier{ + {barrierID: "a-global", barrierTS: 310}, + {barrierID: "z-global", barrierTS: 320, expires: true}, + }) + + if kerneltype.IsNextGen() { + systemState, ok := statesByID[constant.SystemKeyspaceID] + re.True(ok) + re.True(systemState.IsKeyspaceLevelGC) + } else { + unifiedKeyspaceIDString := strconv.FormatUint(uint64(unifiedKeyspaceID), 10) + output, err = tests.ExecuteCommand( + ctl.GetRootCmd(), "-u", pdAddr, "gc-state", "keyspace", unifiedKeyspaceIDString, + ) + re.NoError(err) + re.NoError(json.Unmarshal(output, &single), string(output)) + re.Equal(unifiedKeyspaceID, single.RequestedKeyspaceID) + re.Equal(constant.NullKeyspaceID, single.EffectiveKeyspaceID) + re.False(single.IsKeyspaceLevelGC) + re.Equal(uint64(100), single.TxnSafePoint) + re.Equal(uint64(90), single.GCSafePoint) + requireGCStateCommandBarriers(re, single.GCBarriers, []expectedGCStateCommandBarrier{ + {barrierID: "a-null", barrierTS: 110}, + {barrierID: "z-null", barrierTS: 120, expires: true}, + }) + + unifiedState, ok := statesByID[unifiedKeyspaceID] + re.True(ok) + re.False(unifiedState.IsKeyspaceLevelGC) + re.Zero(unifiedState.TxnSafePoint) + re.Zero(unifiedState.GCSafePoint) + re.NotNil(unifiedState.GCBarriers) + re.Empty(unifiedState.GCBarriers) + } + + output, err = tests.ExecuteCommand(ctl.GetRootCmd(), "-u", pdAddr, "service-gc-safepoint") + re.NoError(err) + re.True(json.Valid(output), string(output)) +} From b25d2cbc7f08bf0be49d950847f00867bc900ed4 Mon Sep 17 00:00:00 2001 From: Wenxuan Zhang Date: Mon, 27 Jul 2026 12:32:40 +0800 Subject: [PATCH 06/31] test: require GC state fixtures Signed-off-by: Wenxuan Zhang --- tests/integrations/client/client_test.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/integrations/client/client_test.go b/tests/integrations/client/client_test.go index 6abb343c3d..9ff5807ea2 100644 --- a/tests/integrations/client/client_test.go +++ b/tests/integrations/client/client_test.go @@ -2872,12 +2872,16 @@ func (s *clientStatefulTestSuite) TestGetAllKeyspaceGCStates() { re.NoError(err) res, err = cli.GetAllKeyspacesGCStates(ctx, gc.ExcludeGCBarriers(false), gc.ExcludeGlobalGCBarriers(false)) re.NoError(err) - re.True(res.GCStates[1].IsKeyspaceLevelGC) - re.True(res.GCStates[2].IsKeyspaceLevelGC) - re.False(res.GCStates[3].IsKeyspaceLevelGC) - state, ok = res.GCStates[2] + state1, ok := res.GCStates[1] re.True(ok) - gcBarriers, err = state.GetGCBarriers() + re.True(state1.IsKeyspaceLevelGC) + state2, ok := res.GCStates[2] + re.True(ok) + re.True(state2.IsKeyspaceLevelGC) + state3, ok := res.GCStates[3] + re.True(ok) + re.False(state3.IsKeyspaceLevelGC) + gcBarriers, err = state2.GetGCBarriers() re.NoError(err) re.Equal("b4", gcBarriers[0].BarrierID) re.Equal(uint64(14), gcBarriers[0].BarrierTS) From 1575e08691c23b66287825c1c1a5499ae7883557 Mon Sep 17 00:00:00 2001 From: Wenxuan Zhang Date: Mon, 27 Jul 2026 14:51:55 +0800 Subject: [PATCH 07/31] docs: document pd-ctl gc-state Signed-off-by: Wenxuan Zhang --- tools/pd-ctl/README.md | 53 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/tools/pd-ctl/README.md b/tools/pd-ctl/README.md index d17cc2621c..f45d9ff9f4 100644 --- a/tools/pd-ctl/README.md +++ b/tools/pd-ctl/README.md @@ -10,3 +10,56 @@ ## Usage The details about how to use `pd-ctl` can be found in [PD Control User Guide](https://docs.pingcap.com/tidb/dev/pd-control). + +## GC state troubleshooting + +Use `gc-state` to inspect the safe points and barriers that can block GC. The +command is read-only and emits deterministic JSON for scripts and diffs. + +Inspect one keyspace by its decimal ID: + +```bash +pd-ctl gc-state keyspace 42 +``` + +```json +{ + "requested_keyspace_id": 42, + "effective_keyspace_id": 4294967295, + "is_keyspace_level_gc": false, + "txn_safe_point": 465000000000000000, + "gc_safe_point": 464900000000000000, + "gc_barriers": [] +} +``` + +The response contains both `requested_keyspace_id` and +`effective_keyspace_id`. They are equal for keyspace-level GC. A keyspace that +uses unified GC returns `4294967295`, the NullKeyspace ID, as its effective +scope. The response contains local `gc_barriers` only. + +Inspect all active GC scopes and cluster-wide barriers: + +```bash +pd-ctl gc-state all +``` + +```json +{ + "gc_states": [ + { + "keyspace_id": 42, + "is_keyspace_level_gc": true, + "txn_safe_point": 465000000000000000, + "gc_safe_point": 464900000000000000, + "gc_barriers": [] + } + ], + "global_gc_barriers": [] +} +``` + +The response sorts `gc_states` by `keyspace_id`. It reports cluster-wide +barriers once in the top-level `global_gc_barriers` array. Barrier TTLs use +remaining seconds, and `9223372036854775807` means that a barrier never +expires. From d71f055b794af61b93d4e0125754e8e121b0b8df Mon Sep 17 00:00:00 2001 From: Wenxuan Zhang Date: Mon, 27 Jul 2026 15:05:54 +0800 Subject: [PATCH 08/31] client: fix GC test license header Signed-off-by: Wenxuan Zhang --- client/gc_client_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/client/gc_client_test.go b/client/gc_client_test.go index 8bee74d5b1..71458d72b8 100644 --- a/client/gc_client_test.go +++ b/client/gc_client_test.go @@ -8,6 +8,7 @@ // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. From 21761f298ceae74c44aa8f335fd1d7d9a0e97a10 Mon Sep 17 00:00:00 2001 From: Wenxuan Zhang Date: Mon, 27 Jul 2026 15:13:17 +0800 Subject: [PATCH 09/31] pd-ctl: fix gc-state static checks Signed-off-by: Wenxuan Zhang --- .../pd-ctl/pdctl/command/gc_state_command.go | 29 ++++++++++--------- .../pdctl/command/gc_state_command_test.go | 16 +++++----- 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/tools/pd-ctl/pdctl/command/gc_state_command.go b/tools/pd-ctl/pdctl/command/gc_state_command.go index f44ed1488d..1ea7981753 100644 --- a/tools/pd-ctl/pdctl/command/gc_state_command.go +++ b/tools/pd-ctl/pdctl/command/gc_state_command.go @@ -22,21 +22,22 @@ import ( "strconv" "time" - "github.com/pingcap/errors" "github.com/spf13/cobra" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "github.com/tikv/pd/client" + "github.com/pingcap/errors" + + pd "github.com/tikv/pd/client" "github.com/tikv/pd/client/clients/gc" "github.com/tikv/pd/client/pkg/caller" "github.com/tikv/pd/pkg/keyspace/constant" ) type gcStateReader interface { - GetGCState(context.Context, uint32) (gc.GCState, error) - GetAllKeyspacesGCStates(context.Context) (gc.ClusterGCStates, error) - Close() + getGCState(context.Context, uint32) (gc.GCState, error) + getAllKeyspacesGCStates(context.Context) (gc.ClusterGCStates, error) + close() } type gcStateReaderFactory func(*cobra.Command) (gcStateReader, error) @@ -45,7 +46,7 @@ type pdGCStateReader struct { client pd.Client } -func (r *pdGCStateReader) GetGCState( +func (r *pdGCStateReader) getGCState( ctx context.Context, keyspaceID uint32, ) (gc.GCState, error) { @@ -55,7 +56,7 @@ func (r *pdGCStateReader) GetGCState( ) } -func (r *pdGCStateReader) GetAllKeyspacesGCStates( +func (r *pdGCStateReader) getAllKeyspacesGCStates( ctx context.Context, ) (gc.ClusterGCStates, error) { return r.client.GetGCStatesClient( @@ -67,7 +68,7 @@ func (r *pdGCStateReader) GetAllKeyspacesGCStates( ) } -func (r *pdGCStateReader) Close() { +func (r *pdGCStateReader) close() { r.client.Close() } @@ -252,10 +253,10 @@ func newAllGCStatesOutput(clusterState gc.ClusterGCStates) (allGCStatesOutput, e // NewGCStateCommand returns the read-only GC state command. func NewGCStateCommand() *cobra.Command { - return newGCStateCommand(newPDGCStateReader) + return buildGCStateCommand(newPDGCStateReader) } -func newGCStateCommand(factory gcStateReaderFactory) *cobra.Command { +func buildGCStateCommand(factory gcStateReaderFactory) *cobra.Command { command := &cobra.Command{ Use: "gc-state", Short: "show keyspace GC state and barriers", @@ -292,9 +293,9 @@ func newGCStateKeyspaceCommand(factory gcStateReaderFactory) *cobra.Command { if err != nil { return errors.Annotate(err, "failed to create PD RPC client") } - defer reader.Close() + defer reader.close() - state, err := reader.GetGCState(cmd.Context(), keyspaceID) + state, err := reader.getGCState(cmd.Context(), keyspaceID) if err != nil { if status.Code(errors.Cause(err)) == codes.Unimplemented { return errors.Annotate(err, @@ -325,9 +326,9 @@ func newGCStateAllCommand(factory gcStateReaderFactory) *cobra.Command { if err != nil { return errors.Annotate(err, "failed to create PD RPC client") } - defer reader.Close() + defer reader.close() - clusterState, err := reader.GetAllKeyspacesGCStates(cmd.Context()) + clusterState, err := reader.getAllKeyspacesGCStates(cmd.Context()) if err != nil { if status.Code(errors.Cause(err)) == codes.Unimplemented { return errors.Annotate(err, diff --git a/tools/pd-ctl/pdctl/command/gc_state_command_test.go b/tools/pd-ctl/pdctl/command/gc_state_command_test.go index faf107cd60..79ebc3b731 100644 --- a/tools/pd-ctl/pdctl/command/gc_state_command_test.go +++ b/tools/pd-ctl/pdctl/command/gc_state_command_test.go @@ -44,7 +44,7 @@ type fakeGCStateReader struct { closed bool } -func (r *fakeGCStateReader) GetGCState( +func (r *fakeGCStateReader) getGCState( _ context.Context, keyspaceID uint32, ) (gc.GCState, error) { @@ -53,14 +53,14 @@ func (r *fakeGCStateReader) GetGCState( return r.state, r.err } -func (r *fakeGCStateReader) GetAllKeyspacesGCStates( +func (r *fakeGCStateReader) getAllKeyspacesGCStates( _ context.Context, ) (gc.ClusterGCStates, error) { r.getAllCalls++ return r.clusterState, r.err } -func (r *fakeGCStateReader) Close() { +func (r *fakeGCStateReader) close() { r.closed = true } @@ -196,7 +196,7 @@ func TestGCStateKeyspaceCommand(t *testing.T) { state.IsKeyspaceLevelGC = true reader := &fakeGCStateReader{state: state} factoryCalls := 0 - cmd := newGCStateCommand(func(*cobra.Command) (gcStateReader, error) { + cmd := buildGCStateCommand(func(*cobra.Command) (gcStateReader, error) { factoryCalls++ return reader, nil }) @@ -226,7 +226,7 @@ func TestGCStateAllCommand(t *testing.T) { nil, ), } - cmd := newGCStateCommand(func(*cobra.Command) (gcStateReader, error) { + cmd := buildGCStateCommand(func(*cobra.Command) (gcStateReader, error) { return reader, nil }) output := new(bytes.Buffer) @@ -257,7 +257,7 @@ func TestGCStateCommandValidatesBeforeCreatingClient(t *testing.T) { } { t.Run(strings.Join(args, "-"), func(t *testing.T) { factoryCalled := false - cmd := newGCStateCommand( + cmd := buildGCStateCommand( func(*cobra.Command) (gcStateReader, error) { factoryCalled = true return nil, errors.New("factory must not run") @@ -353,7 +353,7 @@ func TestGCStateCommandErrors(t *testing.T) { }, } { t.Run(testCase.name, func(t *testing.T) { - cmd := newGCStateCommand(testCase.factory) + cmd := buildGCStateCommand(testCase.factory) cmd.SetOut(io.Discard) cmd.SetErr(io.Discard) cmd.SetArgs(testCase.args) @@ -372,7 +372,7 @@ func (failingWriter) Write([]byte) (int, error) { func TestGCStateCommandReturnsOutputError(t *testing.T) { state := gc.NewGCStateWithGCBarriers(42, 100, 90, nil) reader := &fakeGCStateReader{state: state} - cmd := newGCStateCommand(func(*cobra.Command) (gcStateReader, error) { + cmd := buildGCStateCommand(func(*cobra.Command) (gcStateReader, error) { return reader, nil }) cmd.SetOut(failingWriter{}) From 4901984df9440889fcbf9d930c680d6e21d8b1db Mon Sep 17 00:00:00 2001 From: Wenxuan Zhang Date: Mon, 27 Jul 2026 15:55:58 +0800 Subject: [PATCH 10/31] test: isolate gc-state JSON decodes Signed-off-by: Wenxuan Zhang --- tools/pd-ctl/tests/safepoint/gc_state_test.go | 46 ++++++++++--------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/tools/pd-ctl/tests/safepoint/gc_state_test.go b/tools/pd-ctl/tests/safepoint/gc_state_test.go index 291d98c1eb..bf68755a9c 100644 --- a/tools/pd-ctl/tests/safepoint/gc_state_test.go +++ b/tools/pd-ctl/tests/safepoint/gc_state_test.go @@ -196,14 +196,14 @@ func TestGCState(t *testing.T) { var singleProperties map[string]json.RawMessage re.NoError(json.Unmarshal(output, &singleProperties), string(output)) re.NotContains(singleProperties, "global_gc_barriers") - var single gcStateCommandSingle - re.NoError(json.Unmarshal(output, &single), string(output)) - re.Equal(keyspaceLevel.Id, single.RequestedKeyspaceID) - re.Equal(keyspaceLevel.Id, single.EffectiveKeyspaceID) - re.True(single.IsKeyspaceLevelGC) - re.Equal(uint64(200), single.TxnSafePoint) - re.Equal(uint64(190), single.GCSafePoint) - requireGCStateCommandBarriers(re, single.GCBarriers, []expectedGCStateCommandBarrier{ + var keyspaceLevelResponse gcStateCommandSingle + re.NoError(json.Unmarshal(output, &keyspaceLevelResponse), string(output)) + re.Equal(keyspaceLevel.Id, keyspaceLevelResponse.RequestedKeyspaceID) + re.Equal(keyspaceLevel.Id, keyspaceLevelResponse.EffectiveKeyspaceID) + re.True(keyspaceLevelResponse.IsKeyspaceLevelGC) + re.Equal(uint64(200), keyspaceLevelResponse.TxnSafePoint) + re.Equal(uint64(190), keyspaceLevelResponse.GCSafePoint) + requireGCStateCommandBarriers(re, keyspaceLevelResponse.GCBarriers, []expectedGCStateCommandBarrier{ {barrierID: "a-local", barrierTS: 210}, {barrierID: "z-local", barrierTS: 220, expires: true}, }) @@ -212,13 +212,14 @@ func TestGCState(t *testing.T) { ctl.GetRootCmd(), "-u", pdAddr, "gc-state", "keyspace", "4294967295", ) re.NoError(err) - re.NoError(json.Unmarshal(output, &single), string(output)) - re.Equal(constant.NullKeyspaceID, single.RequestedKeyspaceID) - re.Equal(constant.NullKeyspaceID, single.EffectiveKeyspaceID) - re.False(single.IsKeyspaceLevelGC) - re.Equal(uint64(100), single.TxnSafePoint) - re.Equal(uint64(90), single.GCSafePoint) - requireGCStateCommandBarriers(re, single.GCBarriers, []expectedGCStateCommandBarrier{ + var nullKeyspaceResponse gcStateCommandSingle + re.NoError(json.Unmarshal(output, &nullKeyspaceResponse), string(output)) + re.Equal(constant.NullKeyspaceID, nullKeyspaceResponse.RequestedKeyspaceID) + re.Equal(constant.NullKeyspaceID, nullKeyspaceResponse.EffectiveKeyspaceID) + re.False(nullKeyspaceResponse.IsKeyspaceLevelGC) + re.Equal(uint64(100), nullKeyspaceResponse.TxnSafePoint) + re.Equal(uint64(90), nullKeyspaceResponse.GCSafePoint) + requireGCStateCommandBarriers(re, nullKeyspaceResponse.GCBarriers, []expectedGCStateCommandBarrier{ {barrierID: "a-null", barrierTS: 110}, {barrierID: "z-null", barrierTS: 120, expires: true}, }) @@ -276,13 +277,14 @@ func TestGCState(t *testing.T) { ctl.GetRootCmd(), "-u", pdAddr, "gc-state", "keyspace", unifiedKeyspaceIDString, ) re.NoError(err) - re.NoError(json.Unmarshal(output, &single), string(output)) - re.Equal(unifiedKeyspaceID, single.RequestedKeyspaceID) - re.Equal(constant.NullKeyspaceID, single.EffectiveKeyspaceID) - re.False(single.IsKeyspaceLevelGC) - re.Equal(uint64(100), single.TxnSafePoint) - re.Equal(uint64(90), single.GCSafePoint) - requireGCStateCommandBarriers(re, single.GCBarriers, []expectedGCStateCommandBarrier{ + var unifiedKeyspaceResponse gcStateCommandSingle + re.NoError(json.Unmarshal(output, &unifiedKeyspaceResponse), string(output)) + re.Equal(unifiedKeyspaceID, unifiedKeyspaceResponse.RequestedKeyspaceID) + re.Equal(constant.NullKeyspaceID, unifiedKeyspaceResponse.EffectiveKeyspaceID) + re.False(unifiedKeyspaceResponse.IsKeyspaceLevelGC) + re.Equal(uint64(100), unifiedKeyspaceResponse.TxnSafePoint) + re.Equal(uint64(90), unifiedKeyspaceResponse.GCSafePoint) + requireGCStateCommandBarriers(re, unifiedKeyspaceResponse.GCBarriers, []expectedGCStateCommandBarrier{ {barrierID: "a-null", barrierTS: 110}, {barrierID: "z-null", barrierTS: 120, expires: true}, }) From 1897495a38abd0ae45ad9dc9b2b8c9f903f956a9 Mon Sep 17 00:00:00 2001 From: Wenxuan Zhang Date: Mon, 27 Jul 2026 16:20:29 +0800 Subject: [PATCH 11/31] test: isolate cluster ID initialization Signed-off-by: Wenxuan Zhang --- pkg/storage/endpoint/cluster_id_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/storage/endpoint/cluster_id_test.go b/pkg/storage/endpoint/cluster_id_test.go index 23e840432a..3eb1496ebe 100644 --- a/pkg/storage/endpoint/cluster_id_test.go +++ b/pkg/storage/endpoint/cluster_id_test.go @@ -30,6 +30,7 @@ func TestMain(m *testing.M) { } func TestInitClusterID(t *testing.T) { + t.Cleanup(keypath.ResetClusterID) re := require.New(t) _, client, clean := etcdutil.NewTestEtcdCluster(t, 1, nil) defer clean() From 2ea4924061831242bcdb0444124738c4b908ab3c Mon Sep 17 00:00:00 2001 From: Wenxuan Zhang Date: Mon, 27 Jul 2026 16:50:34 +0800 Subject: [PATCH 12/31] pd-ctl: add global GC state view Signed-off-by: Wenxuan Zhang --- .../pd-ctl/pdctl/command/gc_state_command.go | 106 ++++++++-- .../pdctl/command/gc_state_command_test.go | 185 +++++++++++++++++- 2 files changed, 270 insertions(+), 21 deletions(-) diff --git a/tools/pd-ctl/pdctl/command/gc_state_command.go b/tools/pd-ctl/pdctl/command/gc_state_command.go index 1ea7981753..d27d26ca5e 100644 --- a/tools/pd-ctl/pdctl/command/gc_state_command.go +++ b/tools/pd-ctl/pdctl/command/gc_state_command.go @@ -36,6 +36,7 @@ import ( type gcStateReader interface { getGCState(context.Context, uint32) (gc.GCState, error) + getGlobalGCState(context.Context) (gc.ClusterGCStates, error) getAllKeyspacesGCStates(context.Context) (gc.ClusterGCStates, error) close() } @@ -46,6 +47,13 @@ type pdGCStateReader struct { client pd.Client } +type clusterGCStatesClient interface { + GetAllKeyspacesGCStates( + context.Context, + ...gc.GCStatesAPIOption, + ) (gc.ClusterGCStates, error) +} + func (r *pdGCStateReader) getGCState( ctx context.Context, keyspaceID uint32, @@ -59,11 +67,31 @@ func (r *pdGCStateReader) getGCState( func (r *pdGCStateReader) getAllKeyspacesGCStates( ctx context.Context, ) (gc.ClusterGCStates, error) { - return r.client.GetGCStatesClient( - constant.NullKeyspaceID, - ).GetAllKeyspacesGCStates( + return readClusterGCStates( ctx, - gc.ExcludeGCBarriers(false), + r.client.GetGCStatesClient(constant.NullKeyspaceID), + false, + ) +} + +func (r *pdGCStateReader) getGlobalGCState( + ctx context.Context, +) (gc.ClusterGCStates, error) { + return readClusterGCStates( + ctx, + r.client.GetGCStatesClient(constant.NullKeyspaceID), + true, + ) +} + +func readClusterGCStates( + ctx context.Context, + client clusterGCStatesClient, + excludeGCBarriers bool, +) (gc.ClusterGCStates, error) { + return client.GetAllKeyspacesGCStates( + ctx, + gc.ExcludeGCBarriers(excludeGCBarriers), gc.ExcludeGlobalGCBarriers(false), ) } @@ -129,6 +157,10 @@ type allGCStatesOutput struct { GlobalGCBarriers []gcBarrierOutput `json:"global_gc_barriers"` } +type globalGCStateOutput struct { + GlobalGCBarriers []gcBarrierOutput `json:"global_gc_barriers"` +} + func parseGCStateKeyspaceID(value string) (uint32, error) { parsed, err := strconv.ParseUint(value, 10, 32) if err != nil { @@ -241,12 +273,22 @@ func newAllGCStatesOutput(clusterState gc.ClusterGCStates) (allGCStatesOutput, e return states[i].KeyspaceID < states[j].KeyspaceID }) - globalBarriers, err := clusterState.GetGlobalGCBarriers() + globalOutput, err := newGlobalGCStateOutput(clusterState) if err != nil { - return allGCStatesOutput{}, errors.Annotate(err, "failed to read global GC barriers") + return allGCStatesOutput{}, err } return allGCStatesOutput{ GCStates: states, + GlobalGCBarriers: globalOutput.GlobalGCBarriers, + }, nil +} + +func newGlobalGCStateOutput(clusterState gc.ClusterGCStates) (globalGCStateOutput, error) { + globalBarriers, err := clusterState.GetGlobalGCBarriers() + if err != nil { + return globalGCStateOutput{}, errors.Annotate(err, "failed to read global GC barriers") + } + return globalGCStateOutput{ GlobalGCBarriers: newGlobalGCBarrierOutputs(globalBarriers), }, nil } @@ -259,9 +301,10 @@ func NewGCStateCommand() *cobra.Command { func buildGCStateCommand(factory gcStateReaderFactory) *cobra.Command { command := &cobra.Command{ Use: "gc-state", - Short: "show keyspace GC state and barriers", - Long: "Show effective per-keyspace GC safe points and barriers. " + - "Use the all subcommand to include cluster-wide global barriers.", + Short: "show keyspace and cluster-wide GC state", + Long: "Show effective per-keyspace GC safe points and local barriers, " + + "and cluster-wide GC state. Use keyspace for one effective GC " + + "scope, global for cluster-wide state, or all for a combined view.", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { return cmd.Help() @@ -269,6 +312,7 @@ func buildGCStateCommand(factory gcStateReaderFactory) *cobra.Command { } command.AddCommand( newGCStateKeyspaceCommand(factory), + newGCStateGlobalCommand(factory), newGCStateAllCommand(factory), ) return command @@ -278,8 +322,9 @@ func newGCStateKeyspaceCommand(factory gcStateReaderFactory) *cobra.Command { return &cobra.Command{ Use: "keyspace ", Short: "show one keyspace's effective GC state", - Long: "Show one keyspace's effective GC safe points and local " + - "barriers. Use gc-state all to inspect global barriers. " + + Long: "Show one keyspace's effective GC safe points and local barriers. " + + "Use gc-state global to inspect only cluster-wide state, or " + + "gc-state all for a combined view. " + "The decimal NullKeyspace ID is 4294967295.", Example: " pd-ctl gc-state keyspace 42\n" + " pd-ctl gc-state keyspace 4294967295", @@ -313,12 +358,45 @@ func newGCStateKeyspaceCommand(factory gcStateReaderFactory) *cobra.Command { } } +func newGCStateGlobalCommand(factory gcStateReaderFactory) *cobra.Command { + return &cobra.Command{ + Use: "global", + Short: "show cluster-wide GC state", + Long: "Show cluster-wide GC state without per-keyspace states. The current output contains global GC barriers.", + Example: " pd-ctl gc-state global", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + reader, err := factory(cmd) + if err != nil { + return errors.Annotate(err, "failed to create PD RPC client") + } + defer reader.close() + + clusterState, err := reader.getGlobalGCState(cmd.Context()) + if err != nil { + if status.Code(errors.Cause(err)) == codes.Unimplemented { + return errors.Annotate(err, + "gc-state global requires a PD server that supports "+ + "GetAllKeyspacesGCStates") + } + return errors.Annotate(err, "failed to get global GC state") + } + output, err := newGlobalGCStateOutput(clusterState) + if err != nil { + return err + } + return writeGCStateJSON(cmd, output) + }, + } +} + func newGCStateAllCommand(factory gcStateReaderFactory) *cobra.Command { return &cobra.Command{ Use: "all", - Short: "show all keyspace GC states and global barriers", - Long: "Show all active keyspace GC states and local barriers. " + - "Cluster-wide global barriers appear once at the top level.", + Short: "show combined keyspace and cluster-wide GC state", + Long: "Show all active keyspace GC states and local barriers, with " + + "cluster-wide global barriers once at the top level. Use " + + "gc-state global to inspect only cluster-wide state.", Example: " pd-ctl gc-state all", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { diff --git a/tools/pd-ctl/pdctl/command/gc_state_command_test.go b/tools/pd-ctl/pdctl/command/gc_state_command_test.go index 79ebc3b731..f56c916b79 100644 --- a/tools/pd-ctl/pdctl/command/gc_state_command_test.go +++ b/tools/pd-ctl/pdctl/command/gc_state_command_test.go @@ -35,13 +35,21 @@ import ( ) type fakeGCStateReader struct { - state gc.GCState - clusterState gc.ClusterGCStates - err error - requestedID uint32 - getStateCalls int - getAllCalls int - closed bool + state gc.GCState + clusterState gc.ClusterGCStates + err error + requestedID uint32 + getStateCalls int + getAllCalls int + getGlobalCalls int + closed bool +} + +func (r *fakeGCStateReader) getGlobalGCState( + _ context.Context, +) (gc.ClusterGCStates, error) { + r.getGlobalCalls++ + return r.clusterState, r.err } func (r *fakeGCStateReader) getGCState( @@ -181,6 +189,49 @@ func TestNewAllGCStatesOutputKeepsEmptyGlobalBarrierArray(t *testing.T) { require.Contains(t, string(encoded), `"global_gc_barriers":[]`) } +func TestNewGlobalGCStateOutputSortsAndKeepsEmptyArray(t *testing.T) { + t.Run("sorted", func(t *testing.T) { + clusterState := gc.NewClusterGCStatesWithGlobalGCBarriers( + map[uint32]gc.GCState{42: gc.NewGCStateWithoutGCBarriers(42, 100, 90)}, + []*gc.GlobalGCBarrierInfo{ + gc.NewGlobalGCBarrierInfo("z-global", 60, time.Minute, time.Time{}), + gc.NewGlobalGCBarrierInfo("a-global", 60, gc.TTLNeverExpire, time.Time{}), + gc.NewGlobalGCBarrierInfo("first-global", 50, time.Second, time.Time{}), + }, + ) + + got, err := newGlobalGCStateOutput(clusterState) + require.NoError(t, err) + require.Equal(t, []gcBarrierOutput{ + {BarrierID: "first-global", BarrierTS: 50, TTLSeconds: 1}, + {BarrierID: "a-global", BarrierTS: 60, TTLSeconds: math.MaxInt64}, + {BarrierID: "z-global", BarrierTS: 60, TTLSeconds: 60}, + }, got.GlobalGCBarriers) + + encoded, err := json.Marshal(got) + require.NoError(t, err) + require.JSONEq(t, `{ + "global_gc_barriers": [ + {"barrier_id":"first-global","barrier_ts":50,"ttl_seconds":1}, + {"barrier_id":"a-global","barrier_ts":60,"ttl_seconds":9223372036854775807}, + {"barrier_id":"z-global","barrier_ts":60,"ttl_seconds":60} + ] + }`, string(encoded)) + }) + + t.Run("empty", func(t *testing.T) { + clusterState := gc.NewClusterGCStatesWithGlobalGCBarriers(map[uint32]gc.GCState{}, nil) + got, err := newGlobalGCStateOutput(clusterState) + require.NoError(t, err) + require.NotNil(t, got.GlobalGCBarriers) + require.Empty(t, got.GlobalGCBarriers) + + encoded, err := json.Marshal(got) + require.NoError(t, err) + require.JSONEq(t, `{"global_gc_barriers":[]}`, string(encoded)) + }) +} + func TestGCStateOutputRejectsExcludedBarriers(t *testing.T) { state := gc.NewGCStateWithoutGCBarriers(42, 100, 90) _, err := newKeyspaceGCStateOutput(42, state) @@ -189,6 +240,46 @@ func TestGCStateOutputRejectsExcludedBarriers(t *testing.T) { clusterState := gc.NewClusterGCStatesWithoutGlobalGCBarriers(map[uint32]gc.GCState{}) _, err = newAllGCStatesOutput(clusterState) require.ErrorContains(t, err, "failed to read global GC barriers") + + _, err = newGlobalGCStateOutput(clusterState) + require.ErrorContains(t, err, "failed to read global GC barriers") +} + +type fakeClusterGCStatesClient struct { + options gc.GCStatesAPIOptions + calls int +} + +func (c *fakeClusterGCStatesClient) GetAllKeyspacesGCStates( + _ context.Context, + opts ...gc.GCStatesAPIOption, +) (gc.ClusterGCStates, error) { + c.options = gc.DefaultGCStatesAPIOptions() + for _, opt := range opts { + opt(&c.options) + } + c.calls++ + return gc.NewClusterGCStatesWithGlobalGCBarriers(map[uint32]gc.GCState{}, nil), nil +} + +func TestReadClusterGCStatesOptions(t *testing.T) { + for _, testCase := range []struct { + name string + excludeGCBarriers bool + wantExcludeGCBarriers bool + }{ + {name: "all", excludeGCBarriers: false, wantExcludeGCBarriers: false}, + {name: "global", excludeGCBarriers: true, wantExcludeGCBarriers: true}, + } { + t.Run(testCase.name, func(t *testing.T) { + client := &fakeClusterGCStatesClient{} + _, err := readClusterGCStates(t.Context(), client, testCase.excludeGCBarriers) + require.NoError(t, err) + require.Equal(t, 1, client.calls) + require.Equal(t, testCase.wantExcludeGCBarriers, client.options.ExcludeGCBarriers) + require.False(t, client.options.ExcludeGlobalGCBarriers) + }) + } } func TestGCStateKeyspaceCommand(t *testing.T) { @@ -241,6 +332,32 @@ func TestGCStateAllCommand(t *testing.T) { require.Contains(t, output.String(), `"global_gc_barriers": []`) } +func TestGCStateGlobalCommand(t *testing.T) { + reader := &fakeGCStateReader{clusterState: gc.NewClusterGCStatesWithGlobalGCBarriers( + map[uint32]gc.GCState{42: gc.NewGCStateWithoutGCBarriers(42, 100, 90)}, nil, + )} + cmd := buildGCStateCommand(func(*cobra.Command) (gcStateReader, error) { return reader, nil }) + output := new(bytes.Buffer) + cmd.SetOut(output) + cmd.SetErr(output) + cmd.SetArgs([]string{"global"}) + + require.NoError(t, cmd.Execute()) + require.Equal(t, 1, reader.getGlobalCalls) + require.Zero(t, reader.getAllCalls) + require.Zero(t, reader.getStateCalls) + require.True(t, reader.closed) + + var decoded map[string]json.RawMessage + require.NoError(t, json.Unmarshal(output.Bytes(), &decoded)) + require.Len(t, decoded, 1) + require.Contains(t, decoded, "global_gc_barriers") + require.NotContains(t, decoded, "gc_states") + require.NotContains(t, decoded, "txn_safe_point") + require.NotContains(t, decoded, "gc_safe_point") + require.JSONEq(t, `{"global_gc_barriers":[]}`, output.String()) +} + func TestGCStateCommandValidatesBeforeCreatingClient(t *testing.T) { for _, args := range [][]string{ {}, @@ -254,6 +371,7 @@ func TestGCStateCommandValidatesBeforeCreatingClient(t *testing.T) { {"keyspace", "4294967294"}, {"keyspace", "4294967296"}, {"all", "extra"}, + {"global", "extra"}, } { t.Run(strings.Join(args, "-"), func(t *testing.T) { factoryCalled := false @@ -351,6 +469,30 @@ func TestGCStateCommandErrors(t *testing.T) { }, wantMessage: "failed to read global GC barriers", }, + { + name: "global-rpc-error", + args: []string{"global"}, + factory: func(*cobra.Command) (gcStateReader, error) { + return &fakeGCStateReader{err: errors.New("rpc rejected")}, nil + }, + wantMessage: "failed to get global GC state", + }, + { + name: "global-unimplemented", + args: []string{"global"}, + factory: func(*cobra.Command) (gcStateReader, error) { + return &fakeGCStateReader{err: status.Error(codes.Unimplemented, "method unavailable")}, nil + }, + wantMessage: "gc-state global requires a PD server that supports GetAllKeyspacesGCStates", + }, + { + name: "global-missing-global-barriers", + args: []string{"global"}, + factory: func(*cobra.Command) (gcStateReader, error) { + return &fakeGCStateReader{clusterState: gc.NewClusterGCStatesWithoutGlobalGCBarriers(map[uint32]gc.GCState{})}, nil + }, + wantMessage: "failed to read global GC barriers", + }, } { t.Run(testCase.name, func(t *testing.T) { cmd := buildGCStateCommand(testCase.factory) @@ -363,6 +505,35 @@ func TestGCStateCommandErrors(t *testing.T) { } } +func TestGCStateCommandHelpContract(t *testing.T) { + cmd := buildGCStateCommand(func(*cobra.Command) (gcStateReader, error) { + return nil, errors.New("help must not create a reader") + }) + require.Equal(t, "show keyspace and cluster-wide GC state", cmd.Short) + require.Equal(t, "Show effective per-keyspace GC safe points and local barriers, and cluster-wide GC state. Use keyspace for one effective GC scope, global for cluster-wide state, or all for a combined view.", cmd.Long) + + keyspace, _, err := cmd.Find([]string{"keyspace"}) + require.NoError(t, err) + require.Equal(t, "keyspace ", keyspace.Use) + require.Equal(t, "show one keyspace's effective GC state", keyspace.Short) + require.Equal(t, "Show one keyspace's effective GC safe points and local barriers. Use gc-state global to inspect only cluster-wide state, or gc-state all for a combined view. The decimal NullKeyspace ID is 4294967295.", keyspace.Long) + require.Equal(t, " pd-ctl gc-state keyspace 42\n pd-ctl gc-state keyspace 4294967295", keyspace.Example) + + global, _, err := cmd.Find([]string{"global"}) + require.NoError(t, err) + require.Equal(t, "global", global.Use) + require.Equal(t, "show cluster-wide GC state", global.Short) + require.Equal(t, "Show cluster-wide GC state without per-keyspace states. The current output contains global GC barriers.", global.Long) + require.Equal(t, " pd-ctl gc-state global", global.Example) + + all, _, err := cmd.Find([]string{"all"}) + require.NoError(t, err) + require.Equal(t, "all", all.Use) + require.Equal(t, "show combined keyspace and cluster-wide GC state", all.Short) + require.Equal(t, "Show all active keyspace GC states and local barriers, with cluster-wide global barriers once at the top level. Use gc-state global to inspect only cluster-wide state.", all.Long) + require.Equal(t, " pd-ctl gc-state all", all.Example) +} + type failingWriter struct{} func (failingWriter) Write([]byte) (int, error) { From 188ae40eca9609f0e960f1e285404ccc32cd01ac Mon Sep 17 00:00:00 2001 From: Wenxuan Zhang Date: Mon, 27 Jul 2026 16:55:06 +0800 Subject: [PATCH 13/31] pd-ctl: test and document global GC state Signed-off-by: Wenxuan Zhang --- tools/pd-ctl/README.md | 27 +++++++++++++---- tools/pd-ctl/tests/safepoint/gc_state_test.go | 29 +++++++++++++++++-- 2 files changed, 49 insertions(+), 7 deletions(-) diff --git a/tools/pd-ctl/README.md b/tools/pd-ctl/README.md index f45d9ff9f4..eb2bd65082 100644 --- a/tools/pd-ctl/README.md +++ b/tools/pd-ctl/README.md @@ -38,7 +38,24 @@ The response contains both `requested_keyspace_id` and uses unified GC returns `4294967295`, the NullKeyspace ID, as its effective scope. The response contains local `gc_barriers` only. -Inspect all active GC scopes and cluster-wide barriers: +Inspect cluster-wide state when the local barriers do not explain the +effective safe point: + +```bash +pd-ctl gc-state global +``` + +```json +{ + "global_gc_barriers": [] +} +``` + +The global response does not contain per-keyspace safe points or local +barriers. Its current field is `global_gc_barriers`; other cluster-wide GC +state can be added to the same object in the future. + +Inspect every active GC scope together with cluster-wide state: ```bash pd-ctl gc-state all @@ -59,7 +76,7 @@ pd-ctl gc-state all } ``` -The response sorts `gc_states` by `keyspace_id`. It reports cluster-wide -barriers once in the top-level `global_gc_barriers` array. Barrier TTLs use -remaining seconds, and `9223372036854775807` means that a barrier never -expires. +The combined response sorts `gc_states` by `keyspace_id` and reports +cluster-wide barriers once in the top-level `global_gc_barriers` array. +Barrier TTLs use remaining seconds, and `9223372036854775807` means that a +barrier never expires. diff --git a/tools/pd-ctl/tests/safepoint/gc_state_test.go b/tools/pd-ctl/tests/safepoint/gc_state_test.go index bf68755a9c..83aff513a6 100644 --- a/tools/pd-ctl/tests/safepoint/gc_state_test.go +++ b/tools/pd-ctl/tests/safepoint/gc_state_test.go @@ -61,6 +61,10 @@ type gcStateCommandAll struct { GlobalGCBarriers []gcStateCommandBarrier `json:"global_gc_barriers"` } +type gcStateCommandGlobal struct { + GlobalGCBarriers []gcStateCommandBarrier `json:"global_gc_barriers"` +} + type expectedGCStateCommandBarrier struct { barrierID string barrierTS uint64 @@ -174,7 +178,7 @@ func TestGCState(t *testing.T) { ctx, "z-global", 320, - time.Hour, + time.Duration(math.MaxInt64), now, ) re.NoError(err) @@ -264,9 +268,30 @@ func TestGCState(t *testing.T) { }) requireGCStateCommandBarriers(re, all.GlobalGCBarriers, []expectedGCStateCommandBarrier{ {barrierID: "a-global", barrierTS: 310}, - {barrierID: "z-global", barrierTS: 320, expires: true}, + {barrierID: "z-global", barrierTS: 320}, }) + output, err = tests.ExecuteCommand( + ctl.GetRootCmd(), + "-u", + pdAddr, + "gc-state", + "global", + ) + re.NoError(err) + var globalProperties map[string]json.RawMessage + re.NoError(json.Unmarshal(output, &globalProperties), string(output)) + re.Len(globalProperties, 1) + re.Contains(globalProperties, "global_gc_barriers") + re.NotContains(globalProperties, "gc_states") + re.NotContains(globalProperties, "txn_safe_point") + re.NotContains(globalProperties, "gc_safe_point") + + var global gcStateCommandGlobal + re.NoError(json.Unmarshal(output, &global), string(output)) + re.NotNil(global.GlobalGCBarriers) + re.Equal(all.GlobalGCBarriers, global.GlobalGCBarriers) + if kerneltype.IsNextGen() { systemState, ok := statesByID[constant.SystemKeyspaceID] re.True(ok) From 02b9ba5fab3c7a5de1464a93c1ed6559b67549c3 Mon Sep 17 00:00:00 2001 From: Wenxuan Zhang Date: Tue, 28 Jul 2026 14:33:16 +0800 Subject: [PATCH 14/31] pd-ctl: derive NullKeyspace ID in help Signed-off-by: Wenxuan Zhang --- tools/pd-ctl/pdctl/command/gc_state_command.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/pd-ctl/pdctl/command/gc_state_command.go b/tools/pd-ctl/pdctl/command/gc_state_command.go index d27d26ca5e..a606107cb6 100644 --- a/tools/pd-ctl/pdctl/command/gc_state_command.go +++ b/tools/pd-ctl/pdctl/command/gc_state_command.go @@ -319,15 +319,16 @@ func buildGCStateCommand(factory gcStateReaderFactory) *cobra.Command { } func newGCStateKeyspaceCommand(factory gcStateReaderFactory) *cobra.Command { + nullKeyspaceID := strconv.FormatUint(uint64(constant.NullKeyspaceID), 10) return &cobra.Command{ Use: "keyspace ", Short: "show one keyspace's effective GC state", Long: "Show one keyspace's effective GC safe points and local barriers. " + "Use gc-state global to inspect only cluster-wide state, or " + "gc-state all for a combined view. " + - "The decimal NullKeyspace ID is 4294967295.", + "The decimal NullKeyspace ID is " + nullKeyspaceID + ".", Example: " pd-ctl gc-state keyspace 42\n" + - " pd-ctl gc-state keyspace 4294967295", + " pd-ctl gc-state keyspace " + nullKeyspaceID, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { keyspaceID, err := parseGCStateKeyspaceID(args[0]) From d0245cb024ef0d0bfe7def0f19ca3f47676c70ba Mon Sep 17 00:00:00 2001 From: Wenxuan Zhang Date: Tue, 28 Jul 2026 15:45:26 +0800 Subject: [PATCH 15/31] pd-ctl: omit unified GC placeholders Present only effective GC scopes in gc-state all and use the NullKeyspace scope for unified GC keyspaces. Signed-off-by: Wenxuan Zhang --- tools/pd-ctl/README.md | 8 ++- .../pd-ctl/pdctl/command/gc_state_command.go | 9 ++- .../pdctl/command/gc_state_command_test.go | 59 ++++++++++++++++++- tools/pd-ctl/tests/safepoint/gc_state_test.go | 8 +-- 4 files changed, 70 insertions(+), 14 deletions(-) diff --git a/tools/pd-ctl/README.md b/tools/pd-ctl/README.md index eb2bd65082..7f90c47108 100644 --- a/tools/pd-ctl/README.md +++ b/tools/pd-ctl/README.md @@ -55,7 +55,7 @@ The global response does not contain per-keyspace safe points or local barriers. Its current field is `global_gc_barriers`; other cluster-wide GC state can be added to the same object in the future. -Inspect every active GC scope together with cluster-wide state: +Inspect every effective GC scope together with cluster-wide state: ```bash pd-ctl gc-state all @@ -76,7 +76,9 @@ pd-ctl gc-state all } ``` -The combined response sorts `gc_states` by `keyspace_id` and reports -cluster-wide barriers once in the top-level `global_gc_barriers` array. +The combined response sorts effective `gc_states` by `keyspace_id` and reports +cluster-wide barriers once in the top-level `global_gc_barriers` array. Unified +GC keyspaces share the NullKeyspace scope, so their marker records are not +reported as separate states. Barrier TTLs use remaining seconds, and `9223372036854775807` means that a barrier never expires. diff --git a/tools/pd-ctl/pdctl/command/gc_state_command.go b/tools/pd-ctl/pdctl/command/gc_state_command.go index a606107cb6..bcb4570ddf 100644 --- a/tools/pd-ctl/pdctl/command/gc_state_command.go +++ b/tools/pd-ctl/pdctl/command/gc_state_command.go @@ -263,6 +263,11 @@ func newGCStateOutput(state gc.GCState) (gcStateOutput, error) { func newAllGCStatesOutput(clusterState gc.ClusterGCStates) (allGCStatesOutput, error) { states := make([]gcStateOutput, 0, len(clusterState.GCStates)) for _, state := range clusterState.GCStates { + // Unified-GC keyspaces have marker entries, but their GC state is owned by + // the NullKeyspace scope and must not be presented as a separate scope. + if !state.IsKeyspaceLevelGC && state.KeyspaceID != constant.NullKeyspaceID { + continue + } converted, err := newGCStateOutput(state) if err != nil { return allGCStatesOutput{}, err @@ -394,8 +399,8 @@ func newGCStateGlobalCommand(factory gcStateReaderFactory) *cobra.Command { func newGCStateAllCommand(factory gcStateReaderFactory) *cobra.Command { return &cobra.Command{ Use: "all", - Short: "show combined keyspace and cluster-wide GC state", - Long: "Show all active keyspace GC states and local barriers, with " + + Short: "show effective GC scopes and cluster-wide GC state", + Long: "Show all effective GC scopes and local barriers, with " + "cluster-wide global barriers once at the top level. Use " + "gc-state global to inspect only cluster-wide state.", Example: " pd-ctl gc-state all", diff --git a/tools/pd-ctl/pdctl/command/gc_state_command_test.go b/tools/pd-ctl/pdctl/command/gc_state_command_test.go index f56c916b79..1c6725a016 100644 --- a/tools/pd-ctl/pdctl/command/gc_state_command_test.go +++ b/tools/pd-ctl/pdctl/command/gc_state_command_test.go @@ -173,6 +173,61 @@ func TestNewAllGCStatesOutputSortsAndKeepsEmptyArrays(t *testing.T) { require.Contains(t, string(encoded), `"global_gc_barriers":[`) } +func TestNewAllGCStatesOutputFiltersUnifiedGCPlaceholders(t *testing.T) { + keyspaceLevelState := gc.NewGCStateWithGCBarriers(7, 70, 60, nil) + keyspaceLevelState.IsKeyspaceLevelGC = true + unifiedGCPlaceholder := gc.NewGCStateWithGCBarriers( + 8, + 800, + 700, + []*gc.GCBarrierInfo{ + gc.NewGCBarrierInfo("placeholder", 900, time.Hour, time.Time{}), + }, + ) + // A non-Null state without keyspace-level GC only identifies a unified-GC keyspace. + // Its safe points and barriers are not an effective GC scope. + unifiedGCPlaceholder.IsKeyspaceLevelGC = false + nullKeyspaceState := gc.NewGCStateWithGCBarriers( + constant.NullKeyspaceID, + 100, + 90, + []*gc.GCBarrierInfo{ + gc.NewGCBarrierInfo("null", 110, gc.TTLNeverExpire, time.Time{}), + }, + ) + nullKeyspaceState.IsKeyspaceLevelGC = false + + clusterState := gc.NewClusterGCStatesWithGlobalGCBarriers( + map[uint32]gc.GCState{ + 7: keyspaceLevelState, + 8: unifiedGCPlaceholder, + constant.NullKeyspaceID: nullKeyspaceState, + }, + nil, + ) + + got, err := newAllGCStatesOutput(clusterState) + require.NoError(t, err) + require.Equal(t, []gcStateOutput{ + { + KeyspaceID: 7, + IsKeyspaceLevelGC: true, + TxnSafePoint: 70, + GCSafePoint: 60, + GCBarriers: []gcBarrierOutput{}, + }, + { + KeyspaceID: constant.NullKeyspaceID, + IsKeyspaceLevelGC: false, + TxnSafePoint: 100, + GCSafePoint: 90, + GCBarriers: []gcBarrierOutput{ + {BarrierID: "null", BarrierTS: 110, TTLSeconds: math.MaxInt64}, + }, + }, + }, got.GCStates) +} + func TestNewAllGCStatesOutputKeepsEmptyGlobalBarrierArray(t *testing.T) { clusterState := gc.NewClusterGCStatesWithGlobalGCBarriers( map[uint32]gc.GCState{}, @@ -529,8 +584,8 @@ func TestGCStateCommandHelpContract(t *testing.T) { all, _, err := cmd.Find([]string{"all"}) require.NoError(t, err) require.Equal(t, "all", all.Use) - require.Equal(t, "show combined keyspace and cluster-wide GC state", all.Short) - require.Equal(t, "Show all active keyspace GC states and local barriers, with cluster-wide global barriers once at the top level. Use gc-state global to inspect only cluster-wide state.", all.Long) + require.Equal(t, "show effective GC scopes and cluster-wide GC state", all.Short) + require.Equal(t, "Show all effective GC scopes and local barriers, with cluster-wide global barriers once at the top level. Use gc-state global to inspect only cluster-wide state.", all.Long) require.Equal(t, " pd-ctl gc-state all", all.Example) } diff --git a/tools/pd-ctl/tests/safepoint/gc_state_test.go b/tools/pd-ctl/tests/safepoint/gc_state_test.go index 83aff513a6..7340b6b8c7 100644 --- a/tools/pd-ctl/tests/safepoint/gc_state_test.go +++ b/tools/pd-ctl/tests/safepoint/gc_state_test.go @@ -314,13 +314,7 @@ func TestGCState(t *testing.T) { {barrierID: "z-null", barrierTS: 120, expires: true}, }) - unifiedState, ok := statesByID[unifiedKeyspaceID] - re.True(ok) - re.False(unifiedState.IsKeyspaceLevelGC) - re.Zero(unifiedState.TxnSafePoint) - re.Zero(unifiedState.GCSafePoint) - re.NotNil(unifiedState.GCBarriers) - re.Empty(unifiedState.GCBarriers) + re.NotContains(statesByID, unifiedKeyspaceID) } output, err = tests.ExecuteCommand(ctl.GetRootCmd(), "-u", pdAddr, "service-gc-safepoint") From ae378a19d167015cf887e7cf04fee192abcda62b Mon Sep 17 00:00:00 2001 From: Wenxuan Zhang Date: Tue, 28 Jul 2026 16:10:26 +0800 Subject: [PATCH 16/31] pd-ctl: harden GC state inspection Skip nil local and global barrier entries before projecting JSON, and use gRPC status unwrapping directly for compatibility errors. Relax the finite TTL assertion to avoid slow-CI flakes. Signed-off-by: Wenxuan Zhang --- .../pd-ctl/pdctl/command/gc_state_command.go | 12 ++++-- .../pdctl/command/gc_state_command_test.go | 38 ++++++++++++++++--- tools/pd-ctl/tests/safepoint/gc_state_test.go | 2 +- 3 files changed, 42 insertions(+), 10 deletions(-) diff --git a/tools/pd-ctl/pdctl/command/gc_state_command.go b/tools/pd-ctl/pdctl/command/gc_state_command.go index bcb4570ddf..74a5239598 100644 --- a/tools/pd-ctl/pdctl/command/gc_state_command.go +++ b/tools/pd-ctl/pdctl/command/gc_state_command.go @@ -197,6 +197,9 @@ func sortGCBarrierOutputs(barriers []gcBarrierOutput) { func newLocalGCBarrierOutputs(barriers []*gc.GCBarrierInfo) []gcBarrierOutput { result := make([]gcBarrierOutput, 0, len(barriers)) for _, barrier := range barriers { + if barrier == nil { + continue + } result = append(result, gcBarrierOutput{ BarrierID: barrier.BarrierID, BarrierTS: barrier.BarrierTS, @@ -210,6 +213,9 @@ func newLocalGCBarrierOutputs(barriers []*gc.GCBarrierInfo) []gcBarrierOutput { func newGlobalGCBarrierOutputs(barriers []*gc.GlobalGCBarrierInfo) []gcBarrierOutput { result := make([]gcBarrierOutput, 0, len(barriers)) for _, barrier := range barriers { + if barrier == nil { + continue + } result = append(result, gcBarrierOutput{ BarrierID: barrier.BarrierID, BarrierTS: barrier.BarrierTS, @@ -348,7 +354,7 @@ func newGCStateKeyspaceCommand(factory gcStateReaderFactory) *cobra.Command { state, err := reader.getGCState(cmd.Context(), keyspaceID) if err != nil { - if status.Code(errors.Cause(err)) == codes.Unimplemented { + if status.Code(err) == codes.Unimplemented { return errors.Annotate(err, "gc-state requires a PD server that supports GetGCState") } @@ -380,7 +386,7 @@ func newGCStateGlobalCommand(factory gcStateReaderFactory) *cobra.Command { clusterState, err := reader.getGlobalGCState(cmd.Context()) if err != nil { - if status.Code(errors.Cause(err)) == codes.Unimplemented { + if status.Code(err) == codes.Unimplemented { return errors.Annotate(err, "gc-state global requires a PD server that supports "+ "GetAllKeyspacesGCStates") @@ -414,7 +420,7 @@ func newGCStateAllCommand(factory gcStateReaderFactory) *cobra.Command { clusterState, err := reader.getAllKeyspacesGCStates(cmd.Context()) if err != nil { - if status.Code(errors.Cause(err)) == codes.Unimplemented { + if status.Code(err) == codes.Unimplemented { return errors.Annotate(err, "gc-state all requires a PD server that supports "+ "GetAllKeyspacesGCStates") diff --git a/tools/pd-ctl/pdctl/command/gc_state_command_test.go b/tools/pd-ctl/pdctl/command/gc_state_command_test.go index 1c6725a016..0a2eaac7a8 100644 --- a/tools/pd-ctl/pdctl/command/gc_state_command_test.go +++ b/tools/pd-ctl/pdctl/command/gc_state_command_test.go @@ -19,6 +19,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "io" "math" "strings" @@ -129,6 +130,28 @@ func TestNewKeyspaceGCStateOutput(t *testing.T) { }, got.GCBarriers) } +func TestNewLocalGCBarrierOutputsSkipNilEntries(t *testing.T) { + got := newLocalGCBarrierOutputs([]*gc.GCBarrierInfo{ + nil, + gc.NewGCBarrierInfo("valid-local", 42, 30*time.Second, time.Time{}), + }) + + require.Equal(t, []gcBarrierOutput{ + {BarrierID: "valid-local", BarrierTS: 42, TTLSeconds: 30}, + }, got) +} + +func TestNewGlobalGCBarrierOutputsSkipNilEntries(t *testing.T) { + got := newGlobalGCBarrierOutputs([]*gc.GlobalGCBarrierInfo{ + nil, + gc.NewGlobalGCBarrierInfo("valid-global", 84, time.Minute, time.Time{}), + }) + + require.Equal(t, []gcBarrierOutput{ + {BarrierID: "valid-global", BarrierTS: 84, TTLSeconds: 60}, + }, got) +} + func TestNewAllGCStatesOutputSortsAndKeepsEmptyArrays(t *testing.T) { empty := gc.NewGCStateWithGCBarriers(1, 20, 10, nil) empty.IsKeyspaceLevelGC = true @@ -474,11 +497,12 @@ func TestGCStateCommandErrors(t *testing.T) { wantMessage: "failed to get GC state for keyspace 42", }, { - name: "single-unimplemented", + name: "single-wrapped-unimplemented", args: []string{"keyspace", "42"}, factory: func(*cobra.Command) (gcStateReader, error) { return &fakeGCStateReader{ - err: status.Error(codes.Unimplemented, "method unavailable"), + err: fmt.Errorf("wrapped: %w", + status.Error(codes.Unimplemented, "method unavailable")), }, nil }, wantMessage: "gc-state requires a PD server that supports GetGCState", @@ -502,11 +526,12 @@ func TestGCStateCommandErrors(t *testing.T) { wantMessage: "failed to get all keyspaces GC states", }, { - name: "all-unimplemented", + name: "all-wrapped-unimplemented", args: []string{"all"}, factory: func(*cobra.Command) (gcStateReader, error) { return &fakeGCStateReader{ - err: status.Error(codes.Unimplemented, "method unavailable"), + err: fmt.Errorf("wrapped: %w", + status.Error(codes.Unimplemented, "method unavailable")), }, nil }, wantMessage: "gc-state all requires a PD server that supports " + @@ -533,10 +558,11 @@ func TestGCStateCommandErrors(t *testing.T) { wantMessage: "failed to get global GC state", }, { - name: "global-unimplemented", + name: "global-wrapped-unimplemented", args: []string{"global"}, factory: func(*cobra.Command) (gcStateReader, error) { - return &fakeGCStateReader{err: status.Error(codes.Unimplemented, "method unavailable")}, nil + return &fakeGCStateReader{err: fmt.Errorf("wrapped: %w", + status.Error(codes.Unimplemented, "method unavailable"))}, nil }, wantMessage: "gc-state global requires a PD server that supports GetAllKeyspacesGCStates", }, diff --git a/tools/pd-ctl/tests/safepoint/gc_state_test.go b/tools/pd-ctl/tests/safepoint/gc_state_test.go index 7340b6b8c7..7a4ee4872e 100644 --- a/tools/pd-ctl/tests/safepoint/gc_state_test.go +++ b/tools/pd-ctl/tests/safepoint/gc_state_test.go @@ -81,7 +81,7 @@ func requireGCStateCommandBarriers( re.Equal(want.barrierID, actual[i].BarrierID) re.Equal(want.barrierTS, actual[i].BarrierTS) if want.expires { - re.GreaterOrEqual(actual[i].TTLSeconds, int64(3595)) + re.Greater(actual[i].TTLSeconds, int64(3500)) re.LessOrEqual(actual[i].TTLSeconds, int64(3600)) } else { re.Equal(int64(math.MaxInt64), actual[i].TTLSeconds) From 201ba648e1f85be29f6097572617426d54855e8c Mon Sep 17 00:00:00 2001 From: Wenxuan Zhang Date: Wed, 29 Jul 2026 16:06:48 +0800 Subject: [PATCH 17/31] docs: add GC barrier output examples Show correlated local and global barriers across the keyspace, global, and combined GC-state views. Clarify NullKeyspace placement, empty arrays, and finite versus non-expiring TTLs. Signed-off-by: Wenxuan Zhang --- tools/pd-ctl/README.md | 43 +++++++++++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/tools/pd-ctl/README.md b/tools/pd-ctl/README.md index 7f90c47108..50e8a2fae3 100644 --- a/tools/pd-ctl/README.md +++ b/tools/pd-ctl/README.md @@ -29,7 +29,13 @@ pd-ctl gc-state keyspace 42 "is_keyspace_level_gc": false, "txn_safe_point": 465000000000000000, "gc_safe_point": 464900000000000000, - "gc_barriers": [] + "gc_barriers": [ + { + "barrier_id": "br", + "barrier_ts": 464950000000000000, + "ttl_seconds": 3600 + } + ] } ``` @@ -47,7 +53,13 @@ pd-ctl gc-state global ```json { - "global_gc_barriers": [] + "global_gc_barriers": [ + { + "barrier_id": "native_br", + "barrier_ts": 464940000000000000, + "ttl_seconds": 9223372036854775807 + } + ] } ``` @@ -65,20 +77,33 @@ pd-ctl gc-state all { "gc_states": [ { - "keyspace_id": 42, - "is_keyspace_level_gc": true, + "keyspace_id": 4294967295, + "is_keyspace_level_gc": false, "txn_safe_point": 465000000000000000, "gc_safe_point": 464900000000000000, - "gc_barriers": [] + "gc_barriers": [ + { + "barrier_id": "br", + "barrier_ts": 464950000000000000, + "ttl_seconds": 3600 + } + ] } ], - "global_gc_barriers": [] + "global_gc_barriers": [ + { + "barrier_id": "native_br", + "barrier_ts": 464940000000000000, + "ttl_seconds": 9223372036854775807 + } + ] } ``` The combined response sorts effective `gc_states` by `keyspace_id` and reports cluster-wide barriers once in the top-level `global_gc_barriers` array. Unified GC keyspaces share the NullKeyspace scope, so their marker records are not -reported as separate states. -Barrier TTLs use remaining seconds, and `9223372036854775807` means that a -barrier never expires. +reported as separate states. The real NullKeyspace state appears once with its +safe points and local barriers. When no local or global barriers exist, the +corresponding arrays are encoded as `[]`. Barrier TTLs use remaining seconds, +and `9223372036854775807` means that a barrier never expires. From da195a88d7ea9790b772193792258cbbf4e95620 Mon Sep 17 00:00:00 2001 From: Wenxuan Zhang Date: Thu, 30 Jul 2026 11:52:11 +0800 Subject: [PATCH 18/31] pd-ctl: hide expired GC barriers by default Keep the effective GC state focused on active barriers while allowing operators to inspect zero-TTL entries with --include-expired. Signed-off-by: Wenxuan Zhang --- tools/pd-ctl/README.md | 15 +- .../pd-ctl/pdctl/command/gc_state_command.go | 69 +++++++-- .../pdctl/command/gc_state_command_test.go | 139 ++++++++++++++++-- tools/pd-ctl/tests/safepoint/gc_state_test.go | 71 ++++++++- 4 files changed, 264 insertions(+), 30 deletions(-) diff --git a/tools/pd-ctl/README.md b/tools/pd-ctl/README.md index 50e8a2fae3..3d350394f1 100644 --- a/tools/pd-ctl/README.md +++ b/tools/pd-ctl/README.md @@ -16,6 +16,17 @@ The details about how to use `pd-ctl` can be found in [PD Control User Guide](ht Use `gc-state` to inspect the safe points and barriers that can block GC. The command is read-only and emits deterministic JSON for scripts and diffs. +By default, it omits barriers that PD returns with a zero TTL because they +normally represent expired barriers awaiting lazy deletion. Add +`--include-expired` to any subcommand to include those barriers in the existing +`gc_barriers` or `global_gc_barriers` array with `ttl_seconds` set to `0`. + +For example, inspect one keyspace and include zero-TTL barriers: + +```bash +pd-ctl gc-state keyspace 42 --include-expired +``` + Inspect one keyspace by its decimal ID: ```bash @@ -106,4 +117,6 @@ GC keyspaces share the NullKeyspace scope, so their marker records are not reported as separate states. The real NullKeyspace state appears once with its safe points and local barriers. When no local or global barriers exist, the corresponding arrays are encoded as `[]`. Barrier TTLs use remaining seconds, -and `9223372036854775807` means that a barrier never expires. +and `9223372036854775807` means that a barrier never expires. Because PD rounds +remaining TTLs down to whole seconds, a zero TTL can also represent a barrier +with less than one second remaining. diff --git a/tools/pd-ctl/pdctl/command/gc_state_command.go b/tools/pd-ctl/pdctl/command/gc_state_command.go index 74a5239598..afc3534832 100644 --- a/tools/pd-ctl/pdctl/command/gc_state_command.go +++ b/tools/pd-ctl/pdctl/command/gc_state_command.go @@ -43,6 +43,8 @@ type gcStateReader interface { type gcStateReaderFactory func(*cobra.Command) (gcStateReader, error) +const gcStateIncludeExpiredFlag = "include-expired" + type pdGCStateReader struct { client pd.Client } @@ -194,10 +196,17 @@ func sortGCBarrierOutputs(barriers []gcBarrierOutput) { }) } -func newLocalGCBarrierOutputs(barriers []*gc.GCBarrierInfo) []gcBarrierOutput { +func shouldIncludeGCBarrier(ttl time.Duration, includeExpired bool) bool { + // Expired barriers are lazily deleted and may still be returned by PD with + // a zero TTL. They no longer block GC, so omit them from the effective view + // unless the caller explicitly requests the persisted entries. + return includeExpired || ttl > 0 +} + +func newLocalGCBarrierOutputs(barriers []*gc.GCBarrierInfo, includeExpired bool) []gcBarrierOutput { result := make([]gcBarrierOutput, 0, len(barriers)) for _, barrier := range barriers { - if barrier == nil { + if barrier == nil || !shouldIncludeGCBarrier(barrier.TTL, includeExpired) { continue } result = append(result, gcBarrierOutput{ @@ -210,10 +219,10 @@ func newLocalGCBarrierOutputs(barriers []*gc.GCBarrierInfo) []gcBarrierOutput { return result } -func newGlobalGCBarrierOutputs(barriers []*gc.GlobalGCBarrierInfo) []gcBarrierOutput { +func newGlobalGCBarrierOutputs(barriers []*gc.GlobalGCBarrierInfo, includeExpired bool) []gcBarrierOutput { result := make([]gcBarrierOutput, 0, len(barriers)) for _, barrier := range barriers { - if barrier == nil { + if barrier == nil || !shouldIncludeGCBarrier(barrier.TTL, includeExpired) { continue } result = append(result, gcBarrierOutput{ @@ -229,6 +238,7 @@ func newGlobalGCBarrierOutputs(barriers []*gc.GlobalGCBarrierInfo) []gcBarrierOu func newKeyspaceGCStateOutput( requestedKeyspaceID uint32, state gc.GCState, + includeExpired bool, ) (keyspaceGCStateOutput, error) { barriers, err := state.GetGCBarriers() if err != nil { @@ -244,11 +254,11 @@ func newKeyspaceGCStateOutput( IsKeyspaceLevelGC: state.IsKeyspaceLevelGC, TxnSafePoint: state.TxnSafePoint, GCSafePoint: state.GCSafePoint, - GCBarriers: newLocalGCBarrierOutputs(barriers), + GCBarriers: newLocalGCBarrierOutputs(barriers, includeExpired), }, nil } -func newGCStateOutput(state gc.GCState) (gcStateOutput, error) { +func newGCStateOutput(state gc.GCState, includeExpired bool) (gcStateOutput, error) { barriers, err := state.GetGCBarriers() if err != nil { return gcStateOutput{}, errors.Annotatef( @@ -262,11 +272,11 @@ func newGCStateOutput(state gc.GCState) (gcStateOutput, error) { IsKeyspaceLevelGC: state.IsKeyspaceLevelGC, TxnSafePoint: state.TxnSafePoint, GCSafePoint: state.GCSafePoint, - GCBarriers: newLocalGCBarrierOutputs(barriers), + GCBarriers: newLocalGCBarrierOutputs(barriers, includeExpired), }, nil } -func newAllGCStatesOutput(clusterState gc.ClusterGCStates) (allGCStatesOutput, error) { +func newAllGCStatesOutput(clusterState gc.ClusterGCStates, includeExpired bool) (allGCStatesOutput, error) { states := make([]gcStateOutput, 0, len(clusterState.GCStates)) for _, state := range clusterState.GCStates { // Unified-GC keyspaces have marker entries, but their GC state is owned by @@ -274,7 +284,7 @@ func newAllGCStatesOutput(clusterState gc.ClusterGCStates) (allGCStatesOutput, e if !state.IsKeyspaceLevelGC && state.KeyspaceID != constant.NullKeyspaceID { continue } - converted, err := newGCStateOutput(state) + converted, err := newGCStateOutput(state, includeExpired) if err != nil { return allGCStatesOutput{}, err } @@ -284,7 +294,7 @@ func newAllGCStatesOutput(clusterState gc.ClusterGCStates) (allGCStatesOutput, e return states[i].KeyspaceID < states[j].KeyspaceID }) - globalOutput, err := newGlobalGCStateOutput(clusterState) + globalOutput, err := newGlobalGCStateOutput(clusterState, includeExpired) if err != nil { return allGCStatesOutput{}, err } @@ -294,16 +304,24 @@ func newAllGCStatesOutput(clusterState gc.ClusterGCStates) (allGCStatesOutput, e }, nil } -func newGlobalGCStateOutput(clusterState gc.ClusterGCStates) (globalGCStateOutput, error) { +func newGlobalGCStateOutput(clusterState gc.ClusterGCStates, includeExpired bool) (globalGCStateOutput, error) { globalBarriers, err := clusterState.GetGlobalGCBarriers() if err != nil { return globalGCStateOutput{}, errors.Annotate(err, "failed to read global GC barriers") } return globalGCStateOutput{ - GlobalGCBarriers: newGlobalGCBarrierOutputs(globalBarriers), + GlobalGCBarriers: newGlobalGCBarrierOutputs(globalBarriers, includeExpired), }, nil } +func getGCStateIncludeExpired(cmd *cobra.Command) (bool, error) { + includeExpired, err := cmd.Flags().GetBool(gcStateIncludeExpiredFlag) + if err != nil { + return false, errors.WithStack(err) + } + return includeExpired, nil +} + // NewGCStateCommand returns the read-only GC state command. func NewGCStateCommand() *cobra.Command { return buildGCStateCommand(newPDGCStateReader) @@ -314,13 +332,20 @@ func buildGCStateCommand(factory gcStateReaderFactory) *cobra.Command { Use: "gc-state", Short: "show keyspace and cluster-wide GC state", Long: "Show effective per-keyspace GC safe points and local barriers, " + - "and cluster-wide GC state. Use keyspace for one effective GC " + + "and cluster-wide GC state. Expired barriers awaiting lazy deletion " + + "are hidden by default; use --include-expired to include zero-TTL " + + "barriers returned by PD. Use keyspace for one effective GC " + "scope, global for cluster-wide state, or all for a combined view.", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { return cmd.Help() }, } + command.PersistentFlags().Bool( + gcStateIncludeExpiredFlag, + false, + "include zero-TTL barriers returned by PD, which normally represent expired barriers awaiting lazy deletion", + ) command.AddCommand( newGCStateKeyspaceCommand(factory), newGCStateGlobalCommand(factory), @@ -346,6 +371,10 @@ func newGCStateKeyspaceCommand(factory gcStateReaderFactory) *cobra.Command { if err != nil { return err } + includeExpired, err := getGCStateIncludeExpired(cmd) + if err != nil { + return err + } reader, err := factory(cmd) if err != nil { return errors.Annotate(err, "failed to create PD RPC client") @@ -361,7 +390,7 @@ func newGCStateKeyspaceCommand(factory gcStateReaderFactory) *cobra.Command { return errors.Annotatef(err, "failed to get GC state for keyspace %d", keyspaceID) } - output, err := newKeyspaceGCStateOutput(keyspaceID, state) + output, err := newKeyspaceGCStateOutput(keyspaceID, state, includeExpired) if err != nil { return err } @@ -378,6 +407,10 @@ func newGCStateGlobalCommand(factory gcStateReaderFactory) *cobra.Command { Example: " pd-ctl gc-state global", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { + includeExpired, err := getGCStateIncludeExpired(cmd) + if err != nil { + return err + } reader, err := factory(cmd) if err != nil { return errors.Annotate(err, "failed to create PD RPC client") @@ -393,7 +426,7 @@ func newGCStateGlobalCommand(factory gcStateReaderFactory) *cobra.Command { } return errors.Annotate(err, "failed to get global GC state") } - output, err := newGlobalGCStateOutput(clusterState) + output, err := newGlobalGCStateOutput(clusterState, includeExpired) if err != nil { return err } @@ -412,6 +445,10 @@ func newGCStateAllCommand(factory gcStateReaderFactory) *cobra.Command { Example: " pd-ctl gc-state all", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { + includeExpired, err := getGCStateIncludeExpired(cmd) + if err != nil { + return err + } reader, err := factory(cmd) if err != nil { return errors.Annotate(err, "failed to create PD RPC client") @@ -427,7 +464,7 @@ func newGCStateAllCommand(factory gcStateReaderFactory) *cobra.Command { } return errors.Annotate(err, "failed to get all keyspaces GC states") } - output, err := newAllGCStatesOutput(clusterState) + output, err := newAllGCStatesOutput(clusterState, includeExpired) if err != nil { return err } diff --git a/tools/pd-ctl/pdctl/command/gc_state_command_test.go b/tools/pd-ctl/pdctl/command/gc_state_command_test.go index 0a2eaac7a8..3c44e698a5 100644 --- a/tools/pd-ctl/pdctl/command/gc_state_command_test.go +++ b/tools/pd-ctl/pdctl/command/gc_state_command_test.go @@ -116,7 +116,7 @@ func TestNewKeyspaceGCStateOutput(t *testing.T) { ) state.IsKeyspaceLevelGC = false - got, err := newKeyspaceGCStateOutput(42, state) + got, err := newKeyspaceGCStateOutput(42, state, false) require.NoError(t, err) require.Equal(t, uint32(42), got.RequestedKeyspaceID) require.Equal(t, constant.NullKeyspaceID, got.EffectiveKeyspaceID) @@ -134,7 +134,7 @@ func TestNewLocalGCBarrierOutputsSkipNilEntries(t *testing.T) { got := newLocalGCBarrierOutputs([]*gc.GCBarrierInfo{ nil, gc.NewGCBarrierInfo("valid-local", 42, 30*time.Second, time.Time{}), - }) + }, false) require.Equal(t, []gcBarrierOutput{ {BarrierID: "valid-local", BarrierTS: 42, TTLSeconds: 30}, @@ -145,7 +145,7 @@ func TestNewGlobalGCBarrierOutputsSkipNilEntries(t *testing.T) { got := newGlobalGCBarrierOutputs([]*gc.GlobalGCBarrierInfo{ nil, gc.NewGlobalGCBarrierInfo("valid-global", 84, time.Minute, time.Time{}), - }) + }, false) require.Equal(t, []gcBarrierOutput{ {BarrierID: "valid-global", BarrierTS: 84, TTLSeconds: 60}, @@ -176,7 +176,7 @@ func TestNewAllGCStatesOutputSortsAndKeepsEmptyArrays(t *testing.T) { }, ) - got, err := newAllGCStatesOutput(clusterState) + got, err := newAllGCStatesOutput(clusterState, false) require.NoError(t, err) require.Equal(t, []uint32{1, constant.NullKeyspaceID}, []uint32{ got.GCStates[0].KeyspaceID, @@ -229,7 +229,7 @@ func TestNewAllGCStatesOutputFiltersUnifiedGCPlaceholders(t *testing.T) { nil, ) - got, err := newAllGCStatesOutput(clusterState) + got, err := newAllGCStatesOutput(clusterState, false) require.NoError(t, err) require.Equal(t, []gcStateOutput{ { @@ -257,7 +257,7 @@ func TestNewAllGCStatesOutputKeepsEmptyGlobalBarrierArray(t *testing.T) { nil, ) - got, err := newAllGCStatesOutput(clusterState) + got, err := newAllGCStatesOutput(clusterState, false) require.NoError(t, err) require.NotNil(t, got.GlobalGCBarriers) require.Empty(t, got.GlobalGCBarriers) @@ -278,7 +278,7 @@ func TestNewGlobalGCStateOutputSortsAndKeepsEmptyArray(t *testing.T) { }, ) - got, err := newGlobalGCStateOutput(clusterState) + got, err := newGlobalGCStateOutput(clusterState, false) require.NoError(t, err) require.Equal(t, []gcBarrierOutput{ {BarrierID: "first-global", BarrierTS: 50, TTLSeconds: 1}, @@ -299,7 +299,7 @@ func TestNewGlobalGCStateOutputSortsAndKeepsEmptyArray(t *testing.T) { t.Run("empty", func(t *testing.T) { clusterState := gc.NewClusterGCStatesWithGlobalGCBarriers(map[uint32]gc.GCState{}, nil) - got, err := newGlobalGCStateOutput(clusterState) + got, err := newGlobalGCStateOutput(clusterState, false) require.NoError(t, err) require.NotNil(t, got.GlobalGCBarriers) require.Empty(t, got.GlobalGCBarriers) @@ -312,14 +312,14 @@ func TestNewGlobalGCStateOutputSortsAndKeepsEmptyArray(t *testing.T) { func TestGCStateOutputRejectsExcludedBarriers(t *testing.T) { state := gc.NewGCStateWithoutGCBarriers(42, 100, 90) - _, err := newKeyspaceGCStateOutput(42, state) + _, err := newKeyspaceGCStateOutput(42, state, false) require.ErrorContains(t, err, "failed to read GC barriers for keyspace 42") clusterState := gc.NewClusterGCStatesWithoutGlobalGCBarriers(map[uint32]gc.GCState{}) - _, err = newAllGCStatesOutput(clusterState) + _, err = newAllGCStatesOutput(clusterState, false) require.ErrorContains(t, err, "failed to read global GC barriers") - _, err = newGlobalGCStateOutput(clusterState) + _, err = newGlobalGCStateOutput(clusterState, false) require.ErrorContains(t, err, "failed to read global GC barriers") } @@ -436,6 +436,117 @@ func TestGCStateGlobalCommand(t *testing.T) { require.JSONEq(t, `{"global_gc_barriers":[]}`, output.String()) } +func TestGCStateCommandExpiredBarrierVisibility(t *testing.T) { + state := gc.NewGCStateWithGCBarriers( + 42, + 100, + 90, + []*gc.GCBarrierInfo{ + gc.NewGCBarrierInfo("active-local", 50, gc.TTLNeverExpire, time.Time{}), + gc.NewGCBarrierInfo("expired-local", 40, 0, time.Time{}), + }, + ) + state.IsKeyspaceLevelGC = true + clusterState := gc.NewClusterGCStatesWithGlobalGCBarriers( + map[uint32]gc.GCState{42: state}, + []*gc.GlobalGCBarrierInfo{ + gc.NewGlobalGCBarrierInfo("active-global", 70, gc.TTLNeverExpire, time.Time{}), + gc.NewGlobalGCBarrierInfo("expired-global", 60, 0, time.Time{}), + }, + ) + + for _, testCase := range []struct { + name string + args []string + wantLocal []gcBarrierOutput + wantGlobal []gcBarrierOutput + }{ + { + name: "keyspace-default", + args: []string{"keyspace", "42"}, + wantLocal: []gcBarrierOutput{ + {BarrierID: "active-local", BarrierTS: 50, TTLSeconds: math.MaxInt64}, + }, + }, + { + name: "keyspace-include-expired", + args: []string{"keyspace", "42", "--include-expired"}, + wantLocal: []gcBarrierOutput{ + {BarrierID: "expired-local", BarrierTS: 40, TTLSeconds: 0}, + {BarrierID: "active-local", BarrierTS: 50, TTLSeconds: math.MaxInt64}, + }, + }, + { + name: "all-default", + args: []string{"all"}, + wantLocal: []gcBarrierOutput{ + {BarrierID: "active-local", BarrierTS: 50, TTLSeconds: math.MaxInt64}, + }, + wantGlobal: []gcBarrierOutput{ + {BarrierID: "active-global", BarrierTS: 70, TTLSeconds: math.MaxInt64}, + }, + }, + { + name: "all-include-expired", + args: []string{"all", "--include-expired"}, + wantLocal: []gcBarrierOutput{ + {BarrierID: "expired-local", BarrierTS: 40, TTLSeconds: 0}, + {BarrierID: "active-local", BarrierTS: 50, TTLSeconds: math.MaxInt64}, + }, + wantGlobal: []gcBarrierOutput{ + {BarrierID: "expired-global", BarrierTS: 60, TTLSeconds: 0}, + {BarrierID: "active-global", BarrierTS: 70, TTLSeconds: math.MaxInt64}, + }, + }, + { + name: "global-default", + args: []string{"global"}, + wantGlobal: []gcBarrierOutput{ + {BarrierID: "active-global", BarrierTS: 70, TTLSeconds: math.MaxInt64}, + }, + }, + { + name: "global-include-expired", + args: []string{"global", "--include-expired"}, + wantGlobal: []gcBarrierOutput{ + {BarrierID: "expired-global", BarrierTS: 60, TTLSeconds: 0}, + {BarrierID: "active-global", BarrierTS: 70, TTLSeconds: math.MaxInt64}, + }, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + reader := &fakeGCStateReader{state: state, clusterState: clusterState} + cmd := buildGCStateCommand(func(*cobra.Command) (gcStateReader, error) { + return reader, nil + }) + output := new(bytes.Buffer) + cmd.SetOut(output) + cmd.SetErr(output) + cmd.SetArgs(testCase.args) + + require.NoError(t, cmd.Execute()) + switch testCase.args[0] { + case "keyspace": + var decoded keyspaceGCStateOutput + require.NoError(t, json.Unmarshal(output.Bytes(), &decoded)) + require.Equal(t, testCase.wantLocal, decoded.GCBarriers) + case "all": + var decoded allGCStatesOutput + require.NoError(t, json.Unmarshal(output.Bytes(), &decoded)) + require.Len(t, decoded.GCStates, 1) + require.Equal(t, testCase.wantLocal, decoded.GCStates[0].GCBarriers) + require.Equal(t, testCase.wantGlobal, decoded.GlobalGCBarriers) + case "global": + var decoded globalGCStateOutput + require.NoError(t, json.Unmarshal(output.Bytes(), &decoded)) + require.Equal(t, testCase.wantGlobal, decoded.GlobalGCBarriers) + default: + require.Fail(t, "unexpected subcommand", testCase.args[0]) + } + }) + } +} + func TestGCStateCommandValidatesBeforeCreatingClient(t *testing.T) { for _, args := range [][]string{ {}, @@ -591,7 +702,11 @@ func TestGCStateCommandHelpContract(t *testing.T) { return nil, errors.New("help must not create a reader") }) require.Equal(t, "show keyspace and cluster-wide GC state", cmd.Short) - require.Equal(t, "Show effective per-keyspace GC safe points and local barriers, and cluster-wide GC state. Use keyspace for one effective GC scope, global for cluster-wide state, or all for a combined view.", cmd.Long) + require.Equal(t, "Show effective per-keyspace GC safe points and local barriers, and cluster-wide GC state. Expired barriers awaiting lazy deletion are hidden by default; use --include-expired to include zero-TTL barriers returned by PD. Use keyspace for one effective GC scope, global for cluster-wide state, or all for a combined view.", cmd.Long) + includeExpired := cmd.PersistentFlags().Lookup("include-expired") + require.NotNil(t, includeExpired) + require.Equal(t, "false", includeExpired.DefValue) + require.Equal(t, "include zero-TTL barriers returned by PD, which normally represent expired barriers awaiting lazy deletion", includeExpired.Usage) keyspace, _, err := cmd.Find([]string{"keyspace"}) require.NoError(t, err) diff --git a/tools/pd-ctl/tests/safepoint/gc_state_test.go b/tools/pd-ctl/tests/safepoint/gc_state_test.go index 7a4ee4872e..81d8a1b170 100644 --- a/tools/pd-ctl/tests/safepoint/gc_state_test.go +++ b/tools/pd-ctl/tests/safepoint/gc_state_test.go @@ -69,6 +69,7 @@ type expectedGCStateCommandBarrier struct { barrierID string barrierTS uint64 expires bool + expired bool } func requireGCStateCommandBarriers( @@ -80,7 +81,9 @@ func requireGCStateCommandBarriers( for i, want := range expected { re.Equal(want.barrierID, actual[i].BarrierID) re.Equal(want.barrierTS, actual[i].BarrierTS) - if want.expires { + if want.expired { + re.Zero(actual[i].TTLSeconds) + } else if want.expires { re.Greater(actual[i].TTLSeconds, int64(3500)) re.LessOrEqual(actual[i].TTLSeconds, int64(3600)) } else { @@ -173,6 +176,16 @@ func TestGCState(t *testing.T) { now, ) re.NoError(err) + // Backdate the creation time so the barrier stays persisted until the next + // safe-point advancement but is already inactive when the RPC reads it. + _, err = manager.SetGCBarrier( + keyspaceLevel.Id, + "expired-local", + 230, + time.Hour, + now.Add(-2*time.Hour), + ) + re.NoError(err) _, err = manager.SetGlobalGCBarrier( ctx, @@ -190,6 +203,14 @@ func TestGCState(t *testing.T) { now, ) re.NoError(err) + _, err = manager.SetGlobalGCBarrier( + ctx, + "expired-global", + 330, + time.Hour, + now.Add(-2*time.Hour), + ) + re.NoError(err) pdAddr := cluster.GetConfig().GetClientURL() keyspaceLevelID := strconv.FormatUint(uint64(keyspaceLevel.Id), 10) @@ -212,6 +233,19 @@ func TestGCState(t *testing.T) { {barrierID: "z-local", barrierTS: 220, expires: true}, }) + output, err = tests.ExecuteCommand( + ctl.GetRootCmd(), "-u", pdAddr, "gc-state", "keyspace", keyspaceLevelID, + "--include-expired", + ) + re.NoError(err) + var keyspaceLevelWithExpired gcStateCommandSingle + re.NoError(json.Unmarshal(output, &keyspaceLevelWithExpired), string(output)) + requireGCStateCommandBarriers(re, keyspaceLevelWithExpired.GCBarriers, []expectedGCStateCommandBarrier{ + {barrierID: "a-local", barrierTS: 210}, + {barrierID: "z-local", barrierTS: 220, expires: true}, + {barrierID: "expired-local", barrierTS: 230, expired: true}, + }) + output, err = tests.ExecuteCommand( ctl.GetRootCmd(), "-u", pdAddr, "gc-state", "keyspace", "4294967295", ) @@ -271,6 +305,29 @@ func TestGCState(t *testing.T) { {barrierID: "z-global", barrierTS: 320}, }) + output, err = tests.ExecuteCommand( + ctl.GetRootCmd(), "-u", pdAddr, "gc-state", "all", "--include-expired", + ) + re.NoError(err) + var allWithExpired gcStateCommandAll + re.NoError(json.Unmarshal(output, &allWithExpired), string(output)) + statesByIDWithExpired := make(map[uint32]gcStateCommandState, len(allWithExpired.GCStates)) + for _, state := range allWithExpired.GCStates { + statesByIDWithExpired[state.KeyspaceID] = state + } + keyspaceLevelStateWithExpired, ok := statesByIDWithExpired[keyspaceLevel.Id] + re.True(ok) + requireGCStateCommandBarriers(re, keyspaceLevelStateWithExpired.GCBarriers, []expectedGCStateCommandBarrier{ + {barrierID: "a-local", barrierTS: 210}, + {barrierID: "z-local", barrierTS: 220, expires: true}, + {barrierID: "expired-local", barrierTS: 230, expired: true}, + }) + requireGCStateCommandBarriers(re, allWithExpired.GlobalGCBarriers, []expectedGCStateCommandBarrier{ + {barrierID: "a-global", barrierTS: 310}, + {barrierID: "z-global", barrierTS: 320}, + {barrierID: "expired-global", barrierTS: 330, expired: true}, + }) + output, err = tests.ExecuteCommand( ctl.GetRootCmd(), "-u", @@ -292,6 +349,18 @@ func TestGCState(t *testing.T) { re.NotNil(global.GlobalGCBarriers) re.Equal(all.GlobalGCBarriers, global.GlobalGCBarriers) + output, err = tests.ExecuteCommand( + ctl.GetRootCmd(), "-u", pdAddr, "gc-state", "global", "--include-expired", + ) + re.NoError(err) + var globalWithExpired gcStateCommandGlobal + re.NoError(json.Unmarshal(output, &globalWithExpired), string(output)) + requireGCStateCommandBarriers(re, globalWithExpired.GlobalGCBarriers, []expectedGCStateCommandBarrier{ + {barrierID: "a-global", barrierTS: 310}, + {barrierID: "z-global", barrierTS: 320}, + {barrierID: "expired-global", barrierTS: 330, expired: true}, + }) + if kerneltype.IsNextGen() { systemState, ok := statesByID[constant.SystemKeyspaceID] re.True(ok) From d1c509cd0776663706a4a8c15e5f35e5b749cc3b Mon Sep 17 00:00:00 2001 From: Wenxuan Zhang Date: Wed, 5 Aug 2026 14:44:25 +0800 Subject: [PATCH 19/31] test: adapt GC state tests to API v3 Signed-off-by: Wenxuan Zhang --- client/gc_client_test.go | 2 +- tools/pd-ctl/tests/safepoint/gc_state_test.go | 27 ++++++++++--------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/client/gc_client_test.go b/client/gc_client_test.go index 71458d72b8..e35cc3fa73 100644 --- a/client/gc_client_test.go +++ b/client/gc_client_test.go @@ -45,7 +45,7 @@ func TestPBToGCStatePreservesKeyspaceLevelGC(t *testing.T) { } { t.Run(testCase.name, func(t *testing.T) { pbState := &pdpb.GCState{ - KeyspaceScope: &pdpb.KeyspaceScope{KeyspaceId: 42}, + KeyspaceScope: wrapKeyspaceScope(42), IsKeyspaceLevelGc: testCase.isKeyspaceLevelGC, TxnSafePoint: 100, GcSafePoint: 90, diff --git a/tools/pd-ctl/tests/safepoint/gc_state_test.go b/tools/pd-ctl/tests/safepoint/gc_state_test.go index 81d8a1b170..e181f6aa39 100644 --- a/tools/pd-ctl/tests/safepoint/gc_state_test.go +++ b/tools/pd-ctl/tests/safepoint/gc_state_test.go @@ -117,6 +117,7 @@ func TestGCState(t *testing.T) { }, ) re.NoError(err) + keyspaceLevelID := keyspaceLevel.GetId() var unifiedKeyspaceID uint32 if !kerneltype.IsNextGen() { @@ -130,7 +131,7 @@ func TestGCState(t *testing.T) { }, ) re.NoError(err) - unifiedKeyspaceID = unified.Id + unifiedKeyspaceID = unified.GetId() } manager := leaderServer.GetServer().GetGCStateManager() @@ -156,12 +157,12 @@ func TestGCState(t *testing.T) { ) re.NoError(err) - _, err = manager.AdvanceTxnSafePoint(keyspaceLevel.Id, 200, now) + _, err = manager.AdvanceTxnSafePoint(keyspaceLevelID, 200, now) re.NoError(err) - _, _, err = manager.AdvanceGCSafePoint(keyspaceLevel.Id, 190) + _, _, err = manager.AdvanceGCSafePoint(keyspaceLevelID, 190) re.NoError(err) _, err = manager.SetGCBarrier( - keyspaceLevel.Id, + keyspaceLevelID, "z-local", 220, time.Hour, @@ -169,7 +170,7 @@ func TestGCState(t *testing.T) { ) re.NoError(err) _, err = manager.SetGCBarrier( - keyspaceLevel.Id, + keyspaceLevelID, "a-local", 210, time.Duration(math.MaxInt64), @@ -179,7 +180,7 @@ func TestGCState(t *testing.T) { // Backdate the creation time so the barrier stays persisted until the next // safe-point advancement but is already inactive when the RPC reads it. _, err = manager.SetGCBarrier( - keyspaceLevel.Id, + keyspaceLevelID, "expired-local", 230, time.Hour, @@ -213,9 +214,9 @@ func TestGCState(t *testing.T) { re.NoError(err) pdAddr := cluster.GetConfig().GetClientURL() - keyspaceLevelID := strconv.FormatUint(uint64(keyspaceLevel.Id), 10) + keyspaceLevelIDString := strconv.FormatUint(uint64(keyspaceLevelID), 10) output, err := tests.ExecuteCommand( - ctl.GetRootCmd(), "-u", pdAddr, "gc-state", "keyspace", keyspaceLevelID, + ctl.GetRootCmd(), "-u", pdAddr, "gc-state", "keyspace", keyspaceLevelIDString, ) re.NoError(err) var singleProperties map[string]json.RawMessage @@ -223,8 +224,8 @@ func TestGCState(t *testing.T) { re.NotContains(singleProperties, "global_gc_barriers") var keyspaceLevelResponse gcStateCommandSingle re.NoError(json.Unmarshal(output, &keyspaceLevelResponse), string(output)) - re.Equal(keyspaceLevel.Id, keyspaceLevelResponse.RequestedKeyspaceID) - re.Equal(keyspaceLevel.Id, keyspaceLevelResponse.EffectiveKeyspaceID) + re.Equal(keyspaceLevelID, keyspaceLevelResponse.RequestedKeyspaceID) + re.Equal(keyspaceLevelID, keyspaceLevelResponse.EffectiveKeyspaceID) re.True(keyspaceLevelResponse.IsKeyspaceLevelGC) re.Equal(uint64(200), keyspaceLevelResponse.TxnSafePoint) re.Equal(uint64(190), keyspaceLevelResponse.GCSafePoint) @@ -234,7 +235,7 @@ func TestGCState(t *testing.T) { }) output, err = tests.ExecuteCommand( - ctl.GetRootCmd(), "-u", pdAddr, "gc-state", "keyspace", keyspaceLevelID, + ctl.GetRootCmd(), "-u", pdAddr, "gc-state", "keyspace", keyspaceLevelIDString, "--include-expired", ) re.NoError(err) @@ -291,7 +292,7 @@ func TestGCState(t *testing.T) { {barrierID: "z-null", barrierTS: 120, expires: true}, }) - keyspaceLevelState, ok := statesByID[keyspaceLevel.Id] + keyspaceLevelState, ok := statesByID[keyspaceLevelID] re.True(ok) re.True(keyspaceLevelState.IsKeyspaceLevelGC) re.Equal(uint64(200), keyspaceLevelState.TxnSafePoint) @@ -315,7 +316,7 @@ func TestGCState(t *testing.T) { for _, state := range allWithExpired.GCStates { statesByIDWithExpired[state.KeyspaceID] = state } - keyspaceLevelStateWithExpired, ok := statesByIDWithExpired[keyspaceLevel.Id] + keyspaceLevelStateWithExpired, ok := statesByIDWithExpired[keyspaceLevelID] re.True(ok) requireGCStateCommandBarriers(re, keyspaceLevelStateWithExpired.GCBarriers, []expectedGCStateCommandBarrier{ {barrierID: "a-local", barrierTS: 210}, From 162c550dd7ee9a9c80f851394b46d336d300a08e Mon Sep 17 00:00:00 2001 From: Wenxuan Zhang Date: Tue, 11 Aug 2026 16:17:32 +0800 Subject: [PATCH 20/31] docs: add GC state global barrier refactor design Document the agreed command surface, option semantics, and compatibility behavior, and the test plan for using GetGCState to inspect global barriers. Signed-off-by: Wenxuan Zhang --- ...gc-state-global-barrier-refactor-design.md | 386 ++++++++++++++++++ 1 file changed, 386 insertions(+) create mode 100644 docs/superpowers/specs/2026-08-11-gc-state-global-barrier-refactor-design.md diff --git a/docs/superpowers/specs/2026-08-11-gc-state-global-barrier-refactor-design.md b/docs/superpowers/specs/2026-08-11-gc-state-global-barrier-refactor-design.md new file mode 100644 index 0000000000..83d3b2c85d --- /dev/null +++ b/docs/superpowers/specs/2026-08-11-gc-state-global-barrier-refactor-design.md @@ -0,0 +1,386 @@ +# GC state global barrier refactor design + +This design refactors the `pd-ctl gc-state` command introduced by PR #11054. +The refactor uses the enhanced `GetGCState` API from +[PR #11117](https://github.com/tikv/pd/pull/11117) to include global GC +barriers in single-keyspace diagnostics, removes the standalone `global` +subcommand, and adds an option that omits global barriers from either remaining +view. + +## Context + +PR #11054 currently exposes three views: `keyspace`, `global`, and `all`. The +`global` view calls `GetAllKeyspacesGCStates`, even though it discards every +keyspace state and only emits global barriers. That call still enumerates and +materializes all keyspace states, so it performs unnecessary work on clusters +with many keyspaces. + +PR #11117 adds an opt-in global barrier result to `GetGCState`. The client uses +`gc.ExcludeGlobalGCBarriers(false)` to request the result and preserves whether +the server omitted the result, returned an empty list, or returned a populated +list. The server reads the selected keyspace state and global barriers in one +revision-validated operation. + +## Goals and non-goals + +The refactor keeps the CLI focused on the two diagnostic scopes that have +distinct data requirements. + +The design has these goals: + +- Make `gc-state keyspace` return the selected effective GC state, local + barriers, and global barriers with one `GetGCState` call. +- Keep `gc-state all` as the only command that enumerates every effective GC + scope. +- Remove `gc-state global` and every code, test, help, and documentation path + that exists only for that subcommand. +- Let users skip global barrier reads and output in both remaining subcommands + with `--exclude-global-barriers`. +- Preserve the difference between an omitted global barrier result and a + requested result that contains an empty list. +- Preserve deterministic sorting, expired barrier filtering, effective scope + handling, and actionable compatibility errors. + +The design does not add another PD RPC, automatically retry failed requests, +change GC state semantics, or change how PD stores and expires barriers. + +## CLI contract + +The command tree contains only the two views that correspond to a selected +keyspace or all effective scopes: + +```text +gc-state +├── keyspace +└── all +``` + +The implementation removes `gc-state global` without a deprecated or hidden +alias. PR #11054 has not shipped, so the command does not have a released +compatibility contract. + +### Shared flags + +The parent `gc-state` command defines two persistent flags that both subcommands +inherit: + +- `--include-expired` includes zero-TTL local and requested global barriers. +- `--exclude-global-barriers` skips the global barrier read and omits the + `global_gc_barriers` JSON field. + +Both flags default to `false`. Explicitly setting +`--exclude-global-barriers=false` produces the default complete view. + +### Behavior matrix + +The following matrix defines the request and output behavior for every flag +combination. + +| Command | Local barriers | Global barriers | Global JSON field | +| --- | --- | --- | --- | +| `keyspace 42` | Read; hide expired | Read; hide expired | Present | +| `keyspace 42 --include-expired` | Read; include expired | Read; include expired | Present | +| `keyspace 42 --exclude-global-barriers` | Read; hide expired | Do not read | Omitted | +| `keyspace 42 --exclude-global-barriers --include-expired` | Read; include expired | Do not read | Omitted | +| `all` | Read; hide expired | Read; hide expired | Present | +| `all --include-expired` | Read; include expired | Read; include expired | Present | +| `all --exclude-global-barriers` | Read; hide expired | Do not read | Omitted | +| `all --exclude-global-barriers --include-expired` | Read; include expired | Do not read | Omitted | + +When global barriers are included, the JSON field is present even if the list +is empty. Therefore, `"global_gc_barriers": []` means the command requested +global barriers and PD returned none. A missing field means the user explicitly +excluded them. + +## Request flow + +Each subcommand uses the least expensive RPC that provides all data required by +that view. + +### Keyspace view + +The keyspace view calls `GetGCState` exactly once. Its default client options +are: + +```go +GetGCState( + ctx, + gc.ExcludeGCBarriers(false), + gc.ExcludeGlobalGCBarriers(false), +) +``` + +The returned `gc.GCState` contains the effective keyspace state, local +barriers, and global barriers from the same server-side read. With +`--exclude-global-barriers`, the command changes only the second option to +`gc.ExcludeGlobalGCBarriers(true)`. + +The command never calls `GetAllKeyspacesGCStates` as a fallback for the +keyspace view. + +### All view + +The all view continues to call `GetAllKeyspacesGCStates` because it must +enumerate every effective GC scope. It always requests local barriers and maps +the flag directly to `gc.ExcludeGlobalGCBarriers`. + +The all view does not make an additional `GetGCState` call. No remaining path +calls `GetAllKeyspacesGCStates` solely to obtain global barriers. + +## Internal command structure + +The reader abstraction describes the two command behaviors and carries one +positive boolean that controls global barrier inclusion: + +```go +type gcStateReader interface { + getGCState( + ctx context.Context, + keyspaceID uint32, + includeGlobalGCBarriers bool, + ) (gc.GCState, error) + getAllKeyspacesGCStates( + ctx context.Context, + includeGlobalGCBarriers bool, + ) (gc.ClusterGCStates, error) + close() +} +``` + +The boolean is named `includeGlobalGCBarriers` at every declaration and call +site. The Cobra layer converts the negative flag once: + +```go +includeGlobalGCBarriers := !excludeGlobalGCBarriers +``` + +The concrete reader maps the positive value to the client option with +`gc.ExcludeGlobalGCBarriers(!includeGlobalGCBarriers)`. A shared pure helper +returns the local and global barrier options for both RPCs. This helper keeps +option construction consistent and lets unit tests verify the mapping without +mocking the full PD client. + +The implementation deletes these obsolete elements: + +- `getGlobalGCState` from the reader interface and concrete reader. +- `getGlobalCalls` from the fake reader. +- `clusterGCStatesClient` and `readClusterGCStates`. +- `newGCStateGlobalCommand` and its command registration. +- `globalGCStateOutput` and `newGlobalGCStateOutput`. + +## JSON projection + +The command output must represent the client's three global barrier states: +not requested, requested and empty, and requested and populated. + +A slice with `omitempty` cannot express that contract because JSON encoding +omits both nil and empty slices. The keyspace and all output structs use a +pointer to a slice instead: + +```go +type keyspaceGCStateOutput struct { + RequestedKeyspaceID uint32 `json:"requested_keyspace_id"` + EffectiveKeyspaceID uint32 `json:"effective_keyspace_id"` + IsKeyspaceLevelGC bool `json:"is_keyspace_level_gc"` + TxnSafePoint uint64 `json:"txn_safe_point"` + GCSafePoint uint64 `json:"gc_safe_point"` + GCBarriers []gcBarrierOutput `json:"gc_barriers"` + GlobalGCBarriers *[]gcBarrierOutput `json:"global_gc_barriers,omitempty"` +} + +type allGCStatesOutput struct { + GCStates []gcStateOutput `json:"gc_states"` + GlobalGCBarriers *[]gcBarrierOutput `json:"global_gc_barriers,omitempty"` +} +``` + +When global barriers are excluded, the pointer remains nil, the converter does +not call `GetGlobalGCBarriers`, and JSON encoding omits the field. When global +barriers are included, the pointer references a non-nil slice. The field then +encodes as either `[]` or a populated array. + +`newKeyspaceGCStateOutput` and `newAllGCStatesOutput` accept +`includeGlobalGCBarriers` in addition to `includeExpired`. They obtain global +barriers only when requested and reuse `newGlobalGCBarrierOutputs` for nil +entry filtering, expired entry filtering, TTL conversion, and deterministic +sorting. + +Local barrier fields remain required and encode empty results as +`"gc_barriers": []`. + +## Compatibility and errors + +The command reports capability mismatches explicitly and does not convert an +absent response wrapper into an empty barrier list. + +During a rolling upgrade, an older PD server can ignore the new opt-in request +field and return a successful `GetGCState` response without the global barrier +wrapper. The client then reports `state.HasGlobalGCBarriers() == false`. If the +default keyspace view requires global barriers, the command returns this +actionable error: + +```text +gc-state keyspace requires a PD server whose GetGCState supports global GC barriers; retry with --exclude-global-barriers +``` + +The explicit exclusion flag remains a valid degraded path for reading safe +points and local barriers from that server. + +If the all view requires global barriers but the cluster result does not carry +them, the command returns: + +```text +gc-state all response does not include global GC barriers; retry with --exclude-global-barriers +``` + +The command does not retry automatically. A retry could perform another +server-side read, mix snapshots, hide a rolling-upgrade capability difference, +and reintroduce unnecessary work. + +Existing error behavior remains intact for an unimplemented RPC, client +creation failure, other RPC errors, missing local barriers, JSON encoding, and +output writes. Every path closes the reader after successful creation. + +## Rebase integration + +The implementation starts by rebasing PR #11054 onto `upstream/master`, which +already contains PR #11117. Both changes modify the public GC state model and +protobuf conversion, so conflict resolution must combine their behavior. + +The resolved client model retains all of these elements: + +- `IsKeyspaceLevelGC` from PR #11054. +- `hasGlobalGCBarriers` and `globalGCBarriers` from PR #11117. +- `WithGlobalGCBarriers`, `HasGlobalGCBarriers`, and + `GetGlobalGCBarriers` from PR #11117. + +`pbToGCState` constructs the local state and sets `IsKeyspaceLevelGC`. +`pbToGCStateWithGlobalGCBarriers` then attaches the optional global barrier +result without losing the keyspace-level flag. The client tests cover this +composition without duplicating PR #11117's storage, snapshot, TTL, and rolling +upgrade test coverage. + +## Test design + +The test suite proves the option mapping, command routing, JSON presence +contract, compatibility behavior, and real PD integration in Classic and +NextGen configurations. + +### Unit tests + +The command unit tests cover these cases: + +- Both reader methods always request local barriers. +- Both reader methods include global barriers by default and exclude them only + when requested. +- `keyspace` calls only `getGCState`, and `all` calls only + `getAllKeyspacesGCStates`. +- The fake reader receives the expected `includeGlobalGCBarriers` value. +- `global` is an unknown subcommand and fails before creating a reader. +- Root and subcommand help mention only `keyspace`, `all`, and the two shared + flags. +- Default projections emit an empty or populated global array. +- Excluded projections omit the global field and do not require the client + model to carry global barriers. +- Required but absent global barriers produce the actionable compatibility + errors. +- Local and global nil entries are skipped. +- Local and global barriers use the same active and expired filtering rules. +- Barrier arrays preserve deterministic sorting and TTL conversion. +- Effective scope filtering still omits unified-GC placeholders and retains + the NullKeyspace state. +- Reader closure and output error propagation remain intact. + +Tests inspect encoded JSON maps in addition to Go values so they detect field +presence regressions caused by `omitempty`. + +### Integration tests + +The existing safepoint test fixtures already contain keyspace-level, +NullKeyspace, unified-GC, active, expired, local, and global barrier cases. The +refactor reuses those fixtures and changes the command assertions. + +The integration tests verify these behaviors: + +- Default keyspace output includes the same global barriers for keyspace-level, + NullKeyspace, and unified-GC requests. +- `keyspace --include-expired` includes zero-TTL local and global barriers. +- `keyspace --exclude-global-barriers` preserves safe points and local barriers + while omitting the global field. +- Default all output contains global barriers exactly once at the top level. +- `all --exclude-global-barriers` preserves every effective scope and omits the + top-level global field. +- Combining exclusion with `--include-expired` affects only local barriers. +- Classic unified GC and NextGen keyspace-level GC behavior remain unchanged. + +The tests remove every `gc-state global` invocation, output type, and assertion. +PR #11117 already proves server-side snapshot consistency and verifies that +excluded requests stay on the no-global-read path, so this PR does not duplicate +those failpoint tests. + +## Documentation and PR updates + +The user documentation and PR metadata must describe the final two-command +workflow. + +The `tools/pd-ctl/README.md` update makes these changes: + +- Remove the standalone global view and its JSON example. +- Add `global_gc_barriers` to the keyspace JSON example. +- Explain missing versus present-empty global barrier fields. +- Document `--exclude-global-barriers` for both remaining subcommands. +- Explain its interaction with `--include-expired`. +- Recommend `keyspace` for one scope and reserve `all` for full-cluster + inspection. + +The PR body and release note describe `keyspace` and `all`, the enhanced +`GetGCState` path, and the shared exclusion flag. They do not claim that the PR +adds a standalone global command. + +## Implementation sequence + +The implementation follows this order to isolate rebase work from command +behavior changes: + +1. Rebase the branch onto the `upstream/master` commit that contains PR #11117. +2. Resolve the GC client model and conversion conflicts, and run focused client + tests. +3. Update command unit tests for the two-command tree, shared flag, option + mapping, JSON presence contract, and compatibility errors. +4. Refactor the reader, projections, and Cobra commands until the unit tests + pass. +5. Update the real PD safepoint integration test for the complete flag matrix. +6. Update the README, help contract, PR body, and release note. +7. Run formatting, focused client and pd-ctl tests, Classic and NextGen + integration tests, the `pd-ctl` build, and relevant static checks. +8. Confirm failpoints are disabled and the worktree contains no generated or + unrelated files before updating the PR. + +## Acceptance criteria + +The refactor is complete when the implementation meets every observable +contract in this design. + +- `gc-state global` is absent from command registration, implementation, tests, + help, documentation, and PR metadata. +- Default `gc-state keyspace` obtains local and global barriers with one + `GetGCState` call. +- No code calls `GetAllKeyspacesGCStates` solely to obtain global barriers. +- `gc-state all` remains the only full-keyspace enumeration path. +- `--exclude-global-barriers` controls both the RPC option and JSON field in + `keyspace` and `all`. +- A requested empty list encodes as `[]`, while explicit exclusion omits the + field. +- Expired filtering works consistently for every requested barrier type. +- A server that omits requested global barriers produces an actionable error + and supports the explicit degraded view. +- Client conversion preserves keyspace-level GC metadata and optional global + barriers at the same time. +- Focused unit, Classic integration, NextGen integration, build, and static + checks pass. + +## Next steps + +After this design is reviewed, create a detailed implementation plan with +file-level edits, test-first steps, verification commands, and review +checkpoints. Do not change product code before that plan is approved. From a08163c8a2e5c0e4562517ce0abfe61c4db389ac Mon Sep 17 00:00:00 2001 From: Wenxuan Zhang Date: Tue, 11 Aug 2026 16:31:18 +0800 Subject: [PATCH 21/31] docs: add GC state refactor implementation plan Signed-off-by: Wenxuan Zhang --- ...-08-11-gc-state-global-barrier-refactor.md | 1318 +++++++++++++++++ 1 file changed, 1318 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-11-gc-state-global-barrier-refactor.md diff --git a/docs/superpowers/plans/2026-08-11-gc-state-global-barrier-refactor.md b/docs/superpowers/plans/2026-08-11-gc-state-global-barrier-refactor.md new file mode 100644 index 0000000000..7571e3e99d --- /dev/null +++ b/docs/superpowers/plans/2026-08-11-gc-state-global-barrier-refactor.md @@ -0,0 +1,1318 @@ +# GC State Global Barrier Refactor Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make `pd-ctl gc-state keyspace` use `GetGCState` to return local and +global GC barriers, remove `gc-state global`, and let both remaining views skip +global barriers with `--exclude-global-barriers`. + +**Architecture:** Rebase onto the `GetGCState` global barrier support from PR +#11117, preserve the current PR's keyspace-level GC metadata, and pass a single +positive inclusion boolean through the command reader. Model optional JSON +output with a pointer to a slice so the CLI distinguishes an omitted field from +a requested empty list. + +**Tech Stack:** Go 1.25, Cobra, `github.com/tikv/pd/client/clients/gc`, +`encoding/json`, Testify, PD test clusters, and Make. + +## Global constraints + +These constraints apply to every task in this plan. + +- Read and follow the repository `AGENTS.md` before changing files. +- Use Go 1.25 or later. CI uses Go 1.25. +- Start from a clean worktree with failpoints disabled. +- Rebase onto an `upstream/master` that contains commit `3430f76dc` from PR + #11117 before editing command behavior. +- Do not add dependencies or edit module files beyond conflict resolution with + the versions already merged by PR #11117. +- Keep `gc-state keyspace` to one `GetGCState` RPC. Do not add an automatic + `GetAllKeyspacesGCStates` fallback. +- Keep `GetAllKeyspacesGCStates` only for the `all` view. +- Register only `keyspace` and `all`; do not retain a hidden or deprecated + `global` alias. +- Define `--exclude-global-barriers` on the parent command so both subcommands + inherit it. +- Preserve the three global barrier states: omitted, requested-empty, and + requested-populated. +- Preserve nil barrier filtering, zero-TTL filtering, deterministic sorting, + effective scope filtering, and reader closure behavior. +- Use `status.Code(err)` on the original error for gRPC compatibility checks. +- Use `gofmt` and the repository import order on every touched Go file. +- Never edit files or run non-test commands while failpoints are enabled. +- Disable failpoints immediately after each failpoint-enabled test, including + after a failed test. +- Use signed commits with subjects no longer than 70 characters and bodies + wrapped at 80 characters. +- Do not hard-wrap prose when editing the GitHub PR body. + +The approved design is in +[`../specs/2026-08-11-gc-state-global-barrier-refactor-design.md`](../specs/2026-08-11-gc-state-global-barrier-refactor-design.md). + +--- + +## File map + +The implementation modifies existing GC client, command, test, and user +documentation files. It does not add a new production source file. + +- `client/clients/gc/client.go` retains `IsKeyspaceLevelGC` alongside PR + #11117's optional global barrier fields and accessors. +- `client/gc_client.go` preserves keyspace-level mode through local and global + protobuf conversion. +- `client/gc_client_test.go` locks the composed conversion behavior. +- `tests/integrations/client/client_test.go` retains both sides' integration + assertions while resolving the rebase. +- `tools/pd-ctl/pdctl/command/gc_state_command.go` owns reader options, JSON + projection, flags, routing, and errors. +- `tools/pd-ctl/pdctl/command/gc_state_command_test.go` owns projection, + option, command, error, and help contracts. +- `tools/pd-ctl/tests/safepoint/gc_state_test.go` owns end-to-end Classic and + NextGen command behavior against a real PD server. +- `tools/pd-ctl/README.md` documents the final two-command workflow. + +## Task 1: Rebase and compose the GC client model + +This task establishes the correct baseline and combines PR #11117's optional +global barriers with PR #11054's keyspace-level GC metadata. + +**Files:** + +- Modify during conflict resolution: `client/clients/gc/client.go:285-380` +- Modify during conflict resolution: `client/gc_client.go:298-390` +- Modify: `client/gc_client_test.go:1-90` +- Modify during conflict resolution: + `tests/integrations/client/client_test.go:2069-2780` +- Review only: `go.mod`, `go.sum`, `client/go.mod`, `client/go.sum`, + `tests/integrations/go.mod`, `tests/integrations/go.sum`, `tools/go.mod`, and + `tools/go.sum` + +**Interfaces:** + +- Consumes: PR #11117's `gc.GCState.WithGlobalGCBarriers`, + `gc.GCState.HasGlobalGCBarriers`, and + `gc.GCState.GetGlobalGCBarriers` methods. +- Produces: `gc.GCState.IsKeyspaceLevelGC bool` on states with or without local + and global barriers. +- Produces: `pbToGCStateWithGlobalGCBarriers(*pdpb.GCState, + *pdpb.GlobalGCBarriersInfo, time.Time, bool) gc.GCState` that preserves the + keyspace-level flag. + +- [ ] **Step 1: Verify the pre-rebase state** + +Run these commands before rewriting branch history: + +```bash +make failpoint-disable +git status --short --branch +git log -1 --oneline upstream/master +git merge-base --is-ancestor 3430f76dc upstream/master +``` + +Expected: failpoints are disabled, the worktree is clean, the ancestor check +exits with status 0, and `upstream/master` contains PR #11117. Stop if the +worktree is dirty; do not stash or discard user changes automatically. + +- [ ] **Step 2: Refresh and rebase onto upstream master** + +Run: + +```bash +git fetch upstream master +git rebase upstream/master +``` + +Expected: the rebase can stop in the four client files listed above because +both PRs modify the GC state model and protobuf conversion. Resolve only those +semantic overlaps. Do not resolve an entire file with `--ours` or `--theirs`. + +- [ ] **Step 3: Resolve the public `GCState` model composition** + +Ensure the resolved struct in `client/clients/gc/client.go` contains both the +public mode field and PR #11117's private global barrier state: + +```go +type GCState struct { + // The ID of the keyspace this GC state belongs to. + KeyspaceID uint32 + // IsKeyspaceLevelGC reports whether this state belongs to an independent + // keyspace-level GC scope. + IsKeyspaceLevelGC bool + TxnSafePoint uint64 + GCSafePoint uint64 + hasGCBarriers bool + gcBarriers []*GCBarrierInfo + + hasGlobalGCBarriers bool + globalGCBarriers []*GlobalGCBarrierInfo +} +``` + +Keep `WithGlobalGCBarriers`, `HasGlobalGCBarriers`, and +`GetGlobalGCBarriers` exactly as merged by PR #11117. + +- [ ] **Step 4: Resolve protobuf conversion without losing mode metadata** + +In `client/gc_client.go`, keep PR #11117's local conversion and add the mode +assignment immediately before `pbToGCState` returns: + +```go +func pbToGCState( + pb *pdpb.GCState, + reqStartTime time.Time, + excludeGCBarriers bool, +) gc.GCState { + keyspaceID := constants.NullKeyspaceID + if pb.KeyspaceScope != nil { + keyspaceID = pb.KeyspaceScope.GetKeyspaceId() + } + + var state gc.GCState + if excludeGCBarriers { + state = gc.NewGCStateWithoutGCBarriers( + keyspaceID, + pb.GetTxnSafePoint(), + pb.GetGcSafePoint(), + ) + } else { + gcBarriers := make([]*gc.GCBarrierInfo, 0, len(pb.GetGcBarriers())) + for _, barrier := range pb.GetGcBarriers() { + gcBarriers = append( + gcBarriers, + pbToGCBarrierInfo(barrier, reqStartTime), + ) + } + state = gc.NewGCStateWithGCBarriers( + keyspaceID, + pb.GetTxnSafePoint(), + pb.GetGcSafePoint(), + gcBarriers, + ) + } + state.IsKeyspaceLevelGC = pb.GetIsKeyspaceLevelGc() + return state +} +``` + +Keep `pbToGCStateWithGlobalGCBarriers` based on `result := pbToGCState(...)` +and `return result.WithGlobalGCBarriers(barriers)`. That value-receiver flow +preserves `IsKeyspaceLevelGC`. + +- [ ] **Step 5: Add a focused regression test for the composed conversion** + +Append this test to `client/gc_client_test.go`: + +```go +func TestPBToGCStateWithGlobalBarriersPreservesKeyspaceLevelGC(t *testing.T) { + requestStart := time.Unix(100, 0) + state := pbToGCStateWithGlobalGCBarriers( + &pdpb.GCState{ + KeyspaceScope: wrapKeyspaceScope(42), + IsKeyspaceLevelGc: true, + TxnSafePoint: 100, + GcSafePoint: 90, + }, + &pdpb.GlobalGCBarriersInfo{ + Barriers: []*pdpb.GlobalGCBarrierInfo{ + {BarrierId: "snapshot", BarrierTs: 95, TtlSeconds: 60}, + }, + }, + requestStart, + true, + ) + + require.True(t, state.IsKeyspaceLevelGC) + require.False(t, state.HasGCBarriers()) + require.True(t, state.HasGlobalGCBarriers()) + barriers, err := state.GetGlobalGCBarriers() + require.NoError(t, err) + require.Len(t, barriers, 1) + require.Equal(t, "snapshot", barriers[0].BarrierID) +} +``` + +- [ ] **Step 6: Run the focused client tests** + +Run: + +```bash +cd client && go test . -run '^(TestPBToGCStatePreservesKeyspaceLevelGC|TestPBToGCStateWithGlobalBarriersPreservesKeyspaceLevelGC)$' -count=1 +``` + +Expected: PASS. A missing `IsKeyspaceLevelGC` field or a converter that +reconstructs the state after setting the field causes a compile or assertion +failure. + +- [ ] **Step 7: Review dependency conflict resolution** + +Run: + +```bash +git diff upstream/master...HEAD -- go.mod go.sum client/go.mod client/go.sum tests/integrations/go.mod tests/integrations/go.sum tools/go.mod tools/go.sum +``` + +Expected: the branch uses the kvproto version already present in +`upstream/master`; this task adds no new dependency version. + +- [ ] **Step 8: Commit the focused composition regression** + +Stage only the intentional post-rebase client changes and test: + +```bash +git add client/clients/gc/client.go client/gc_client.go client/gc_client_test.go tests/integrations/client/client_test.go +git diff --cached --check +git diff --cached +git commit -s -m "client: preserve GC mode with global barriers" +``` + +Expected: the commit contains the composed model/converter and focused +regression only. Replayed rebase commits remain separate history. + +## Task 2: Add optional global barriers to JSON projections + +This task changes only output construction. It keeps the existing command tree +temporarily so projection behavior can be reviewed independently from routing. + +**Files:** + +- Modify: `tools/pd-ctl/pdctl/command/gc_state_command.go:148-315` +- Test: `tools/pd-ctl/pdctl/command/gc_state_command_test.go:106-325` + +**Interfaces:** + +- Consumes: `gc.GCState.HasGlobalGCBarriers()` and + `gc.GCState.GetGlobalGCBarriers()` from Task 1. +- Produces: `newKeyspaceGCStateOutput(uint32, gc.GCState, bool, bool) + (keyspaceGCStateOutput, error)`. +- Produces: `newAllGCStatesOutput(gc.ClusterGCStates, bool, bool) + (allGCStatesOutput, error)`. +- Produces: optional `GlobalGCBarriers *[]gcBarrierOutput` fields on both + top-level output types. + +- [ ] **Step 1: Write failing keyspace projection tests** + +Add this test next to `TestNewKeyspaceGCStateOutput`: + +```go +func TestNewKeyspaceGCStateOutputGlobalBarrierPresence(t *testing.T) { + t.Run("requested-empty", func(t *testing.T) { + state := gc.NewGCStateWithGCBarriers(42, 100, 90, nil). + WithGlobalGCBarriers(nil) + got, err := newKeyspaceGCStateOutput(42, state, false, true) + require.NoError(t, err) + require.NotNil(t, got.GlobalGCBarriers) + require.Empty(t, *got.GlobalGCBarriers) + encoded, err := json.Marshal(got) + require.NoError(t, err) + require.Contains(t, string(encoded), `"global_gc_barriers":[]`) + }) + + t.Run("excluded", func(t *testing.T) { + state := gc.NewGCStateWithGCBarriers(42, 100, 90, nil) + got, err := newKeyspaceGCStateOutput(42, state, false, false) + require.NoError(t, err) + require.Nil(t, got.GlobalGCBarriers) + encoded, err := json.Marshal(got) + require.NoError(t, err) + require.NotContains(t, string(encoded), "global_gc_barriers") + }) + + t.Run("requested-missing", func(t *testing.T) { + state := gc.NewGCStateWithGCBarriers(42, 100, 90, nil) + _, err := newKeyspaceGCStateOutput(42, state, false, true) + require.ErrorContains(t, err, + "retry with --exclude-global-barriers") + }) +} +``` + +- [ ] **Step 2: Write failing all-view projection tests** + +Add this test next to `TestNewAllGCStatesOutputKeepsEmptyGlobalBarrierArray`: + +```go +func TestNewAllGCStatesOutputGlobalBarrierPresence(t *testing.T) { + t.Run("requested-empty", func(t *testing.T) { + state := gc.NewClusterGCStatesWithGlobalGCBarriers( + map[uint32]gc.GCState{}, + nil, + ) + got, err := newAllGCStatesOutput(state, false, true) + require.NoError(t, err) + require.NotNil(t, got.GlobalGCBarriers) + require.Empty(t, *got.GlobalGCBarriers) + encoded, err := json.Marshal(got) + require.NoError(t, err) + require.Contains(t, string(encoded), `"global_gc_barriers":[]`) + }) + + t.Run("excluded", func(t *testing.T) { + state := gc.NewClusterGCStatesWithoutGlobalGCBarriers( + map[uint32]gc.GCState{}, + ) + got, err := newAllGCStatesOutput(state, false, false) + require.NoError(t, err) + require.Nil(t, got.GlobalGCBarriers) + encoded, err := json.Marshal(got) + require.NoError(t, err) + require.NotContains(t, string(encoded), "global_gc_barriers") + }) + + t.Run("requested-missing", func(t *testing.T) { + state := gc.NewClusterGCStatesWithoutGlobalGCBarriers( + map[uint32]gc.GCState{}, + ) + _, err := newAllGCStatesOutput(state, false, true) + require.ErrorContains(t, err, + "retry with --exclude-global-barriers") + }) +} +``` + +- [ ] **Step 3: Run the projection tests to verify they fail** + +Run: + +```bash +cd tools && go test ./pd-ctl/pdctl/command -run '^(TestNewKeyspaceGCStateOutputGlobalBarrierPresence|TestNewAllGCStatesOutputGlobalBarrierPresence)$' -count=1 +``` + +Expected: FAIL to compile because the output structs lack +`GlobalGCBarriers` and the converters accept only three and two arguments. + +- [ ] **Step 4: Add optional fields to both output structs** + +Replace the two output types with: + +```go +type keyspaceGCStateOutput struct { + RequestedKeyspaceID uint32 `json:"requested_keyspace_id"` + EffectiveKeyspaceID uint32 `json:"effective_keyspace_id"` + IsKeyspaceLevelGC bool `json:"is_keyspace_level_gc"` + TxnSafePoint uint64 `json:"txn_safe_point"` + GCSafePoint uint64 `json:"gc_safe_point"` + GCBarriers []gcBarrierOutput `json:"gc_barriers"` + GlobalGCBarriers *[]gcBarrierOutput `json:"global_gc_barriers,omitempty"` +} + +type allGCStatesOutput struct { + GCStates []gcStateOutput `json:"gc_states"` + GlobalGCBarriers *[]gcBarrierOutput `json:"global_gc_barriers,omitempty"` +} +``` + +Do not add `omitempty` to local barrier fields. + +- [ ] **Step 5: Implement keyspace global barrier projection** + +Replace `newKeyspaceGCStateOutput` with: + +```go +func newKeyspaceGCStateOutput( + requestedKeyspaceID uint32, + state gc.GCState, + includeExpired bool, + includeGlobalGCBarriers bool, +) (keyspaceGCStateOutput, error) { + barriers, err := state.GetGCBarriers() + if err != nil { + return keyspaceGCStateOutput{}, errors.Annotatef( + err, + "failed to read GC barriers for keyspace %d", + requestedKeyspaceID, + ) + } + + var globalOutput *[]gcBarrierOutput + if includeGlobalGCBarriers { + if !state.HasGlobalGCBarriers() { + return keyspaceGCStateOutput{}, errors.Errorf( + "gc-state keyspace requires a PD server whose GetGCState " + + "supports global GC barriers; retry with " + + "--exclude-global-barriers", + ) + } + globalBarriers, err := state.GetGlobalGCBarriers() + if err != nil { + return keyspaceGCStateOutput{}, errors.WithStack(err) + } + converted := newGlobalGCBarrierOutputs( + globalBarriers, + includeExpired, + ) + globalOutput = &converted + } + + return keyspaceGCStateOutput{ + RequestedKeyspaceID: requestedKeyspaceID, + EffectiveKeyspaceID: state.KeyspaceID, + IsKeyspaceLevelGC: state.IsKeyspaceLevelGC, + TxnSafePoint: state.TxnSafePoint, + GCSafePoint: state.GCSafePoint, + GCBarriers: newLocalGCBarrierOutputs( + barriers, + includeExpired, + ), + GlobalGCBarriers: globalOutput, + }, nil +} +``` + +- [ ] **Step 6: Implement all-view global barrier projection** + +Keep the existing state conversion and sorting loop in +`newAllGCStatesOutput`. Replace its final global conversion and return block +with: + +```go +var globalOutput *[]gcBarrierOutput +if includeGlobalGCBarriers { + if !clusterState.HasGlobalGCBarriers() { + return allGCStatesOutput{}, errors.New( + "gc-state all response does not include global GC barriers; " + + "retry with --exclude-global-barriers", + ) + } + globalBarriers, err := clusterState.GetGlobalGCBarriers() + if err != nil { + return allGCStatesOutput{}, errors.WithStack(err) + } + converted := newGlobalGCBarrierOutputs(globalBarriers, includeExpired) + globalOutput = &converted +} + +return allGCStatesOutput{ + GCStates: states, + GlobalGCBarriers: globalOutput, +}, nil +``` + +Add `includeGlobalGCBarriers bool` as the third parameter. Until Task 3 removes +the global subcommand and adds the flag, pass `true` from the existing +`keyspace` and `all` command call sites so the package compiles. + +- [ ] **Step 7: Update existing projection assertions** + +Update existing calls to the converters with the new boolean. Dereference +`GlobalGCBarriers` only after `require.NotNil`. Delete +`TestNewGlobalGCStateOutputSortsAndKeepsEmptyArray` only in Task 3, when its +production converter is removed. For existing included keyspace cases, attach +the expected globals with `state.WithGlobalGCBarriers(...)`; for excluded +cases, deliberately use a state without global data and pass `false`. + +Update `TestGCStateOutputRejectsExcludedBarriers` to keep the missing-local +assertion, assert the exact actionable error for a requested but missing global +result, and add successful excluded projections that omit the field. This +locks the distinction between missing server capability and explicit user +exclusion. + +- [ ] **Step 8: Run all projection tests** + +Run: + +```bash +cd tools && go test ./pd-ctl/pdctl/command -run '^(TestNew(Keyspace|All)GCStates?Output.*|TestGCStateOutputRejectsExcludedBarriers)$' -count=1 +``` + +Expected: PASS. The encoded present-empty cases contain `[]`, and excluded +cases omit the field. + +- [ ] **Step 9: Commit the projection change** + +Run: + +```bash +git add tools/pd-ctl/pdctl/command/gc_state_command.go tools/pd-ctl/pdctl/command/gc_state_command_test.go +git diff --cached --check +git diff --cached +git commit -s -m "pd-ctl: project optional global GC barriers" +``` + +Expected: the commit changes projection types and converters but does not yet +remove command routing. + +## Task 3: Add the shared flag and remove the global command + +This task changes command routing and reader options after projection behavior +is independently covered. + +**Files:** + +- Modify: `tools/pd-ctl/pdctl/command/gc_state_command.go:37-103` +- Modify: `tools/pd-ctl/pdctl/command/gc_state_command.go:317-474` +- Test: `tools/pd-ctl/pdctl/command/gc_state_command_test.go:38-72` +- Test: `tools/pd-ctl/pdctl/command/gc_state_command_test.go:326-731` + +**Interfaces:** + +- Consumes: both optional projection signatures from Task 2. +- Produces: `gcStateAPIOptions(bool) []gc.GCStatesAPIOption`. +- Produces: `gcStateReader.getGCState(context.Context, uint32, bool)`. +- Produces: `gcStateReader.getAllKeyspacesGCStates(context.Context, bool)`. +- Produces: parent flag `--exclude-global-barriers`, default `false`. + +- [ ] **Step 1: Refactor the fake reader for failing routing tests** + +Replace its global call counter with one inclusion field, delete +`getGlobalGCState`, and use these signatures: + +```go +type fakeGCStateReader struct { + state gc.GCState + clusterState gc.ClusterGCStates + err error + requestedID uint32 + includeGlobalGCBarriers bool + getStateCalls int + getAllCalls int + closed bool +} + +func (r *fakeGCStateReader) getGCState( + _ context.Context, + keyspaceID uint32, + includeGlobalGCBarriers bool, +) (gc.GCState, error) { + r.requestedID = keyspaceID + r.includeGlobalGCBarriers = includeGlobalGCBarriers + r.getStateCalls++ + return r.state, r.err +} + +func (r *fakeGCStateReader) getAllKeyspacesGCStates( + _ context.Context, + includeGlobalGCBarriers bool, +) (gc.ClusterGCStates, error) { + r.includeGlobalGCBarriers = includeGlobalGCBarriers + r.getAllCalls++ + return r.clusterState, r.err +} +``` + +The production package now fails to compile until its interface and call sites +match. + +- [ ] **Step 2: Write a failing client option test** + +Replace `TestReadClusterGCStatesOptions` and its fake client with: + +```go +func TestGCStateAPIOptions(t *testing.T) { + for _, includeGlobalGCBarriers := range []bool{false, true} { + t.Run(strconv.FormatBool(includeGlobalGCBarriers), func(t *testing.T) { + options := gc.DefaultGCStatesAPIOptions() + for _, option := range gcStateAPIOptions( + includeGlobalGCBarriers, + ) { + option(&options) + } + require.False(t, options.ExcludeGCBarriers) + require.Equal(t, !includeGlobalGCBarriers, + options.ExcludeGlobalGCBarriers) + }) + } +} +``` + +Add `strconv` to the standard-library import block before adding this test. + +- [ ] **Step 3: Write failing flag and command-tree tests** + +Add a table test that executes both subcommands with and without the flag: + +```go +func TestGCStateCommandGlobalBarrierFlag(t *testing.T) { + for _, testCase := range []struct { + name string + args []string + wantInclude bool + }{ + {name: "keyspace-default", args: []string{"keyspace", "42"}, wantInclude: true}, + {name: "keyspace-excluded", args: []string{"keyspace", "42", "--exclude-global-barriers"}}, + {name: "all-default", args: []string{"all"}, wantInclude: true}, + {name: "all-excluded", args: []string{"all", "--exclude-global-barriers"}}, + } { + t.Run(testCase.name, func(t *testing.T) { + reader := &fakeGCStateReader{ + state: gc.NewGCStateWithGCBarriers(42, 100, 90, nil). + WithGlobalGCBarriers(nil), + clusterState: gc.NewClusterGCStatesWithGlobalGCBarriers( + map[uint32]gc.GCState{}, + nil, + ), + } + cmd := buildGCStateCommand(func(*cobra.Command) (gcStateReader, error) { + return reader, nil + }) + output := new(bytes.Buffer) + cmd.SetOut(output) + cmd.SetErr(output) + cmd.SetArgs(testCase.args) + + require.NoError(t, cmd.Execute()) + require.Equal(t, testCase.wantInclude, + reader.includeGlobalGCBarriers) + if testCase.wantInclude { + require.Contains(t, output.String(), "global_gc_barriers") + } else { + require.NotContains(t, output.String(), "global_gc_barriers") + } + }) + } +} +``` + +Add an explicit removal test: + +```go +func TestGCStateGlobalCommandIsRemoved(t *testing.T) { + factoryCalled := false + cmd := buildGCStateCommand(func(*cobra.Command) (gcStateReader, error) { + factoryCalled = true + return nil, errors.New("factory must not run") + }) + cmd.SetOut(io.Discard) + cmd.SetErr(io.Discard) + cmd.SetArgs([]string{"global"}) + + err := cmd.Execute() + require.ErrorContains(t, err, `unknown command "global"`) + require.False(t, factoryCalled) +} +``` + +- [ ] **Step 4: Run the routing tests to verify they fail** + +Run: + +```bash +cd tools && go test ./pd-ctl/pdctl/command -run '^(TestGCStateAPIOptions|TestGCStateCommandGlobalBarrierFlag|TestGCStateGlobalCommandIsRemoved)$' -count=1 +``` + +Expected: FAIL because `gcStateAPIOptions` and the exclusion flag do not exist, +the production reader uses old signatures, and `global` remains registered. + +- [ ] **Step 5: Replace the reader interface and option construction** + +Use this interface and helper in `gc_state_command.go`: + +```go +type gcStateReader interface { + getGCState( + ctx context.Context, + keyspaceID uint32, + includeGlobalGCBarriers bool, + ) (gc.GCState, error) + getAllKeyspacesGCStates( + ctx context.Context, + includeGlobalGCBarriers bool, + ) (gc.ClusterGCStates, error) + close() +} + +func gcStateAPIOptions( + includeGlobalGCBarriers bool, +) []gc.GCStatesAPIOption { + return []gc.GCStatesAPIOption{ + gc.ExcludeGCBarriers(false), + gc.ExcludeGlobalGCBarriers(!includeGlobalGCBarriers), + } +} +``` + +Update both concrete reader methods to accept the boolean and call the bound +client with `gcStateAPIOptions(includeGlobalGCBarriers)...`. Delete +`clusterGCStatesClient`, `readClusterGCStates`, and `getGlobalGCState`. + +- [ ] **Step 6: Add and read the persistent flag** + +Define the flag next to `gcStateIncludeExpiredFlag`: + +```go +const ( + gcStateIncludeExpiredFlag = "include-expired" + gcStateExcludeGlobalBarriersFlag = "exclude-global-barriers" +) +``` + +Add this getter: + +```go +func getGCStateIncludeGlobalGCBarriers( + cmd *cobra.Command, +) (bool, error) { + excludeGlobalGCBarriers, err := cmd.Flags().GetBool( + gcStateExcludeGlobalBarriersFlag, + ) + if err != nil { + return false, errors.WithStack(err) + } + return !excludeGlobalGCBarriers, nil +} +``` + +Register it on the parent command: + +```go +command.PersistentFlags().Bool( + gcStateExcludeGlobalBarriersFlag, + false, + "exclude global GC barriers from the PD request and JSON output", +) +``` + +- [ ] **Step 7: Route the boolean through both remaining commands** + +In both `RunE` functions, read `includeExpired` and then +`includeGlobalGCBarriers` before creating the reader. Use these exact call +shapes: + +```go +state, err := reader.getGCState( + cmd.Context(), + keyspaceID, + includeGlobalGCBarriers, +) +``` + +```go +output, err := newKeyspaceGCStateOutput( + keyspaceID, + state, + includeExpired, + includeGlobalGCBarriers, +) +``` + +```go +clusterState, err := reader.getAllKeyspacesGCStates( + cmd.Context(), + includeGlobalGCBarriers, +) +``` + +```go +output, err := newAllGCStatesOutput( + clusterState, + includeExpired, + includeGlobalGCBarriers, +) +``` + +Keep existing RPC error annotations and original-error `status.Code` checks. + +- [ ] **Step 8: Remove the standalone global command** + +Delete `newGCStateGlobalCommand`, `globalGCStateOutput`, and +`newGlobalGCStateOutput`. Register only: + +```go +command.AddCommand( + newGCStateKeyspaceCommand(factory), + newGCStateAllCommand(factory), +) +``` + +Delete `TestGCStateGlobalCommand`, the obsolete global converter test, every +`global-*` expired-visibility row, every `global-*` error row, and the +`{"global", "extra"}` validation row. Do not retain a test invocation that +could make the removed subcommand look supported. + +Update the remaining command tests as follows: + +- `TestGCStateKeyspaceCommand` supplies a state with requested-empty globals, + requires `global_gc_barriers`, and proves one `getStateCalls` and zero + `getAllCalls`. +- `TestGCStateCommandExpiredBarrierVisibility` attaches the active and expired + global fixtures to the keyspace state, then checks globals for both keyspace + rows as well as both all rows. +- `TestGCStateCommandErrors` adds a `single-missing-global-barriers` case whose + state has local barriers but no global wrapper and expects + `retry with --exclude-global-barriers`. Update the all-view missing-global + case to expect the same remediation. +- `TestGCStateCommandReturnsOutputError` supplies requested-empty globals so + execution reaches the failing writer. + +- [ ] **Step 9: Update the help contract** + +Use help copy that states the final behavior: + +```go +Short: "show keyspace and cluster-wide GC state", +Long: "Show effective per-keyspace GC safe points and local and global " + + "barriers. Expired barriers awaiting lazy deletion are hidden by " + + "default; use --include-expired to include zero-TTL barriers returned " + + "by PD. Use keyspace for one effective GC scope or all for every " + + "effective GC scope.", +``` + +The keyspace long help must say that it includes local and global barriers and +that `--exclude-global-barriers` omits cluster-wide barriers. The all long help +must say that global barriers appear once at the top level and that the same +flag omits them. Remove every recommendation to run `gc-state global`. + +In `TestGCStateCommandHelpContract`, assert both persistent flags have default +`false`, assert the exact exclusion usage string, and assert that the command's +children are exactly `all` and `keyspace` after Cobra sorts them. + +- [ ] **Step 10: Run the complete command unit suite** + +Run: + +```bash +cd tools && go test ./pd-ctl/pdctl/command -run 'GCState' -count=1 +``` + +Expected: PASS. The default cases include an empty global array, exclusion +cases omit it, and `global` is rejected before client creation. + +- [ ] **Step 11: Commit command routing and flag behavior** + +Run: + +```bash +git add tools/pd-ctl/pdctl/command/gc_state_command.go tools/pd-ctl/pdctl/command/gc_state_command_test.go +git diff --cached --check +git diff --cached +git commit -s -m "pd-ctl: use GetGCState for global barriers" +``` + +Expected: this commit contains the reader refactor, shared flag, command +removal, compatibility paths, and command tests. + +## Task 4: Update real PD command coverage + +This task verifies the complete behavior against a real Classic or NextGen PD +server without duplicating PR #11117's server failpoint tests. + +**Files:** + +- Test: `tools/pd-ctl/tests/safepoint/gc_state_test.go:42-66` +- Test: `tools/pd-ctl/tests/safepoint/gc_state_test.go:218-388` + +**Interfaces:** + +- Consumes: the final `keyspace`, `all`, `--include-expired`, and + `--exclude-global-barriers` command contracts from Task 3. +- Produces: end-to-end coverage for default, expired, excluded, Classic, and + NextGen output. + +- [ ] **Step 1: Update integration output types** + +Add global barriers to the single-keyspace decoder: + +```go +type gcStateCommandSingle struct { + RequestedKeyspaceID uint32 `json:"requested_keyspace_id"` + EffectiveKeyspaceID uint32 `json:"effective_keyspace_id"` + IsKeyspaceLevelGC bool `json:"is_keyspace_level_gc"` + TxnSafePoint uint64 `json:"txn_safe_point"` + GCSafePoint uint64 `json:"gc_safe_point"` + GCBarriers []gcStateCommandBarrier `json:"gc_barriers"` + GlobalGCBarriers []gcStateCommandBarrier `json:"global_gc_barriers"` +} +``` + +Delete `gcStateCommandGlobal`. + +- [ ] **Step 2: Make the default keyspace assertions require global barriers** + +Replace the default `NotContains` assertion with: + +```go +re.Contains(singleProperties, "global_gc_barriers") +requireGCStateCommandBarriers( + re, + keyspaceLevelResponse.GlobalGCBarriers, + []expectedGCStateCommandBarrier{ + {barrierID: "a-global", barrierTS: 310}, + {barrierID: "z-global", barrierTS: 320}, + }, +) +``` + +Add the same active global barrier expectation to the NullKeyspace and Classic +unified-GC keyspace responses. This proves the global result is independent of +the requested keyspace. + +- [ ] **Step 3: Extend the keyspace expired assertion** + +After decoding `keyspaceLevelWithExpired`, assert: + +```go +requireGCStateCommandBarriers( + re, + keyspaceLevelWithExpired.GlobalGCBarriers, + []expectedGCStateCommandBarrier{ + {barrierID: "a-global", barrierTS: 310}, + {barrierID: "z-global", barrierTS: 320}, + {barrierID: "expired-global", barrierTS: 330, expired: true}, + }, +) +``` + +- [ ] **Step 4: Add keyspace exclusion coverage** + +Execute the excluded command and inspect raw field presence: + +```go +output, err = tests.ExecuteCommand( + ctl.GetRootCmd(), + "-u", + pdAddr, + "gc-state", + "keyspace", + keyspaceLevelIDString, + "--exclude-global-barriers", +) +re.NoError(err) +var excludedKeyspaceProperties map[string]json.RawMessage +re.NoError(json.Unmarshal(output, &excludedKeyspaceProperties), string(output)) +re.NotContains(excludedKeyspaceProperties, "global_gc_barriers") +var excludedKeyspace gcStateCommandSingle +re.NoError(json.Unmarshal(output, &excludedKeyspace), string(output)) +requireGCStateCommandBarriers( + re, + excludedKeyspace.GCBarriers, + []expectedGCStateCommandBarrier{ + {barrierID: "a-local", barrierTS: 210}, + {barrierID: "z-local", barrierTS: 220, expires: true}, + }, +) +``` + +Repeat with both flags and assert `expired-local` appears while the global +field remains absent. + +- [ ] **Step 5: Add all-view exclusion coverage** + +Execute: + +```go +output, err = tests.ExecuteCommand( + ctl.GetRootCmd(), + "-u", + pdAddr, + "gc-state", + "all", + "--exclude-global-barriers", +) +re.NoError(err) +var excludedAllProperties map[string]json.RawMessage +re.NoError(json.Unmarshal(output, &excludedAllProperties), string(output)) +re.Contains(excludedAllProperties, "gc_states") +re.NotContains(excludedAllProperties, "global_gc_barriers") +``` + +Decode `gc_states` and compare the NullKeyspace and keyspace-level entries to +the default all view. Repeat with `--include-expired` and assert only local +expired barriers are added. + +- [ ] **Step 6: Delete standalone global command coverage** + +Delete the command calls and assertions from the current global block, +including `globalProperties`, `global`, `globalWithExpired`, and every +`gcStateCommandGlobal` use. + +- [ ] **Step 7: Run the Classic integration test** + +Enable failpoints only around the test: + +```bash +make failpoint-enable +cd tools && go test ./pd-ctl/tests/safepoint -run '^TestGCState$' -count=1 +cd .. && make failpoint-disable +``` + +Expected: PASS. If the test fails, return to the repository root and run +`make failpoint-disable` before diagnosing or editing. + +- [ ] **Step 8: Run the NextGen integration test** + +Run: + +```bash +make failpoint-enable +cd tools && go test -tags nextgen ./pd-ctl/tests/safepoint -run '^TestGCState$' -count=1 +cd .. && make failpoint-disable +``` + +Expected: PASS with the system keyspace retained as a keyspace-level GC scope. +If the test fails, disable failpoints before any other action. + +- [ ] **Step 9: Commit real PD coverage** + +Run: + +```bash +git add tools/pd-ctl/tests/safepoint/gc_state_test.go +git diff --cached --check +git diff --cached +git commit -s -m "test: cover optional global GC state output" +``` + +Expected: the commit changes only the real PD test and removes all standalone +global command coverage. + +## Task 5: Update the user workflow documentation + +This task updates user-facing documentation after the executable behavior and +integration tests are stable. Use the `docs-writer` skill for this task. + +**Files:** + +- Modify: `tools/pd-ctl/README.md:14-130` + +**Interfaces:** + +- Consumes: final command and JSON contracts from Tasks 2 through 4. +- Produces: one documented workflow for a selected keyspace and one for all + effective scopes. + +- [ ] **Step 1: Remove the standalone global workflow** + +Delete the `pd-ctl gc-state global` command example, its JSON object, and prose +that recommends a second command when local barriers do not explain a safe +point. + +- [ ] **Step 2: Add global barriers to the keyspace example** + +Extend the existing keyspace JSON object with this top-level field: + +```json +"global_gc_barriers": [ + { + "barrier_id": "native_br", + "barrier_ts": 464940000000000000, + "ttl_seconds": 9223372036854775807 + } +] +``` + +State that local and global barriers come from one `GetGCState` read and that +the same global list applies to every keyspace. + +- [ ] **Step 3: Document presence and exclusion semantics** + +Add prose with these exact behavioral claims: + +- The default `keyspace` and `all` views request global barriers. +- An empty `global_gc_barriers` array means PD returned no global barriers. +- `--exclude-global-barriers` skips the read and removes the JSON field. +- The flag applies to both remaining subcommands. +- When combined with `--include-expired`, exclusion wins for global barriers, + while expired local barriers remain visible. + +Include these examples: + +```bash +pd-ctl gc-state keyspace 42 --exclude-global-barriers +pd-ctl gc-state all --exclude-global-barriers --include-expired +``` + +- [ ] **Step 4: Clarify command selection** + +Recommend `keyspace` when diagnosing one scope. Reserve `all` for cases that +require every effective GC scope because it enumerates all keyspaces. + +- [ ] **Step 5: Verify documentation references** + +Run: + +```bash +rg -n 'gc-state global|getGlobalGCState|global command' tools/pd-ctl/README.md tools/pd-ctl/pdctl/command +git diff --check +``` + +Expected: `rg` returns no matches and `git diff --check` passes. + +- [ ] **Step 6: Commit the documentation update** + +Run: + +```bash +git add tools/pd-ctl/README.md +git diff --cached --check +git diff --cached +git commit -s -m "docs: update GC state troubleshooting workflow" +``` + +Expected: the commit contains only user documentation that matches the tested +command behavior. + +## Task 6: Run final verification and update PR metadata + +This task proves the refactor across modules and updates PR #11054 only after +the user authorizes the external GitHub change. + +**Files:** + +- Verify: all files modified by Tasks 1 through 5 +- External update after authorization: PR #11054 body + +**Interfaces:** + +- Consumes: all implementation and documentation commits. +- Produces: a clean worktree, passing focused and repository checks, and PR + metadata that describes only `keyspace` and `all`. + +- [ ] **Step 1: Format and verify no dependency drift** + +Run with failpoints disabled: + +```bash +make failpoint-disable +gofmt -w client/gc_client.go client/gc_client_test.go client/clients/gc/client.go tools/pd-ctl/pdctl/command/gc_state_command.go tools/pd-ctl/pdctl/command/gc_state_command_test.go tools/pd-ctl/tests/safepoint/gc_state_test.go +git diff --check +git diff -- go.mod go.sum client/go.mod client/go.sum tests/integrations/go.mod tests/integrations/go.sum tools/go.mod tools/go.sum +``` + +Expected: formatting produces no new semantic diff, whitespace checks pass, +and module files contain only versions inherited from the rebased master. + +- [ ] **Step 2: Run focused client and command unit tests** + +Run: + +```bash +cd client && go test . -run 'GCState' -count=1 +cd ../tools && go test ./pd-ctl/pdctl/command -run 'GCState' -count=1 +cd .. +``` + +Expected: PASS in both modules. + +- [ ] **Step 3: Run Classic and NextGen real PD tests** + +Run only test commands while failpoints are enabled: + +```bash +make failpoint-enable +cd tools && go test ./pd-ctl/tests/safepoint -run '^TestGCState$' -count=1 +go test -tags nextgen ./pd-ctl/tests/safepoint -run '^TestGCState$' -count=1 +cd .. && make failpoint-disable +``` + +Expected: both runs pass. Regardless of either result, run +`make failpoint-disable` before continuing. + +- [ ] **Step 4: Build both pd-ctl variants** + +Run: + +```bash +make pd-ctl +NEXT_GEN=1 make pd-ctl +``` + +Expected: both builds succeed. + +- [ ] **Step 5: Run repository checks** + +Run: + +```bash +make check +make basic-test +cd client && make +cd .. +``` + +Expected: formatting, lint, tidy, error documentation, root tests, and the +client module pipeline pass. If a command enables failpoints internally and +fails, run `make failpoint-disable` before inspecting or editing files. + +- [ ] **Step 6: Prove obsolete paths are absent** + +Run: + +```bash +rg -n 'gc-state global|getGlobalGCState|newGCStateGlobalCommand|globalGCStateOutput' tools/pd-ctl +rg -n 'GetAllKeyspacesGCStates' tools/pd-ctl/pdctl/command/gc_state_command.go +``` + +Expected: the first command returns no matches. The second command shows only +the all-view reader interface, concrete method, and all command call path. + +- [ ] **Step 7: Review the final branch diff and worktree** + +Run: + +```bash +make failpoint-disable +git status --short --branch +git diff --check upstream/master...HEAD +git diff --stat upstream/master...HEAD +git log --oneline upstream/master..HEAD +``` + +Expected: no unstaged or untracked artifacts remain, the diff contains only +the approved PR scope, and every new commit has a signed repository-style +message. If formatting changed tracked files in Step 1, fold those changes into +the task that owns them instead of creating an unrelated cleanup commit. + +- [ ] **Step 8: Prepare the unwrapped PR body update** + +After the user authorizes editing PR #11054, create +`/tmp/pd-11054-refactor-body.md` with this content. Keep prose paragraphs on +single lines because PR Markdown must not be hard-wrapped: + +````markdown +### What problem does this PR solve? + +PD exposes per-keyspace and cluster-wide GC state through RPCs, but operators cannot inspect that state through `pd-ctl`. This makes it difficult to identify whether GC advancement is blocked by a keyspace-local barrier or a cluster-wide global barrier. + +Issue Number: close #11013, ref #8978 + +### What is changed and how does it work? + +```commit-message +Add read-only `pd-ctl gc-state keyspace` and `gc-state all` commands. The keyspace view uses `GetGCState` to return the effective safe points, local barriers, and global barriers in one read. The all view uses `GetAllKeyspacesGCStates` only when every effective GC scope is required. Both views support `--exclude-global-barriers` to skip the global barrier read and omit the JSON field, while `--include-expired` controls zero-TTL barrier visibility. + +Expose the server-provided keyspace-level GC mode through the public GC client model so the command can distinguish independent keyspace GC from unified GC. Return deterministic JSON and preserve the distinction between an omitted global barrier result and a requested empty list. +``` + +### Check List + +Tests + +- Unit test +- Integration test +- Manual test + +### Release note + +```release-note +Add `pd-ctl gc-state keyspace` and `gc-state all` commands for inspecting GC safe points and the local and global barriers that can block GC. +``` +```` + +Use `apply_patch` to create the temporary file; do not use shell redirection. + +- [ ] **Step 9: Update and verify PR #11054 after authorization** + +Run: + +```bash +gh pr edit 11054 --repo tikv/pd --body-file /tmp/pd-11054-refactor-body.md +gh pr view 11054 --repo tikv/pd --json title,body,url +``` + +Expected: the title remains `pd-ctl: add GC state inspection commands`; the +body names only `keyspace` and `all`, includes both flags, and contains the +required issue and release-note blocks. If authorization is not granted, skip +the external update and return the prepared body to the user. + +- [ ] **Step 10: Report verification evidence** + +Report every command run and its result, the final commit list, any tests that +were skipped with a reason, and whether PR metadata was updated. Do not claim +the refactor is complete if any required check failed or failpoints remain +enabled. + +## Execution handoff + +Implementation starts only after the user selects an execution approach. +Follow the sub-skill named in the agentic worker header for that approach, keep +the task checkpoints in order, and leave the PR body update approval-gated +because it changes external GitHub state. From 682f5ab32fc250f231bc3afc4ff7d2cfd6339cbd Mon Sep 17 00:00:00 2001 From: Wenxuan Zhang Date: Tue, 11 Aug 2026 16:40:58 +0800 Subject: [PATCH 22/31] docs: merge GC state command refactor stages Signed-off-by: Wenxuan Zhang --- ...-08-11-gc-state-global-barrier-refactor.md | 99 +++++++------------ 1 file changed, 38 insertions(+), 61 deletions(-) diff --git a/docs/superpowers/plans/2026-08-11-gc-state-global-barrier-refactor.md b/docs/superpowers/plans/2026-08-11-gc-state-global-barrier-refactor.md index 7571e3e99d..8f53aa8a22 100644 --- a/docs/superpowers/plans/2026-08-11-gc-state-global-barrier-refactor.md +++ b/docs/superpowers/plans/2026-08-11-gc-state-global-barrier-refactor.md @@ -268,15 +268,17 @@ git commit -s -m "client: preserve GC mode with global barriers" Expected: the commit contains the composed model/converter and focused regression only. Replayed rebase commits remain separate history. -## Task 2: Add optional global barriers to JSON projections +## Task 2: Refactor command output, options, and routing -This task changes only output construction. It keeps the existing command tree -temporarily so projection behavior can be reviewed independently from routing. +This task changes JSON projection and command routing as one independently +testable unit. It locks the three-state output contract first, then connects +the shared option to both RPC paths and removes the standalone global command +before creating a commit. **Files:** -- Modify: `tools/pd-ctl/pdctl/command/gc_state_command.go:148-315` -- Test: `tools/pd-ctl/pdctl/command/gc_state_command_test.go:106-325` +- Modify: `tools/pd-ctl/pdctl/command/gc_state_command.go:37-474` +- Test: `tools/pd-ctl/pdctl/command/gc_state_command_test.go:38-731` **Interfaces:** @@ -288,6 +290,10 @@ temporarily so projection behavior can be reviewed independently from routing. (allGCStatesOutput, error)`. - Produces: optional `GlobalGCBarriers *[]gcBarrierOutput` fields on both top-level output types. +- Produces: `gcStateAPIOptions(bool) []gc.GCStatesAPIOption`. +- Produces: `gcStateReader.getGCState(context.Context, uint32, bool)`. +- Produces: `gcStateReader.getAllKeyspacesGCStates(context.Context, bool)`. +- Produces: parent flag `--exclude-global-barriers`, default `false`. - [ ] **Step 1: Write failing keyspace projection tests** @@ -487,15 +493,17 @@ return allGCStatesOutput{ }, nil ``` -Add `includeGlobalGCBarriers bool` as the third parameter. Until Task 3 removes -the global subcommand and adds the flag, pass `true` from the existing -`keyspace` and `all` command call sites so the package compiles. +Add `includeGlobalGCBarriers bool` as the third parameter. During the focused +projection cycle in Steps 4 through 8, pass `true` from the existing `keyspace` +and `all` call sites so the package compiles. Do not commit that intermediate +state; Steps 9 through 16 replace those temporary calls with flag-driven +routing and remove the global subcommand. - [ ] **Step 7: Update existing projection assertions** Update existing calls to the converters with the new boolean. Dereference `GlobalGCBarriers` only after `require.NotNil`. Delete -`TestNewGlobalGCStateOutputSortsAndKeepsEmptyArray` only in Task 3, when its +`TestNewGlobalGCStateOutputSortsAndKeepsEmptyArray` only in Step 16, when its production converter is removed. For existing included keyspace cases, attach the expected globals with `state.WithGlobalGCBarriers(...)`; for excluded cases, deliberately use a state without global data and pass `false`. @@ -517,41 +525,10 @@ cd tools && go test ./pd-ctl/pdctl/command -run '^(TestNew(Keyspace|All)GCStates Expected: PASS. The encoded present-empty cases contain `[]`, and excluded cases omit the field. -- [ ] **Step 9: Commit the projection change** - -Run: - -```bash -git add tools/pd-ctl/pdctl/command/gc_state_command.go tools/pd-ctl/pdctl/command/gc_state_command_test.go -git diff --cached --check -git diff --cached -git commit -s -m "pd-ctl: project optional global GC barriers" -``` - -Expected: the commit changes projection types and converters but does not yet -remove command routing. - -## Task 3: Add the shared flag and remove the global command - -This task changes command routing and reader options after projection behavior -is independently covered. - -**Files:** - -- Modify: `tools/pd-ctl/pdctl/command/gc_state_command.go:37-103` -- Modify: `tools/pd-ctl/pdctl/command/gc_state_command.go:317-474` -- Test: `tools/pd-ctl/pdctl/command/gc_state_command_test.go:38-72` -- Test: `tools/pd-ctl/pdctl/command/gc_state_command_test.go:326-731` - -**Interfaces:** - -- Consumes: both optional projection signatures from Task 2. -- Produces: `gcStateAPIOptions(bool) []gc.GCStatesAPIOption`. -- Produces: `gcStateReader.getGCState(context.Context, uint32, bool)`. -- Produces: `gcStateReader.getAllKeyspacesGCStates(context.Context, bool)`. -- Produces: parent flag `--exclude-global-barriers`, default `false`. +The remaining steps connect the tested projection to the command reader and +finish the command-tree refactor before the task-level review and commit. -- [ ] **Step 1: Refactor the fake reader for failing routing tests** +- [ ] **Step 9: Refactor the fake reader for failing routing tests** Replace its global call counter with one inclusion field, delete `getGlobalGCState`, and use these signatures: @@ -592,7 +569,7 @@ func (r *fakeGCStateReader) getAllKeyspacesGCStates( The production package now fails to compile until its interface and call sites match. -- [ ] **Step 2: Write a failing client option test** +- [ ] **Step 10: Write a failing client option test** Replace `TestReadClusterGCStatesOptions` and its fake client with: @@ -616,7 +593,7 @@ func TestGCStateAPIOptions(t *testing.T) { Add `strconv` to the standard-library import block before adding this test. -- [ ] **Step 3: Write failing flag and command-tree tests** +- [ ] **Step 11: Write failing flag and command-tree tests** Add a table test that executes both subcommands with and without the flag: @@ -681,7 +658,7 @@ func TestGCStateGlobalCommandIsRemoved(t *testing.T) { } ``` -- [ ] **Step 4: Run the routing tests to verify they fail** +- [ ] **Step 12: Run the routing tests to verify they fail** Run: @@ -692,7 +669,7 @@ cd tools && go test ./pd-ctl/pdctl/command -run '^(TestGCStateAPIOptions|TestGCS Expected: FAIL because `gcStateAPIOptions` and the exclusion flag do not exist, the production reader uses old signatures, and `global` remains registered. -- [ ] **Step 5: Replace the reader interface and option construction** +- [ ] **Step 13: Replace the reader interface and option construction** Use this interface and helper in `gc_state_command.go`: @@ -724,7 +701,7 @@ Update both concrete reader methods to accept the boolean and call the bound client with `gcStateAPIOptions(includeGlobalGCBarriers)...`. Delete `clusterGCStatesClient`, `readClusterGCStates`, and `getGlobalGCState`. -- [ ] **Step 6: Add and read the persistent flag** +- [ ] **Step 14: Add and read the persistent flag** Define the flag next to `gcStateIncludeExpiredFlag`: @@ -761,7 +738,7 @@ command.PersistentFlags().Bool( ) ``` -- [ ] **Step 7: Route the boolean through both remaining commands** +- [ ] **Step 15: Route the boolean through both remaining commands** In both `RunE` functions, read `includeExpired` and then `includeGlobalGCBarriers` before creating the reader. Use these exact call @@ -801,7 +778,7 @@ output, err := newAllGCStatesOutput( Keep existing RPC error annotations and original-error `status.Code` checks. -- [ ] **Step 8: Remove the standalone global command** +- [ ] **Step 16: Remove the standalone global command** Delete `newGCStateGlobalCommand`, `globalGCStateOutput`, and `newGlobalGCStateOutput`. Register only: @@ -833,7 +810,7 @@ Update the remaining command tests as follows: - `TestGCStateCommandReturnsOutputError` supplies requested-empty globals so execution reaches the failing writer. -- [ ] **Step 9: Update the help contract** +- [ ] **Step 17: Update the help contract** Use help copy that states the final behavior: @@ -855,7 +832,7 @@ In `TestGCStateCommandHelpContract`, assert both persistent flags have default `false`, assert the exact exclusion usage string, and assert that the command's children are exactly `all` and `keyspace` after Cobra sorts them. -- [ ] **Step 10: Run the complete command unit suite** +- [ ] **Step 18: Run the complete command unit suite** Run: @@ -866,7 +843,7 @@ cd tools && go test ./pd-ctl/pdctl/command -run 'GCState' -count=1 Expected: PASS. The default cases include an empty global array, exclusion cases omit it, and `global` is rejected before client creation. -- [ ] **Step 11: Commit command routing and flag behavior** +- [ ] **Step 19: Commit the complete command refactor** Run: @@ -877,10 +854,10 @@ git diff --cached git commit -s -m "pd-ctl: use GetGCState for global barriers" ``` -Expected: this commit contains the reader refactor, shared flag, command -removal, compatibility paths, and command tests. +Expected: this commit contains optional output projection, reader refactoring, +the shared flag, command removal, compatibility paths, and all command tests. -## Task 4: Update real PD command coverage +## Task 3: Update real PD command coverage This task verifies the complete behavior against a real Classic or NextGen PD server without duplicating PR #11117's server failpoint tests. @@ -893,7 +870,7 @@ server without duplicating PR #11117's server failpoint tests. **Interfaces:** - Consumes: the final `keyspace`, `all`, `--include-expired`, and - `--exclude-global-barriers` command contracts from Task 3. + `--exclude-global-barriers` command contracts from Task 2. - Produces: end-to-end coverage for default, expired, excluded, Classic, and NextGen output. @@ -1054,7 +1031,7 @@ git commit -s -m "test: cover optional global GC state output" Expected: the commit changes only the real PD test and removes all standalone global command coverage. -## Task 5: Update the user workflow documentation +## Task 4: Update the user workflow documentation This task updates user-facing documentation after the executable behavior and integration tests are stable. Use the `docs-writer` skill for this task. @@ -1065,7 +1042,7 @@ integration tests are stable. Use the `docs-writer` skill for this task. **Interfaces:** -- Consumes: final command and JSON contracts from Tasks 2 through 4. +- Consumes: final command and JSON contracts from Tasks 2 and 3. - Produces: one documented workflow for a selected keyspace and one for all effective scopes. @@ -1140,14 +1117,14 @@ git commit -s -m "docs: update GC state troubleshooting workflow" Expected: the commit contains only user documentation that matches the tested command behavior. -## Task 6: Run final verification and update PR metadata +## Task 5: Run final verification and update PR metadata This task proves the refactor across modules and updates PR #11054 only after the user authorizes the external GitHub change. **Files:** -- Verify: all files modified by Tasks 1 through 5 +- Verify: all files modified by Tasks 1 through 4 - External update after authorization: PR #11054 body **Interfaces:** From bc9a7d860319813c51cdb32b50adc6ae484baec4 Mon Sep 17 00:00:00 2001 From: Wenxuan Zhang Date: Tue, 11 Aug 2026 16:48:39 +0800 Subject: [PATCH 23/31] client: preserve GC mode with global barriers Signed-off-by: Wenxuan Zhang --- client/gc_client_test.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/client/gc_client_test.go b/client/gc_client_test.go index e35cc3fa73..ee233ef1fa 100644 --- a/client/gc_client_test.go +++ b/client/gc_client_test.go @@ -69,3 +69,30 @@ func TestPBToGCStatePreservesKeyspaceLevelGC(t *testing.T) { }) } } + +func TestPBToGCStateWithGlobalBarriersPreservesKeyspaceLevelGC(t *testing.T) { + requestStart := time.Unix(100, 0) + state := pbToGCStateWithGlobalGCBarriers( + &pdpb.GCState{ + KeyspaceScope: wrapKeyspaceScope(42), + IsKeyspaceLevelGc: true, + TxnSafePoint: 100, + GcSafePoint: 90, + }, + &pdpb.GlobalGCBarriersInfo{ + Barriers: []*pdpb.GlobalGCBarrierInfo{ + {BarrierId: "snapshot", BarrierTs: 95, TtlSeconds: 60}, + }, + }, + requestStart, + true, + ) + + require.True(t, state.IsKeyspaceLevelGC) + require.False(t, state.HasGCBarriers()) + require.True(t, state.HasGlobalGCBarriers()) + barriers, err := state.GetGlobalGCBarriers() + require.NoError(t, err) + require.Len(t, barriers, 1) + require.Equal(t, "snapshot", barriers[0].BarrierID) +} From 90d04538fbbe9cc3085e1ecb777380a323235a57 Mon Sep 17 00:00:00 2001 From: Wenxuan Zhang Date: Tue, 11 Aug 2026 16:58:49 +0800 Subject: [PATCH 24/31] pd-ctl: use GetGCState for global barriers Signed-off-by: Wenxuan Zhang --- .../pd-ctl/pdctl/command/gc_state_command.go | 251 ++++++----- .../pdctl/command/gc_state_command_test.go | 412 ++++++++++-------- 2 files changed, 370 insertions(+), 293 deletions(-) diff --git a/tools/pd-ctl/pdctl/command/gc_state_command.go b/tools/pd-ctl/pdctl/command/gc_state_command.go index afc3534832..e56662e54c 100644 --- a/tools/pd-ctl/pdctl/command/gc_state_command.go +++ b/tools/pd-ctl/pdctl/command/gc_state_command.go @@ -35,67 +35,57 @@ import ( ) type gcStateReader interface { - getGCState(context.Context, uint32) (gc.GCState, error) - getGlobalGCState(context.Context) (gc.ClusterGCStates, error) - getAllKeyspacesGCStates(context.Context) (gc.ClusterGCStates, error) + getGCState( + ctx context.Context, + keyspaceID uint32, + includeGlobalGCBarriers bool, + ) (gc.GCState, error) + getAllKeyspacesGCStates( + ctx context.Context, + includeGlobalGCBarriers bool, + ) (gc.ClusterGCStates, error) close() } type gcStateReaderFactory func(*cobra.Command) (gcStateReader, error) -const gcStateIncludeExpiredFlag = "include-expired" +const ( + gcStateIncludeExpiredFlag = "include-expired" + gcStateExcludeGlobalBarriersFlag = "exclude-global-barriers" +) type pdGCStateReader struct { client pd.Client } -type clusterGCStatesClient interface { - GetAllKeyspacesGCStates( - context.Context, - ...gc.GCStatesAPIOption, - ) (gc.ClusterGCStates, error) -} - func (r *pdGCStateReader) getGCState( ctx context.Context, keyspaceID uint32, + includeGlobalGCBarriers bool, ) (gc.GCState, error) { return r.client.GetGCStatesClient(keyspaceID).GetGCState( ctx, - gc.ExcludeGCBarriers(false), + gcStateAPIOptions(includeGlobalGCBarriers)..., ) } func (r *pdGCStateReader) getAllKeyspacesGCStates( ctx context.Context, + includeGlobalGCBarriers bool, ) (gc.ClusterGCStates, error) { - return readClusterGCStates( - ctx, - r.client.GetGCStatesClient(constant.NullKeyspaceID), - false, - ) -} - -func (r *pdGCStateReader) getGlobalGCState( - ctx context.Context, -) (gc.ClusterGCStates, error) { - return readClusterGCStates( + return r.client.GetGCStatesClient(constant.NullKeyspaceID).GetAllKeyspacesGCStates( ctx, - r.client.GetGCStatesClient(constant.NullKeyspaceID), - true, + gcStateAPIOptions(includeGlobalGCBarriers)..., ) } -func readClusterGCStates( - ctx context.Context, - client clusterGCStatesClient, - excludeGCBarriers bool, -) (gc.ClusterGCStates, error) { - return client.GetAllKeyspacesGCStates( - ctx, - gc.ExcludeGCBarriers(excludeGCBarriers), - gc.ExcludeGlobalGCBarriers(false), - ) +func gcStateAPIOptions( + includeGlobalGCBarriers bool, +) []gc.GCStatesAPIOption { + return []gc.GCStatesAPIOption{ + gc.ExcludeGCBarriers(false), + gc.ExcludeGlobalGCBarriers(!includeGlobalGCBarriers), + } } func (r *pdGCStateReader) close() { @@ -146,21 +136,18 @@ type gcStateOutput struct { } type keyspaceGCStateOutput struct { - RequestedKeyspaceID uint32 `json:"requested_keyspace_id"` - EffectiveKeyspaceID uint32 `json:"effective_keyspace_id"` - IsKeyspaceLevelGC bool `json:"is_keyspace_level_gc"` - TxnSafePoint uint64 `json:"txn_safe_point"` - GCSafePoint uint64 `json:"gc_safe_point"` - GCBarriers []gcBarrierOutput `json:"gc_barriers"` + RequestedKeyspaceID uint32 `json:"requested_keyspace_id"` + EffectiveKeyspaceID uint32 `json:"effective_keyspace_id"` + IsKeyspaceLevelGC bool `json:"is_keyspace_level_gc"` + TxnSafePoint uint64 `json:"txn_safe_point"` + GCSafePoint uint64 `json:"gc_safe_point"` + GCBarriers []gcBarrierOutput `json:"gc_barriers"` + GlobalGCBarriers *[]gcBarrierOutput `json:"global_gc_barriers,omitempty"` } type allGCStatesOutput struct { - GCStates []gcStateOutput `json:"gc_states"` - GlobalGCBarriers []gcBarrierOutput `json:"global_gc_barriers"` -} - -type globalGCStateOutput struct { - GlobalGCBarriers []gcBarrierOutput `json:"global_gc_barriers"` + GCStates []gcStateOutput `json:"gc_states"` + GlobalGCBarriers *[]gcBarrierOutput `json:"global_gc_barriers,omitempty"` } func parseGCStateKeyspaceID(value string) (uint32, error) { @@ -239,6 +226,7 @@ func newKeyspaceGCStateOutput( requestedKeyspaceID uint32, state gc.GCState, includeExpired bool, + includeGlobalGCBarriers bool, ) (keyspaceGCStateOutput, error) { barriers, err := state.GetGCBarriers() if err != nil { @@ -248,13 +236,38 @@ func newKeyspaceGCStateOutput( requestedKeyspaceID, ) } + + var globalOutput *[]gcBarrierOutput + if includeGlobalGCBarriers { + if !state.HasGlobalGCBarriers() { + return keyspaceGCStateOutput{}, errors.Errorf( + "gc-state keyspace requires a PD server whose GetGCState " + + "supports global GC barriers; retry with " + + "--exclude-global-barriers", + ) + } + globalBarriers, err := state.GetGlobalGCBarriers() + if err != nil { + return keyspaceGCStateOutput{}, errors.WithStack(err) + } + converted := newGlobalGCBarrierOutputs( + globalBarriers, + includeExpired, + ) + globalOutput = &converted + } + return keyspaceGCStateOutput{ RequestedKeyspaceID: requestedKeyspaceID, EffectiveKeyspaceID: state.KeyspaceID, IsKeyspaceLevelGC: state.IsKeyspaceLevelGC, TxnSafePoint: state.TxnSafePoint, GCSafePoint: state.GCSafePoint, - GCBarriers: newLocalGCBarrierOutputs(barriers, includeExpired), + GCBarriers: newLocalGCBarrierOutputs( + barriers, + includeExpired, + ), + GlobalGCBarriers: globalOutput, }, nil } @@ -276,7 +289,11 @@ func newGCStateOutput(state gc.GCState, includeExpired bool) (gcStateOutput, err }, nil } -func newAllGCStatesOutput(clusterState gc.ClusterGCStates, includeExpired bool) (allGCStatesOutput, error) { +func newAllGCStatesOutput( + clusterState gc.ClusterGCStates, + includeExpired bool, + includeGlobalGCBarriers bool, +) (allGCStatesOutput, error) { states := make([]gcStateOutput, 0, len(clusterState.GCStates)) for _, state := range clusterState.GCStates { // Unified-GC keyspaces have marker entries, but their GC state is owned by @@ -294,32 +311,46 @@ func newAllGCStatesOutput(clusterState gc.ClusterGCStates, includeExpired bool) return states[i].KeyspaceID < states[j].KeyspaceID }) - globalOutput, err := newGlobalGCStateOutput(clusterState, includeExpired) - if err != nil { - return allGCStatesOutput{}, err + var globalOutput *[]gcBarrierOutput + if includeGlobalGCBarriers { + if !clusterState.HasGlobalGCBarriers() { + return allGCStatesOutput{}, errors.New( + "gc-state all response does not include global GC barriers; " + + "retry with --exclude-global-barriers", + ) + } + globalBarriers, err := clusterState.GetGlobalGCBarriers() + if err != nil { + return allGCStatesOutput{}, errors.WithStack(err) + } + converted := newGlobalGCBarrierOutputs(globalBarriers, includeExpired) + globalOutput = &converted } + return allGCStatesOutput{ GCStates: states, - GlobalGCBarriers: globalOutput.GlobalGCBarriers, + GlobalGCBarriers: globalOutput, }, nil } -func newGlobalGCStateOutput(clusterState gc.ClusterGCStates, includeExpired bool) (globalGCStateOutput, error) { - globalBarriers, err := clusterState.GetGlobalGCBarriers() +func getGCStateIncludeExpired(cmd *cobra.Command) (bool, error) { + includeExpired, err := cmd.Flags().GetBool(gcStateIncludeExpiredFlag) if err != nil { - return globalGCStateOutput{}, errors.Annotate(err, "failed to read global GC barriers") + return false, errors.WithStack(err) } - return globalGCStateOutput{ - GlobalGCBarriers: newGlobalGCBarrierOutputs(globalBarriers, includeExpired), - }, nil + return includeExpired, nil } -func getGCStateIncludeExpired(cmd *cobra.Command) (bool, error) { - includeExpired, err := cmd.Flags().GetBool(gcStateIncludeExpiredFlag) +func getGCStateIncludeGlobalGCBarriers( + cmd *cobra.Command, +) (bool, error) { + excludeGlobalGCBarriers, err := cmd.Flags().GetBool( + gcStateExcludeGlobalBarriersFlag, + ) if err != nil { return false, errors.WithStack(err) } - return includeExpired, nil + return !excludeGlobalGCBarriers, nil } // NewGCStateCommand returns the read-only GC state command. @@ -331,11 +362,11 @@ func buildGCStateCommand(factory gcStateReaderFactory) *cobra.Command { command := &cobra.Command{ Use: "gc-state", Short: "show keyspace and cluster-wide GC state", - Long: "Show effective per-keyspace GC safe points and local barriers, " + - "and cluster-wide GC state. Expired barriers awaiting lazy deletion " + - "are hidden by default; use --include-expired to include zero-TTL " + - "barriers returned by PD. Use keyspace for one effective GC " + - "scope, global for cluster-wide state, or all for a combined view.", + Long: "Show effective per-keyspace GC safe points and local and global " + + "barriers. Expired barriers awaiting lazy deletion are hidden by " + + "default; use --include-expired to include zero-TTL barriers returned " + + "by PD. Use keyspace for one effective GC scope or all for every " + + "effective GC scope.", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { return cmd.Help() @@ -346,9 +377,13 @@ func buildGCStateCommand(factory gcStateReaderFactory) *cobra.Command { false, "include zero-TTL barriers returned by PD, which normally represent expired barriers awaiting lazy deletion", ) + command.PersistentFlags().Bool( + gcStateExcludeGlobalBarriersFlag, + false, + "exclude global GC barriers from the PD request and JSON output", + ) command.AddCommand( newGCStateKeyspaceCommand(factory), - newGCStateGlobalCommand(factory), newGCStateAllCommand(factory), ) return command @@ -359,9 +394,9 @@ func newGCStateKeyspaceCommand(factory gcStateReaderFactory) *cobra.Command { return &cobra.Command{ Use: "keyspace ", Short: "show one keyspace's effective GC state", - Long: "Show one keyspace's effective GC safe points and local barriers. " + - "Use gc-state global to inspect only cluster-wide state, or " + - "gc-state all for a combined view. " + + Long: "Show one keyspace's effective GC safe points and local and global barriers. " + + "Use --exclude-global-barriers to omit cluster-wide barriers. " + + "Use gc-state all to inspect every effective GC scope. " + "The decimal NullKeyspace ID is " + nullKeyspaceID + ".", Example: " pd-ctl gc-state keyspace 42\n" + " pd-ctl gc-state keyspace " + nullKeyspaceID, @@ -375,39 +410,7 @@ func newGCStateKeyspaceCommand(factory gcStateReaderFactory) *cobra.Command { if err != nil { return err } - reader, err := factory(cmd) - if err != nil { - return errors.Annotate(err, "failed to create PD RPC client") - } - defer reader.close() - - state, err := reader.getGCState(cmd.Context(), keyspaceID) - if err != nil { - if status.Code(err) == codes.Unimplemented { - return errors.Annotate(err, - "gc-state requires a PD server that supports GetGCState") - } - return errors.Annotatef(err, - "failed to get GC state for keyspace %d", keyspaceID) - } - output, err := newKeyspaceGCStateOutput(keyspaceID, state, includeExpired) - if err != nil { - return err - } - return writeGCStateJSON(cmd, output) - }, - } -} - -func newGCStateGlobalCommand(factory gcStateReaderFactory) *cobra.Command { - return &cobra.Command{ - Use: "global", - Short: "show cluster-wide GC state", - Long: "Show cluster-wide GC state without per-keyspace states. The current output contains global GC barriers.", - Example: " pd-ctl gc-state global", - Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, _ []string) error { - includeExpired, err := getGCStateIncludeExpired(cmd) + includeGlobalGCBarriers, err := getGCStateIncludeGlobalGCBarriers(cmd) if err != nil { return err } @@ -417,16 +420,25 @@ func newGCStateGlobalCommand(factory gcStateReaderFactory) *cobra.Command { } defer reader.close() - clusterState, err := reader.getGlobalGCState(cmd.Context()) + state, err := reader.getGCState( + cmd.Context(), + keyspaceID, + includeGlobalGCBarriers, + ) if err != nil { if status.Code(err) == codes.Unimplemented { return errors.Annotate(err, - "gc-state global requires a PD server that supports "+ - "GetAllKeyspacesGCStates") + "gc-state requires a PD server that supports GetGCState") } - return errors.Annotate(err, "failed to get global GC state") + return errors.Annotatef(err, + "failed to get GC state for keyspace %d", keyspaceID) } - output, err := newGlobalGCStateOutput(clusterState, includeExpired) + output, err := newKeyspaceGCStateOutput( + keyspaceID, + state, + includeExpired, + includeGlobalGCBarriers, + ) if err != nil { return err } @@ -439,9 +451,9 @@ func newGCStateAllCommand(factory gcStateReaderFactory) *cobra.Command { return &cobra.Command{ Use: "all", Short: "show effective GC scopes and cluster-wide GC state", - Long: "Show all effective GC scopes and local barriers, with " + - "cluster-wide global barriers once at the top level. Use " + - "gc-state global to inspect only cluster-wide state.", + Long: "Show all effective GC scopes and local barriers, with global " + + "barriers once at the top level. Use --exclude-global-barriers to " + + "omit cluster-wide barriers.", Example: " pd-ctl gc-state all", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { @@ -449,13 +461,20 @@ func newGCStateAllCommand(factory gcStateReaderFactory) *cobra.Command { if err != nil { return err } + includeGlobalGCBarriers, err := getGCStateIncludeGlobalGCBarriers(cmd) + if err != nil { + return err + } reader, err := factory(cmd) if err != nil { return errors.Annotate(err, "failed to create PD RPC client") } defer reader.close() - clusterState, err := reader.getAllKeyspacesGCStates(cmd.Context()) + clusterState, err := reader.getAllKeyspacesGCStates( + cmd.Context(), + includeGlobalGCBarriers, + ) if err != nil { if status.Code(err) == codes.Unimplemented { return errors.Annotate(err, @@ -464,7 +483,11 @@ func newGCStateAllCommand(factory gcStateReaderFactory) *cobra.Command { } return errors.Annotate(err, "failed to get all keyspaces GC states") } - output, err := newAllGCStatesOutput(clusterState, includeExpired) + output, err := newAllGCStatesOutput( + clusterState, + includeExpired, + includeGlobalGCBarriers, + ) if err != nil { return err } diff --git a/tools/pd-ctl/pdctl/command/gc_state_command_test.go b/tools/pd-ctl/pdctl/command/gc_state_command_test.go index 3c44e698a5..b9f0d0c3fd 100644 --- a/tools/pd-ctl/pdctl/command/gc_state_command_test.go +++ b/tools/pd-ctl/pdctl/command/gc_state_command_test.go @@ -22,6 +22,7 @@ import ( "fmt" "io" "math" + "strconv" "strings" "testing" "time" @@ -36,35 +37,32 @@ import ( ) type fakeGCStateReader struct { - state gc.GCState - clusterState gc.ClusterGCStates - err error - requestedID uint32 - getStateCalls int - getAllCalls int - getGlobalCalls int - closed bool -} - -func (r *fakeGCStateReader) getGlobalGCState( - _ context.Context, -) (gc.ClusterGCStates, error) { - r.getGlobalCalls++ - return r.clusterState, r.err + state gc.GCState + clusterState gc.ClusterGCStates + err error + requestedID uint32 + includeGlobalGCBarriers bool + getStateCalls int + getAllCalls int + closed bool } func (r *fakeGCStateReader) getGCState( _ context.Context, keyspaceID uint32, + includeGlobalGCBarriers bool, ) (gc.GCState, error) { r.requestedID = keyspaceID + r.includeGlobalGCBarriers = includeGlobalGCBarriers r.getStateCalls++ return r.state, r.err } func (r *fakeGCStateReader) getAllKeyspacesGCStates( _ context.Context, + includeGlobalGCBarriers bool, ) (gc.ClusterGCStates, error) { + r.includeGlobalGCBarriers = includeGlobalGCBarriers r.getAllCalls++ return r.clusterState, r.err } @@ -113,10 +111,10 @@ func TestNewKeyspaceGCStateOutput(t *testing.T) { gc.NewGCBarrierInfo("b-backup", 105, gc.TTLNeverExpire, time.Time{}), gc.NewGCBarrierInfo("a-backup", 110, 30*time.Second, time.Time{}), }, - ) + ).WithGlobalGCBarriers(nil) state.IsKeyspaceLevelGC = false - got, err := newKeyspaceGCStateOutput(42, state, false) + got, err := newKeyspaceGCStateOutput(42, state, false, true) require.NoError(t, err) require.Equal(t, uint32(42), got.RequestedKeyspaceID) require.Equal(t, constant.NullKeyspaceID, got.EffectiveKeyspaceID) @@ -128,6 +126,39 @@ func TestNewKeyspaceGCStateOutput(t *testing.T) { {BarrierID: "a-backup", BarrierTS: 110, TTLSeconds: 30}, {BarrierID: "z-backup", BarrierTS: 110, TTLSeconds: 3600}, }, got.GCBarriers) + require.NotNil(t, got.GlobalGCBarriers) + require.Empty(t, *got.GlobalGCBarriers) +} + +func TestNewKeyspaceGCStateOutputGlobalBarrierPresence(t *testing.T) { + t.Run("requested-empty", func(t *testing.T) { + state := gc.NewGCStateWithGCBarriers(42, 100, 90, nil). + WithGlobalGCBarriers(nil) + got, err := newKeyspaceGCStateOutput(42, state, false, true) + require.NoError(t, err) + require.NotNil(t, got.GlobalGCBarriers) + require.Empty(t, *got.GlobalGCBarriers) + encoded, err := json.Marshal(got) + require.NoError(t, err) + require.Contains(t, string(encoded), `"global_gc_barriers":[]`) + }) + + t.Run("excluded", func(t *testing.T) { + state := gc.NewGCStateWithGCBarriers(42, 100, 90, nil) + got, err := newKeyspaceGCStateOutput(42, state, false, false) + require.NoError(t, err) + require.Nil(t, got.GlobalGCBarriers) + encoded, err := json.Marshal(got) + require.NoError(t, err) + require.NotContains(t, string(encoded), "global_gc_barriers") + }) + + t.Run("requested-missing", func(t *testing.T) { + state := gc.NewGCStateWithGCBarriers(42, 100, 90, nil) + _, err := newKeyspaceGCStateOutput(42, state, false, true) + require.ErrorContains(t, err, + "retry with --exclude-global-barriers") + }) } func TestNewLocalGCBarrierOutputsSkipNilEntries(t *testing.T) { @@ -176,7 +207,7 @@ func TestNewAllGCStatesOutputSortsAndKeepsEmptyArrays(t *testing.T) { }, ) - got, err := newAllGCStatesOutput(clusterState, false) + got, err := newAllGCStatesOutput(clusterState, false, true) require.NoError(t, err) require.Equal(t, []uint32{1, constant.NullKeyspaceID}, []uint32{ got.GCStates[0].KeyspaceID, @@ -184,10 +215,11 @@ func TestNewAllGCStatesOutputSortsAndKeepsEmptyArrays(t *testing.T) { }) require.NotNil(t, got.GCStates[0].GCBarriers) require.Empty(t, got.GCStates[0].GCBarriers) + require.NotNil(t, got.GlobalGCBarriers) require.Equal(t, []string{"first-global", "a-global", "z-global"}, []string{ - got.GlobalGCBarriers[0].BarrierID, - got.GlobalGCBarriers[1].BarrierID, - got.GlobalGCBarriers[2].BarrierID, + (*got.GlobalGCBarriers)[0].BarrierID, + (*got.GlobalGCBarriers)[1].BarrierID, + (*got.GlobalGCBarriers)[2].BarrierID, }) encoded, err := json.Marshal(got) @@ -229,7 +261,7 @@ func TestNewAllGCStatesOutputFiltersUnifiedGCPlaceholders(t *testing.T) { nil, ) - got, err := newAllGCStatesOutput(clusterState, false) + got, err := newAllGCStatesOutput(clusterState, false, true) require.NoError(t, err) require.Equal(t, []gcStateOutput{ { @@ -257,111 +289,176 @@ func TestNewAllGCStatesOutputKeepsEmptyGlobalBarrierArray(t *testing.T) { nil, ) - got, err := newAllGCStatesOutput(clusterState, false) + got, err := newAllGCStatesOutput(clusterState, false, true) require.NoError(t, err) require.NotNil(t, got.GlobalGCBarriers) - require.Empty(t, got.GlobalGCBarriers) + require.Empty(t, *got.GlobalGCBarriers) encoded, err := json.Marshal(got) require.NoError(t, err) require.Contains(t, string(encoded), `"global_gc_barriers":[]`) } -func TestNewGlobalGCStateOutputSortsAndKeepsEmptyArray(t *testing.T) { - t.Run("sorted", func(t *testing.T) { - clusterState := gc.NewClusterGCStatesWithGlobalGCBarriers( - map[uint32]gc.GCState{42: gc.NewGCStateWithoutGCBarriers(42, 100, 90)}, - []*gc.GlobalGCBarrierInfo{ - gc.NewGlobalGCBarrierInfo("z-global", 60, time.Minute, time.Time{}), - gc.NewGlobalGCBarrierInfo("a-global", 60, gc.TTLNeverExpire, time.Time{}), - gc.NewGlobalGCBarrierInfo("first-global", 50, time.Second, time.Time{}), - }, +func TestNewAllGCStatesOutputGlobalBarrierPresence(t *testing.T) { + t.Run("requested-empty", func(t *testing.T) { + state := gc.NewClusterGCStatesWithGlobalGCBarriers( + map[uint32]gc.GCState{}, + nil, ) - - got, err := newGlobalGCStateOutput(clusterState, false) + got, err := newAllGCStatesOutput(state, false, true) require.NoError(t, err) - require.Equal(t, []gcBarrierOutput{ - {BarrierID: "first-global", BarrierTS: 50, TTLSeconds: 1}, - {BarrierID: "a-global", BarrierTS: 60, TTLSeconds: math.MaxInt64}, - {BarrierID: "z-global", BarrierTS: 60, TTLSeconds: 60}, - }, got.GlobalGCBarriers) - + require.NotNil(t, got.GlobalGCBarriers) + require.Empty(t, *got.GlobalGCBarriers) encoded, err := json.Marshal(got) require.NoError(t, err) - require.JSONEq(t, `{ - "global_gc_barriers": [ - {"barrier_id":"first-global","barrier_ts":50,"ttl_seconds":1}, - {"barrier_id":"a-global","barrier_ts":60,"ttl_seconds":9223372036854775807}, - {"barrier_id":"z-global","barrier_ts":60,"ttl_seconds":60} - ] - }`, string(encoded)) + require.Contains(t, string(encoded), `"global_gc_barriers":[]`) }) - t.Run("empty", func(t *testing.T) { - clusterState := gc.NewClusterGCStatesWithGlobalGCBarriers(map[uint32]gc.GCState{}, nil) - got, err := newGlobalGCStateOutput(clusterState, false) + t.Run("excluded", func(t *testing.T) { + state := gc.NewClusterGCStatesWithoutGlobalGCBarriers( + map[uint32]gc.GCState{}, + ) + got, err := newAllGCStatesOutput(state, false, false) require.NoError(t, err) - require.NotNil(t, got.GlobalGCBarriers) - require.Empty(t, got.GlobalGCBarriers) - + require.Nil(t, got.GlobalGCBarriers) encoded, err := json.Marshal(got) require.NoError(t, err) - require.JSONEq(t, `{"global_gc_barriers":[]}`, string(encoded)) + require.NotContains(t, string(encoded), "global_gc_barriers") + }) + + t.Run("requested-missing", func(t *testing.T) { + state := gc.NewClusterGCStatesWithoutGlobalGCBarriers( + map[uint32]gc.GCState{}, + ) + _, err := newAllGCStatesOutput(state, false, true) + require.ErrorContains(t, err, + "retry with --exclude-global-barriers") }) } func TestGCStateOutputRejectsExcludedBarriers(t *testing.T) { - state := gc.NewGCStateWithoutGCBarriers(42, 100, 90) - _, err := newKeyspaceGCStateOutput(42, state, false) - require.ErrorContains(t, err, "failed to read GC barriers for keyspace 42") + t.Run("missing-local", func(t *testing.T) { + state := gc.NewGCStateWithoutGCBarriers(42, 100, 90) + _, err := newKeyspaceGCStateOutput(42, state, false, false) + require.ErrorContains(t, err, "failed to read GC barriers for keyspace 42") + }) - clusterState := gc.NewClusterGCStatesWithoutGlobalGCBarriers(map[uint32]gc.GCState{}) - _, err = newAllGCStatesOutput(clusterState, false) - require.ErrorContains(t, err, "failed to read global GC barriers") + t.Run("keyspace-requested-missing-global", func(t *testing.T) { + state := gc.NewGCStateWithGCBarriers(42, 100, 90, nil) + _, err := newKeyspaceGCStateOutput(42, state, false, true) + require.EqualError(t, err, + "gc-state keyspace requires a PD server whose GetGCState supports global GC barriers; "+ + "retry with --exclude-global-barriers") + }) - _, err = newGlobalGCStateOutput(clusterState, false) - require.ErrorContains(t, err, "failed to read global GC barriers") -} + t.Run("keyspace-excluded-global", func(t *testing.T) { + state := gc.NewGCStateWithGCBarriers(42, 100, 90, nil) + got, err := newKeyspaceGCStateOutput(42, state, false, false) + require.NoError(t, err) + require.Nil(t, got.GlobalGCBarriers) + encoded, err := json.Marshal(got) + require.NoError(t, err) + require.NotContains(t, string(encoded), "global_gc_barriers") + }) + + t.Run("all-requested-missing-global", func(t *testing.T) { + clusterState := gc.NewClusterGCStatesWithoutGlobalGCBarriers( + map[uint32]gc.GCState{}, + ) + _, err := newAllGCStatesOutput(clusterState, false, true) + require.EqualError(t, err, + "gc-state all response does not include global GC barriers; "+ + "retry with --exclude-global-barriers") + }) + + t.Run("all-excluded-global", func(t *testing.T) { + clusterState := gc.NewClusterGCStatesWithoutGlobalGCBarriers( + map[uint32]gc.GCState{}, + ) + got, err := newAllGCStatesOutput(clusterState, false, false) + require.NoError(t, err) + require.Nil(t, got.GlobalGCBarriers) + encoded, err := json.Marshal(got) + require.NoError(t, err) + require.NotContains(t, string(encoded), "global_gc_barriers") + }) -type fakeClusterGCStatesClient struct { - options gc.GCStatesAPIOptions - calls int } -func (c *fakeClusterGCStatesClient) GetAllKeyspacesGCStates( - _ context.Context, - opts ...gc.GCStatesAPIOption, -) (gc.ClusterGCStates, error) { - c.options = gc.DefaultGCStatesAPIOptions() - for _, opt := range opts { - opt(&c.options) +func TestGCStateAPIOptions(t *testing.T) { + for _, includeGlobalGCBarriers := range []bool{false, true} { + t.Run(strconv.FormatBool(includeGlobalGCBarriers), func(t *testing.T) { + options := gc.DefaultGCStatesAPIOptions() + for _, option := range gcStateAPIOptions( + includeGlobalGCBarriers, + ) { + option(&options) + } + require.False(t, options.ExcludeGCBarriers) + require.Equal(t, !includeGlobalGCBarriers, + options.ExcludeGlobalGCBarriers) + }) } - c.calls++ - return gc.NewClusterGCStatesWithGlobalGCBarriers(map[uint32]gc.GCState{}, nil), nil } -func TestReadClusterGCStatesOptions(t *testing.T) { +func TestGCStateCommandGlobalBarrierFlag(t *testing.T) { for _, testCase := range []struct { - name string - excludeGCBarriers bool - wantExcludeGCBarriers bool + name string + args []string + wantInclude bool }{ - {name: "all", excludeGCBarriers: false, wantExcludeGCBarriers: false}, - {name: "global", excludeGCBarriers: true, wantExcludeGCBarriers: true}, + {name: "keyspace-default", args: []string{"keyspace", "42"}, wantInclude: true}, + {name: "keyspace-excluded", args: []string{"keyspace", "42", "--exclude-global-barriers"}}, + {name: "all-default", args: []string{"all"}, wantInclude: true}, + {name: "all-excluded", args: []string{"all", "--exclude-global-barriers"}}, } { t.Run(testCase.name, func(t *testing.T) { - client := &fakeClusterGCStatesClient{} - _, err := readClusterGCStates(t.Context(), client, testCase.excludeGCBarriers) - require.NoError(t, err) - require.Equal(t, 1, client.calls) - require.Equal(t, testCase.wantExcludeGCBarriers, client.options.ExcludeGCBarriers) - require.False(t, client.options.ExcludeGlobalGCBarriers) + reader := &fakeGCStateReader{ + state: gc.NewGCStateWithGCBarriers(42, 100, 90, nil). + WithGlobalGCBarriers(nil), + clusterState: gc.NewClusterGCStatesWithGlobalGCBarriers( + map[uint32]gc.GCState{}, + nil, + ), + } + cmd := buildGCStateCommand(func(*cobra.Command) (gcStateReader, error) { + return reader, nil + }) + output := new(bytes.Buffer) + cmd.SetOut(output) + cmd.SetErr(output) + cmd.SetArgs(testCase.args) + + require.NoError(t, cmd.Execute()) + require.Equal(t, testCase.wantInclude, + reader.includeGlobalGCBarriers) + if testCase.wantInclude { + require.Contains(t, output.String(), "global_gc_barriers") + } else { + require.NotContains(t, output.String(), "global_gc_barriers") + } }) } } +func TestGCStateGlobalCommandIsRemoved(t *testing.T) { + factoryCalled := false + cmd := buildGCStateCommand(func(*cobra.Command) (gcStateReader, error) { + factoryCalled = true + return nil, errors.New("factory must not run") + }) + cmd.SetOut(io.Discard) + cmd.SetErr(io.Discard) + cmd.SetArgs([]string{"global"}) + + err := cmd.Execute() + require.ErrorContains(t, err, `unknown command "global"`) + require.False(t, factoryCalled) +} + func TestGCStateKeyspaceCommand(t *testing.T) { - state := gc.NewGCStateWithGCBarriers(42, 100, 90, nil) + state := gc.NewGCStateWithGCBarriers(42, 100, 90, nil). + WithGlobalGCBarriers(nil) state.IsKeyspaceLevelGC = true reader := &fakeGCStateReader{state: state} factoryCalls := 0 @@ -377,6 +474,7 @@ func TestGCStateKeyspaceCommand(t *testing.T) { require.NoError(t, cmd.Execute()) require.Equal(t, 1, factoryCalls) require.Equal(t, 1, reader.getStateCalls) + require.Zero(t, reader.getAllCalls) require.Equal(t, uint32(42), reader.requestedID) require.True(t, reader.closed) @@ -385,7 +483,7 @@ func TestGCStateKeyspaceCommand(t *testing.T) { require.Contains(t, decoded, "requested_keyspace_id") require.Contains(t, decoded, "effective_keyspace_id") require.Contains(t, decoded, "gc_barriers") - require.NotContains(t, decoded, "global_gc_barriers") + require.Contains(t, decoded, "global_gc_barriers") } func TestGCStateAllCommand(t *testing.T) { @@ -405,38 +503,17 @@ func TestGCStateAllCommand(t *testing.T) { require.NoError(t, cmd.Execute()) require.Equal(t, 1, reader.getAllCalls) + require.Zero(t, reader.getStateCalls) require.True(t, reader.closed) require.Contains(t, output.String(), `"gc_states": []`) require.Contains(t, output.String(), `"global_gc_barriers": []`) } -func TestGCStateGlobalCommand(t *testing.T) { - reader := &fakeGCStateReader{clusterState: gc.NewClusterGCStatesWithGlobalGCBarriers( - map[uint32]gc.GCState{42: gc.NewGCStateWithoutGCBarriers(42, 100, 90)}, nil, - )} - cmd := buildGCStateCommand(func(*cobra.Command) (gcStateReader, error) { return reader, nil }) - output := new(bytes.Buffer) - cmd.SetOut(output) - cmd.SetErr(output) - cmd.SetArgs([]string{"global"}) - - require.NoError(t, cmd.Execute()) - require.Equal(t, 1, reader.getGlobalCalls) - require.Zero(t, reader.getAllCalls) - require.Zero(t, reader.getStateCalls) - require.True(t, reader.closed) - - var decoded map[string]json.RawMessage - require.NoError(t, json.Unmarshal(output.Bytes(), &decoded)) - require.Len(t, decoded, 1) - require.Contains(t, decoded, "global_gc_barriers") - require.NotContains(t, decoded, "gc_states") - require.NotContains(t, decoded, "txn_safe_point") - require.NotContains(t, decoded, "gc_safe_point") - require.JSONEq(t, `{"global_gc_barriers":[]}`, output.String()) -} - func TestGCStateCommandExpiredBarrierVisibility(t *testing.T) { + globalBarriers := []*gc.GlobalGCBarrierInfo{ + gc.NewGlobalGCBarrierInfo("active-global", 70, gc.TTLNeverExpire, time.Time{}), + gc.NewGlobalGCBarrierInfo("expired-global", 60, 0, time.Time{}), + } state := gc.NewGCStateWithGCBarriers( 42, 100, @@ -445,14 +522,11 @@ func TestGCStateCommandExpiredBarrierVisibility(t *testing.T) { gc.NewGCBarrierInfo("active-local", 50, gc.TTLNeverExpire, time.Time{}), gc.NewGCBarrierInfo("expired-local", 40, 0, time.Time{}), }, - ) + ).WithGlobalGCBarriers(globalBarriers) state.IsKeyspaceLevelGC = true clusterState := gc.NewClusterGCStatesWithGlobalGCBarriers( map[uint32]gc.GCState{42: state}, - []*gc.GlobalGCBarrierInfo{ - gc.NewGlobalGCBarrierInfo("active-global", 70, gc.TTLNeverExpire, time.Time{}), - gc.NewGlobalGCBarrierInfo("expired-global", 60, 0, time.Time{}), - }, + globalBarriers, ) for _, testCase := range []struct { @@ -467,6 +541,9 @@ func TestGCStateCommandExpiredBarrierVisibility(t *testing.T) { wantLocal: []gcBarrierOutput{ {BarrierID: "active-local", BarrierTS: 50, TTLSeconds: math.MaxInt64}, }, + wantGlobal: []gcBarrierOutput{ + {BarrierID: "active-global", BarrierTS: 70, TTLSeconds: math.MaxInt64}, + }, }, { name: "keyspace-include-expired", @@ -475,6 +552,10 @@ func TestGCStateCommandExpiredBarrierVisibility(t *testing.T) { {BarrierID: "expired-local", BarrierTS: 40, TTLSeconds: 0}, {BarrierID: "active-local", BarrierTS: 50, TTLSeconds: math.MaxInt64}, }, + wantGlobal: []gcBarrierOutput{ + {BarrierID: "expired-global", BarrierTS: 60, TTLSeconds: 0}, + {BarrierID: "active-global", BarrierTS: 70, TTLSeconds: math.MaxInt64}, + }, }, { name: "all-default", @@ -498,21 +579,6 @@ func TestGCStateCommandExpiredBarrierVisibility(t *testing.T) { {BarrierID: "active-global", BarrierTS: 70, TTLSeconds: math.MaxInt64}, }, }, - { - name: "global-default", - args: []string{"global"}, - wantGlobal: []gcBarrierOutput{ - {BarrierID: "active-global", BarrierTS: 70, TTLSeconds: math.MaxInt64}, - }, - }, - { - name: "global-include-expired", - args: []string{"global", "--include-expired"}, - wantGlobal: []gcBarrierOutput{ - {BarrierID: "expired-global", BarrierTS: 60, TTLSeconds: 0}, - {BarrierID: "active-global", BarrierTS: 70, TTLSeconds: math.MaxInt64}, - }, - }, } { t.Run(testCase.name, func(t *testing.T) { reader := &fakeGCStateReader{state: state, clusterState: clusterState} @@ -530,16 +596,15 @@ func TestGCStateCommandExpiredBarrierVisibility(t *testing.T) { var decoded keyspaceGCStateOutput require.NoError(t, json.Unmarshal(output.Bytes(), &decoded)) require.Equal(t, testCase.wantLocal, decoded.GCBarriers) + require.NotNil(t, decoded.GlobalGCBarriers) + require.Equal(t, testCase.wantGlobal, *decoded.GlobalGCBarriers) case "all": var decoded allGCStatesOutput require.NoError(t, json.Unmarshal(output.Bytes(), &decoded)) require.Len(t, decoded.GCStates, 1) require.Equal(t, testCase.wantLocal, decoded.GCStates[0].GCBarriers) - require.Equal(t, testCase.wantGlobal, decoded.GlobalGCBarriers) - case "global": - var decoded globalGCStateOutput - require.NoError(t, json.Unmarshal(output.Bytes(), &decoded)) - require.Equal(t, testCase.wantGlobal, decoded.GlobalGCBarriers) + require.NotNil(t, decoded.GlobalGCBarriers) + require.Equal(t, testCase.wantGlobal, *decoded.GlobalGCBarriers) default: require.Fail(t, "unexpected subcommand", testCase.args[0]) } @@ -560,7 +625,6 @@ func TestGCStateCommandValidatesBeforeCreatingClient(t *testing.T) { {"keyspace", "4294967294"}, {"keyspace", "4294967296"}, {"all", "extra"}, - {"global", "extra"}, } { t.Run(strings.Join(args, "-"), func(t *testing.T) { factoryCalled := false @@ -628,6 +692,16 @@ func TestGCStateCommandErrors(t *testing.T) { }, wantMessage: "failed to read GC barriers for keyspace 42", }, + { + name: "single-missing-global-barriers", + args: []string{"keyspace", "42"}, + factory: func(*cobra.Command) (gcStateReader, error) { + return &fakeGCStateReader{ + state: gc.NewGCStateWithGCBarriers(42, 100, 90, nil), + }, nil + }, + wantMessage: "retry with --exclude-global-barriers", + }, { name: "all-rpc-error", args: []string{"all"}, @@ -658,32 +732,7 @@ func TestGCStateCommandErrors(t *testing.T) { ), }, nil }, - wantMessage: "failed to read global GC barriers", - }, - { - name: "global-rpc-error", - args: []string{"global"}, - factory: func(*cobra.Command) (gcStateReader, error) { - return &fakeGCStateReader{err: errors.New("rpc rejected")}, nil - }, - wantMessage: "failed to get global GC state", - }, - { - name: "global-wrapped-unimplemented", - args: []string{"global"}, - factory: func(*cobra.Command) (gcStateReader, error) { - return &fakeGCStateReader{err: fmt.Errorf("wrapped: %w", - status.Error(codes.Unimplemented, "method unavailable"))}, nil - }, - wantMessage: "gc-state global requires a PD server that supports GetAllKeyspacesGCStates", - }, - { - name: "global-missing-global-barriers", - args: []string{"global"}, - factory: func(*cobra.Command) (gcStateReader, error) { - return &fakeGCStateReader{clusterState: gc.NewClusterGCStatesWithoutGlobalGCBarriers(map[uint32]gc.GCState{})}, nil - }, - wantMessage: "failed to read global GC barriers", + wantMessage: "retry with --exclude-global-barriers", }, } { t.Run(testCase.name, func(t *testing.T) { @@ -702,31 +751,35 @@ func TestGCStateCommandHelpContract(t *testing.T) { return nil, errors.New("help must not create a reader") }) require.Equal(t, "show keyspace and cluster-wide GC state", cmd.Short) - require.Equal(t, "Show effective per-keyspace GC safe points and local barriers, and cluster-wide GC state. Expired barriers awaiting lazy deletion are hidden by default; use --include-expired to include zero-TTL barriers returned by PD. Use keyspace for one effective GC scope, global for cluster-wide state, or all for a combined view.", cmd.Long) + require.Equal(t, "Show effective per-keyspace GC safe points and local and global barriers. Expired barriers awaiting lazy deletion are hidden by default; use --include-expired to include zero-TTL barriers returned by PD. Use keyspace for one effective GC scope or all for every effective GC scope.", cmd.Long) includeExpired := cmd.PersistentFlags().Lookup("include-expired") require.NotNil(t, includeExpired) require.Equal(t, "false", includeExpired.DefValue) require.Equal(t, "include zero-TTL barriers returned by PD, which normally represent expired barriers awaiting lazy deletion", includeExpired.Usage) + excludeGlobalBarriers := cmd.PersistentFlags().Lookup("exclude-global-barriers") + require.NotNil(t, excludeGlobalBarriers) + require.Equal(t, "false", excludeGlobalBarriers.DefValue) + require.Equal(t, "exclude global GC barriers from the PD request and JSON output", excludeGlobalBarriers.Usage) + + commands := cmd.Commands() + require.Len(t, commands, 2) + require.Equal(t, []string{"all", "keyspace"}, []string{ + commands[0].Name(), + commands[1].Name(), + }) keyspace, _, err := cmd.Find([]string{"keyspace"}) require.NoError(t, err) require.Equal(t, "keyspace ", keyspace.Use) require.Equal(t, "show one keyspace's effective GC state", keyspace.Short) - require.Equal(t, "Show one keyspace's effective GC safe points and local barriers. Use gc-state global to inspect only cluster-wide state, or gc-state all for a combined view. The decimal NullKeyspace ID is 4294967295.", keyspace.Long) + require.Equal(t, "Show one keyspace's effective GC safe points and local and global barriers. Use --exclude-global-barriers to omit cluster-wide barriers. Use gc-state all to inspect every effective GC scope. The decimal NullKeyspace ID is 4294967295.", keyspace.Long) require.Equal(t, " pd-ctl gc-state keyspace 42\n pd-ctl gc-state keyspace 4294967295", keyspace.Example) - global, _, err := cmd.Find([]string{"global"}) - require.NoError(t, err) - require.Equal(t, "global", global.Use) - require.Equal(t, "show cluster-wide GC state", global.Short) - require.Equal(t, "Show cluster-wide GC state without per-keyspace states. The current output contains global GC barriers.", global.Long) - require.Equal(t, " pd-ctl gc-state global", global.Example) - all, _, err := cmd.Find([]string{"all"}) require.NoError(t, err) require.Equal(t, "all", all.Use) require.Equal(t, "show effective GC scopes and cluster-wide GC state", all.Short) - require.Equal(t, "Show all effective GC scopes and local barriers, with cluster-wide global barriers once at the top level. Use gc-state global to inspect only cluster-wide state.", all.Long) + require.Equal(t, "Show all effective GC scopes and local barriers, with global barriers once at the top level. Use --exclude-global-barriers to omit cluster-wide barriers.", all.Long) require.Equal(t, " pd-ctl gc-state all", all.Example) } @@ -737,7 +790,8 @@ func (failingWriter) Write([]byte) (int, error) { } func TestGCStateCommandReturnsOutputError(t *testing.T) { - state := gc.NewGCStateWithGCBarriers(42, 100, 90, nil) + state := gc.NewGCStateWithGCBarriers(42, 100, 90, nil). + WithGlobalGCBarriers(nil) reader := &fakeGCStateReader{state: state} cmd := buildGCStateCommand(func(*cobra.Command) (gcStateReader, error) { return reader, nil From cfc81e877abdfe00d00dbb1220977e8213b14f79 Mon Sep 17 00:00:00 2001 From: Wenxuan Zhang Date: Tue, 11 Aug 2026 17:06:37 +0800 Subject: [PATCH 25/31] test: cover optional global GC state output Signed-off-by: Wenxuan Zhang --- tools/pd-ctl/tests/safepoint/gc_state_test.go | 155 +++++++++++++++--- 1 file changed, 129 insertions(+), 26 deletions(-) diff --git a/tools/pd-ctl/tests/safepoint/gc_state_test.go b/tools/pd-ctl/tests/safepoint/gc_state_test.go index e181f6aa39..207a51a634 100644 --- a/tools/pd-ctl/tests/safepoint/gc_state_test.go +++ b/tools/pd-ctl/tests/safepoint/gc_state_test.go @@ -46,6 +46,7 @@ type gcStateCommandSingle struct { TxnSafePoint uint64 `json:"txn_safe_point"` GCSafePoint uint64 `json:"gc_safe_point"` GCBarriers []gcStateCommandBarrier `json:"gc_barriers"` + GlobalGCBarriers []gcStateCommandBarrier `json:"global_gc_barriers"` } type gcStateCommandState struct { @@ -61,10 +62,6 @@ type gcStateCommandAll struct { GlobalGCBarriers []gcStateCommandBarrier `json:"global_gc_barriers"` } -type gcStateCommandGlobal struct { - GlobalGCBarriers []gcStateCommandBarrier `json:"global_gc_barriers"` -} - type expectedGCStateCommandBarrier struct { barrierID string barrierTS uint64 @@ -221,7 +218,7 @@ func TestGCState(t *testing.T) { re.NoError(err) var singleProperties map[string]json.RawMessage re.NoError(json.Unmarshal(output, &singleProperties), string(output)) - re.NotContains(singleProperties, "global_gc_barriers") + re.Contains(singleProperties, "global_gc_barriers") var keyspaceLevelResponse gcStateCommandSingle re.NoError(json.Unmarshal(output, &keyspaceLevelResponse), string(output)) re.Equal(keyspaceLevelID, keyspaceLevelResponse.RequestedKeyspaceID) @@ -233,6 +230,14 @@ func TestGCState(t *testing.T) { {barrierID: "a-local", barrierTS: 210}, {barrierID: "z-local", barrierTS: 220, expires: true}, }) + requireGCStateCommandBarriers( + re, + keyspaceLevelResponse.GlobalGCBarriers, + []expectedGCStateCommandBarrier{ + {barrierID: "a-global", barrierTS: 310}, + {barrierID: "z-global", barrierTS: 320}, + }, + ) output, err = tests.ExecuteCommand( ctl.GetRootCmd(), "-u", pdAddr, "gc-state", "keyspace", keyspaceLevelIDString, @@ -246,6 +251,65 @@ func TestGCState(t *testing.T) { {barrierID: "z-local", barrierTS: 220, expires: true}, {barrierID: "expired-local", barrierTS: 230, expired: true}, }) + requireGCStateCommandBarriers( + re, + keyspaceLevelWithExpired.GlobalGCBarriers, + []expectedGCStateCommandBarrier{ + {barrierID: "a-global", barrierTS: 310}, + {barrierID: "z-global", barrierTS: 320}, + {barrierID: "expired-global", barrierTS: 330, expired: true}, + }, + ) + + output, err = tests.ExecuteCommand( + ctl.GetRootCmd(), + "-u", + pdAddr, + "gc-state", + "keyspace", + keyspaceLevelIDString, + "--exclude-global-barriers", + ) + re.NoError(err) + var excludedKeyspaceProperties map[string]json.RawMessage + re.NoError(json.Unmarshal(output, &excludedKeyspaceProperties), string(output)) + re.NotContains(excludedKeyspaceProperties, "global_gc_barriers") + var excludedKeyspace gcStateCommandSingle + re.NoError(json.Unmarshal(output, &excludedKeyspace), string(output)) + requireGCStateCommandBarriers( + re, + excludedKeyspace.GCBarriers, + []expectedGCStateCommandBarrier{ + {barrierID: "a-local", barrierTS: 210}, + {barrierID: "z-local", barrierTS: 220, expires: true}, + }, + ) + + output, err = tests.ExecuteCommand( + ctl.GetRootCmd(), + "-u", + pdAddr, + "gc-state", + "keyspace", + keyspaceLevelIDString, + "--include-expired", + "--exclude-global-barriers", + ) + re.NoError(err) + var excludedKeyspaceWithExpiredProperties map[string]json.RawMessage + re.NoError(json.Unmarshal(output, &excludedKeyspaceWithExpiredProperties), string(output)) + re.NotContains(excludedKeyspaceWithExpiredProperties, "global_gc_barriers") + var excludedKeyspaceWithExpired gcStateCommandSingle + re.NoError(json.Unmarshal(output, &excludedKeyspaceWithExpired), string(output)) + requireGCStateCommandBarriers( + re, + excludedKeyspaceWithExpired.GCBarriers, + []expectedGCStateCommandBarrier{ + {barrierID: "a-local", barrierTS: 210}, + {barrierID: "z-local", barrierTS: 220, expires: true}, + {barrierID: "expired-local", barrierTS: 230, expired: true}, + }, + ) output, err = tests.ExecuteCommand( ctl.GetRootCmd(), "-u", pdAddr, "gc-state", "keyspace", "4294967295", @@ -262,6 +326,14 @@ func TestGCState(t *testing.T) { {barrierID: "a-null", barrierTS: 110}, {barrierID: "z-null", barrierTS: 120, expires: true}, }) + requireGCStateCommandBarriers( + re, + nullKeyspaceResponse.GlobalGCBarriers, + []expectedGCStateCommandBarrier{ + {barrierID: "a-global", barrierTS: 310}, + {barrierID: "z-global", barrierTS: 320}, + }, + ) _, err = tests.ExecuteCommand( ctl.GetRootCmd(), "-u", pdAddr, "gc-state", "keyspace", "16770000", @@ -334,33 +406,56 @@ func TestGCState(t *testing.T) { "-u", pdAddr, "gc-state", - "global", + "all", + "--exclude-global-barriers", ) re.NoError(err) - var globalProperties map[string]json.RawMessage - re.NoError(json.Unmarshal(output, &globalProperties), string(output)) - re.Len(globalProperties, 1) - re.Contains(globalProperties, "global_gc_barriers") - re.NotContains(globalProperties, "gc_states") - re.NotContains(globalProperties, "txn_safe_point") - re.NotContains(globalProperties, "gc_safe_point") - - var global gcStateCommandGlobal - re.NoError(json.Unmarshal(output, &global), string(output)) - re.NotNil(global.GlobalGCBarriers) - re.Equal(all.GlobalGCBarriers, global.GlobalGCBarriers) + var excludedAllProperties map[string]json.RawMessage + re.NoError(json.Unmarshal(output, &excludedAllProperties), string(output)) + re.Contains(excludedAllProperties, "gc_states") + re.NotContains(excludedAllProperties, "global_gc_barriers") + var excludedAll gcStateCommandAll + re.NoError(json.Unmarshal(output, &excludedAll), string(output)) + excludedStatesByID := make(map[uint32]gcStateCommandState, len(excludedAll.GCStates)) + for _, state := range excludedAll.GCStates { + excludedStatesByID[state.KeyspaceID] = state + } + excludedNullState, ok := excludedStatesByID[constant.NullKeyspaceID] + re.True(ok) + re.Equal(nullState, excludedNullState) + excludedKeyspaceLevelState, ok := excludedStatesByID[keyspaceLevelID] + re.True(ok) + re.Equal(keyspaceLevelState, excludedKeyspaceLevelState) output, err = tests.ExecuteCommand( - ctl.GetRootCmd(), "-u", pdAddr, "gc-state", "global", "--include-expired", + ctl.GetRootCmd(), + "-u", + pdAddr, + "gc-state", + "all", + "--include-expired", + "--exclude-global-barriers", ) re.NoError(err) - var globalWithExpired gcStateCommandGlobal - re.NoError(json.Unmarshal(output, &globalWithExpired), string(output)) - requireGCStateCommandBarriers(re, globalWithExpired.GlobalGCBarriers, []expectedGCStateCommandBarrier{ - {barrierID: "a-global", barrierTS: 310}, - {barrierID: "z-global", barrierTS: 320}, - {barrierID: "expired-global", barrierTS: 330, expired: true}, - }) + var excludedAllWithExpiredProperties map[string]json.RawMessage + re.NoError(json.Unmarshal(output, &excludedAllWithExpiredProperties), string(output)) + re.Contains(excludedAllWithExpiredProperties, "gc_states") + re.NotContains(excludedAllWithExpiredProperties, "global_gc_barriers") + var excludedAllWithExpired gcStateCommandAll + re.NoError(json.Unmarshal(output, &excludedAllWithExpired), string(output)) + excludedStatesByIDWithExpired := make( + map[uint32]gcStateCommandState, + len(excludedAllWithExpired.GCStates), + ) + for _, state := range excludedAllWithExpired.GCStates { + excludedStatesByIDWithExpired[state.KeyspaceID] = state + } + excludedNullStateWithExpired, ok := excludedStatesByIDWithExpired[constant.NullKeyspaceID] + re.True(ok) + re.Equal(nullState, excludedNullStateWithExpired) + excludedKeyspaceLevelStateWithExpired, ok := excludedStatesByIDWithExpired[keyspaceLevelID] + re.True(ok) + re.Equal(keyspaceLevelStateWithExpired, excludedKeyspaceLevelStateWithExpired) if kerneltype.IsNextGen() { systemState, ok := statesByID[constant.SystemKeyspaceID] @@ -383,6 +478,14 @@ func TestGCState(t *testing.T) { {barrierID: "a-null", barrierTS: 110}, {barrierID: "z-null", barrierTS: 120, expires: true}, }) + requireGCStateCommandBarriers( + re, + unifiedKeyspaceResponse.GlobalGCBarriers, + []expectedGCStateCommandBarrier{ + {barrierID: "a-global", barrierTS: 310}, + {barrierID: "z-global", barrierTS: 320}, + }, + ) re.NotContains(statesByID, unifiedKeyspaceID) } From 78c56dcc2e0bf352d032b42232ab726632861383 Mon Sep 17 00:00:00 2001 From: Wenxuan Zhang Date: Tue, 11 Aug 2026 17:13:47 +0800 Subject: [PATCH 26/31] docs: update GC state troubleshooting workflow Signed-off-by: Wenxuan Zhang --- tools/pd-ctl/README.md | 55 +++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/tools/pd-ctl/README.md b/tools/pd-ctl/README.md index 3d350394f1..e8a0bf772b 100644 --- a/tools/pd-ctl/README.md +++ b/tools/pd-ctl/README.md @@ -16,10 +16,12 @@ The details about how to use `pd-ctl` can be found in [PD Control User Guide](ht Use `gc-state` to inspect the safe points and barriers that can block GC. The command is read-only and emits deterministic JSON for scripts and diffs. -By default, it omits barriers that PD returns with a zero TTL because they -normally represent expired barriers awaiting lazy deletion. Add -`--include-expired` to any subcommand to include those barriers in the existing -`gc_barriers` or `global_gc_barriers` array with `ttl_seconds` set to `0`. +The default `keyspace` and `all` views request global barriers. They omit +barriers that PD returns with a zero TTL because those barriers normally +represent expired barriers awaiting lazy deletion. Add `--include-expired` to +include those barriers in the existing `gc_barriers` or `global_gc_barriers` +array with `ttl_seconds` set to `0`. An empty `global_gc_barriers` array means +PD returned no global barriers. For example, inspect one keyspace and include zero-TTL barriers: @@ -27,7 +29,8 @@ For example, inspect one keyspace and include zero-TTL barriers: pd-ctl gc-state keyspace 42 --include-expired ``` -Inspect one keyspace by its decimal ID: +Use `keyspace` when diagnosing one GC scope. Inspect a keyspace by its decimal +ID: ```bash pd-ctl gc-state keyspace 42 @@ -46,24 +49,7 @@ pd-ctl gc-state keyspace 42 "barrier_ts": 464950000000000000, "ttl_seconds": 3600 } - ] -} -``` - -The response contains both `requested_keyspace_id` and -`effective_keyspace_id`. They are equal for keyspace-level GC. A keyspace that -uses unified GC returns `4294967295`, the NullKeyspace ID, as its effective -scope. The response contains local `gc_barriers` only. - -Inspect cluster-wide state when the local barriers do not explain the -effective safe point: - -```bash -pd-ctl gc-state global -``` - -```json -{ + ], "global_gc_barriers": [ { "barrier_id": "native_br", @@ -74,11 +60,14 @@ pd-ctl gc-state global } ``` -The global response does not contain per-keyspace safe points or local -barriers. Its current field is `global_gc_barriers`; other cluster-wide GC -state can be added to the same object in the future. +The response contains both `requested_keyspace_id` and +`effective_keyspace_id`. They are equal for keyspace-level GC. A keyspace that +uses unified GC returns `4294967295`, the NullKeyspace ID, as its effective +scope. The local and global barriers come from one `GetGCState` read. The same +global list applies to every keyspace. -Inspect every effective GC scope together with cluster-wide state: +Use `all` only when you need every effective GC scope because it enumerates all +keyspaces. Inspect every effective GC scope together with cluster-wide state: ```bash pd-ctl gc-state all @@ -120,3 +109,15 @@ corresponding arrays are encoded as `[]`. Barrier TTLs use remaining seconds, and `9223372036854775807` means that a barrier never expires. Because PD rounds remaining TTLs down to whole seconds, a zero TTL can also represent a barrier with less than one second remaining. + +Use `--exclude-global-barriers` to skip the global-barrier read and remove the +`global_gc_barriers` field from the JSON output. The flag applies to both +remaining subcommands: + +```bash +pd-ctl gc-state keyspace 42 --exclude-global-barriers +pd-ctl gc-state all --exclude-global-barriers --include-expired +``` + +When combined with `--include-expired`, exclusion wins for global barriers, +while expired local barriers remain visible. From 0696343c4d65b301affef84b8b0edc6c17d0d351 Mon Sep 17 00:00:00 2001 From: Wenxuan Zhang Date: Tue, 11 Aug 2026 17:31:06 +0800 Subject: [PATCH 27/31] pd-ctl: fix GC state test formatting Signed-off-by: Wenxuan Zhang --- tools/pd-ctl/pdctl/command/gc_state_command_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/pd-ctl/pdctl/command/gc_state_command_test.go b/tools/pd-ctl/pdctl/command/gc_state_command_test.go index b9f0d0c3fd..efb3540bb1 100644 --- a/tools/pd-ctl/pdctl/command/gc_state_command_test.go +++ b/tools/pd-ctl/pdctl/command/gc_state_command_test.go @@ -382,7 +382,6 @@ func TestGCStateOutputRejectsExcludedBarriers(t *testing.T) { require.NoError(t, err) require.NotContains(t, string(encoded), "global_gc_barriers") }) - } func TestGCStateAPIOptions(t *testing.T) { From 4badda98d8a289f5f6e0a651ec6acae74e3e7028 Mon Sep 17 00:00:00 2001 From: Wenxuan Zhang Date: Wed, 12 Aug 2026 09:32:30 +0800 Subject: [PATCH 28/31] docs: remove GC state agent workflow artifacts Remove the one-time design and implementation workflow documents now that the GC state command changes are implemented. Keep the PR focused on durable product code, tests, and user documentation. Signed-off-by: Wenxuan Zhang --- ...-08-11-gc-state-global-barrier-refactor.md | 1295 ----------------- ...gc-state-global-barrier-refactor-design.md | 386 ----- 2 files changed, 1681 deletions(-) delete mode 100644 docs/superpowers/plans/2026-08-11-gc-state-global-barrier-refactor.md delete mode 100644 docs/superpowers/specs/2026-08-11-gc-state-global-barrier-refactor-design.md diff --git a/docs/superpowers/plans/2026-08-11-gc-state-global-barrier-refactor.md b/docs/superpowers/plans/2026-08-11-gc-state-global-barrier-refactor.md deleted file mode 100644 index 8f53aa8a22..0000000000 --- a/docs/superpowers/plans/2026-08-11-gc-state-global-barrier-refactor.md +++ /dev/null @@ -1,1295 +0,0 @@ -# GC State Global Barrier Refactor Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make `pd-ctl gc-state keyspace` use `GetGCState` to return local and -global GC barriers, remove `gc-state global`, and let both remaining views skip -global barriers with `--exclude-global-barriers`. - -**Architecture:** Rebase onto the `GetGCState` global barrier support from PR -#11117, preserve the current PR's keyspace-level GC metadata, and pass a single -positive inclusion boolean through the command reader. Model optional JSON -output with a pointer to a slice so the CLI distinguishes an omitted field from -a requested empty list. - -**Tech Stack:** Go 1.25, Cobra, `github.com/tikv/pd/client/clients/gc`, -`encoding/json`, Testify, PD test clusters, and Make. - -## Global constraints - -These constraints apply to every task in this plan. - -- Read and follow the repository `AGENTS.md` before changing files. -- Use Go 1.25 or later. CI uses Go 1.25. -- Start from a clean worktree with failpoints disabled. -- Rebase onto an `upstream/master` that contains commit `3430f76dc` from PR - #11117 before editing command behavior. -- Do not add dependencies or edit module files beyond conflict resolution with - the versions already merged by PR #11117. -- Keep `gc-state keyspace` to one `GetGCState` RPC. Do not add an automatic - `GetAllKeyspacesGCStates` fallback. -- Keep `GetAllKeyspacesGCStates` only for the `all` view. -- Register only `keyspace` and `all`; do not retain a hidden or deprecated - `global` alias. -- Define `--exclude-global-barriers` on the parent command so both subcommands - inherit it. -- Preserve the three global barrier states: omitted, requested-empty, and - requested-populated. -- Preserve nil barrier filtering, zero-TTL filtering, deterministic sorting, - effective scope filtering, and reader closure behavior. -- Use `status.Code(err)` on the original error for gRPC compatibility checks. -- Use `gofmt` and the repository import order on every touched Go file. -- Never edit files or run non-test commands while failpoints are enabled. -- Disable failpoints immediately after each failpoint-enabled test, including - after a failed test. -- Use signed commits with subjects no longer than 70 characters and bodies - wrapped at 80 characters. -- Do not hard-wrap prose when editing the GitHub PR body. - -The approved design is in -[`../specs/2026-08-11-gc-state-global-barrier-refactor-design.md`](../specs/2026-08-11-gc-state-global-barrier-refactor-design.md). - ---- - -## File map - -The implementation modifies existing GC client, command, test, and user -documentation files. It does not add a new production source file. - -- `client/clients/gc/client.go` retains `IsKeyspaceLevelGC` alongside PR - #11117's optional global barrier fields and accessors. -- `client/gc_client.go` preserves keyspace-level mode through local and global - protobuf conversion. -- `client/gc_client_test.go` locks the composed conversion behavior. -- `tests/integrations/client/client_test.go` retains both sides' integration - assertions while resolving the rebase. -- `tools/pd-ctl/pdctl/command/gc_state_command.go` owns reader options, JSON - projection, flags, routing, and errors. -- `tools/pd-ctl/pdctl/command/gc_state_command_test.go` owns projection, - option, command, error, and help contracts. -- `tools/pd-ctl/tests/safepoint/gc_state_test.go` owns end-to-end Classic and - NextGen command behavior against a real PD server. -- `tools/pd-ctl/README.md` documents the final two-command workflow. - -## Task 1: Rebase and compose the GC client model - -This task establishes the correct baseline and combines PR #11117's optional -global barriers with PR #11054's keyspace-level GC metadata. - -**Files:** - -- Modify during conflict resolution: `client/clients/gc/client.go:285-380` -- Modify during conflict resolution: `client/gc_client.go:298-390` -- Modify: `client/gc_client_test.go:1-90` -- Modify during conflict resolution: - `tests/integrations/client/client_test.go:2069-2780` -- Review only: `go.mod`, `go.sum`, `client/go.mod`, `client/go.sum`, - `tests/integrations/go.mod`, `tests/integrations/go.sum`, `tools/go.mod`, and - `tools/go.sum` - -**Interfaces:** - -- Consumes: PR #11117's `gc.GCState.WithGlobalGCBarriers`, - `gc.GCState.HasGlobalGCBarriers`, and - `gc.GCState.GetGlobalGCBarriers` methods. -- Produces: `gc.GCState.IsKeyspaceLevelGC bool` on states with or without local - and global barriers. -- Produces: `pbToGCStateWithGlobalGCBarriers(*pdpb.GCState, - *pdpb.GlobalGCBarriersInfo, time.Time, bool) gc.GCState` that preserves the - keyspace-level flag. - -- [ ] **Step 1: Verify the pre-rebase state** - -Run these commands before rewriting branch history: - -```bash -make failpoint-disable -git status --short --branch -git log -1 --oneline upstream/master -git merge-base --is-ancestor 3430f76dc upstream/master -``` - -Expected: failpoints are disabled, the worktree is clean, the ancestor check -exits with status 0, and `upstream/master` contains PR #11117. Stop if the -worktree is dirty; do not stash or discard user changes automatically. - -- [ ] **Step 2: Refresh and rebase onto upstream master** - -Run: - -```bash -git fetch upstream master -git rebase upstream/master -``` - -Expected: the rebase can stop in the four client files listed above because -both PRs modify the GC state model and protobuf conversion. Resolve only those -semantic overlaps. Do not resolve an entire file with `--ours` or `--theirs`. - -- [ ] **Step 3: Resolve the public `GCState` model composition** - -Ensure the resolved struct in `client/clients/gc/client.go` contains both the -public mode field and PR #11117's private global barrier state: - -```go -type GCState struct { - // The ID of the keyspace this GC state belongs to. - KeyspaceID uint32 - // IsKeyspaceLevelGC reports whether this state belongs to an independent - // keyspace-level GC scope. - IsKeyspaceLevelGC bool - TxnSafePoint uint64 - GCSafePoint uint64 - hasGCBarriers bool - gcBarriers []*GCBarrierInfo - - hasGlobalGCBarriers bool - globalGCBarriers []*GlobalGCBarrierInfo -} -``` - -Keep `WithGlobalGCBarriers`, `HasGlobalGCBarriers`, and -`GetGlobalGCBarriers` exactly as merged by PR #11117. - -- [ ] **Step 4: Resolve protobuf conversion without losing mode metadata** - -In `client/gc_client.go`, keep PR #11117's local conversion and add the mode -assignment immediately before `pbToGCState` returns: - -```go -func pbToGCState( - pb *pdpb.GCState, - reqStartTime time.Time, - excludeGCBarriers bool, -) gc.GCState { - keyspaceID := constants.NullKeyspaceID - if pb.KeyspaceScope != nil { - keyspaceID = pb.KeyspaceScope.GetKeyspaceId() - } - - var state gc.GCState - if excludeGCBarriers { - state = gc.NewGCStateWithoutGCBarriers( - keyspaceID, - pb.GetTxnSafePoint(), - pb.GetGcSafePoint(), - ) - } else { - gcBarriers := make([]*gc.GCBarrierInfo, 0, len(pb.GetGcBarriers())) - for _, barrier := range pb.GetGcBarriers() { - gcBarriers = append( - gcBarriers, - pbToGCBarrierInfo(barrier, reqStartTime), - ) - } - state = gc.NewGCStateWithGCBarriers( - keyspaceID, - pb.GetTxnSafePoint(), - pb.GetGcSafePoint(), - gcBarriers, - ) - } - state.IsKeyspaceLevelGC = pb.GetIsKeyspaceLevelGc() - return state -} -``` - -Keep `pbToGCStateWithGlobalGCBarriers` based on `result := pbToGCState(...)` -and `return result.WithGlobalGCBarriers(barriers)`. That value-receiver flow -preserves `IsKeyspaceLevelGC`. - -- [ ] **Step 5: Add a focused regression test for the composed conversion** - -Append this test to `client/gc_client_test.go`: - -```go -func TestPBToGCStateWithGlobalBarriersPreservesKeyspaceLevelGC(t *testing.T) { - requestStart := time.Unix(100, 0) - state := pbToGCStateWithGlobalGCBarriers( - &pdpb.GCState{ - KeyspaceScope: wrapKeyspaceScope(42), - IsKeyspaceLevelGc: true, - TxnSafePoint: 100, - GcSafePoint: 90, - }, - &pdpb.GlobalGCBarriersInfo{ - Barriers: []*pdpb.GlobalGCBarrierInfo{ - {BarrierId: "snapshot", BarrierTs: 95, TtlSeconds: 60}, - }, - }, - requestStart, - true, - ) - - require.True(t, state.IsKeyspaceLevelGC) - require.False(t, state.HasGCBarriers()) - require.True(t, state.HasGlobalGCBarriers()) - barriers, err := state.GetGlobalGCBarriers() - require.NoError(t, err) - require.Len(t, barriers, 1) - require.Equal(t, "snapshot", barriers[0].BarrierID) -} -``` - -- [ ] **Step 6: Run the focused client tests** - -Run: - -```bash -cd client && go test . -run '^(TestPBToGCStatePreservesKeyspaceLevelGC|TestPBToGCStateWithGlobalBarriersPreservesKeyspaceLevelGC)$' -count=1 -``` - -Expected: PASS. A missing `IsKeyspaceLevelGC` field or a converter that -reconstructs the state after setting the field causes a compile or assertion -failure. - -- [ ] **Step 7: Review dependency conflict resolution** - -Run: - -```bash -git diff upstream/master...HEAD -- go.mod go.sum client/go.mod client/go.sum tests/integrations/go.mod tests/integrations/go.sum tools/go.mod tools/go.sum -``` - -Expected: the branch uses the kvproto version already present in -`upstream/master`; this task adds no new dependency version. - -- [ ] **Step 8: Commit the focused composition regression** - -Stage only the intentional post-rebase client changes and test: - -```bash -git add client/clients/gc/client.go client/gc_client.go client/gc_client_test.go tests/integrations/client/client_test.go -git diff --cached --check -git diff --cached -git commit -s -m "client: preserve GC mode with global barriers" -``` - -Expected: the commit contains the composed model/converter and focused -regression only. Replayed rebase commits remain separate history. - -## Task 2: Refactor command output, options, and routing - -This task changes JSON projection and command routing as one independently -testable unit. It locks the three-state output contract first, then connects -the shared option to both RPC paths and removes the standalone global command -before creating a commit. - -**Files:** - -- Modify: `tools/pd-ctl/pdctl/command/gc_state_command.go:37-474` -- Test: `tools/pd-ctl/pdctl/command/gc_state_command_test.go:38-731` - -**Interfaces:** - -- Consumes: `gc.GCState.HasGlobalGCBarriers()` and - `gc.GCState.GetGlobalGCBarriers()` from Task 1. -- Produces: `newKeyspaceGCStateOutput(uint32, gc.GCState, bool, bool) - (keyspaceGCStateOutput, error)`. -- Produces: `newAllGCStatesOutput(gc.ClusterGCStates, bool, bool) - (allGCStatesOutput, error)`. -- Produces: optional `GlobalGCBarriers *[]gcBarrierOutput` fields on both - top-level output types. -- Produces: `gcStateAPIOptions(bool) []gc.GCStatesAPIOption`. -- Produces: `gcStateReader.getGCState(context.Context, uint32, bool)`. -- Produces: `gcStateReader.getAllKeyspacesGCStates(context.Context, bool)`. -- Produces: parent flag `--exclude-global-barriers`, default `false`. - -- [ ] **Step 1: Write failing keyspace projection tests** - -Add this test next to `TestNewKeyspaceGCStateOutput`: - -```go -func TestNewKeyspaceGCStateOutputGlobalBarrierPresence(t *testing.T) { - t.Run("requested-empty", func(t *testing.T) { - state := gc.NewGCStateWithGCBarriers(42, 100, 90, nil). - WithGlobalGCBarriers(nil) - got, err := newKeyspaceGCStateOutput(42, state, false, true) - require.NoError(t, err) - require.NotNil(t, got.GlobalGCBarriers) - require.Empty(t, *got.GlobalGCBarriers) - encoded, err := json.Marshal(got) - require.NoError(t, err) - require.Contains(t, string(encoded), `"global_gc_barriers":[]`) - }) - - t.Run("excluded", func(t *testing.T) { - state := gc.NewGCStateWithGCBarriers(42, 100, 90, nil) - got, err := newKeyspaceGCStateOutput(42, state, false, false) - require.NoError(t, err) - require.Nil(t, got.GlobalGCBarriers) - encoded, err := json.Marshal(got) - require.NoError(t, err) - require.NotContains(t, string(encoded), "global_gc_barriers") - }) - - t.Run("requested-missing", func(t *testing.T) { - state := gc.NewGCStateWithGCBarriers(42, 100, 90, nil) - _, err := newKeyspaceGCStateOutput(42, state, false, true) - require.ErrorContains(t, err, - "retry with --exclude-global-barriers") - }) -} -``` - -- [ ] **Step 2: Write failing all-view projection tests** - -Add this test next to `TestNewAllGCStatesOutputKeepsEmptyGlobalBarrierArray`: - -```go -func TestNewAllGCStatesOutputGlobalBarrierPresence(t *testing.T) { - t.Run("requested-empty", func(t *testing.T) { - state := gc.NewClusterGCStatesWithGlobalGCBarriers( - map[uint32]gc.GCState{}, - nil, - ) - got, err := newAllGCStatesOutput(state, false, true) - require.NoError(t, err) - require.NotNil(t, got.GlobalGCBarriers) - require.Empty(t, *got.GlobalGCBarriers) - encoded, err := json.Marshal(got) - require.NoError(t, err) - require.Contains(t, string(encoded), `"global_gc_barriers":[]`) - }) - - t.Run("excluded", func(t *testing.T) { - state := gc.NewClusterGCStatesWithoutGlobalGCBarriers( - map[uint32]gc.GCState{}, - ) - got, err := newAllGCStatesOutput(state, false, false) - require.NoError(t, err) - require.Nil(t, got.GlobalGCBarriers) - encoded, err := json.Marshal(got) - require.NoError(t, err) - require.NotContains(t, string(encoded), "global_gc_barriers") - }) - - t.Run("requested-missing", func(t *testing.T) { - state := gc.NewClusterGCStatesWithoutGlobalGCBarriers( - map[uint32]gc.GCState{}, - ) - _, err := newAllGCStatesOutput(state, false, true) - require.ErrorContains(t, err, - "retry with --exclude-global-barriers") - }) -} -``` - -- [ ] **Step 3: Run the projection tests to verify they fail** - -Run: - -```bash -cd tools && go test ./pd-ctl/pdctl/command -run '^(TestNewKeyspaceGCStateOutputGlobalBarrierPresence|TestNewAllGCStatesOutputGlobalBarrierPresence)$' -count=1 -``` - -Expected: FAIL to compile because the output structs lack -`GlobalGCBarriers` and the converters accept only three and two arguments. - -- [ ] **Step 4: Add optional fields to both output structs** - -Replace the two output types with: - -```go -type keyspaceGCStateOutput struct { - RequestedKeyspaceID uint32 `json:"requested_keyspace_id"` - EffectiveKeyspaceID uint32 `json:"effective_keyspace_id"` - IsKeyspaceLevelGC bool `json:"is_keyspace_level_gc"` - TxnSafePoint uint64 `json:"txn_safe_point"` - GCSafePoint uint64 `json:"gc_safe_point"` - GCBarriers []gcBarrierOutput `json:"gc_barriers"` - GlobalGCBarriers *[]gcBarrierOutput `json:"global_gc_barriers,omitempty"` -} - -type allGCStatesOutput struct { - GCStates []gcStateOutput `json:"gc_states"` - GlobalGCBarriers *[]gcBarrierOutput `json:"global_gc_barriers,omitempty"` -} -``` - -Do not add `omitempty` to local barrier fields. - -- [ ] **Step 5: Implement keyspace global barrier projection** - -Replace `newKeyspaceGCStateOutput` with: - -```go -func newKeyspaceGCStateOutput( - requestedKeyspaceID uint32, - state gc.GCState, - includeExpired bool, - includeGlobalGCBarriers bool, -) (keyspaceGCStateOutput, error) { - barriers, err := state.GetGCBarriers() - if err != nil { - return keyspaceGCStateOutput{}, errors.Annotatef( - err, - "failed to read GC barriers for keyspace %d", - requestedKeyspaceID, - ) - } - - var globalOutput *[]gcBarrierOutput - if includeGlobalGCBarriers { - if !state.HasGlobalGCBarriers() { - return keyspaceGCStateOutput{}, errors.Errorf( - "gc-state keyspace requires a PD server whose GetGCState " + - "supports global GC barriers; retry with " + - "--exclude-global-barriers", - ) - } - globalBarriers, err := state.GetGlobalGCBarriers() - if err != nil { - return keyspaceGCStateOutput{}, errors.WithStack(err) - } - converted := newGlobalGCBarrierOutputs( - globalBarriers, - includeExpired, - ) - globalOutput = &converted - } - - return keyspaceGCStateOutput{ - RequestedKeyspaceID: requestedKeyspaceID, - EffectiveKeyspaceID: state.KeyspaceID, - IsKeyspaceLevelGC: state.IsKeyspaceLevelGC, - TxnSafePoint: state.TxnSafePoint, - GCSafePoint: state.GCSafePoint, - GCBarriers: newLocalGCBarrierOutputs( - barriers, - includeExpired, - ), - GlobalGCBarriers: globalOutput, - }, nil -} -``` - -- [ ] **Step 6: Implement all-view global barrier projection** - -Keep the existing state conversion and sorting loop in -`newAllGCStatesOutput`. Replace its final global conversion and return block -with: - -```go -var globalOutput *[]gcBarrierOutput -if includeGlobalGCBarriers { - if !clusterState.HasGlobalGCBarriers() { - return allGCStatesOutput{}, errors.New( - "gc-state all response does not include global GC barriers; " + - "retry with --exclude-global-barriers", - ) - } - globalBarriers, err := clusterState.GetGlobalGCBarriers() - if err != nil { - return allGCStatesOutput{}, errors.WithStack(err) - } - converted := newGlobalGCBarrierOutputs(globalBarriers, includeExpired) - globalOutput = &converted -} - -return allGCStatesOutput{ - GCStates: states, - GlobalGCBarriers: globalOutput, -}, nil -``` - -Add `includeGlobalGCBarriers bool` as the third parameter. During the focused -projection cycle in Steps 4 through 8, pass `true` from the existing `keyspace` -and `all` call sites so the package compiles. Do not commit that intermediate -state; Steps 9 through 16 replace those temporary calls with flag-driven -routing and remove the global subcommand. - -- [ ] **Step 7: Update existing projection assertions** - -Update existing calls to the converters with the new boolean. Dereference -`GlobalGCBarriers` only after `require.NotNil`. Delete -`TestNewGlobalGCStateOutputSortsAndKeepsEmptyArray` only in Step 16, when its -production converter is removed. For existing included keyspace cases, attach -the expected globals with `state.WithGlobalGCBarriers(...)`; for excluded -cases, deliberately use a state without global data and pass `false`. - -Update `TestGCStateOutputRejectsExcludedBarriers` to keep the missing-local -assertion, assert the exact actionable error for a requested but missing global -result, and add successful excluded projections that omit the field. This -locks the distinction between missing server capability and explicit user -exclusion. - -- [ ] **Step 8: Run all projection tests** - -Run: - -```bash -cd tools && go test ./pd-ctl/pdctl/command -run '^(TestNew(Keyspace|All)GCStates?Output.*|TestGCStateOutputRejectsExcludedBarriers)$' -count=1 -``` - -Expected: PASS. The encoded present-empty cases contain `[]`, and excluded -cases omit the field. - -The remaining steps connect the tested projection to the command reader and -finish the command-tree refactor before the task-level review and commit. - -- [ ] **Step 9: Refactor the fake reader for failing routing tests** - -Replace its global call counter with one inclusion field, delete -`getGlobalGCState`, and use these signatures: - -```go -type fakeGCStateReader struct { - state gc.GCState - clusterState gc.ClusterGCStates - err error - requestedID uint32 - includeGlobalGCBarriers bool - getStateCalls int - getAllCalls int - closed bool -} - -func (r *fakeGCStateReader) getGCState( - _ context.Context, - keyspaceID uint32, - includeGlobalGCBarriers bool, -) (gc.GCState, error) { - r.requestedID = keyspaceID - r.includeGlobalGCBarriers = includeGlobalGCBarriers - r.getStateCalls++ - return r.state, r.err -} - -func (r *fakeGCStateReader) getAllKeyspacesGCStates( - _ context.Context, - includeGlobalGCBarriers bool, -) (gc.ClusterGCStates, error) { - r.includeGlobalGCBarriers = includeGlobalGCBarriers - r.getAllCalls++ - return r.clusterState, r.err -} -``` - -The production package now fails to compile until its interface and call sites -match. - -- [ ] **Step 10: Write a failing client option test** - -Replace `TestReadClusterGCStatesOptions` and its fake client with: - -```go -func TestGCStateAPIOptions(t *testing.T) { - for _, includeGlobalGCBarriers := range []bool{false, true} { - t.Run(strconv.FormatBool(includeGlobalGCBarriers), func(t *testing.T) { - options := gc.DefaultGCStatesAPIOptions() - for _, option := range gcStateAPIOptions( - includeGlobalGCBarriers, - ) { - option(&options) - } - require.False(t, options.ExcludeGCBarriers) - require.Equal(t, !includeGlobalGCBarriers, - options.ExcludeGlobalGCBarriers) - }) - } -} -``` - -Add `strconv` to the standard-library import block before adding this test. - -- [ ] **Step 11: Write failing flag and command-tree tests** - -Add a table test that executes both subcommands with and without the flag: - -```go -func TestGCStateCommandGlobalBarrierFlag(t *testing.T) { - for _, testCase := range []struct { - name string - args []string - wantInclude bool - }{ - {name: "keyspace-default", args: []string{"keyspace", "42"}, wantInclude: true}, - {name: "keyspace-excluded", args: []string{"keyspace", "42", "--exclude-global-barriers"}}, - {name: "all-default", args: []string{"all"}, wantInclude: true}, - {name: "all-excluded", args: []string{"all", "--exclude-global-barriers"}}, - } { - t.Run(testCase.name, func(t *testing.T) { - reader := &fakeGCStateReader{ - state: gc.NewGCStateWithGCBarriers(42, 100, 90, nil). - WithGlobalGCBarriers(nil), - clusterState: gc.NewClusterGCStatesWithGlobalGCBarriers( - map[uint32]gc.GCState{}, - nil, - ), - } - cmd := buildGCStateCommand(func(*cobra.Command) (gcStateReader, error) { - return reader, nil - }) - output := new(bytes.Buffer) - cmd.SetOut(output) - cmd.SetErr(output) - cmd.SetArgs(testCase.args) - - require.NoError(t, cmd.Execute()) - require.Equal(t, testCase.wantInclude, - reader.includeGlobalGCBarriers) - if testCase.wantInclude { - require.Contains(t, output.String(), "global_gc_barriers") - } else { - require.NotContains(t, output.String(), "global_gc_barriers") - } - }) - } -} -``` - -Add an explicit removal test: - -```go -func TestGCStateGlobalCommandIsRemoved(t *testing.T) { - factoryCalled := false - cmd := buildGCStateCommand(func(*cobra.Command) (gcStateReader, error) { - factoryCalled = true - return nil, errors.New("factory must not run") - }) - cmd.SetOut(io.Discard) - cmd.SetErr(io.Discard) - cmd.SetArgs([]string{"global"}) - - err := cmd.Execute() - require.ErrorContains(t, err, `unknown command "global"`) - require.False(t, factoryCalled) -} -``` - -- [ ] **Step 12: Run the routing tests to verify they fail** - -Run: - -```bash -cd tools && go test ./pd-ctl/pdctl/command -run '^(TestGCStateAPIOptions|TestGCStateCommandGlobalBarrierFlag|TestGCStateGlobalCommandIsRemoved)$' -count=1 -``` - -Expected: FAIL because `gcStateAPIOptions` and the exclusion flag do not exist, -the production reader uses old signatures, and `global` remains registered. - -- [ ] **Step 13: Replace the reader interface and option construction** - -Use this interface and helper in `gc_state_command.go`: - -```go -type gcStateReader interface { - getGCState( - ctx context.Context, - keyspaceID uint32, - includeGlobalGCBarriers bool, - ) (gc.GCState, error) - getAllKeyspacesGCStates( - ctx context.Context, - includeGlobalGCBarriers bool, - ) (gc.ClusterGCStates, error) - close() -} - -func gcStateAPIOptions( - includeGlobalGCBarriers bool, -) []gc.GCStatesAPIOption { - return []gc.GCStatesAPIOption{ - gc.ExcludeGCBarriers(false), - gc.ExcludeGlobalGCBarriers(!includeGlobalGCBarriers), - } -} -``` - -Update both concrete reader methods to accept the boolean and call the bound -client with `gcStateAPIOptions(includeGlobalGCBarriers)...`. Delete -`clusterGCStatesClient`, `readClusterGCStates`, and `getGlobalGCState`. - -- [ ] **Step 14: Add and read the persistent flag** - -Define the flag next to `gcStateIncludeExpiredFlag`: - -```go -const ( - gcStateIncludeExpiredFlag = "include-expired" - gcStateExcludeGlobalBarriersFlag = "exclude-global-barriers" -) -``` - -Add this getter: - -```go -func getGCStateIncludeGlobalGCBarriers( - cmd *cobra.Command, -) (bool, error) { - excludeGlobalGCBarriers, err := cmd.Flags().GetBool( - gcStateExcludeGlobalBarriersFlag, - ) - if err != nil { - return false, errors.WithStack(err) - } - return !excludeGlobalGCBarriers, nil -} -``` - -Register it on the parent command: - -```go -command.PersistentFlags().Bool( - gcStateExcludeGlobalBarriersFlag, - false, - "exclude global GC barriers from the PD request and JSON output", -) -``` - -- [ ] **Step 15: Route the boolean through both remaining commands** - -In both `RunE` functions, read `includeExpired` and then -`includeGlobalGCBarriers` before creating the reader. Use these exact call -shapes: - -```go -state, err := reader.getGCState( - cmd.Context(), - keyspaceID, - includeGlobalGCBarriers, -) -``` - -```go -output, err := newKeyspaceGCStateOutput( - keyspaceID, - state, - includeExpired, - includeGlobalGCBarriers, -) -``` - -```go -clusterState, err := reader.getAllKeyspacesGCStates( - cmd.Context(), - includeGlobalGCBarriers, -) -``` - -```go -output, err := newAllGCStatesOutput( - clusterState, - includeExpired, - includeGlobalGCBarriers, -) -``` - -Keep existing RPC error annotations and original-error `status.Code` checks. - -- [ ] **Step 16: Remove the standalone global command** - -Delete `newGCStateGlobalCommand`, `globalGCStateOutput`, and -`newGlobalGCStateOutput`. Register only: - -```go -command.AddCommand( - newGCStateKeyspaceCommand(factory), - newGCStateAllCommand(factory), -) -``` - -Delete `TestGCStateGlobalCommand`, the obsolete global converter test, every -`global-*` expired-visibility row, every `global-*` error row, and the -`{"global", "extra"}` validation row. Do not retain a test invocation that -could make the removed subcommand look supported. - -Update the remaining command tests as follows: - -- `TestGCStateKeyspaceCommand` supplies a state with requested-empty globals, - requires `global_gc_barriers`, and proves one `getStateCalls` and zero - `getAllCalls`. -- `TestGCStateCommandExpiredBarrierVisibility` attaches the active and expired - global fixtures to the keyspace state, then checks globals for both keyspace - rows as well as both all rows. -- `TestGCStateCommandErrors` adds a `single-missing-global-barriers` case whose - state has local barriers but no global wrapper and expects - `retry with --exclude-global-barriers`. Update the all-view missing-global - case to expect the same remediation. -- `TestGCStateCommandReturnsOutputError` supplies requested-empty globals so - execution reaches the failing writer. - -- [ ] **Step 17: Update the help contract** - -Use help copy that states the final behavior: - -```go -Short: "show keyspace and cluster-wide GC state", -Long: "Show effective per-keyspace GC safe points and local and global " + - "barriers. Expired barriers awaiting lazy deletion are hidden by " + - "default; use --include-expired to include zero-TTL barriers returned " + - "by PD. Use keyspace for one effective GC scope or all for every " + - "effective GC scope.", -``` - -The keyspace long help must say that it includes local and global barriers and -that `--exclude-global-barriers` omits cluster-wide barriers. The all long help -must say that global barriers appear once at the top level and that the same -flag omits them. Remove every recommendation to run `gc-state global`. - -In `TestGCStateCommandHelpContract`, assert both persistent flags have default -`false`, assert the exact exclusion usage string, and assert that the command's -children are exactly `all` and `keyspace` after Cobra sorts them. - -- [ ] **Step 18: Run the complete command unit suite** - -Run: - -```bash -cd tools && go test ./pd-ctl/pdctl/command -run 'GCState' -count=1 -``` - -Expected: PASS. The default cases include an empty global array, exclusion -cases omit it, and `global` is rejected before client creation. - -- [ ] **Step 19: Commit the complete command refactor** - -Run: - -```bash -git add tools/pd-ctl/pdctl/command/gc_state_command.go tools/pd-ctl/pdctl/command/gc_state_command_test.go -git diff --cached --check -git diff --cached -git commit -s -m "pd-ctl: use GetGCState for global barriers" -``` - -Expected: this commit contains optional output projection, reader refactoring, -the shared flag, command removal, compatibility paths, and all command tests. - -## Task 3: Update real PD command coverage - -This task verifies the complete behavior against a real Classic or NextGen PD -server without duplicating PR #11117's server failpoint tests. - -**Files:** - -- Test: `tools/pd-ctl/tests/safepoint/gc_state_test.go:42-66` -- Test: `tools/pd-ctl/tests/safepoint/gc_state_test.go:218-388` - -**Interfaces:** - -- Consumes: the final `keyspace`, `all`, `--include-expired`, and - `--exclude-global-barriers` command contracts from Task 2. -- Produces: end-to-end coverage for default, expired, excluded, Classic, and - NextGen output. - -- [ ] **Step 1: Update integration output types** - -Add global barriers to the single-keyspace decoder: - -```go -type gcStateCommandSingle struct { - RequestedKeyspaceID uint32 `json:"requested_keyspace_id"` - EffectiveKeyspaceID uint32 `json:"effective_keyspace_id"` - IsKeyspaceLevelGC bool `json:"is_keyspace_level_gc"` - TxnSafePoint uint64 `json:"txn_safe_point"` - GCSafePoint uint64 `json:"gc_safe_point"` - GCBarriers []gcStateCommandBarrier `json:"gc_barriers"` - GlobalGCBarriers []gcStateCommandBarrier `json:"global_gc_barriers"` -} -``` - -Delete `gcStateCommandGlobal`. - -- [ ] **Step 2: Make the default keyspace assertions require global barriers** - -Replace the default `NotContains` assertion with: - -```go -re.Contains(singleProperties, "global_gc_barriers") -requireGCStateCommandBarriers( - re, - keyspaceLevelResponse.GlobalGCBarriers, - []expectedGCStateCommandBarrier{ - {barrierID: "a-global", barrierTS: 310}, - {barrierID: "z-global", barrierTS: 320}, - }, -) -``` - -Add the same active global barrier expectation to the NullKeyspace and Classic -unified-GC keyspace responses. This proves the global result is independent of -the requested keyspace. - -- [ ] **Step 3: Extend the keyspace expired assertion** - -After decoding `keyspaceLevelWithExpired`, assert: - -```go -requireGCStateCommandBarriers( - re, - keyspaceLevelWithExpired.GlobalGCBarriers, - []expectedGCStateCommandBarrier{ - {barrierID: "a-global", barrierTS: 310}, - {barrierID: "z-global", barrierTS: 320}, - {barrierID: "expired-global", barrierTS: 330, expired: true}, - }, -) -``` - -- [ ] **Step 4: Add keyspace exclusion coverage** - -Execute the excluded command and inspect raw field presence: - -```go -output, err = tests.ExecuteCommand( - ctl.GetRootCmd(), - "-u", - pdAddr, - "gc-state", - "keyspace", - keyspaceLevelIDString, - "--exclude-global-barriers", -) -re.NoError(err) -var excludedKeyspaceProperties map[string]json.RawMessage -re.NoError(json.Unmarshal(output, &excludedKeyspaceProperties), string(output)) -re.NotContains(excludedKeyspaceProperties, "global_gc_barriers") -var excludedKeyspace gcStateCommandSingle -re.NoError(json.Unmarshal(output, &excludedKeyspace), string(output)) -requireGCStateCommandBarriers( - re, - excludedKeyspace.GCBarriers, - []expectedGCStateCommandBarrier{ - {barrierID: "a-local", barrierTS: 210}, - {barrierID: "z-local", barrierTS: 220, expires: true}, - }, -) -``` - -Repeat with both flags and assert `expired-local` appears while the global -field remains absent. - -- [ ] **Step 5: Add all-view exclusion coverage** - -Execute: - -```go -output, err = tests.ExecuteCommand( - ctl.GetRootCmd(), - "-u", - pdAddr, - "gc-state", - "all", - "--exclude-global-barriers", -) -re.NoError(err) -var excludedAllProperties map[string]json.RawMessage -re.NoError(json.Unmarshal(output, &excludedAllProperties), string(output)) -re.Contains(excludedAllProperties, "gc_states") -re.NotContains(excludedAllProperties, "global_gc_barriers") -``` - -Decode `gc_states` and compare the NullKeyspace and keyspace-level entries to -the default all view. Repeat with `--include-expired` and assert only local -expired barriers are added. - -- [ ] **Step 6: Delete standalone global command coverage** - -Delete the command calls and assertions from the current global block, -including `globalProperties`, `global`, `globalWithExpired`, and every -`gcStateCommandGlobal` use. - -- [ ] **Step 7: Run the Classic integration test** - -Enable failpoints only around the test: - -```bash -make failpoint-enable -cd tools && go test ./pd-ctl/tests/safepoint -run '^TestGCState$' -count=1 -cd .. && make failpoint-disable -``` - -Expected: PASS. If the test fails, return to the repository root and run -`make failpoint-disable` before diagnosing or editing. - -- [ ] **Step 8: Run the NextGen integration test** - -Run: - -```bash -make failpoint-enable -cd tools && go test -tags nextgen ./pd-ctl/tests/safepoint -run '^TestGCState$' -count=1 -cd .. && make failpoint-disable -``` - -Expected: PASS with the system keyspace retained as a keyspace-level GC scope. -If the test fails, disable failpoints before any other action. - -- [ ] **Step 9: Commit real PD coverage** - -Run: - -```bash -git add tools/pd-ctl/tests/safepoint/gc_state_test.go -git diff --cached --check -git diff --cached -git commit -s -m "test: cover optional global GC state output" -``` - -Expected: the commit changes only the real PD test and removes all standalone -global command coverage. - -## Task 4: Update the user workflow documentation - -This task updates user-facing documentation after the executable behavior and -integration tests are stable. Use the `docs-writer` skill for this task. - -**Files:** - -- Modify: `tools/pd-ctl/README.md:14-130` - -**Interfaces:** - -- Consumes: final command and JSON contracts from Tasks 2 and 3. -- Produces: one documented workflow for a selected keyspace and one for all - effective scopes. - -- [ ] **Step 1: Remove the standalone global workflow** - -Delete the `pd-ctl gc-state global` command example, its JSON object, and prose -that recommends a second command when local barriers do not explain a safe -point. - -- [ ] **Step 2: Add global barriers to the keyspace example** - -Extend the existing keyspace JSON object with this top-level field: - -```json -"global_gc_barriers": [ - { - "barrier_id": "native_br", - "barrier_ts": 464940000000000000, - "ttl_seconds": 9223372036854775807 - } -] -``` - -State that local and global barriers come from one `GetGCState` read and that -the same global list applies to every keyspace. - -- [ ] **Step 3: Document presence and exclusion semantics** - -Add prose with these exact behavioral claims: - -- The default `keyspace` and `all` views request global barriers. -- An empty `global_gc_barriers` array means PD returned no global barriers. -- `--exclude-global-barriers` skips the read and removes the JSON field. -- The flag applies to both remaining subcommands. -- When combined with `--include-expired`, exclusion wins for global barriers, - while expired local barriers remain visible. - -Include these examples: - -```bash -pd-ctl gc-state keyspace 42 --exclude-global-barriers -pd-ctl gc-state all --exclude-global-barriers --include-expired -``` - -- [ ] **Step 4: Clarify command selection** - -Recommend `keyspace` when diagnosing one scope. Reserve `all` for cases that -require every effective GC scope because it enumerates all keyspaces. - -- [ ] **Step 5: Verify documentation references** - -Run: - -```bash -rg -n 'gc-state global|getGlobalGCState|global command' tools/pd-ctl/README.md tools/pd-ctl/pdctl/command -git diff --check -``` - -Expected: `rg` returns no matches and `git diff --check` passes. - -- [ ] **Step 6: Commit the documentation update** - -Run: - -```bash -git add tools/pd-ctl/README.md -git diff --cached --check -git diff --cached -git commit -s -m "docs: update GC state troubleshooting workflow" -``` - -Expected: the commit contains only user documentation that matches the tested -command behavior. - -## Task 5: Run final verification and update PR metadata - -This task proves the refactor across modules and updates PR #11054 only after -the user authorizes the external GitHub change. - -**Files:** - -- Verify: all files modified by Tasks 1 through 4 -- External update after authorization: PR #11054 body - -**Interfaces:** - -- Consumes: all implementation and documentation commits. -- Produces: a clean worktree, passing focused and repository checks, and PR - metadata that describes only `keyspace` and `all`. - -- [ ] **Step 1: Format and verify no dependency drift** - -Run with failpoints disabled: - -```bash -make failpoint-disable -gofmt -w client/gc_client.go client/gc_client_test.go client/clients/gc/client.go tools/pd-ctl/pdctl/command/gc_state_command.go tools/pd-ctl/pdctl/command/gc_state_command_test.go tools/pd-ctl/tests/safepoint/gc_state_test.go -git diff --check -git diff -- go.mod go.sum client/go.mod client/go.sum tests/integrations/go.mod tests/integrations/go.sum tools/go.mod tools/go.sum -``` - -Expected: formatting produces no new semantic diff, whitespace checks pass, -and module files contain only versions inherited from the rebased master. - -- [ ] **Step 2: Run focused client and command unit tests** - -Run: - -```bash -cd client && go test . -run 'GCState' -count=1 -cd ../tools && go test ./pd-ctl/pdctl/command -run 'GCState' -count=1 -cd .. -``` - -Expected: PASS in both modules. - -- [ ] **Step 3: Run Classic and NextGen real PD tests** - -Run only test commands while failpoints are enabled: - -```bash -make failpoint-enable -cd tools && go test ./pd-ctl/tests/safepoint -run '^TestGCState$' -count=1 -go test -tags nextgen ./pd-ctl/tests/safepoint -run '^TestGCState$' -count=1 -cd .. && make failpoint-disable -``` - -Expected: both runs pass. Regardless of either result, run -`make failpoint-disable` before continuing. - -- [ ] **Step 4: Build both pd-ctl variants** - -Run: - -```bash -make pd-ctl -NEXT_GEN=1 make pd-ctl -``` - -Expected: both builds succeed. - -- [ ] **Step 5: Run repository checks** - -Run: - -```bash -make check -make basic-test -cd client && make -cd .. -``` - -Expected: formatting, lint, tidy, error documentation, root tests, and the -client module pipeline pass. If a command enables failpoints internally and -fails, run `make failpoint-disable` before inspecting or editing files. - -- [ ] **Step 6: Prove obsolete paths are absent** - -Run: - -```bash -rg -n 'gc-state global|getGlobalGCState|newGCStateGlobalCommand|globalGCStateOutput' tools/pd-ctl -rg -n 'GetAllKeyspacesGCStates' tools/pd-ctl/pdctl/command/gc_state_command.go -``` - -Expected: the first command returns no matches. The second command shows only -the all-view reader interface, concrete method, and all command call path. - -- [ ] **Step 7: Review the final branch diff and worktree** - -Run: - -```bash -make failpoint-disable -git status --short --branch -git diff --check upstream/master...HEAD -git diff --stat upstream/master...HEAD -git log --oneline upstream/master..HEAD -``` - -Expected: no unstaged or untracked artifacts remain, the diff contains only -the approved PR scope, and every new commit has a signed repository-style -message. If formatting changed tracked files in Step 1, fold those changes into -the task that owns them instead of creating an unrelated cleanup commit. - -- [ ] **Step 8: Prepare the unwrapped PR body update** - -After the user authorizes editing PR #11054, create -`/tmp/pd-11054-refactor-body.md` with this content. Keep prose paragraphs on -single lines because PR Markdown must not be hard-wrapped: - -````markdown -### What problem does this PR solve? - -PD exposes per-keyspace and cluster-wide GC state through RPCs, but operators cannot inspect that state through `pd-ctl`. This makes it difficult to identify whether GC advancement is blocked by a keyspace-local barrier or a cluster-wide global barrier. - -Issue Number: close #11013, ref #8978 - -### What is changed and how does it work? - -```commit-message -Add read-only `pd-ctl gc-state keyspace` and `gc-state all` commands. The keyspace view uses `GetGCState` to return the effective safe points, local barriers, and global barriers in one read. The all view uses `GetAllKeyspacesGCStates` only when every effective GC scope is required. Both views support `--exclude-global-barriers` to skip the global barrier read and omit the JSON field, while `--include-expired` controls zero-TTL barrier visibility. - -Expose the server-provided keyspace-level GC mode through the public GC client model so the command can distinguish independent keyspace GC from unified GC. Return deterministic JSON and preserve the distinction between an omitted global barrier result and a requested empty list. -``` - -### Check List - -Tests - -- Unit test -- Integration test -- Manual test - -### Release note - -```release-note -Add `pd-ctl gc-state keyspace` and `gc-state all` commands for inspecting GC safe points and the local and global barriers that can block GC. -``` -```` - -Use `apply_patch` to create the temporary file; do not use shell redirection. - -- [ ] **Step 9: Update and verify PR #11054 after authorization** - -Run: - -```bash -gh pr edit 11054 --repo tikv/pd --body-file /tmp/pd-11054-refactor-body.md -gh pr view 11054 --repo tikv/pd --json title,body,url -``` - -Expected: the title remains `pd-ctl: add GC state inspection commands`; the -body names only `keyspace` and `all`, includes both flags, and contains the -required issue and release-note blocks. If authorization is not granted, skip -the external update and return the prepared body to the user. - -- [ ] **Step 10: Report verification evidence** - -Report every command run and its result, the final commit list, any tests that -were skipped with a reason, and whether PR metadata was updated. Do not claim -the refactor is complete if any required check failed or failpoints remain -enabled. - -## Execution handoff - -Implementation starts only after the user selects an execution approach. -Follow the sub-skill named in the agentic worker header for that approach, keep -the task checkpoints in order, and leave the PR body update approval-gated -because it changes external GitHub state. diff --git a/docs/superpowers/specs/2026-08-11-gc-state-global-barrier-refactor-design.md b/docs/superpowers/specs/2026-08-11-gc-state-global-barrier-refactor-design.md deleted file mode 100644 index 83d3b2c85d..0000000000 --- a/docs/superpowers/specs/2026-08-11-gc-state-global-barrier-refactor-design.md +++ /dev/null @@ -1,386 +0,0 @@ -# GC state global barrier refactor design - -This design refactors the `pd-ctl gc-state` command introduced by PR #11054. -The refactor uses the enhanced `GetGCState` API from -[PR #11117](https://github.com/tikv/pd/pull/11117) to include global GC -barriers in single-keyspace diagnostics, removes the standalone `global` -subcommand, and adds an option that omits global barriers from either remaining -view. - -## Context - -PR #11054 currently exposes three views: `keyspace`, `global`, and `all`. The -`global` view calls `GetAllKeyspacesGCStates`, even though it discards every -keyspace state and only emits global barriers. That call still enumerates and -materializes all keyspace states, so it performs unnecessary work on clusters -with many keyspaces. - -PR #11117 adds an opt-in global barrier result to `GetGCState`. The client uses -`gc.ExcludeGlobalGCBarriers(false)` to request the result and preserves whether -the server omitted the result, returned an empty list, or returned a populated -list. The server reads the selected keyspace state and global barriers in one -revision-validated operation. - -## Goals and non-goals - -The refactor keeps the CLI focused on the two diagnostic scopes that have -distinct data requirements. - -The design has these goals: - -- Make `gc-state keyspace` return the selected effective GC state, local - barriers, and global barriers with one `GetGCState` call. -- Keep `gc-state all` as the only command that enumerates every effective GC - scope. -- Remove `gc-state global` and every code, test, help, and documentation path - that exists only for that subcommand. -- Let users skip global barrier reads and output in both remaining subcommands - with `--exclude-global-barriers`. -- Preserve the difference between an omitted global barrier result and a - requested result that contains an empty list. -- Preserve deterministic sorting, expired barrier filtering, effective scope - handling, and actionable compatibility errors. - -The design does not add another PD RPC, automatically retry failed requests, -change GC state semantics, or change how PD stores and expires barriers. - -## CLI contract - -The command tree contains only the two views that correspond to a selected -keyspace or all effective scopes: - -```text -gc-state -├── keyspace -└── all -``` - -The implementation removes `gc-state global` without a deprecated or hidden -alias. PR #11054 has not shipped, so the command does not have a released -compatibility contract. - -### Shared flags - -The parent `gc-state` command defines two persistent flags that both subcommands -inherit: - -- `--include-expired` includes zero-TTL local and requested global barriers. -- `--exclude-global-barriers` skips the global barrier read and omits the - `global_gc_barriers` JSON field. - -Both flags default to `false`. Explicitly setting -`--exclude-global-barriers=false` produces the default complete view. - -### Behavior matrix - -The following matrix defines the request and output behavior for every flag -combination. - -| Command | Local barriers | Global barriers | Global JSON field | -| --- | --- | --- | --- | -| `keyspace 42` | Read; hide expired | Read; hide expired | Present | -| `keyspace 42 --include-expired` | Read; include expired | Read; include expired | Present | -| `keyspace 42 --exclude-global-barriers` | Read; hide expired | Do not read | Omitted | -| `keyspace 42 --exclude-global-barriers --include-expired` | Read; include expired | Do not read | Omitted | -| `all` | Read; hide expired | Read; hide expired | Present | -| `all --include-expired` | Read; include expired | Read; include expired | Present | -| `all --exclude-global-barriers` | Read; hide expired | Do not read | Omitted | -| `all --exclude-global-barriers --include-expired` | Read; include expired | Do not read | Omitted | - -When global barriers are included, the JSON field is present even if the list -is empty. Therefore, `"global_gc_barriers": []` means the command requested -global barriers and PD returned none. A missing field means the user explicitly -excluded them. - -## Request flow - -Each subcommand uses the least expensive RPC that provides all data required by -that view. - -### Keyspace view - -The keyspace view calls `GetGCState` exactly once. Its default client options -are: - -```go -GetGCState( - ctx, - gc.ExcludeGCBarriers(false), - gc.ExcludeGlobalGCBarriers(false), -) -``` - -The returned `gc.GCState` contains the effective keyspace state, local -barriers, and global barriers from the same server-side read. With -`--exclude-global-barriers`, the command changes only the second option to -`gc.ExcludeGlobalGCBarriers(true)`. - -The command never calls `GetAllKeyspacesGCStates` as a fallback for the -keyspace view. - -### All view - -The all view continues to call `GetAllKeyspacesGCStates` because it must -enumerate every effective GC scope. It always requests local barriers and maps -the flag directly to `gc.ExcludeGlobalGCBarriers`. - -The all view does not make an additional `GetGCState` call. No remaining path -calls `GetAllKeyspacesGCStates` solely to obtain global barriers. - -## Internal command structure - -The reader abstraction describes the two command behaviors and carries one -positive boolean that controls global barrier inclusion: - -```go -type gcStateReader interface { - getGCState( - ctx context.Context, - keyspaceID uint32, - includeGlobalGCBarriers bool, - ) (gc.GCState, error) - getAllKeyspacesGCStates( - ctx context.Context, - includeGlobalGCBarriers bool, - ) (gc.ClusterGCStates, error) - close() -} -``` - -The boolean is named `includeGlobalGCBarriers` at every declaration and call -site. The Cobra layer converts the negative flag once: - -```go -includeGlobalGCBarriers := !excludeGlobalGCBarriers -``` - -The concrete reader maps the positive value to the client option with -`gc.ExcludeGlobalGCBarriers(!includeGlobalGCBarriers)`. A shared pure helper -returns the local and global barrier options for both RPCs. This helper keeps -option construction consistent and lets unit tests verify the mapping without -mocking the full PD client. - -The implementation deletes these obsolete elements: - -- `getGlobalGCState` from the reader interface and concrete reader. -- `getGlobalCalls` from the fake reader. -- `clusterGCStatesClient` and `readClusterGCStates`. -- `newGCStateGlobalCommand` and its command registration. -- `globalGCStateOutput` and `newGlobalGCStateOutput`. - -## JSON projection - -The command output must represent the client's three global barrier states: -not requested, requested and empty, and requested and populated. - -A slice with `omitempty` cannot express that contract because JSON encoding -omits both nil and empty slices. The keyspace and all output structs use a -pointer to a slice instead: - -```go -type keyspaceGCStateOutput struct { - RequestedKeyspaceID uint32 `json:"requested_keyspace_id"` - EffectiveKeyspaceID uint32 `json:"effective_keyspace_id"` - IsKeyspaceLevelGC bool `json:"is_keyspace_level_gc"` - TxnSafePoint uint64 `json:"txn_safe_point"` - GCSafePoint uint64 `json:"gc_safe_point"` - GCBarriers []gcBarrierOutput `json:"gc_barriers"` - GlobalGCBarriers *[]gcBarrierOutput `json:"global_gc_barriers,omitempty"` -} - -type allGCStatesOutput struct { - GCStates []gcStateOutput `json:"gc_states"` - GlobalGCBarriers *[]gcBarrierOutput `json:"global_gc_barriers,omitempty"` -} -``` - -When global barriers are excluded, the pointer remains nil, the converter does -not call `GetGlobalGCBarriers`, and JSON encoding omits the field. When global -barriers are included, the pointer references a non-nil slice. The field then -encodes as either `[]` or a populated array. - -`newKeyspaceGCStateOutput` and `newAllGCStatesOutput` accept -`includeGlobalGCBarriers` in addition to `includeExpired`. They obtain global -barriers only when requested and reuse `newGlobalGCBarrierOutputs` for nil -entry filtering, expired entry filtering, TTL conversion, and deterministic -sorting. - -Local barrier fields remain required and encode empty results as -`"gc_barriers": []`. - -## Compatibility and errors - -The command reports capability mismatches explicitly and does not convert an -absent response wrapper into an empty barrier list. - -During a rolling upgrade, an older PD server can ignore the new opt-in request -field and return a successful `GetGCState` response without the global barrier -wrapper. The client then reports `state.HasGlobalGCBarriers() == false`. If the -default keyspace view requires global barriers, the command returns this -actionable error: - -```text -gc-state keyspace requires a PD server whose GetGCState supports global GC barriers; retry with --exclude-global-barriers -``` - -The explicit exclusion flag remains a valid degraded path for reading safe -points and local barriers from that server. - -If the all view requires global barriers but the cluster result does not carry -them, the command returns: - -```text -gc-state all response does not include global GC barriers; retry with --exclude-global-barriers -``` - -The command does not retry automatically. A retry could perform another -server-side read, mix snapshots, hide a rolling-upgrade capability difference, -and reintroduce unnecessary work. - -Existing error behavior remains intact for an unimplemented RPC, client -creation failure, other RPC errors, missing local barriers, JSON encoding, and -output writes. Every path closes the reader after successful creation. - -## Rebase integration - -The implementation starts by rebasing PR #11054 onto `upstream/master`, which -already contains PR #11117. Both changes modify the public GC state model and -protobuf conversion, so conflict resolution must combine their behavior. - -The resolved client model retains all of these elements: - -- `IsKeyspaceLevelGC` from PR #11054. -- `hasGlobalGCBarriers` and `globalGCBarriers` from PR #11117. -- `WithGlobalGCBarriers`, `HasGlobalGCBarriers`, and - `GetGlobalGCBarriers` from PR #11117. - -`pbToGCState` constructs the local state and sets `IsKeyspaceLevelGC`. -`pbToGCStateWithGlobalGCBarriers` then attaches the optional global barrier -result without losing the keyspace-level flag. The client tests cover this -composition without duplicating PR #11117's storage, snapshot, TTL, and rolling -upgrade test coverage. - -## Test design - -The test suite proves the option mapping, command routing, JSON presence -contract, compatibility behavior, and real PD integration in Classic and -NextGen configurations. - -### Unit tests - -The command unit tests cover these cases: - -- Both reader methods always request local barriers. -- Both reader methods include global barriers by default and exclude them only - when requested. -- `keyspace` calls only `getGCState`, and `all` calls only - `getAllKeyspacesGCStates`. -- The fake reader receives the expected `includeGlobalGCBarriers` value. -- `global` is an unknown subcommand and fails before creating a reader. -- Root and subcommand help mention only `keyspace`, `all`, and the two shared - flags. -- Default projections emit an empty or populated global array. -- Excluded projections omit the global field and do not require the client - model to carry global barriers. -- Required but absent global barriers produce the actionable compatibility - errors. -- Local and global nil entries are skipped. -- Local and global barriers use the same active and expired filtering rules. -- Barrier arrays preserve deterministic sorting and TTL conversion. -- Effective scope filtering still omits unified-GC placeholders and retains - the NullKeyspace state. -- Reader closure and output error propagation remain intact. - -Tests inspect encoded JSON maps in addition to Go values so they detect field -presence regressions caused by `omitempty`. - -### Integration tests - -The existing safepoint test fixtures already contain keyspace-level, -NullKeyspace, unified-GC, active, expired, local, and global barrier cases. The -refactor reuses those fixtures and changes the command assertions. - -The integration tests verify these behaviors: - -- Default keyspace output includes the same global barriers for keyspace-level, - NullKeyspace, and unified-GC requests. -- `keyspace --include-expired` includes zero-TTL local and global barriers. -- `keyspace --exclude-global-barriers` preserves safe points and local barriers - while omitting the global field. -- Default all output contains global barriers exactly once at the top level. -- `all --exclude-global-barriers` preserves every effective scope and omits the - top-level global field. -- Combining exclusion with `--include-expired` affects only local barriers. -- Classic unified GC and NextGen keyspace-level GC behavior remain unchanged. - -The tests remove every `gc-state global` invocation, output type, and assertion. -PR #11117 already proves server-side snapshot consistency and verifies that -excluded requests stay on the no-global-read path, so this PR does not duplicate -those failpoint tests. - -## Documentation and PR updates - -The user documentation and PR metadata must describe the final two-command -workflow. - -The `tools/pd-ctl/README.md` update makes these changes: - -- Remove the standalone global view and its JSON example. -- Add `global_gc_barriers` to the keyspace JSON example. -- Explain missing versus present-empty global barrier fields. -- Document `--exclude-global-barriers` for both remaining subcommands. -- Explain its interaction with `--include-expired`. -- Recommend `keyspace` for one scope and reserve `all` for full-cluster - inspection. - -The PR body and release note describe `keyspace` and `all`, the enhanced -`GetGCState` path, and the shared exclusion flag. They do not claim that the PR -adds a standalone global command. - -## Implementation sequence - -The implementation follows this order to isolate rebase work from command -behavior changes: - -1. Rebase the branch onto the `upstream/master` commit that contains PR #11117. -2. Resolve the GC client model and conversion conflicts, and run focused client - tests. -3. Update command unit tests for the two-command tree, shared flag, option - mapping, JSON presence contract, and compatibility errors. -4. Refactor the reader, projections, and Cobra commands until the unit tests - pass. -5. Update the real PD safepoint integration test for the complete flag matrix. -6. Update the README, help contract, PR body, and release note. -7. Run formatting, focused client and pd-ctl tests, Classic and NextGen - integration tests, the `pd-ctl` build, and relevant static checks. -8. Confirm failpoints are disabled and the worktree contains no generated or - unrelated files before updating the PR. - -## Acceptance criteria - -The refactor is complete when the implementation meets every observable -contract in this design. - -- `gc-state global` is absent from command registration, implementation, tests, - help, documentation, and PR metadata. -- Default `gc-state keyspace` obtains local and global barriers with one - `GetGCState` call. -- No code calls `GetAllKeyspacesGCStates` solely to obtain global barriers. -- `gc-state all` remains the only full-keyspace enumeration path. -- `--exclude-global-barriers` controls both the RPC option and JSON field in - `keyspace` and `all`. -- A requested empty list encodes as `[]`, while explicit exclusion omits the - field. -- Expired filtering works consistently for every requested barrier type. -- A server that omits requested global barriers produces an actionable error - and supports the explicit degraded view. -- Client conversion preserves keyspace-level GC metadata and optional global - barriers at the same time. -- Focused unit, Classic integration, NextGen integration, build, and static - checks pass. - -## Next steps - -After this design is reviewed, create a detailed implementation plan with -file-level edits, test-first steps, verification commands, and review -checkpoints. Do not change product code before that plan is approved. From 2b3ff09d5ed1c9f14d972efb94928cb8dd9ccbfe Mon Sep 17 00:00:00 2001 From: Wenxuan Zhang Date: Wed, 12 Aug 2026 17:42:22 +0800 Subject: [PATCH 29/31] pd-ctl: make GC state RPC timeout configurable Use a 30-second default and expose --timeout on both GC state commands. This lets full-cluster inspection tolerate large keyspace and barrier sets. Signed-off-by: Wenxuan Zhang --- tools/pd-ctl/README.md | 8 ++ .../pd-ctl/pdctl/command/gc_state_command.go | 70 ++++++++- .../pdctl/command/gc_state_command_test.go | 135 +++++++++++++++++- 3 files changed, 207 insertions(+), 6 deletions(-) diff --git a/tools/pd-ctl/README.md b/tools/pd-ctl/README.md index e8a0bf772b..8aad448f68 100644 --- a/tools/pd-ctl/README.md +++ b/tools/pd-ctl/README.md @@ -110,6 +110,14 @@ and `9223372036854775807` means that a barrier never expires. Because PD rounds remaining TTLs down to whole seconds, a zero TTL can also represent a barrier with less than one second remaining. +GC state RPCs time out after 30 seconds by default. The timeout applies to both +subcommands and must be a positive duration. If a large cluster takes longer to +enumerate, increase it with `--timeout` using Go duration syntax: + +```bash +pd-ctl gc-state all --timeout 2m +``` + Use `--exclude-global-barriers` to skip the global-barrier read and remove the `global_gc_barriers` field from the JSON output. The flag applies to both remaining subcommands: diff --git a/tools/pd-ctl/pdctl/command/gc_state_command.go b/tools/pd-ctl/pdctl/command/gc_state_command.go index e56662e54c..fed4d49b7b 100644 --- a/tools/pd-ctl/pdctl/command/gc_state_command.go +++ b/tools/pd-ctl/pdctl/command/gc_state_command.go @@ -30,6 +30,7 @@ import ( pd "github.com/tikv/pd/client" "github.com/tikv/pd/client/clients/gc" + "github.com/tikv/pd/client/opt" "github.com/tikv/pd/client/pkg/caller" "github.com/tikv/pd/pkg/keyspace/constant" ) @@ -49,9 +50,19 @@ type gcStateReader interface { type gcStateReaderFactory func(*cobra.Command) (gcStateReader, error) +type pdClientFactory func( + ctx context.Context, + callerComponent caller.Component, + svrAddrs []string, + security pd.SecurityOption, + opts ...opt.ClientOption, +) (pd.Client, error) + const ( gcStateIncludeExpiredFlag = "include-expired" gcStateExcludeGlobalBarriersFlag = "exclude-global-barriers" + gcStateTimeoutFlag = "timeout" + defaultGCStateTimeout = 30 * time.Second ) type pdGCStateReader struct { @@ -93,6 +104,13 @@ func (r *pdGCStateReader) close() { } func newPDGCStateReader(cmd *cobra.Command) (gcStateReader, error) { + return newPDGCStateReaderWithClientFactory(cmd, pd.NewClientWithContext) +} + +func newPDGCStateReaderWithClientFactory( + cmd *cobra.Command, + clientFactory pdClientFactory, +) (gcStateReader, error) { caPath, err := cmd.Flags().GetString("cacert") if err != nil { return nil, errors.WithStack(err) @@ -105,7 +123,11 @@ func newPDGCStateReader(cmd *cobra.Command) (gcStateReader, error) { if err != nil { return nil, errors.WithStack(err) } - client, err := pd.NewClientWithContext( + timeout, err := getGCStateTimeout(cmd) + if err != nil { + return nil, err + } + client, err := clientFactory( cmd.Context(), caller.Component(PDControlCallerID), getEndpoints(cmd), @@ -114,6 +136,7 @@ func newPDGCStateReader(cmd *cobra.Command) (gcStateReader, error) { CertPath: certPath, KeyPath: keyPath, }, + opt.WithCustomTimeoutOption(timeout), ) if err != nil { return nil, err @@ -341,6 +364,17 @@ func getGCStateIncludeExpired(cmd *cobra.Command) (bool, error) { return includeExpired, nil } +func getGCStateTimeout(cmd *cobra.Command) (time.Duration, error) { + timeout, err := cmd.Flags().GetDuration(gcStateTimeoutFlag) + if err != nil { + return 0, errors.WithStack(err) + } + if timeout <= 0 { + return 0, errors.New("timeout must be a positive duration") + } + return timeout, nil +} + func getGCStateIncludeGlobalGCBarriers( cmd *cobra.Command, ) (bool, error) { @@ -382,6 +416,11 @@ func buildGCStateCommand(factory gcStateReaderFactory) *cobra.Command { false, "exclude global GC barriers from the PD request and JSON output", ) + command.PersistentFlags().Duration( + gcStateTimeoutFlag, + defaultGCStateTimeout, + "timeout for GC state RPCs", + ) command.AddCommand( newGCStateKeyspaceCommand(factory), newGCStateAllCommand(factory), @@ -406,6 +445,10 @@ func newGCStateKeyspaceCommand(factory gcStateReaderFactory) *cobra.Command { if err != nil { return err } + timeout, err := getGCStateTimeout(cmd) + if err != nil { + return err + } includeExpired, err := getGCStateIncludeExpired(cmd) if err != nil { return err @@ -426,7 +469,13 @@ func newGCStateKeyspaceCommand(factory gcStateReaderFactory) *cobra.Command { includeGlobalGCBarriers, ) if err != nil { - if status.Code(err) == codes.Unimplemented { + switch status.Code(err) { + case codes.DeadlineExceeded: + return errors.Annotatef(err, + "gc-state keyspace timed out after %s; "+ + "retry with a longer --%s", + timeout, gcStateTimeoutFlag) + case codes.Unimplemented: return errors.Annotate(err, "gc-state requires a PD server that supports GetGCState") } @@ -454,9 +503,14 @@ func newGCStateAllCommand(factory gcStateReaderFactory) *cobra.Command { Long: "Show all effective GC scopes and local barriers, with global " + "barriers once at the top level. Use --exclude-global-barriers to " + "omit cluster-wide barriers.", - Example: " pd-ctl gc-state all", - Args: cobra.NoArgs, + Example: " pd-ctl gc-state all\n" + + " pd-ctl gc-state all --timeout 2m", + Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { + timeout, err := getGCStateTimeout(cmd) + if err != nil { + return err + } includeExpired, err := getGCStateIncludeExpired(cmd) if err != nil { return err @@ -476,7 +530,13 @@ func newGCStateAllCommand(factory gcStateReaderFactory) *cobra.Command { includeGlobalGCBarriers, ) if err != nil { - if status.Code(err) == codes.Unimplemented { + switch status.Code(err) { + case codes.DeadlineExceeded: + return errors.Annotatef(err, + "gc-state all timed out after %s; "+ + "retry with a longer --%s", + timeout, gcStateTimeoutFlag) + case codes.Unimplemented: return errors.Annotate(err, "gc-state all requires a PD server that supports "+ "GetAllKeyspacesGCStates") diff --git a/tools/pd-ctl/pdctl/command/gc_state_command_test.go b/tools/pd-ctl/pdctl/command/gc_state_command_test.go index efb3540bb1..d01a68f76d 100644 --- a/tools/pd-ctl/pdctl/command/gc_state_command_test.go +++ b/tools/pd-ctl/pdctl/command/gc_state_command_test.go @@ -32,7 +32,10 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + pd "github.com/tikv/pd/client" "github.com/tikv/pd/client/clients/gc" + "github.com/tikv/pd/client/opt" + "github.com/tikv/pd/client/pkg/caller" "github.com/tikv/pd/pkg/keyspace/constant" ) @@ -47,6 +50,10 @@ type fakeGCStateReader struct { closed bool } +type unusedPDClient struct { + pd.Client +} + func (r *fakeGCStateReader) getGCState( _ context.Context, keyspaceID uint32, @@ -400,6 +407,108 @@ func TestGCStateAPIOptions(t *testing.T) { } } +func TestNewPDGCStateReaderUsesConfiguredTimeout(t *testing.T) { + for _, testCase := range []struct { + name string + flagValue string + wantTimeout time.Duration + }{ + {name: "default", wantTimeout: 30 * time.Second}, + {name: "custom", flagValue: "45s", wantTimeout: 45 * time.Second}, + } { + t.Run(testCase.name, func(t *testing.T) { + wantErr := errors.New("stop after inspecting client options") + clientFactory := func( + _ context.Context, + _ caller.Component, + _ []string, + _ pd.SecurityOption, + clientOptions ...opt.ClientOption, + ) (pd.Client, error) { + options := opt.NewOption() + for _, apply := range clientOptions { + apply(options) + } + require.Equal(t, testCase.wantTimeout, options.Timeout) + return &unusedPDClient{}, wantErr + } + gcStateCommand := buildGCStateCommand(func(cmd *cobra.Command) (gcStateReader, error) { + return newPDGCStateReaderWithClientFactory(cmd, clientFactory) + }) + rootCommand := &cobra.Command{Use: "pd-ctl", SilenceUsage: true} + rootCommand.PersistentFlags().String("pd", "http://127.0.0.1:2379", "") + rootCommand.PersistentFlags().String("cacert", "", "") + rootCommand.PersistentFlags().String("cert", "", "") + rootCommand.PersistentFlags().String("key", "", "") + rootCommand.AddCommand(gcStateCommand) + args := []string{"gc-state", "all"} + if testCase.flagValue != "" { + args = append(args, "--timeout", testCase.flagValue) + } + rootCommand.SetArgs(args) + rootCommand.SetOut(io.Discard) + rootCommand.SetErr(io.Discard) + + err := rootCommand.Execute() + require.ErrorIs(t, err, wantErr) + }) + } +} + +func TestGCStateCommandTimeout(t *testing.T) { + for _, testCase := range []struct { + name string + args []string + wantTimeout time.Duration + }{ + {name: "all-default", args: []string{"all"}, wantTimeout: 30 * time.Second}, + {name: "all-custom", args: []string{"all", "--timeout", "2m"}, wantTimeout: 2 * time.Minute}, + {name: "keyspace-custom", args: []string{"keyspace", "42", "--timeout", "45s"}, wantTimeout: 45 * time.Second}, + } { + t.Run(testCase.name, func(t *testing.T) { + reader := &fakeGCStateReader{ + state: gc.NewGCStateWithGCBarriers(42, 100, 90, nil). + WithGlobalGCBarriers(nil), + clusterState: gc.NewClusterGCStatesWithGlobalGCBarriers( + map[uint32]gc.GCState{}, + nil, + ), + } + var gotTimeout time.Duration + cmd := buildGCStateCommand(func(cmd *cobra.Command) (gcStateReader, error) { + var err error + gotTimeout, err = cmd.Flags().GetDuration(gcStateTimeoutFlag) + return reader, err + }) + cmd.SetOut(io.Discard) + cmd.SetErr(io.Discard) + cmd.SetArgs(testCase.args) + + require.NoError(t, cmd.Execute()) + require.Equal(t, testCase.wantTimeout, gotTimeout) + }) + } +} + +func TestGCStateCommandRejectsNonPositiveTimeout(t *testing.T) { + for _, timeout := range []string{"0s", "-1s"} { + t.Run(timeout, func(t *testing.T) { + factoryCalled := false + cmd := buildGCStateCommand(func(*cobra.Command) (gcStateReader, error) { + factoryCalled = true + return nil, errors.New("factory must not run") + }) + cmd.SetOut(io.Discard) + cmd.SetErr(io.Discard) + cmd.SetArgs([]string{"all", "--timeout", timeout}) + + err := cmd.Execute() + require.EqualError(t, err, "timeout must be a positive duration") + require.False(t, factoryCalled) + }) + } +} + func TestGCStateCommandGlobalBarrierFlag(t *testing.T) { for _, testCase := range []struct { name string @@ -670,6 +779,16 @@ func TestGCStateCommandErrors(t *testing.T) { }, wantMessage: "failed to get GC state for keyspace 42", }, + { + name: "single-rpc-timeout", + args: []string{"keyspace", "42", "--timeout", "45s"}, + factory: func(*cobra.Command) (gcStateReader, error) { + return &fakeGCStateReader{ + err: status.Error(codes.DeadlineExceeded, "deadline exceeded"), + }, nil + }, + wantMessage: "gc-state keyspace timed out after 45s; retry with a longer --timeout", + }, { name: "single-wrapped-unimplemented", args: []string{"keyspace", "42"}, @@ -709,6 +828,16 @@ func TestGCStateCommandErrors(t *testing.T) { }, wantMessage: "failed to get all keyspaces GC states", }, + { + name: "all-rpc-timeout", + args: []string{"all", "--timeout", "2m"}, + factory: func(*cobra.Command) (gcStateReader, error) { + return &fakeGCStateReader{ + err: status.Error(codes.DeadlineExceeded, "deadline exceeded"), + }, nil + }, + wantMessage: "gc-state all timed out after 2m0s; retry with a longer --timeout", + }, { name: "all-wrapped-unimplemented", args: []string{"all"}, @@ -759,6 +888,10 @@ func TestGCStateCommandHelpContract(t *testing.T) { require.NotNil(t, excludeGlobalBarriers) require.Equal(t, "false", excludeGlobalBarriers.DefValue) require.Equal(t, "exclude global GC barriers from the PD request and JSON output", excludeGlobalBarriers.Usage) + timeout := cmd.PersistentFlags().Lookup("timeout") + require.NotNil(t, timeout) + require.Equal(t, "30s", timeout.DefValue) + require.Equal(t, "timeout for GC state RPCs", timeout.Usage) commands := cmd.Commands() require.Len(t, commands, 2) @@ -779,7 +912,7 @@ func TestGCStateCommandHelpContract(t *testing.T) { require.Equal(t, "all", all.Use) require.Equal(t, "show effective GC scopes and cluster-wide GC state", all.Short) require.Equal(t, "Show all effective GC scopes and local barriers, with global barriers once at the top level. Use --exclude-global-barriers to omit cluster-wide barriers.", all.Long) - require.Equal(t, " pd-ctl gc-state all", all.Example) + require.Equal(t, " pd-ctl gc-state all\n pd-ctl gc-state all --timeout 2m", all.Example) } type failingWriter struct{} From a43f633e0a1a6021779ccf1cb55686a004caeac1 Mon Sep 17 00:00:00 2001 From: Wenxuan Zhang Date: Thu, 13 Aug 2026 09:47:12 +0800 Subject: [PATCH 30/31] pd-ctl: address GC state review comments Share local and global barrier output conversion to prevent behavior drift. Update the guide for the final command surface and remove the history-only test. Signed-off-by: Wenxuan Zhang --- tools/pd-ctl/README.md | 43 +++------------ .../pd-ctl/pdctl/command/gc_state_command.go | 55 +++++++++++++------ .../pdctl/command/gc_state_command_test.go | 15 ----- 3 files changed, 47 insertions(+), 66 deletions(-) diff --git a/tools/pd-ctl/README.md b/tools/pd-ctl/README.md index 8aad448f68..805b97e90e 100644 --- a/tools/pd-ctl/README.md +++ b/tools/pd-ctl/README.md @@ -13,15 +13,9 @@ The details about how to use `pd-ctl` can be found in [PD Control User Guide](ht ## GC state troubleshooting -Use `gc-state` to inspect the safe points and barriers that can block GC. The -command is read-only and emits deterministic JSON for scripts and diffs. +Use `gc-state` to inspect the safe points and barriers that can block GC. The command is read-only and emits deterministic JSON for scripts and diffs. -The default `keyspace` and `all` views request global barriers. They omit -barriers that PD returns with a zero TTL because those barriers normally -represent expired barriers awaiting lazy deletion. Add `--include-expired` to -include those barriers in the existing `gc_barriers` or `global_gc_barriers` -array with `ttl_seconds` set to `0`. An empty `global_gc_barriers` array means -PD returned no global barriers. +The default `keyspace` and `all` views request global barriers. They omit barriers that PD returns with a zero TTL because those barriers normally represent expired barriers awaiting lazy deletion. Add `--include-expired` to include those barriers in the existing `gc_barriers` or `global_gc_barriers` array with `ttl_seconds` set to `0`. An empty `global_gc_barriers` array means PD returned no global barriers. For example, inspect one keyspace and include zero-TTL barriers: @@ -29,8 +23,7 @@ For example, inspect one keyspace and include zero-TTL barriers: pd-ctl gc-state keyspace 42 --include-expired ``` -Use `keyspace` when diagnosing one GC scope. Inspect a keyspace by its decimal -ID: +Use `keyspace` when diagnosing one GC scope. Inspect a keyspace by its decimal ID: ```bash pd-ctl gc-state keyspace 42 @@ -60,14 +53,9 @@ pd-ctl gc-state keyspace 42 } ``` -The response contains both `requested_keyspace_id` and -`effective_keyspace_id`. They are equal for keyspace-level GC. A keyspace that -uses unified GC returns `4294967295`, the NullKeyspace ID, as its effective -scope. The local and global barriers come from one `GetGCState` read. The same -global list applies to every keyspace. +The response contains both `requested_keyspace_id` and `effective_keyspace_id`. They are equal for keyspace-level GC. A keyspace that uses unified GC returns `4294967295`, the NullKeyspace ID, as its effective scope. The local and global barriers come from one `GetGCState` read. The same global list applies to every keyspace. -Use `all` only when you need every effective GC scope because it enumerates all -keyspaces. Inspect every effective GC scope together with cluster-wide state: +Use `all` only when you need every effective GC scope because it enumerates all keyspaces. Inspect every effective GC scope together with cluster-wide state: ```bash pd-ctl gc-state all @@ -100,32 +88,19 @@ pd-ctl gc-state all } ``` -The combined response sorts effective `gc_states` by `keyspace_id` and reports -cluster-wide barriers once in the top-level `global_gc_barriers` array. Unified -GC keyspaces share the NullKeyspace scope, so their marker records are not -reported as separate states. The real NullKeyspace state appears once with its -safe points and local barriers. When no local or global barriers exist, the -corresponding arrays are encoded as `[]`. Barrier TTLs use remaining seconds, -and `9223372036854775807` means that a barrier never expires. Because PD rounds -remaining TTLs down to whole seconds, a zero TTL can also represent a barrier -with less than one second remaining. +The combined response sorts effective `gc_states` by `keyspace_id` and reports cluster-wide barriers once in the top-level `global_gc_barriers` array. Unified GC keyspaces share the NullKeyspace scope, so their marker records are not reported as separate states. The real NullKeyspace state appears once with its safe points and local barriers. When no local or global barriers exist, the corresponding arrays are encoded as `[]`. Barrier TTLs use remaining seconds, and `9223372036854775807` means that a barrier never expires. Because PD rounds remaining TTLs down to whole seconds, a zero TTL can also represent a barrier with less than one second remaining. -GC state RPCs time out after 30 seconds by default. The timeout applies to both -subcommands and must be a positive duration. If a large cluster takes longer to -enumerate, increase it with `--timeout` using Go duration syntax: +GC state RPCs time out after 30 seconds by default. The timeout applies to both subcommands and must be a positive duration. If a large cluster takes longer to enumerate, increase it with `--timeout` using Go duration syntax: ```bash pd-ctl gc-state all --timeout 2m ``` -Use `--exclude-global-barriers` to skip the global-barrier read and remove the -`global_gc_barriers` field from the JSON output. The flag applies to both -remaining subcommands: +Use `--exclude-global-barriers` to skip the global-barrier read and remove the `global_gc_barriers` field from the JSON output. The flag applies to both subcommands: ```bash pd-ctl gc-state keyspace 42 --exclude-global-barriers pd-ctl gc-state all --exclude-global-barriers --include-expired ``` -When combined with `--include-expired`, exclusion wins for global barriers, -while expired local barriers remain visible. +When combined with `--include-expired`, exclusion wins for global barriers, while expired local barriers remain visible. diff --git a/tools/pd-ctl/pdctl/command/gc_state_command.go b/tools/pd-ctl/pdctl/command/gc_state_command.go index fed4d49b7b..906819891a 100644 --- a/tools/pd-ctl/pdctl/command/gc_state_command.go +++ b/tools/pd-ctl/pdctl/command/gc_state_command.go @@ -213,36 +213,57 @@ func shouldIncludeGCBarrier(ttl time.Duration, includeExpired bool) bool { return includeExpired || ttl > 0 } -func newLocalGCBarrierOutputs(barriers []*gc.GCBarrierInfo, includeExpired bool) []gcBarrierOutput { +type gcBarrierFields struct { + barrierID string + barrierTS uint64 + ttl time.Duration +} + +func newGCBarrierOutputs[T any]( + barriers []T, + includeExpired bool, + extract func(T) (gcBarrierFields, bool), +) []gcBarrierOutput { result := make([]gcBarrierOutput, 0, len(barriers)) for _, barrier := range barriers { - if barrier == nil || !shouldIncludeGCBarrier(barrier.TTL, includeExpired) { + fields, ok := extract(barrier) + if !ok || !shouldIncludeGCBarrier(fields.ttl, includeExpired) { continue } result = append(result, gcBarrierOutput{ - BarrierID: barrier.BarrierID, - BarrierTS: barrier.BarrierTS, - TTLSeconds: gcStateTTLSeconds(barrier.TTL), + BarrierID: fields.barrierID, + BarrierTS: fields.barrierTS, + TTLSeconds: gcStateTTLSeconds(fields.ttl), }) } sortGCBarrierOutputs(result) return result } +func newLocalGCBarrierOutputs(barriers []*gc.GCBarrierInfo, includeExpired bool) []gcBarrierOutput { + return newGCBarrierOutputs(barriers, includeExpired, func(barrier *gc.GCBarrierInfo) (gcBarrierFields, bool) { + if barrier == nil { + return gcBarrierFields{}, false + } + return gcBarrierFields{ + barrierID: barrier.BarrierID, + barrierTS: barrier.BarrierTS, + ttl: barrier.TTL, + }, true + }) +} + func newGlobalGCBarrierOutputs(barriers []*gc.GlobalGCBarrierInfo, includeExpired bool) []gcBarrierOutput { - result := make([]gcBarrierOutput, 0, len(barriers)) - for _, barrier := range barriers { - if barrier == nil || !shouldIncludeGCBarrier(barrier.TTL, includeExpired) { - continue + return newGCBarrierOutputs(barriers, includeExpired, func(barrier *gc.GlobalGCBarrierInfo) (gcBarrierFields, bool) { + if barrier == nil { + return gcBarrierFields{}, false } - result = append(result, gcBarrierOutput{ - BarrierID: barrier.BarrierID, - BarrierTS: barrier.BarrierTS, - TTLSeconds: gcStateTTLSeconds(barrier.TTL), - }) - } - sortGCBarrierOutputs(result) - return result + return gcBarrierFields{ + barrierID: barrier.BarrierID, + barrierTS: barrier.BarrierTS, + ttl: barrier.TTL, + }, true + }) } func newKeyspaceGCStateOutput( diff --git a/tools/pd-ctl/pdctl/command/gc_state_command_test.go b/tools/pd-ctl/pdctl/command/gc_state_command_test.go index d01a68f76d..bfa12ece59 100644 --- a/tools/pd-ctl/pdctl/command/gc_state_command_test.go +++ b/tools/pd-ctl/pdctl/command/gc_state_command_test.go @@ -549,21 +549,6 @@ func TestGCStateCommandGlobalBarrierFlag(t *testing.T) { } } -func TestGCStateGlobalCommandIsRemoved(t *testing.T) { - factoryCalled := false - cmd := buildGCStateCommand(func(*cobra.Command) (gcStateReader, error) { - factoryCalled = true - return nil, errors.New("factory must not run") - }) - cmd.SetOut(io.Discard) - cmd.SetErr(io.Discard) - cmd.SetArgs([]string{"global"}) - - err := cmd.Execute() - require.ErrorContains(t, err, `unknown command "global"`) - require.False(t, factoryCalled) -} - func TestGCStateKeyspaceCommand(t *testing.T) { state := gc.NewGCStateWithGCBarriers(42, 100, 90, nil). WithGlobalGCBarriers(nil) From c239a4a39fe6262e0b5e11c3daa00f5aba6d04b3 Mon Sep 17 00:00:00 2001 From: Wenxuan Zhang Date: Thu, 13 Aug 2026 11:47:26 +0800 Subject: [PATCH 31/31] pd-ctl: stabilize GC state snapshot tests Signed-off-by: Wenxuan Zhang --- tools/pd-ctl/tests/safepoint/gc_state_test.go | 54 +++++++++++++++++-- 1 file changed, 50 insertions(+), 4 deletions(-) diff --git a/tools/pd-ctl/tests/safepoint/gc_state_test.go b/tools/pd-ctl/tests/safepoint/gc_state_test.go index 207a51a634..acb3912a68 100644 --- a/tools/pd-ctl/tests/safepoint/gc_state_test.go +++ b/tools/pd-ctl/tests/safepoint/gc_state_test.go @@ -89,6 +89,19 @@ func requireGCStateCommandBarriers( } } +func requireGCStateCommandState( + re *require.Assertions, + actual gcStateCommandState, + expected gcStateCommandState, + expectedBarriers []expectedGCStateCommandBarrier, +) { + re.Equal(expected.KeyspaceID, actual.KeyspaceID) + re.Equal(expected.IsKeyspaceLevelGC, actual.IsKeyspaceLevelGC) + re.Equal(expected.TxnSafePoint, actual.TxnSafePoint) + re.Equal(expected.GCSafePoint, actual.GCSafePoint) + requireGCStateCommandBarriers(re, actual.GCBarriers, expectedBarriers) +} + func TestGCState(t *testing.T) { re := require.New(t) ctx := t.Context() @@ -422,10 +435,26 @@ func TestGCState(t *testing.T) { } excludedNullState, ok := excludedStatesByID[constant.NullKeyspaceID] re.True(ok) - re.Equal(nullState, excludedNullState) + requireGCStateCommandState( + re, + excludedNullState, + nullState, + []expectedGCStateCommandBarrier{ + {barrierID: "a-null", barrierTS: 110}, + {barrierID: "z-null", barrierTS: 120, expires: true}, + }, + ) excludedKeyspaceLevelState, ok := excludedStatesByID[keyspaceLevelID] re.True(ok) - re.Equal(keyspaceLevelState, excludedKeyspaceLevelState) + requireGCStateCommandState( + re, + excludedKeyspaceLevelState, + keyspaceLevelState, + []expectedGCStateCommandBarrier{ + {barrierID: "a-local", barrierTS: 210}, + {barrierID: "z-local", barrierTS: 220, expires: true}, + }, + ) output, err = tests.ExecuteCommand( ctl.GetRootCmd(), @@ -452,10 +481,27 @@ func TestGCState(t *testing.T) { } excludedNullStateWithExpired, ok := excludedStatesByIDWithExpired[constant.NullKeyspaceID] re.True(ok) - re.Equal(nullState, excludedNullStateWithExpired) + requireGCStateCommandState( + re, + excludedNullStateWithExpired, + nullState, + []expectedGCStateCommandBarrier{ + {barrierID: "a-null", barrierTS: 110}, + {barrierID: "z-null", barrierTS: 120, expires: true}, + }, + ) excludedKeyspaceLevelStateWithExpired, ok := excludedStatesByIDWithExpired[keyspaceLevelID] re.True(ok) - re.Equal(keyspaceLevelStateWithExpired, excludedKeyspaceLevelStateWithExpired) + requireGCStateCommandState( + re, + excludedKeyspaceLevelStateWithExpired, + keyspaceLevelStateWithExpired, + []expectedGCStateCommandBarrier{ + {barrierID: "a-local", barrierTS: 210}, + {barrierID: "z-local", barrierTS: 220, expires: true}, + {barrierID: "expired-local", barrierTS: 230, expired: true}, + }, + ) if kerneltype.IsNextGen() { systemState, ok := statesByID[constant.SystemKeyspaceID]