diff --git a/client/clients/gc/client.go b/client/clients/gc/client.go index 9149def7e5..c1ed1f7984 100644 --- a/client/clients/gc/client.go +++ b/client/clients/gc/client.go @@ -286,11 +286,14 @@ func (b *GlobalGCBarrierInfo) isExpiredImpl(now time.Time) bool { //nolint:revive type GCState struct { // The ID of the keyspace this GC state belongs to. - KeyspaceID uint32 - TxnSafePoint uint64 - GCSafePoint uint64 - hasGCBarriers bool - gcBarriers []*GCBarrierInfo + KeyspaceID uint32 + // IsKeyspaceLevelGC reports whether this state belongs to an independent + // keyspace-level GC scope. + IsKeyspaceLevelGC bool + TxnSafePoint uint64 + GCSafePoint uint64 + hasGCBarriers bool + gcBarriers []*GCBarrierInfo hasGlobalGCBarriers bool globalGCBarriers []*GlobalGCBarrierInfo diff --git a/client/gc_client.go b/client/gc_client.go index ac8154e1c8..f89b1c98be 100644 --- a/client/gc_client.go +++ b/client/gc_client.go @@ -339,15 +339,18 @@ func pbToGCState(pb *pdpb.GCState, reqStartTime time.Time, excludeGCBarriers boo if pb.KeyspaceScope != nil { keyspaceID = pb.KeyspaceScope.GetKeyspaceId() } + var state gc.GCState if excludeGCBarriers { - return gc.NewGCStateWithoutGCBarriers(keyspaceID, pb.GetTxnSafePoint(), pb.GetGcSafePoint()) - } - - gcBarriers := make([]*gc.GCBarrierInfo, 0, len(pb.GetGcBarriers())) - for _, b := range pb.GetGcBarriers() { - gcBarriers = append(gcBarriers, pbToGCBarrierInfo(b, reqStartTime)) + state = gc.NewGCStateWithoutGCBarriers(keyspaceID, pb.GetTxnSafePoint(), pb.GetGcSafePoint()) + } else { + gcBarriers := make([]*gc.GCBarrierInfo, 0, len(pb.GetGcBarriers())) + for _, b := range pb.GetGcBarriers() { + gcBarriers = append(gcBarriers, pbToGCBarrierInfo(b, reqStartTime)) + } + state = gc.NewGCStateWithGCBarriers(keyspaceID, pb.GetTxnSafePoint(), pb.GetGcSafePoint(), gcBarriers) } - return gc.NewGCStateWithGCBarriers(keyspaceID, pb.GetTxnSafePoint(), pb.GetGcSafePoint(), gcBarriers) + state.IsKeyspaceLevelGC = pb.GetIsKeyspaceLevelGc() + return state } func pbToGCStateWithGlobalGCBarriers( diff --git a/client/gc_client_test.go b/client/gc_client_test.go new file mode 100644 index 0000000000..ee233ef1fa --- /dev/null +++ b/client/gc_client_test.go @@ -0,0 +1,98 @@ +// Copyright 2026 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pd + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/pingcap/kvproto/pkg/pdpb" +) + +func TestPBToGCStatePreservesKeyspaceLevelGC(t *testing.T) { + requestStart := time.Unix(100, 0) + for _, testCase := range []struct { + name string + isKeyspaceLevelGC bool + excludeBarriers bool + }{ + {name: "keyspace-level-with-barriers", isKeyspaceLevelGC: true}, + { + name: "keyspace-level-without-barriers", + isKeyspaceLevelGC: true, + excludeBarriers: true, + }, + {name: "unified-with-barriers", isKeyspaceLevelGC: false}, + { + name: "unified-without-barriers", + isKeyspaceLevelGC: false, + excludeBarriers: true, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + pbState := &pdpb.GCState{ + KeyspaceScope: wrapKeyspaceScope(42), + IsKeyspaceLevelGc: testCase.isKeyspaceLevelGC, + TxnSafePoint: 100, + GcSafePoint: 90, + GcBarriers: []*pdpb.GCBarrierInfo{ + {BarrierId: "backup", BarrierTs: 95, TtlSeconds: 60}, + }, + } + + state := pbToGCState( + pbState, + requestStart, + testCase.excludeBarriers, + ) + require.Equal(t, testCase.isKeyspaceLevelGC, + state.IsKeyspaceLevelGC) + require.Equal(t, uint32(42), state.KeyspaceID) + require.Equal(t, uint64(100), state.TxnSafePoint) + require.Equal(t, uint64(90), state.GCSafePoint) + require.Equal(t, !testCase.excludeBarriers, + state.HasGCBarriers()) + }) + } +} + +func TestPBToGCStateWithGlobalBarriersPreservesKeyspaceLevelGC(t *testing.T) { + requestStart := time.Unix(100, 0) + state := pbToGCStateWithGlobalGCBarriers( + &pdpb.GCState{ + KeyspaceScope: wrapKeyspaceScope(42), + IsKeyspaceLevelGc: true, + TxnSafePoint: 100, + GcSafePoint: 90, + }, + &pdpb.GlobalGCBarriersInfo{ + Barriers: []*pdpb.GlobalGCBarrierInfo{ + {BarrierId: "snapshot", BarrierTs: 95, TtlSeconds: 60}, + }, + }, + requestStart, + true, + ) + + require.True(t, state.IsKeyspaceLevelGC) + require.False(t, state.HasGCBarriers()) + require.True(t, state.HasGlobalGCBarriers()) + barriers, err := state.GetGlobalGCBarriers() + require.NoError(t, err) + require.Len(t, barriers, 1) + require.Equal(t, "snapshot", barriers[0].BarrierID) +} diff --git a/pkg/storage/endpoint/cluster_id_test.go b/pkg/storage/endpoint/cluster_id_test.go index 23e840432a..3eb1496ebe 100644 --- a/pkg/storage/endpoint/cluster_id_test.go +++ b/pkg/storage/endpoint/cluster_id_test.go @@ -30,6 +30,7 @@ func TestMain(m *testing.M) { } func TestInitClusterID(t *testing.T) { + t.Cleanup(keypath.ResetClusterID) re := require.New(t) _, client, clean := etcdutil.NewTestEtcdCluster(t, 1, nil) defer clean() diff --git a/tests/integrations/client/client_test.go b/tests/integrations/client/client_test.go index 97e5f3aeeb..9ff5807ea2 100644 --- a/tests/integrations/client/client_test.go +++ b/tests/integrations/client/client_test.go @@ -2872,9 +2872,16 @@ func (s *clientStatefulTestSuite) TestGetAllKeyspaceGCStates() { re.NoError(err) res, err = cli.GetAllKeyspacesGCStates(ctx, gc.ExcludeGCBarriers(false), gc.ExcludeGlobalGCBarriers(false)) re.NoError(err) - state, ok = res.GCStates[2] + state1, ok := res.GCStates[1] re.True(ok) - gcBarriers, err = state.GetGCBarriers() + re.True(state1.IsKeyspaceLevelGC) + state2, ok := res.GCStates[2] + re.True(ok) + re.True(state2.IsKeyspaceLevelGC) + state3, ok := res.GCStates[3] + re.True(ok) + re.False(state3.IsKeyspaceLevelGC) + gcBarriers, err = state2.GetGCBarriers() re.NoError(err) re.Equal("b4", gcBarriers[0].BarrierID) re.Equal(uint64(14), gcBarriers[0].BarrierTS) diff --git a/tools/pd-ctl/README.md b/tools/pd-ctl/README.md index d17cc2621c..805b97e90e 100644 --- a/tools/pd-ctl/README.md +++ b/tools/pd-ctl/README.md @@ -10,3 +10,97 @@ ## Usage The details about how to use `pd-ctl` can be found in [PD Control User Guide](https://docs.pingcap.com/tidb/dev/pd-control). + +## GC state troubleshooting + +Use `gc-state` to inspect the safe points and barriers that can block GC. The command is read-only and emits deterministic JSON for scripts and diffs. + +The default `keyspace` and `all` views request global barriers. They omit barriers that PD returns with a zero TTL because those barriers normally represent expired barriers awaiting lazy deletion. Add `--include-expired` to include those barriers in the existing `gc_barriers` or `global_gc_barriers` array with `ttl_seconds` set to `0`. An empty `global_gc_barriers` array means PD returned no global barriers. + +For example, inspect one keyspace and include zero-TTL barriers: + +```bash +pd-ctl gc-state keyspace 42 --include-expired +``` + +Use `keyspace` when diagnosing one GC scope. Inspect a keyspace by its decimal ID: + +```bash +pd-ctl gc-state keyspace 42 +``` + +```json +{ + "requested_keyspace_id": 42, + "effective_keyspace_id": 4294967295, + "is_keyspace_level_gc": false, + "txn_safe_point": 465000000000000000, + "gc_safe_point": 464900000000000000, + "gc_barriers": [ + { + "barrier_id": "br", + "barrier_ts": 464950000000000000, + "ttl_seconds": 3600 + } + ], + "global_gc_barriers": [ + { + "barrier_id": "native_br", + "barrier_ts": 464940000000000000, + "ttl_seconds": 9223372036854775807 + } + ] +} +``` + +The response contains both `requested_keyspace_id` and `effective_keyspace_id`. They are equal for keyspace-level GC. A keyspace that uses unified GC returns `4294967295`, the NullKeyspace ID, as its effective scope. The local and global barriers come from one `GetGCState` read. The same global list applies to every keyspace. + +Use `all` only when you need every effective GC scope because it enumerates all keyspaces. Inspect every effective GC scope together with cluster-wide state: + +```bash +pd-ctl gc-state all +``` + +```json +{ + "gc_states": [ + { + "keyspace_id": 4294967295, + "is_keyspace_level_gc": false, + "txn_safe_point": 465000000000000000, + "gc_safe_point": 464900000000000000, + "gc_barriers": [ + { + "barrier_id": "br", + "barrier_ts": 464950000000000000, + "ttl_seconds": 3600 + } + ] + } + ], + "global_gc_barriers": [ + { + "barrier_id": "native_br", + "barrier_ts": 464940000000000000, + "ttl_seconds": 9223372036854775807 + } + ] +} +``` + +The combined response sorts effective `gc_states` by `keyspace_id` and reports cluster-wide barriers once in the top-level `global_gc_barriers` array. Unified GC keyspaces share the NullKeyspace scope, so their marker records are not reported as separate states. The real NullKeyspace state appears once with its safe points and local barriers. When no local or global barriers exist, the corresponding arrays are encoded as `[]`. Barrier TTLs use remaining seconds, and `9223372036854775807` means that a barrier never expires. Because PD rounds remaining TTLs down to whole seconds, a zero TTL can also represent a barrier with less than one second remaining. + +GC state RPCs time out after 30 seconds by default. The timeout applies to both subcommands and must be a positive duration. If a large cluster takes longer to enumerate, increase it with `--timeout` using Go duration syntax: + +```bash +pd-ctl gc-state all --timeout 2m +``` + +Use `--exclude-global-barriers` to skip the global-barrier read and remove the `global_gc_barriers` field from the JSON output. The flag applies to both subcommands: + +```bash +pd-ctl gc-state keyspace 42 --exclude-global-barriers +pd-ctl gc-state all --exclude-global-barriers --include-expired +``` + +When combined with `--include-expired`, exclusion wins for global barriers, while expired local barriers remain visible. diff --git a/tools/pd-ctl/pdctl/command/gc_state_command.go b/tools/pd-ctl/pdctl/command/gc_state_command.go new file mode 100644 index 0000000000..906819891a --- /dev/null +++ b/tools/pd-ctl/pdctl/command/gc_state_command.go @@ -0,0 +1,590 @@ +// Copyright 2026 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package command + +import ( + "context" + "encoding/json" + "math" + "sort" + "strconv" + "time" + + "github.com/spf13/cobra" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/pingcap/errors" + + pd "github.com/tikv/pd/client" + "github.com/tikv/pd/client/clients/gc" + "github.com/tikv/pd/client/opt" + "github.com/tikv/pd/client/pkg/caller" + "github.com/tikv/pd/pkg/keyspace/constant" +) + +type gcStateReader interface { + getGCState( + ctx context.Context, + keyspaceID uint32, + includeGlobalGCBarriers bool, + ) (gc.GCState, error) + getAllKeyspacesGCStates( + ctx context.Context, + includeGlobalGCBarriers bool, + ) (gc.ClusterGCStates, error) + close() +} + +type gcStateReaderFactory func(*cobra.Command) (gcStateReader, error) + +type pdClientFactory func( + ctx context.Context, + callerComponent caller.Component, + svrAddrs []string, + security pd.SecurityOption, + opts ...opt.ClientOption, +) (pd.Client, error) + +const ( + gcStateIncludeExpiredFlag = "include-expired" + gcStateExcludeGlobalBarriersFlag = "exclude-global-barriers" + gcStateTimeoutFlag = "timeout" + defaultGCStateTimeout = 30 * time.Second +) + +type pdGCStateReader struct { + client pd.Client +} + +func (r *pdGCStateReader) getGCState( + ctx context.Context, + keyspaceID uint32, + includeGlobalGCBarriers bool, +) (gc.GCState, error) { + return r.client.GetGCStatesClient(keyspaceID).GetGCState( + ctx, + gcStateAPIOptions(includeGlobalGCBarriers)..., + ) +} + +func (r *pdGCStateReader) getAllKeyspacesGCStates( + ctx context.Context, + includeGlobalGCBarriers bool, +) (gc.ClusterGCStates, error) { + return r.client.GetGCStatesClient(constant.NullKeyspaceID).GetAllKeyspacesGCStates( + ctx, + gcStateAPIOptions(includeGlobalGCBarriers)..., + ) +} + +func gcStateAPIOptions( + includeGlobalGCBarriers bool, +) []gc.GCStatesAPIOption { + return []gc.GCStatesAPIOption{ + gc.ExcludeGCBarriers(false), + gc.ExcludeGlobalGCBarriers(!includeGlobalGCBarriers), + } +} + +func (r *pdGCStateReader) close() { + r.client.Close() +} + +func newPDGCStateReader(cmd *cobra.Command) (gcStateReader, error) { + return newPDGCStateReaderWithClientFactory(cmd, pd.NewClientWithContext) +} + +func newPDGCStateReaderWithClientFactory( + cmd *cobra.Command, + clientFactory pdClientFactory, +) (gcStateReader, error) { + caPath, err := cmd.Flags().GetString("cacert") + if err != nil { + return nil, errors.WithStack(err) + } + certPath, err := cmd.Flags().GetString("cert") + if err != nil { + return nil, errors.WithStack(err) + } + keyPath, err := cmd.Flags().GetString("key") + if err != nil { + return nil, errors.WithStack(err) + } + timeout, err := getGCStateTimeout(cmd) + if err != nil { + return nil, err + } + client, err := clientFactory( + cmd.Context(), + caller.Component(PDControlCallerID), + getEndpoints(cmd), + pd.SecurityOption{ + CAPath: caPath, + CertPath: certPath, + KeyPath: keyPath, + }, + opt.WithCustomTimeoutOption(timeout), + ) + if err != nil { + return nil, err + } + return &pdGCStateReader{client: client}, nil +} + +type gcBarrierOutput struct { + BarrierID string `json:"barrier_id"` + BarrierTS uint64 `json:"barrier_ts"` + TTLSeconds int64 `json:"ttl_seconds"` +} + +type gcStateOutput struct { + KeyspaceID uint32 `json:"keyspace_id"` + IsKeyspaceLevelGC bool `json:"is_keyspace_level_gc"` + TxnSafePoint uint64 `json:"txn_safe_point"` + GCSafePoint uint64 `json:"gc_safe_point"` + GCBarriers []gcBarrierOutput `json:"gc_barriers"` +} + +type keyspaceGCStateOutput struct { + RequestedKeyspaceID uint32 `json:"requested_keyspace_id"` + EffectiveKeyspaceID uint32 `json:"effective_keyspace_id"` + IsKeyspaceLevelGC bool `json:"is_keyspace_level_gc"` + TxnSafePoint uint64 `json:"txn_safe_point"` + GCSafePoint uint64 `json:"gc_safe_point"` + GCBarriers []gcBarrierOutput `json:"gc_barriers"` + GlobalGCBarriers *[]gcBarrierOutput `json:"global_gc_barriers,omitempty"` +} + +type allGCStatesOutput struct { + GCStates []gcStateOutput `json:"gc_states"` + GlobalGCBarriers *[]gcBarrierOutput `json:"global_gc_barriers,omitempty"` +} + +func parseGCStateKeyspaceID(value string) (uint32, error) { + parsed, err := strconv.ParseUint(value, 10, 32) + if err != nil { + return 0, errors.Annotatef(err, "invalid keyspace ID %q", value) + } + keyspaceID := uint32(parsed) + if keyspaceID > constant.MaxValidKeyspaceID && keyspaceID != constant.NullKeyspaceID { + return 0, errors.Errorf( + "invalid keyspace ID %q: expected 0 through %d or %d", + value, + constant.MaxValidKeyspaceID, + constant.NullKeyspaceID, + ) + } + return keyspaceID, nil +} + +func gcStateTTLSeconds(ttl time.Duration) int64 { + if ttl == gc.TTLNeverExpire { + return math.MaxInt64 + } + return int64(ttl / time.Second) +} + +func sortGCBarrierOutputs(barriers []gcBarrierOutput) { + sort.Slice(barriers, func(i, j int) bool { + if barriers[i].BarrierTS != barriers[j].BarrierTS { + return barriers[i].BarrierTS < barriers[j].BarrierTS + } + return barriers[i].BarrierID < barriers[j].BarrierID + }) +} + +func shouldIncludeGCBarrier(ttl time.Duration, includeExpired bool) bool { + // Expired barriers are lazily deleted and may still be returned by PD with + // a zero TTL. They no longer block GC, so omit them from the effective view + // unless the caller explicitly requests the persisted entries. + return includeExpired || ttl > 0 +} + +type gcBarrierFields struct { + barrierID string + barrierTS uint64 + ttl time.Duration +} + +func newGCBarrierOutputs[T any]( + barriers []T, + includeExpired bool, + extract func(T) (gcBarrierFields, bool), +) []gcBarrierOutput { + result := make([]gcBarrierOutput, 0, len(barriers)) + for _, barrier := range barriers { + fields, ok := extract(barrier) + if !ok || !shouldIncludeGCBarrier(fields.ttl, includeExpired) { + continue + } + result = append(result, gcBarrierOutput{ + BarrierID: fields.barrierID, + BarrierTS: fields.barrierTS, + TTLSeconds: gcStateTTLSeconds(fields.ttl), + }) + } + sortGCBarrierOutputs(result) + return result +} + +func newLocalGCBarrierOutputs(barriers []*gc.GCBarrierInfo, includeExpired bool) []gcBarrierOutput { + return newGCBarrierOutputs(barriers, includeExpired, func(barrier *gc.GCBarrierInfo) (gcBarrierFields, bool) { + if barrier == nil { + return gcBarrierFields{}, false + } + return gcBarrierFields{ + barrierID: barrier.BarrierID, + barrierTS: barrier.BarrierTS, + ttl: barrier.TTL, + }, true + }) +} + +func newGlobalGCBarrierOutputs(barriers []*gc.GlobalGCBarrierInfo, includeExpired bool) []gcBarrierOutput { + return newGCBarrierOutputs(barriers, includeExpired, func(barrier *gc.GlobalGCBarrierInfo) (gcBarrierFields, bool) { + if barrier == nil { + return gcBarrierFields{}, false + } + return gcBarrierFields{ + barrierID: barrier.BarrierID, + barrierTS: barrier.BarrierTS, + ttl: barrier.TTL, + }, true + }) +} + +func newKeyspaceGCStateOutput( + requestedKeyspaceID uint32, + state gc.GCState, + includeExpired bool, + includeGlobalGCBarriers bool, +) (keyspaceGCStateOutput, error) { + barriers, err := state.GetGCBarriers() + if err != nil { + return keyspaceGCStateOutput{}, errors.Annotatef( + err, + "failed to read GC barriers for keyspace %d", + requestedKeyspaceID, + ) + } + + var globalOutput *[]gcBarrierOutput + if includeGlobalGCBarriers { + if !state.HasGlobalGCBarriers() { + return keyspaceGCStateOutput{}, errors.Errorf( + "gc-state keyspace requires a PD server whose GetGCState " + + "supports global GC barriers; retry with " + + "--exclude-global-barriers", + ) + } + globalBarriers, err := state.GetGlobalGCBarriers() + if err != nil { + return keyspaceGCStateOutput{}, errors.WithStack(err) + } + converted := newGlobalGCBarrierOutputs( + globalBarriers, + includeExpired, + ) + globalOutput = &converted + } + + return keyspaceGCStateOutput{ + RequestedKeyspaceID: requestedKeyspaceID, + EffectiveKeyspaceID: state.KeyspaceID, + IsKeyspaceLevelGC: state.IsKeyspaceLevelGC, + TxnSafePoint: state.TxnSafePoint, + GCSafePoint: state.GCSafePoint, + GCBarriers: newLocalGCBarrierOutputs( + barriers, + includeExpired, + ), + GlobalGCBarriers: globalOutput, + }, nil +} + +func newGCStateOutput(state gc.GCState, includeExpired bool) (gcStateOutput, error) { + barriers, err := state.GetGCBarriers() + if err != nil { + return gcStateOutput{}, errors.Annotatef( + err, + "failed to read GC barriers for keyspace %d", + state.KeyspaceID, + ) + } + return gcStateOutput{ + KeyspaceID: state.KeyspaceID, + IsKeyspaceLevelGC: state.IsKeyspaceLevelGC, + TxnSafePoint: state.TxnSafePoint, + GCSafePoint: state.GCSafePoint, + GCBarriers: newLocalGCBarrierOutputs(barriers, includeExpired), + }, nil +} + +func newAllGCStatesOutput( + clusterState gc.ClusterGCStates, + includeExpired bool, + includeGlobalGCBarriers bool, +) (allGCStatesOutput, error) { + states := make([]gcStateOutput, 0, len(clusterState.GCStates)) + for _, state := range clusterState.GCStates { + // Unified-GC keyspaces have marker entries, but their GC state is owned by + // the NullKeyspace scope and must not be presented as a separate scope. + if !state.IsKeyspaceLevelGC && state.KeyspaceID != constant.NullKeyspaceID { + continue + } + converted, err := newGCStateOutput(state, includeExpired) + if err != nil { + return allGCStatesOutput{}, err + } + states = append(states, converted) + } + sort.Slice(states, func(i, j int) bool { + return states[i].KeyspaceID < states[j].KeyspaceID + }) + + var globalOutput *[]gcBarrierOutput + if includeGlobalGCBarriers { + if !clusterState.HasGlobalGCBarriers() { + return allGCStatesOutput{}, errors.New( + "gc-state all response does not include global GC barriers; " + + "retry with --exclude-global-barriers", + ) + } + globalBarriers, err := clusterState.GetGlobalGCBarriers() + if err != nil { + return allGCStatesOutput{}, errors.WithStack(err) + } + converted := newGlobalGCBarrierOutputs(globalBarriers, includeExpired) + globalOutput = &converted + } + + return allGCStatesOutput{ + GCStates: states, + GlobalGCBarriers: globalOutput, + }, nil +} + +func getGCStateIncludeExpired(cmd *cobra.Command) (bool, error) { + includeExpired, err := cmd.Flags().GetBool(gcStateIncludeExpiredFlag) + if err != nil { + return false, errors.WithStack(err) + } + return includeExpired, nil +} + +func getGCStateTimeout(cmd *cobra.Command) (time.Duration, error) { + timeout, err := cmd.Flags().GetDuration(gcStateTimeoutFlag) + if err != nil { + return 0, errors.WithStack(err) + } + if timeout <= 0 { + return 0, errors.New("timeout must be a positive duration") + } + return timeout, nil +} + +func getGCStateIncludeGlobalGCBarriers( + cmd *cobra.Command, +) (bool, error) { + excludeGlobalGCBarriers, err := cmd.Flags().GetBool( + gcStateExcludeGlobalBarriersFlag, + ) + if err != nil { + return false, errors.WithStack(err) + } + return !excludeGlobalGCBarriers, nil +} + +// NewGCStateCommand returns the read-only GC state command. +func NewGCStateCommand() *cobra.Command { + return buildGCStateCommand(newPDGCStateReader) +} + +func buildGCStateCommand(factory gcStateReaderFactory) *cobra.Command { + command := &cobra.Command{ + Use: "gc-state", + Short: "show keyspace and cluster-wide GC state", + Long: "Show effective per-keyspace GC safe points and local and global " + + "barriers. Expired barriers awaiting lazy deletion are hidden by " + + "default; use --include-expired to include zero-TTL barriers returned " + + "by PD. Use keyspace for one effective GC scope or all for every " + + "effective GC scope.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + return cmd.Help() + }, + } + command.PersistentFlags().Bool( + gcStateIncludeExpiredFlag, + false, + "include zero-TTL barriers returned by PD, which normally represent expired barriers awaiting lazy deletion", + ) + command.PersistentFlags().Bool( + gcStateExcludeGlobalBarriersFlag, + false, + "exclude global GC barriers from the PD request and JSON output", + ) + command.PersistentFlags().Duration( + gcStateTimeoutFlag, + defaultGCStateTimeout, + "timeout for GC state RPCs", + ) + command.AddCommand( + newGCStateKeyspaceCommand(factory), + newGCStateAllCommand(factory), + ) + return command +} + +func newGCStateKeyspaceCommand(factory gcStateReaderFactory) *cobra.Command { + nullKeyspaceID := strconv.FormatUint(uint64(constant.NullKeyspaceID), 10) + return &cobra.Command{ + Use: "keyspace ", + Short: "show one keyspace's effective GC state", + Long: "Show one keyspace's effective GC safe points and local and global barriers. " + + "Use --exclude-global-barriers to omit cluster-wide barriers. " + + "Use gc-state all to inspect every effective GC scope. " + + "The decimal NullKeyspace ID is " + nullKeyspaceID + ".", + Example: " pd-ctl gc-state keyspace 42\n" + + " pd-ctl gc-state keyspace " + nullKeyspaceID, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + keyspaceID, err := parseGCStateKeyspaceID(args[0]) + if err != nil { + return err + } + timeout, err := getGCStateTimeout(cmd) + if err != nil { + return err + } + includeExpired, err := getGCStateIncludeExpired(cmd) + if err != nil { + return err + } + includeGlobalGCBarriers, err := getGCStateIncludeGlobalGCBarriers(cmd) + if err != nil { + return err + } + reader, err := factory(cmd) + if err != nil { + return errors.Annotate(err, "failed to create PD RPC client") + } + defer reader.close() + + state, err := reader.getGCState( + cmd.Context(), + keyspaceID, + includeGlobalGCBarriers, + ) + if err != nil { + switch status.Code(err) { + case codes.DeadlineExceeded: + return errors.Annotatef(err, + "gc-state keyspace timed out after %s; "+ + "retry with a longer --%s", + timeout, gcStateTimeoutFlag) + case codes.Unimplemented: + return errors.Annotate(err, + "gc-state requires a PD server that supports GetGCState") + } + return errors.Annotatef(err, + "failed to get GC state for keyspace %d", keyspaceID) + } + output, err := newKeyspaceGCStateOutput( + keyspaceID, + state, + includeExpired, + includeGlobalGCBarriers, + ) + if err != nil { + return err + } + return writeGCStateJSON(cmd, output) + }, + } +} + +func newGCStateAllCommand(factory gcStateReaderFactory) *cobra.Command { + return &cobra.Command{ + Use: "all", + Short: "show effective GC scopes and cluster-wide GC state", + Long: "Show all effective GC scopes and local barriers, with global " + + "barriers once at the top level. Use --exclude-global-barriers to " + + "omit cluster-wide barriers.", + Example: " pd-ctl gc-state all\n" + + " pd-ctl gc-state all --timeout 2m", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + timeout, err := getGCStateTimeout(cmd) + if err != nil { + return err + } + includeExpired, err := getGCStateIncludeExpired(cmd) + if err != nil { + return err + } + includeGlobalGCBarriers, err := getGCStateIncludeGlobalGCBarriers(cmd) + if err != nil { + return err + } + reader, err := factory(cmd) + if err != nil { + return errors.Annotate(err, "failed to create PD RPC client") + } + defer reader.close() + + clusterState, err := reader.getAllKeyspacesGCStates( + cmd.Context(), + includeGlobalGCBarriers, + ) + if err != nil { + switch status.Code(err) { + case codes.DeadlineExceeded: + return errors.Annotatef(err, + "gc-state all timed out after %s; "+ + "retry with a longer --%s", + timeout, gcStateTimeoutFlag) + case codes.Unimplemented: + return errors.Annotate(err, + "gc-state all requires a PD server that supports "+ + "GetAllKeyspacesGCStates") + } + return errors.Annotate(err, "failed to get all keyspaces GC states") + } + output, err := newAllGCStatesOutput( + clusterState, + includeExpired, + includeGlobalGCBarriers, + ) + if err != nil { + return err + } + return writeGCStateJSON(cmd, output) + }, + } +} + +func writeGCStateJSON(cmd *cobra.Command, value any) error { + data, err := json.MarshalIndent(value, "", " ") + if err != nil { + return errors.Annotate(err, "failed to marshal GC state JSON") + } + data = append(data, '\n') + if _, err := cmd.OutOrStdout().Write(data); err != nil { + return errors.Annotate(err, "failed to write GC state JSON") + } + return nil +} diff --git a/tools/pd-ctl/pdctl/command/gc_state_command_test.go b/tools/pd-ctl/pdctl/command/gc_state_command_test.go new file mode 100644 index 0000000000..bfa12ece59 --- /dev/null +++ b/tools/pd-ctl/pdctl/command/gc_state_command_test.go @@ -0,0 +1,923 @@ +// Copyright 2026 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package command + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "strconv" + "strings" + "testing" + "time" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + pd "github.com/tikv/pd/client" + "github.com/tikv/pd/client/clients/gc" + "github.com/tikv/pd/client/opt" + "github.com/tikv/pd/client/pkg/caller" + "github.com/tikv/pd/pkg/keyspace/constant" +) + +type fakeGCStateReader struct { + state gc.GCState + clusterState gc.ClusterGCStates + err error + requestedID uint32 + includeGlobalGCBarriers bool + getStateCalls int + getAllCalls int + closed bool +} + +type unusedPDClient struct { + pd.Client +} + +func (r *fakeGCStateReader) getGCState( + _ context.Context, + keyspaceID uint32, + includeGlobalGCBarriers bool, +) (gc.GCState, error) { + r.requestedID = keyspaceID + r.includeGlobalGCBarriers = includeGlobalGCBarriers + r.getStateCalls++ + return r.state, r.err +} + +func (r *fakeGCStateReader) getAllKeyspacesGCStates( + _ context.Context, + includeGlobalGCBarriers bool, +) (gc.ClusterGCStates, error) { + r.includeGlobalGCBarriers = includeGlobalGCBarriers + r.getAllCalls++ + return r.clusterState, r.err +} + +func (r *fakeGCStateReader) close() { + r.closed = true +} + +func TestParseGCStateKeyspaceID(t *testing.T) { + for _, testCase := range []struct { + name string + input string + want uint32 + wantErr bool + }{ + {name: "zero", input: "0", want: 0}, + {name: "maximum-normal", input: "16777215", want: 16777215}, + {name: "null-keyspace", input: "4294967295", want: 4294967295}, + {name: "empty", input: "", wantErr: true}, + {name: "negative", input: "-1", wantErr: true}, + {name: "text", input: "tenant-a", wantErr: true}, + {name: "hexadecimal", input: "0xffffff", wantErr: true}, + {name: "first-invalid", input: "16777216", wantErr: true}, + {name: "null-synonym", input: "4294967294", wantErr: true}, + {name: "uint32-overflow", input: "4294967296", wantErr: true}, + } { + t.Run(testCase.name, func(t *testing.T) { + got, err := parseGCStateKeyspaceID(testCase.input) + if testCase.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + require.Equal(t, testCase.want, got) + }) + } +} + +func TestNewKeyspaceGCStateOutput(t *testing.T) { + state := gc.NewGCStateWithGCBarriers( + constant.NullKeyspaceID, + 100, + 90, + []*gc.GCBarrierInfo{ + gc.NewGCBarrierInfo("z-backup", 110, time.Hour, time.Time{}), + gc.NewGCBarrierInfo("b-backup", 105, gc.TTLNeverExpire, time.Time{}), + gc.NewGCBarrierInfo("a-backup", 110, 30*time.Second, time.Time{}), + }, + ).WithGlobalGCBarriers(nil) + state.IsKeyspaceLevelGC = false + + got, err := newKeyspaceGCStateOutput(42, state, false, true) + require.NoError(t, err) + require.Equal(t, uint32(42), got.RequestedKeyspaceID) + require.Equal(t, constant.NullKeyspaceID, got.EffectiveKeyspaceID) + require.False(t, got.IsKeyspaceLevelGC) + require.Equal(t, uint64(100), got.TxnSafePoint) + require.Equal(t, uint64(90), got.GCSafePoint) + require.Equal(t, []gcBarrierOutput{ + {BarrierID: "b-backup", BarrierTS: 105, TTLSeconds: math.MaxInt64}, + {BarrierID: "a-backup", BarrierTS: 110, TTLSeconds: 30}, + {BarrierID: "z-backup", BarrierTS: 110, TTLSeconds: 3600}, + }, got.GCBarriers) + require.NotNil(t, got.GlobalGCBarriers) + require.Empty(t, *got.GlobalGCBarriers) +} + +func TestNewKeyspaceGCStateOutputGlobalBarrierPresence(t *testing.T) { + t.Run("requested-empty", func(t *testing.T) { + state := gc.NewGCStateWithGCBarriers(42, 100, 90, nil). + WithGlobalGCBarriers(nil) + got, err := newKeyspaceGCStateOutput(42, state, false, true) + require.NoError(t, err) + require.NotNil(t, got.GlobalGCBarriers) + require.Empty(t, *got.GlobalGCBarriers) + encoded, err := json.Marshal(got) + require.NoError(t, err) + require.Contains(t, string(encoded), `"global_gc_barriers":[]`) + }) + + t.Run("excluded", func(t *testing.T) { + state := gc.NewGCStateWithGCBarriers(42, 100, 90, nil) + got, err := newKeyspaceGCStateOutput(42, state, false, false) + require.NoError(t, err) + require.Nil(t, got.GlobalGCBarriers) + encoded, err := json.Marshal(got) + require.NoError(t, err) + require.NotContains(t, string(encoded), "global_gc_barriers") + }) + + t.Run("requested-missing", func(t *testing.T) { + state := gc.NewGCStateWithGCBarriers(42, 100, 90, nil) + _, err := newKeyspaceGCStateOutput(42, state, false, true) + require.ErrorContains(t, err, + "retry with --exclude-global-barriers") + }) +} + +func TestNewLocalGCBarrierOutputsSkipNilEntries(t *testing.T) { + got := newLocalGCBarrierOutputs([]*gc.GCBarrierInfo{ + nil, + gc.NewGCBarrierInfo("valid-local", 42, 30*time.Second, time.Time{}), + }, false) + + require.Equal(t, []gcBarrierOutput{ + {BarrierID: "valid-local", BarrierTS: 42, TTLSeconds: 30}, + }, got) +} + +func TestNewGlobalGCBarrierOutputsSkipNilEntries(t *testing.T) { + got := newGlobalGCBarrierOutputs([]*gc.GlobalGCBarrierInfo{ + nil, + gc.NewGlobalGCBarrierInfo("valid-global", 84, time.Minute, time.Time{}), + }, false) + + require.Equal(t, []gcBarrierOutput{ + {BarrierID: "valid-global", BarrierTS: 84, TTLSeconds: 60}, + }, got) +} + +func TestNewAllGCStatesOutputSortsAndKeepsEmptyArrays(t *testing.T) { + empty := gc.NewGCStateWithGCBarriers(1, 20, 10, nil) + empty.IsKeyspaceLevelGC = true + nullState := gc.NewGCStateWithGCBarriers( + constant.NullKeyspaceID, + 40, + 30, + []*gc.GCBarrierInfo{ + gc.NewGCBarrierInfo("local", 45, gc.TTLNeverExpire, time.Time{}), + }, + ) + + clusterState := gc.NewClusterGCStatesWithGlobalGCBarriers( + map[uint32]gc.GCState{ + constant.NullKeyspaceID: nullState, + 1: empty, + }, + []*gc.GlobalGCBarrierInfo{ + gc.NewGlobalGCBarrierInfo("z-global", 60, time.Minute, time.Time{}), + gc.NewGlobalGCBarrierInfo("a-global", 60, gc.TTLNeverExpire, time.Time{}), + gc.NewGlobalGCBarrierInfo("first-global", 50, time.Second, time.Time{}), + }, + ) + + got, err := newAllGCStatesOutput(clusterState, false, true) + require.NoError(t, err) + require.Equal(t, []uint32{1, constant.NullKeyspaceID}, []uint32{ + got.GCStates[0].KeyspaceID, + got.GCStates[1].KeyspaceID, + }) + require.NotNil(t, got.GCStates[0].GCBarriers) + require.Empty(t, got.GCStates[0].GCBarriers) + require.NotNil(t, got.GlobalGCBarriers) + require.Equal(t, []string{"first-global", "a-global", "z-global"}, []string{ + (*got.GlobalGCBarriers)[0].BarrierID, + (*got.GlobalGCBarriers)[1].BarrierID, + (*got.GlobalGCBarriers)[2].BarrierID, + }) + + encoded, err := json.Marshal(got) + require.NoError(t, err) + require.Contains(t, string(encoded), `"gc_barriers":[]`) + require.Contains(t, string(encoded), `"global_gc_barriers":[`) +} + +func TestNewAllGCStatesOutputFiltersUnifiedGCPlaceholders(t *testing.T) { + keyspaceLevelState := gc.NewGCStateWithGCBarriers(7, 70, 60, nil) + keyspaceLevelState.IsKeyspaceLevelGC = true + unifiedGCPlaceholder := gc.NewGCStateWithGCBarriers( + 8, + 800, + 700, + []*gc.GCBarrierInfo{ + gc.NewGCBarrierInfo("placeholder", 900, time.Hour, time.Time{}), + }, + ) + // A non-Null state without keyspace-level GC only identifies a unified-GC keyspace. + // Its safe points and barriers are not an effective GC scope. + unifiedGCPlaceholder.IsKeyspaceLevelGC = false + nullKeyspaceState := gc.NewGCStateWithGCBarriers( + constant.NullKeyspaceID, + 100, + 90, + []*gc.GCBarrierInfo{ + gc.NewGCBarrierInfo("null", 110, gc.TTLNeverExpire, time.Time{}), + }, + ) + nullKeyspaceState.IsKeyspaceLevelGC = false + + clusterState := gc.NewClusterGCStatesWithGlobalGCBarriers( + map[uint32]gc.GCState{ + 7: keyspaceLevelState, + 8: unifiedGCPlaceholder, + constant.NullKeyspaceID: nullKeyspaceState, + }, + nil, + ) + + got, err := newAllGCStatesOutput(clusterState, false, true) + require.NoError(t, err) + require.Equal(t, []gcStateOutput{ + { + KeyspaceID: 7, + IsKeyspaceLevelGC: true, + TxnSafePoint: 70, + GCSafePoint: 60, + GCBarriers: []gcBarrierOutput{}, + }, + { + KeyspaceID: constant.NullKeyspaceID, + IsKeyspaceLevelGC: false, + TxnSafePoint: 100, + GCSafePoint: 90, + GCBarriers: []gcBarrierOutput{ + {BarrierID: "null", BarrierTS: 110, TTLSeconds: math.MaxInt64}, + }, + }, + }, got.GCStates) +} + +func TestNewAllGCStatesOutputKeepsEmptyGlobalBarrierArray(t *testing.T) { + clusterState := gc.NewClusterGCStatesWithGlobalGCBarriers( + map[uint32]gc.GCState{}, + nil, + ) + + got, err := newAllGCStatesOutput(clusterState, false, true) + require.NoError(t, err) + require.NotNil(t, got.GlobalGCBarriers) + require.Empty(t, *got.GlobalGCBarriers) + + encoded, err := json.Marshal(got) + require.NoError(t, err) + require.Contains(t, string(encoded), `"global_gc_barriers":[]`) +} + +func TestNewAllGCStatesOutputGlobalBarrierPresence(t *testing.T) { + t.Run("requested-empty", func(t *testing.T) { + state := gc.NewClusterGCStatesWithGlobalGCBarriers( + map[uint32]gc.GCState{}, + nil, + ) + got, err := newAllGCStatesOutput(state, false, true) + require.NoError(t, err) + require.NotNil(t, got.GlobalGCBarriers) + require.Empty(t, *got.GlobalGCBarriers) + encoded, err := json.Marshal(got) + require.NoError(t, err) + require.Contains(t, string(encoded), `"global_gc_barriers":[]`) + }) + + t.Run("excluded", func(t *testing.T) { + state := gc.NewClusterGCStatesWithoutGlobalGCBarriers( + map[uint32]gc.GCState{}, + ) + got, err := newAllGCStatesOutput(state, false, false) + require.NoError(t, err) + require.Nil(t, got.GlobalGCBarriers) + encoded, err := json.Marshal(got) + require.NoError(t, err) + require.NotContains(t, string(encoded), "global_gc_barriers") + }) + + t.Run("requested-missing", func(t *testing.T) { + state := gc.NewClusterGCStatesWithoutGlobalGCBarriers( + map[uint32]gc.GCState{}, + ) + _, err := newAllGCStatesOutput(state, false, true) + require.ErrorContains(t, err, + "retry with --exclude-global-barriers") + }) +} + +func TestGCStateOutputRejectsExcludedBarriers(t *testing.T) { + t.Run("missing-local", func(t *testing.T) { + state := gc.NewGCStateWithoutGCBarriers(42, 100, 90) + _, err := newKeyspaceGCStateOutput(42, state, false, false) + require.ErrorContains(t, err, "failed to read GC barriers for keyspace 42") + }) + + t.Run("keyspace-requested-missing-global", func(t *testing.T) { + state := gc.NewGCStateWithGCBarriers(42, 100, 90, nil) + _, err := newKeyspaceGCStateOutput(42, state, false, true) + require.EqualError(t, err, + "gc-state keyspace requires a PD server whose GetGCState supports global GC barriers; "+ + "retry with --exclude-global-barriers") + }) + + t.Run("keyspace-excluded-global", func(t *testing.T) { + state := gc.NewGCStateWithGCBarriers(42, 100, 90, nil) + got, err := newKeyspaceGCStateOutput(42, state, false, false) + require.NoError(t, err) + require.Nil(t, got.GlobalGCBarriers) + encoded, err := json.Marshal(got) + require.NoError(t, err) + require.NotContains(t, string(encoded), "global_gc_barriers") + }) + + t.Run("all-requested-missing-global", func(t *testing.T) { + clusterState := gc.NewClusterGCStatesWithoutGlobalGCBarriers( + map[uint32]gc.GCState{}, + ) + _, err := newAllGCStatesOutput(clusterState, false, true) + require.EqualError(t, err, + "gc-state all response does not include global GC barriers; "+ + "retry with --exclude-global-barriers") + }) + + t.Run("all-excluded-global", func(t *testing.T) { + clusterState := gc.NewClusterGCStatesWithoutGlobalGCBarriers( + map[uint32]gc.GCState{}, + ) + got, err := newAllGCStatesOutput(clusterState, false, false) + require.NoError(t, err) + require.Nil(t, got.GlobalGCBarriers) + encoded, err := json.Marshal(got) + require.NoError(t, err) + require.NotContains(t, string(encoded), "global_gc_barriers") + }) +} + +func TestGCStateAPIOptions(t *testing.T) { + for _, includeGlobalGCBarriers := range []bool{false, true} { + t.Run(strconv.FormatBool(includeGlobalGCBarriers), func(t *testing.T) { + options := gc.DefaultGCStatesAPIOptions() + for _, option := range gcStateAPIOptions( + includeGlobalGCBarriers, + ) { + option(&options) + } + require.False(t, options.ExcludeGCBarriers) + require.Equal(t, !includeGlobalGCBarriers, + options.ExcludeGlobalGCBarriers) + }) + } +} + +func TestNewPDGCStateReaderUsesConfiguredTimeout(t *testing.T) { + for _, testCase := range []struct { + name string + flagValue string + wantTimeout time.Duration + }{ + {name: "default", wantTimeout: 30 * time.Second}, + {name: "custom", flagValue: "45s", wantTimeout: 45 * time.Second}, + } { + t.Run(testCase.name, func(t *testing.T) { + wantErr := errors.New("stop after inspecting client options") + clientFactory := func( + _ context.Context, + _ caller.Component, + _ []string, + _ pd.SecurityOption, + clientOptions ...opt.ClientOption, + ) (pd.Client, error) { + options := opt.NewOption() + for _, apply := range clientOptions { + apply(options) + } + require.Equal(t, testCase.wantTimeout, options.Timeout) + return &unusedPDClient{}, wantErr + } + gcStateCommand := buildGCStateCommand(func(cmd *cobra.Command) (gcStateReader, error) { + return newPDGCStateReaderWithClientFactory(cmd, clientFactory) + }) + rootCommand := &cobra.Command{Use: "pd-ctl", SilenceUsage: true} + rootCommand.PersistentFlags().String("pd", "http://127.0.0.1:2379", "") + rootCommand.PersistentFlags().String("cacert", "", "") + rootCommand.PersistentFlags().String("cert", "", "") + rootCommand.PersistentFlags().String("key", "", "") + rootCommand.AddCommand(gcStateCommand) + args := []string{"gc-state", "all"} + if testCase.flagValue != "" { + args = append(args, "--timeout", testCase.flagValue) + } + rootCommand.SetArgs(args) + rootCommand.SetOut(io.Discard) + rootCommand.SetErr(io.Discard) + + err := rootCommand.Execute() + require.ErrorIs(t, err, wantErr) + }) + } +} + +func TestGCStateCommandTimeout(t *testing.T) { + for _, testCase := range []struct { + name string + args []string + wantTimeout time.Duration + }{ + {name: "all-default", args: []string{"all"}, wantTimeout: 30 * time.Second}, + {name: "all-custom", args: []string{"all", "--timeout", "2m"}, wantTimeout: 2 * time.Minute}, + {name: "keyspace-custom", args: []string{"keyspace", "42", "--timeout", "45s"}, wantTimeout: 45 * time.Second}, + } { + t.Run(testCase.name, func(t *testing.T) { + reader := &fakeGCStateReader{ + state: gc.NewGCStateWithGCBarriers(42, 100, 90, nil). + WithGlobalGCBarriers(nil), + clusterState: gc.NewClusterGCStatesWithGlobalGCBarriers( + map[uint32]gc.GCState{}, + nil, + ), + } + var gotTimeout time.Duration + cmd := buildGCStateCommand(func(cmd *cobra.Command) (gcStateReader, error) { + var err error + gotTimeout, err = cmd.Flags().GetDuration(gcStateTimeoutFlag) + return reader, err + }) + cmd.SetOut(io.Discard) + cmd.SetErr(io.Discard) + cmd.SetArgs(testCase.args) + + require.NoError(t, cmd.Execute()) + require.Equal(t, testCase.wantTimeout, gotTimeout) + }) + } +} + +func TestGCStateCommandRejectsNonPositiveTimeout(t *testing.T) { + for _, timeout := range []string{"0s", "-1s"} { + t.Run(timeout, func(t *testing.T) { + factoryCalled := false + cmd := buildGCStateCommand(func(*cobra.Command) (gcStateReader, error) { + factoryCalled = true + return nil, errors.New("factory must not run") + }) + cmd.SetOut(io.Discard) + cmd.SetErr(io.Discard) + cmd.SetArgs([]string{"all", "--timeout", timeout}) + + err := cmd.Execute() + require.EqualError(t, err, "timeout must be a positive duration") + require.False(t, factoryCalled) + }) + } +} + +func TestGCStateCommandGlobalBarrierFlag(t *testing.T) { + for _, testCase := range []struct { + name string + args []string + wantInclude bool + }{ + {name: "keyspace-default", args: []string{"keyspace", "42"}, wantInclude: true}, + {name: "keyspace-excluded", args: []string{"keyspace", "42", "--exclude-global-barriers"}}, + {name: "all-default", args: []string{"all"}, wantInclude: true}, + {name: "all-excluded", args: []string{"all", "--exclude-global-barriers"}}, + } { + t.Run(testCase.name, func(t *testing.T) { + reader := &fakeGCStateReader{ + state: gc.NewGCStateWithGCBarriers(42, 100, 90, nil). + WithGlobalGCBarriers(nil), + clusterState: gc.NewClusterGCStatesWithGlobalGCBarriers( + map[uint32]gc.GCState{}, + nil, + ), + } + cmd := buildGCStateCommand(func(*cobra.Command) (gcStateReader, error) { + return reader, nil + }) + output := new(bytes.Buffer) + cmd.SetOut(output) + cmd.SetErr(output) + cmd.SetArgs(testCase.args) + + require.NoError(t, cmd.Execute()) + require.Equal(t, testCase.wantInclude, + reader.includeGlobalGCBarriers) + if testCase.wantInclude { + require.Contains(t, output.String(), "global_gc_barriers") + } else { + require.NotContains(t, output.String(), "global_gc_barriers") + } + }) + } +} + +func TestGCStateKeyspaceCommand(t *testing.T) { + state := gc.NewGCStateWithGCBarriers(42, 100, 90, nil). + WithGlobalGCBarriers(nil) + state.IsKeyspaceLevelGC = true + reader := &fakeGCStateReader{state: state} + factoryCalls := 0 + cmd := buildGCStateCommand(func(*cobra.Command) (gcStateReader, error) { + factoryCalls++ + return reader, nil + }) + output := new(bytes.Buffer) + cmd.SetOut(output) + cmd.SetErr(output) + cmd.SetArgs([]string{"keyspace", "42"}) + + require.NoError(t, cmd.Execute()) + require.Equal(t, 1, factoryCalls) + require.Equal(t, 1, reader.getStateCalls) + require.Zero(t, reader.getAllCalls) + require.Equal(t, uint32(42), reader.requestedID) + require.True(t, reader.closed) + + var decoded map[string]json.RawMessage + require.NoError(t, json.Unmarshal(output.Bytes(), &decoded)) + require.Contains(t, decoded, "requested_keyspace_id") + require.Contains(t, decoded, "effective_keyspace_id") + require.Contains(t, decoded, "gc_barriers") + require.Contains(t, decoded, "global_gc_barriers") +} + +func TestGCStateAllCommand(t *testing.T) { + reader := &fakeGCStateReader{ + clusterState: gc.NewClusterGCStatesWithGlobalGCBarriers( + map[uint32]gc.GCState{}, + nil, + ), + } + cmd := buildGCStateCommand(func(*cobra.Command) (gcStateReader, error) { + return reader, nil + }) + output := new(bytes.Buffer) + cmd.SetOut(output) + cmd.SetErr(output) + cmd.SetArgs([]string{"all"}) + + require.NoError(t, cmd.Execute()) + require.Equal(t, 1, reader.getAllCalls) + require.Zero(t, reader.getStateCalls) + require.True(t, reader.closed) + require.Contains(t, output.String(), `"gc_states": []`) + require.Contains(t, output.String(), `"global_gc_barriers": []`) +} + +func TestGCStateCommandExpiredBarrierVisibility(t *testing.T) { + globalBarriers := []*gc.GlobalGCBarrierInfo{ + gc.NewGlobalGCBarrierInfo("active-global", 70, gc.TTLNeverExpire, time.Time{}), + gc.NewGlobalGCBarrierInfo("expired-global", 60, 0, time.Time{}), + } + state := gc.NewGCStateWithGCBarriers( + 42, + 100, + 90, + []*gc.GCBarrierInfo{ + gc.NewGCBarrierInfo("active-local", 50, gc.TTLNeverExpire, time.Time{}), + gc.NewGCBarrierInfo("expired-local", 40, 0, time.Time{}), + }, + ).WithGlobalGCBarriers(globalBarriers) + state.IsKeyspaceLevelGC = true + clusterState := gc.NewClusterGCStatesWithGlobalGCBarriers( + map[uint32]gc.GCState{42: state}, + globalBarriers, + ) + + for _, testCase := range []struct { + name string + args []string + wantLocal []gcBarrierOutput + wantGlobal []gcBarrierOutput + }{ + { + name: "keyspace-default", + args: []string{"keyspace", "42"}, + wantLocal: []gcBarrierOutput{ + {BarrierID: "active-local", BarrierTS: 50, TTLSeconds: math.MaxInt64}, + }, + wantGlobal: []gcBarrierOutput{ + {BarrierID: "active-global", BarrierTS: 70, TTLSeconds: math.MaxInt64}, + }, + }, + { + name: "keyspace-include-expired", + args: []string{"keyspace", "42", "--include-expired"}, + wantLocal: []gcBarrierOutput{ + {BarrierID: "expired-local", BarrierTS: 40, TTLSeconds: 0}, + {BarrierID: "active-local", BarrierTS: 50, TTLSeconds: math.MaxInt64}, + }, + wantGlobal: []gcBarrierOutput{ + {BarrierID: "expired-global", BarrierTS: 60, TTLSeconds: 0}, + {BarrierID: "active-global", BarrierTS: 70, TTLSeconds: math.MaxInt64}, + }, + }, + { + name: "all-default", + args: []string{"all"}, + wantLocal: []gcBarrierOutput{ + {BarrierID: "active-local", BarrierTS: 50, TTLSeconds: math.MaxInt64}, + }, + wantGlobal: []gcBarrierOutput{ + {BarrierID: "active-global", BarrierTS: 70, TTLSeconds: math.MaxInt64}, + }, + }, + { + name: "all-include-expired", + args: []string{"all", "--include-expired"}, + wantLocal: []gcBarrierOutput{ + {BarrierID: "expired-local", BarrierTS: 40, TTLSeconds: 0}, + {BarrierID: "active-local", BarrierTS: 50, TTLSeconds: math.MaxInt64}, + }, + wantGlobal: []gcBarrierOutput{ + {BarrierID: "expired-global", BarrierTS: 60, TTLSeconds: 0}, + {BarrierID: "active-global", BarrierTS: 70, TTLSeconds: math.MaxInt64}, + }, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + reader := &fakeGCStateReader{state: state, clusterState: clusterState} + cmd := buildGCStateCommand(func(*cobra.Command) (gcStateReader, error) { + return reader, nil + }) + output := new(bytes.Buffer) + cmd.SetOut(output) + cmd.SetErr(output) + cmd.SetArgs(testCase.args) + + require.NoError(t, cmd.Execute()) + switch testCase.args[0] { + case "keyspace": + var decoded keyspaceGCStateOutput + require.NoError(t, json.Unmarshal(output.Bytes(), &decoded)) + require.Equal(t, testCase.wantLocal, decoded.GCBarriers) + require.NotNil(t, decoded.GlobalGCBarriers) + require.Equal(t, testCase.wantGlobal, *decoded.GlobalGCBarriers) + case "all": + var decoded allGCStatesOutput + require.NoError(t, json.Unmarshal(output.Bytes(), &decoded)) + require.Len(t, decoded.GCStates, 1) + require.Equal(t, testCase.wantLocal, decoded.GCStates[0].GCBarriers) + require.NotNil(t, decoded.GlobalGCBarriers) + require.Equal(t, testCase.wantGlobal, *decoded.GlobalGCBarriers) + default: + require.Fail(t, "unexpected subcommand", testCase.args[0]) + } + }) + } +} + +func TestGCStateCommandValidatesBeforeCreatingClient(t *testing.T) { + for _, args := range [][]string{ + {}, + {"keyspace"}, + {"keyspace", "42", "extra"}, + {"keyspace", ""}, + {"keyspace", "-1"}, + {"keyspace", "tenant-a"}, + {"keyspace", "0xffffff"}, + {"keyspace", "16777216"}, + {"keyspace", "4294967294"}, + {"keyspace", "4294967296"}, + {"all", "extra"}, + } { + t.Run(strings.Join(args, "-"), func(t *testing.T) { + factoryCalled := false + cmd := buildGCStateCommand( + func(*cobra.Command) (gcStateReader, error) { + factoryCalled = true + return nil, errors.New("factory must not run") + }, + ) + cmd.SetOut(io.Discard) + cmd.SetErr(io.Discard) + cmd.SetArgs(args) + err := cmd.Execute() + if len(args) == 0 { + require.NoError(t, err) + } else { + require.Error(t, err) + } + require.False(t, factoryCalled) + }) + } +} + +func TestGCStateCommandErrors(t *testing.T) { + for _, testCase := range []struct { + name string + args []string + factory gcStateReaderFactory + wantMessage string + }{ + { + name: "client-creation", + args: []string{"keyspace", "42"}, + factory: func(*cobra.Command) (gcStateReader, error) { + return nil, errors.New("dial rejected") + }, + wantMessage: "failed to create PD RPC client", + }, + { + name: "single-rpc-error", + args: []string{"keyspace", "42"}, + factory: func(*cobra.Command) (gcStateReader, error) { + return &fakeGCStateReader{err: errors.New("rpc rejected")}, nil + }, + wantMessage: "failed to get GC state for keyspace 42", + }, + { + name: "single-rpc-timeout", + args: []string{"keyspace", "42", "--timeout", "45s"}, + factory: func(*cobra.Command) (gcStateReader, error) { + return &fakeGCStateReader{ + err: status.Error(codes.DeadlineExceeded, "deadline exceeded"), + }, nil + }, + wantMessage: "gc-state keyspace timed out after 45s; retry with a longer --timeout", + }, + { + name: "single-wrapped-unimplemented", + args: []string{"keyspace", "42"}, + factory: func(*cobra.Command) (gcStateReader, error) { + return &fakeGCStateReader{ + err: fmt.Errorf("wrapped: %w", + status.Error(codes.Unimplemented, "method unavailable")), + }, nil + }, + wantMessage: "gc-state requires a PD server that supports GetGCState", + }, + { + name: "single-missing-barriers", + args: []string{"keyspace", "42"}, + factory: func(*cobra.Command) (gcStateReader, error) { + return &fakeGCStateReader{ + state: gc.NewGCStateWithoutGCBarriers(42, 100, 90), + }, nil + }, + wantMessage: "failed to read GC barriers for keyspace 42", + }, + { + name: "single-missing-global-barriers", + args: []string{"keyspace", "42"}, + factory: func(*cobra.Command) (gcStateReader, error) { + return &fakeGCStateReader{ + state: gc.NewGCStateWithGCBarriers(42, 100, 90, nil), + }, nil + }, + wantMessage: "retry with --exclude-global-barriers", + }, + { + name: "all-rpc-error", + args: []string{"all"}, + factory: func(*cobra.Command) (gcStateReader, error) { + return &fakeGCStateReader{err: errors.New("rpc rejected")}, nil + }, + wantMessage: "failed to get all keyspaces GC states", + }, + { + name: "all-rpc-timeout", + args: []string{"all", "--timeout", "2m"}, + factory: func(*cobra.Command) (gcStateReader, error) { + return &fakeGCStateReader{ + err: status.Error(codes.DeadlineExceeded, "deadline exceeded"), + }, nil + }, + wantMessage: "gc-state all timed out after 2m0s; retry with a longer --timeout", + }, + { + name: "all-wrapped-unimplemented", + args: []string{"all"}, + factory: func(*cobra.Command) (gcStateReader, error) { + return &fakeGCStateReader{ + err: fmt.Errorf("wrapped: %w", + status.Error(codes.Unimplemented, "method unavailable")), + }, nil + }, + wantMessage: "gc-state all requires a PD server that supports " + + "GetAllKeyspacesGCStates", + }, + { + name: "all-missing-global-barriers", + args: []string{"all"}, + factory: func(*cobra.Command) (gcStateReader, error) { + return &fakeGCStateReader{ + clusterState: gc.NewClusterGCStatesWithoutGlobalGCBarriers( + map[uint32]gc.GCState{}, + ), + }, nil + }, + wantMessage: "retry with --exclude-global-barriers", + }, + } { + t.Run(testCase.name, func(t *testing.T) { + cmd := buildGCStateCommand(testCase.factory) + cmd.SetOut(io.Discard) + cmd.SetErr(io.Discard) + cmd.SetArgs(testCase.args) + err := cmd.Execute() + require.ErrorContains(t, err, testCase.wantMessage) + }) + } +} + +func TestGCStateCommandHelpContract(t *testing.T) { + cmd := buildGCStateCommand(func(*cobra.Command) (gcStateReader, error) { + return nil, errors.New("help must not create a reader") + }) + require.Equal(t, "show keyspace and cluster-wide GC state", cmd.Short) + require.Equal(t, "Show effective per-keyspace GC safe points and local and global barriers. Expired barriers awaiting lazy deletion are hidden by default; use --include-expired to include zero-TTL barriers returned by PD. Use keyspace for one effective GC scope or all for every effective GC scope.", cmd.Long) + includeExpired := cmd.PersistentFlags().Lookup("include-expired") + require.NotNil(t, includeExpired) + require.Equal(t, "false", includeExpired.DefValue) + require.Equal(t, "include zero-TTL barriers returned by PD, which normally represent expired barriers awaiting lazy deletion", includeExpired.Usage) + excludeGlobalBarriers := cmd.PersistentFlags().Lookup("exclude-global-barriers") + require.NotNil(t, excludeGlobalBarriers) + require.Equal(t, "false", excludeGlobalBarriers.DefValue) + require.Equal(t, "exclude global GC barriers from the PD request and JSON output", excludeGlobalBarriers.Usage) + timeout := cmd.PersistentFlags().Lookup("timeout") + require.NotNil(t, timeout) + require.Equal(t, "30s", timeout.DefValue) + require.Equal(t, "timeout for GC state RPCs", timeout.Usage) + + commands := cmd.Commands() + require.Len(t, commands, 2) + require.Equal(t, []string{"all", "keyspace"}, []string{ + commands[0].Name(), + commands[1].Name(), + }) + + keyspace, _, err := cmd.Find([]string{"keyspace"}) + require.NoError(t, err) + require.Equal(t, "keyspace ", keyspace.Use) + require.Equal(t, "show one keyspace's effective GC state", keyspace.Short) + require.Equal(t, "Show one keyspace's effective GC safe points and local and global barriers. Use --exclude-global-barriers to omit cluster-wide barriers. Use gc-state all to inspect every effective GC scope. The decimal NullKeyspace ID is 4294967295.", keyspace.Long) + require.Equal(t, " pd-ctl gc-state keyspace 42\n pd-ctl gc-state keyspace 4294967295", keyspace.Example) + + all, _, err := cmd.Find([]string{"all"}) + require.NoError(t, err) + require.Equal(t, "all", all.Use) + require.Equal(t, "show effective GC scopes and cluster-wide GC state", all.Short) + require.Equal(t, "Show all effective GC scopes and local barriers, with global barriers once at the top level. Use --exclude-global-barriers to omit cluster-wide barriers.", all.Long) + require.Equal(t, " pd-ctl gc-state all\n pd-ctl gc-state all --timeout 2m", all.Example) +} + +type failingWriter struct{} + +func (failingWriter) Write([]byte) (int, error) { + return 0, errors.New("output rejected") +} + +func TestGCStateCommandReturnsOutputError(t *testing.T) { + state := gc.NewGCStateWithGCBarriers(42, 100, 90, nil). + WithGlobalGCBarriers(nil) + reader := &fakeGCStateReader{state: state} + cmd := buildGCStateCommand(func(*cobra.Command) (gcStateReader, error) { + return reader, nil + }) + cmd.SetOut(failingWriter{}) + cmd.SetErr(io.Discard) + cmd.SetArgs([]string{"keyspace", "42"}) + + err := cmd.Execute() + require.ErrorContains(t, err, "failed to write GC state JSON") + require.True(t, reader.closed) +} diff --git a/tools/pd-ctl/pdctl/ctl.go b/tools/pd-ctl/pdctl/ctl.go index be4be12ff9..393fc82f67 100644 --- a/tools/pd-ctl/pdctl/ctl.go +++ b/tools/pd-ctl/pdctl/ctl.go @@ -68,6 +68,7 @@ func GetRootCmd() *cobra.Command { command.NewLogCommand(), command.NewPluginCommand(), command.NewServiceGCSafepointCommand(), + command.NewGCStateCommand(), command.NewMinResolvedTSCommand(), command.NewCompletionCommand(), command.NewUnsafeCommand(), diff --git a/tools/pd-ctl/tests/safepoint/gc_state_test.go b/tools/pd-ctl/tests/safepoint/gc_state_test.go new file mode 100644 index 0000000000..acb3912a68 --- /dev/null +++ b/tools/pd-ctl/tests/safepoint/gc_state_test.go @@ -0,0 +1,542 @@ +// Copyright 2026 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package safepoint_test + +import ( + "bytes" + "encoding/json" + "math" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/tikv/pd/pkg/keyspace" + "github.com/tikv/pd/pkg/keyspace/constant" + "github.com/tikv/pd/pkg/versioninfo/kerneltype" + "github.com/tikv/pd/server/config" + pdTests "github.com/tikv/pd/tests" + ctl "github.com/tikv/pd/tools/pd-ctl/pdctl" + "github.com/tikv/pd/tools/pd-ctl/tests" +) + +type gcStateCommandBarrier struct { + BarrierID string `json:"barrier_id"` + BarrierTS uint64 `json:"barrier_ts"` + TTLSeconds int64 `json:"ttl_seconds"` +} + +type gcStateCommandSingle struct { + RequestedKeyspaceID uint32 `json:"requested_keyspace_id"` + EffectiveKeyspaceID uint32 `json:"effective_keyspace_id"` + IsKeyspaceLevelGC bool `json:"is_keyspace_level_gc"` + TxnSafePoint uint64 `json:"txn_safe_point"` + GCSafePoint uint64 `json:"gc_safe_point"` + GCBarriers []gcStateCommandBarrier `json:"gc_barriers"` + GlobalGCBarriers []gcStateCommandBarrier `json:"global_gc_barriers"` +} + +type gcStateCommandState struct { + KeyspaceID uint32 `json:"keyspace_id"` + IsKeyspaceLevelGC bool `json:"is_keyspace_level_gc"` + TxnSafePoint uint64 `json:"txn_safe_point"` + GCSafePoint uint64 `json:"gc_safe_point"` + GCBarriers []gcStateCommandBarrier `json:"gc_barriers"` +} + +type gcStateCommandAll struct { + GCStates []gcStateCommandState `json:"gc_states"` + GlobalGCBarriers []gcStateCommandBarrier `json:"global_gc_barriers"` +} + +type expectedGCStateCommandBarrier struct { + barrierID string + barrierTS uint64 + expires bool + expired bool +} + +func requireGCStateCommandBarriers( + re *require.Assertions, + actual []gcStateCommandBarrier, + expected []expectedGCStateCommandBarrier, +) { + re.Len(actual, len(expected)) + for i, want := range expected { + re.Equal(want.barrierID, actual[i].BarrierID) + re.Equal(want.barrierTS, actual[i].BarrierTS) + if want.expired { + re.Zero(actual[i].TTLSeconds) + } else if want.expires { + re.Greater(actual[i].TTLSeconds, int64(3500)) + re.LessOrEqual(actual[i].TTLSeconds, int64(3600)) + } else { + re.Equal(int64(math.MaxInt64), actual[i].TTLSeconds) + } + } +} + +func requireGCStateCommandState( + re *require.Assertions, + actual gcStateCommandState, + expected gcStateCommandState, + expectedBarriers []expectedGCStateCommandBarrier, +) { + re.Equal(expected.KeyspaceID, actual.KeyspaceID) + re.Equal(expected.IsKeyspaceLevelGC, actual.IsKeyspaceLevelGC) + re.Equal(expected.TxnSafePoint, actual.TxnSafePoint) + re.Equal(expected.GCSafePoint, actual.GCSafePoint) + requireGCStateCommandBarriers(re, actual.GCBarriers, expectedBarriers) +} + +func TestGCState(t *testing.T) { + re := require.New(t) + ctx := t.Context() + cluster, err := pdTests.NewTestCluster(ctx, 1, + func(conf *config.Config, _ string) { + conf.Keyspace.WaitRegionSplit = false + }, + ) + re.NoError(err) + defer cluster.Destroy() + re.NoError(cluster.RunInitialServers()) + re.NotEmpty(cluster.WaitLeader()) + leaderServer := cluster.GetLeaderServer() + re.NoError(leaderServer.BootstrapCluster()) + + keyspaceLevel, err := leaderServer.GetKeyspaceManager().CreateKeyspace( + &keyspace.CreateKeyspaceRequest{ + Name: "gc_state_ks_level", + Config: map[string]string{ + keyspace.GCManagementType: keyspace.KeyspaceLevelGC, + }, + CreateTime: time.Now().Unix(), + }, + ) + re.NoError(err) + keyspaceLevelID := keyspaceLevel.GetId() + + var unifiedKeyspaceID uint32 + if !kerneltype.IsNextGen() { + unified, err := leaderServer.GetKeyspaceManager().CreateKeyspace( + &keyspace.CreateKeyspaceRequest{ + Name: "gc_state_unified", + Config: map[string]string{ + keyspace.GCManagementType: keyspace.UnifiedGC, + }, + CreateTime: time.Now().Unix(), + }, + ) + re.NoError(err) + unifiedKeyspaceID = unified.GetId() + } + + manager := leaderServer.GetServer().GetGCStateManager() + now := time.Now() + _, err = manager.AdvanceTxnSafePoint(constant.NullKeyspaceID, 100, now) + re.NoError(err) + _, _, err = manager.AdvanceGCSafePoint(constant.NullKeyspaceID, 90) + re.NoError(err) + _, err = manager.SetGCBarrier( + constant.NullKeyspaceID, + "z-null", + 120, + time.Hour, + now, + ) + re.NoError(err) + _, err = manager.SetGCBarrier( + constant.NullKeyspaceID, + "a-null", + 110, + time.Duration(math.MaxInt64), + now, + ) + re.NoError(err) + + _, err = manager.AdvanceTxnSafePoint(keyspaceLevelID, 200, now) + re.NoError(err) + _, _, err = manager.AdvanceGCSafePoint(keyspaceLevelID, 190) + re.NoError(err) + _, err = manager.SetGCBarrier( + keyspaceLevelID, + "z-local", + 220, + time.Hour, + now, + ) + re.NoError(err) + _, err = manager.SetGCBarrier( + keyspaceLevelID, + "a-local", + 210, + time.Duration(math.MaxInt64), + now, + ) + re.NoError(err) + // Backdate the creation time so the barrier stays persisted until the next + // safe-point advancement but is already inactive when the RPC reads it. + _, err = manager.SetGCBarrier( + keyspaceLevelID, + "expired-local", + 230, + time.Hour, + now.Add(-2*time.Hour), + ) + re.NoError(err) + + _, err = manager.SetGlobalGCBarrier( + ctx, + "z-global", + 320, + time.Duration(math.MaxInt64), + now, + ) + re.NoError(err) + _, err = manager.SetGlobalGCBarrier( + ctx, + "a-global", + 310, + time.Duration(math.MaxInt64), + now, + ) + re.NoError(err) + _, err = manager.SetGlobalGCBarrier( + ctx, + "expired-global", + 330, + time.Hour, + now.Add(-2*time.Hour), + ) + re.NoError(err) + + pdAddr := cluster.GetConfig().GetClientURL() + keyspaceLevelIDString := strconv.FormatUint(uint64(keyspaceLevelID), 10) + output, err := tests.ExecuteCommand( + ctl.GetRootCmd(), "-u", pdAddr, "gc-state", "keyspace", keyspaceLevelIDString, + ) + re.NoError(err) + var singleProperties map[string]json.RawMessage + re.NoError(json.Unmarshal(output, &singleProperties), string(output)) + re.Contains(singleProperties, "global_gc_barriers") + var keyspaceLevelResponse gcStateCommandSingle + re.NoError(json.Unmarshal(output, &keyspaceLevelResponse), string(output)) + re.Equal(keyspaceLevelID, keyspaceLevelResponse.RequestedKeyspaceID) + re.Equal(keyspaceLevelID, keyspaceLevelResponse.EffectiveKeyspaceID) + re.True(keyspaceLevelResponse.IsKeyspaceLevelGC) + re.Equal(uint64(200), keyspaceLevelResponse.TxnSafePoint) + re.Equal(uint64(190), keyspaceLevelResponse.GCSafePoint) + requireGCStateCommandBarriers(re, keyspaceLevelResponse.GCBarriers, []expectedGCStateCommandBarrier{ + {barrierID: "a-local", barrierTS: 210}, + {barrierID: "z-local", barrierTS: 220, expires: true}, + }) + requireGCStateCommandBarriers( + re, + keyspaceLevelResponse.GlobalGCBarriers, + []expectedGCStateCommandBarrier{ + {barrierID: "a-global", barrierTS: 310}, + {barrierID: "z-global", barrierTS: 320}, + }, + ) + + output, err = tests.ExecuteCommand( + ctl.GetRootCmd(), "-u", pdAddr, "gc-state", "keyspace", keyspaceLevelIDString, + "--include-expired", + ) + re.NoError(err) + var keyspaceLevelWithExpired gcStateCommandSingle + re.NoError(json.Unmarshal(output, &keyspaceLevelWithExpired), string(output)) + requireGCStateCommandBarriers(re, keyspaceLevelWithExpired.GCBarriers, []expectedGCStateCommandBarrier{ + {barrierID: "a-local", barrierTS: 210}, + {barrierID: "z-local", barrierTS: 220, expires: true}, + {barrierID: "expired-local", barrierTS: 230, expired: true}, + }) + requireGCStateCommandBarriers( + re, + keyspaceLevelWithExpired.GlobalGCBarriers, + []expectedGCStateCommandBarrier{ + {barrierID: "a-global", barrierTS: 310}, + {barrierID: "z-global", barrierTS: 320}, + {barrierID: "expired-global", barrierTS: 330, expired: true}, + }, + ) + + output, err = tests.ExecuteCommand( + ctl.GetRootCmd(), + "-u", + pdAddr, + "gc-state", + "keyspace", + keyspaceLevelIDString, + "--exclude-global-barriers", + ) + re.NoError(err) + var excludedKeyspaceProperties map[string]json.RawMessage + re.NoError(json.Unmarshal(output, &excludedKeyspaceProperties), string(output)) + re.NotContains(excludedKeyspaceProperties, "global_gc_barriers") + var excludedKeyspace gcStateCommandSingle + re.NoError(json.Unmarshal(output, &excludedKeyspace), string(output)) + requireGCStateCommandBarriers( + re, + excludedKeyspace.GCBarriers, + []expectedGCStateCommandBarrier{ + {barrierID: "a-local", barrierTS: 210}, + {barrierID: "z-local", barrierTS: 220, expires: true}, + }, + ) + + output, err = tests.ExecuteCommand( + ctl.GetRootCmd(), + "-u", + pdAddr, + "gc-state", + "keyspace", + keyspaceLevelIDString, + "--include-expired", + "--exclude-global-barriers", + ) + re.NoError(err) + var excludedKeyspaceWithExpiredProperties map[string]json.RawMessage + re.NoError(json.Unmarshal(output, &excludedKeyspaceWithExpiredProperties), string(output)) + re.NotContains(excludedKeyspaceWithExpiredProperties, "global_gc_barriers") + var excludedKeyspaceWithExpired gcStateCommandSingle + re.NoError(json.Unmarshal(output, &excludedKeyspaceWithExpired), string(output)) + requireGCStateCommandBarriers( + re, + excludedKeyspaceWithExpired.GCBarriers, + []expectedGCStateCommandBarrier{ + {barrierID: "a-local", barrierTS: 210}, + {barrierID: "z-local", barrierTS: 220, expires: true}, + {barrierID: "expired-local", barrierTS: 230, expired: true}, + }, + ) + + output, err = tests.ExecuteCommand( + ctl.GetRootCmd(), "-u", pdAddr, "gc-state", "keyspace", "4294967295", + ) + re.NoError(err) + var nullKeyspaceResponse gcStateCommandSingle + re.NoError(json.Unmarshal(output, &nullKeyspaceResponse), string(output)) + re.Equal(constant.NullKeyspaceID, nullKeyspaceResponse.RequestedKeyspaceID) + re.Equal(constant.NullKeyspaceID, nullKeyspaceResponse.EffectiveKeyspaceID) + re.False(nullKeyspaceResponse.IsKeyspaceLevelGC) + re.Equal(uint64(100), nullKeyspaceResponse.TxnSafePoint) + re.Equal(uint64(90), nullKeyspaceResponse.GCSafePoint) + requireGCStateCommandBarriers(re, nullKeyspaceResponse.GCBarriers, []expectedGCStateCommandBarrier{ + {barrierID: "a-null", barrierTS: 110}, + {barrierID: "z-null", barrierTS: 120, expires: true}, + }) + requireGCStateCommandBarriers( + re, + nullKeyspaceResponse.GlobalGCBarriers, + []expectedGCStateCommandBarrier{ + {barrierID: "a-global", barrierTS: 310}, + {barrierID: "z-global", barrierTS: 320}, + }, + ) + + _, err = tests.ExecuteCommand( + ctl.GetRootCmd(), "-u", pdAddr, "gc-state", "keyspace", "16770000", + ) + re.ErrorContains(err, "failed to get GC state for keyspace 16770000") + + output, err = tests.ExecuteCommand(ctl.GetRootCmd(), "-u", pdAddr, "gc-state", "all") + re.NoError(err) + re.Equal(1, bytes.Count(output, []byte(`"global_gc_barriers"`)), string(output)) + var all gcStateCommandAll + re.NoError(json.Unmarshal(output, &all), string(output)) + for i := 1; i < len(all.GCStates); i++ { + re.Less(all.GCStates[i-1].KeyspaceID, all.GCStates[i].KeyspaceID) + } + statesByID := make(map[uint32]gcStateCommandState, len(all.GCStates)) + for _, state := range all.GCStates { + re.NotNil(state.GCBarriers) + statesByID[state.KeyspaceID] = state + } + + nullState, ok := statesByID[constant.NullKeyspaceID] + re.True(ok) + re.False(nullState.IsKeyspaceLevelGC) + re.Equal(uint64(100), nullState.TxnSafePoint) + re.Equal(uint64(90), nullState.GCSafePoint) + requireGCStateCommandBarriers(re, nullState.GCBarriers, []expectedGCStateCommandBarrier{ + {barrierID: "a-null", barrierTS: 110}, + {barrierID: "z-null", barrierTS: 120, expires: true}, + }) + + keyspaceLevelState, ok := statesByID[keyspaceLevelID] + re.True(ok) + re.True(keyspaceLevelState.IsKeyspaceLevelGC) + re.Equal(uint64(200), keyspaceLevelState.TxnSafePoint) + re.Equal(uint64(190), keyspaceLevelState.GCSafePoint) + requireGCStateCommandBarriers(re, keyspaceLevelState.GCBarriers, []expectedGCStateCommandBarrier{ + {barrierID: "a-local", barrierTS: 210}, + {barrierID: "z-local", barrierTS: 220, expires: true}, + }) + requireGCStateCommandBarriers(re, all.GlobalGCBarriers, []expectedGCStateCommandBarrier{ + {barrierID: "a-global", barrierTS: 310}, + {barrierID: "z-global", barrierTS: 320}, + }) + + output, err = tests.ExecuteCommand( + ctl.GetRootCmd(), "-u", pdAddr, "gc-state", "all", "--include-expired", + ) + re.NoError(err) + var allWithExpired gcStateCommandAll + re.NoError(json.Unmarshal(output, &allWithExpired), string(output)) + statesByIDWithExpired := make(map[uint32]gcStateCommandState, len(allWithExpired.GCStates)) + for _, state := range allWithExpired.GCStates { + statesByIDWithExpired[state.KeyspaceID] = state + } + keyspaceLevelStateWithExpired, ok := statesByIDWithExpired[keyspaceLevelID] + re.True(ok) + requireGCStateCommandBarriers(re, keyspaceLevelStateWithExpired.GCBarriers, []expectedGCStateCommandBarrier{ + {barrierID: "a-local", barrierTS: 210}, + {barrierID: "z-local", barrierTS: 220, expires: true}, + {barrierID: "expired-local", barrierTS: 230, expired: true}, + }) + requireGCStateCommandBarriers(re, allWithExpired.GlobalGCBarriers, []expectedGCStateCommandBarrier{ + {barrierID: "a-global", barrierTS: 310}, + {barrierID: "z-global", barrierTS: 320}, + {barrierID: "expired-global", barrierTS: 330, expired: true}, + }) + + output, err = tests.ExecuteCommand( + ctl.GetRootCmd(), + "-u", + pdAddr, + "gc-state", + "all", + "--exclude-global-barriers", + ) + re.NoError(err) + var excludedAllProperties map[string]json.RawMessage + re.NoError(json.Unmarshal(output, &excludedAllProperties), string(output)) + re.Contains(excludedAllProperties, "gc_states") + re.NotContains(excludedAllProperties, "global_gc_barriers") + var excludedAll gcStateCommandAll + re.NoError(json.Unmarshal(output, &excludedAll), string(output)) + excludedStatesByID := make(map[uint32]gcStateCommandState, len(excludedAll.GCStates)) + for _, state := range excludedAll.GCStates { + excludedStatesByID[state.KeyspaceID] = state + } + excludedNullState, ok := excludedStatesByID[constant.NullKeyspaceID] + re.True(ok) + requireGCStateCommandState( + re, + excludedNullState, + nullState, + []expectedGCStateCommandBarrier{ + {barrierID: "a-null", barrierTS: 110}, + {barrierID: "z-null", barrierTS: 120, expires: true}, + }, + ) + excludedKeyspaceLevelState, ok := excludedStatesByID[keyspaceLevelID] + re.True(ok) + requireGCStateCommandState( + re, + excludedKeyspaceLevelState, + keyspaceLevelState, + []expectedGCStateCommandBarrier{ + {barrierID: "a-local", barrierTS: 210}, + {barrierID: "z-local", barrierTS: 220, expires: true}, + }, + ) + + output, err = tests.ExecuteCommand( + ctl.GetRootCmd(), + "-u", + pdAddr, + "gc-state", + "all", + "--include-expired", + "--exclude-global-barriers", + ) + re.NoError(err) + var excludedAllWithExpiredProperties map[string]json.RawMessage + re.NoError(json.Unmarshal(output, &excludedAllWithExpiredProperties), string(output)) + re.Contains(excludedAllWithExpiredProperties, "gc_states") + re.NotContains(excludedAllWithExpiredProperties, "global_gc_barriers") + var excludedAllWithExpired gcStateCommandAll + re.NoError(json.Unmarshal(output, &excludedAllWithExpired), string(output)) + excludedStatesByIDWithExpired := make( + map[uint32]gcStateCommandState, + len(excludedAllWithExpired.GCStates), + ) + for _, state := range excludedAllWithExpired.GCStates { + excludedStatesByIDWithExpired[state.KeyspaceID] = state + } + excludedNullStateWithExpired, ok := excludedStatesByIDWithExpired[constant.NullKeyspaceID] + re.True(ok) + requireGCStateCommandState( + re, + excludedNullStateWithExpired, + nullState, + []expectedGCStateCommandBarrier{ + {barrierID: "a-null", barrierTS: 110}, + {barrierID: "z-null", barrierTS: 120, expires: true}, + }, + ) + excludedKeyspaceLevelStateWithExpired, ok := excludedStatesByIDWithExpired[keyspaceLevelID] + re.True(ok) + requireGCStateCommandState( + re, + excludedKeyspaceLevelStateWithExpired, + keyspaceLevelStateWithExpired, + []expectedGCStateCommandBarrier{ + {barrierID: "a-local", barrierTS: 210}, + {barrierID: "z-local", barrierTS: 220, expires: true}, + {barrierID: "expired-local", barrierTS: 230, expired: true}, + }, + ) + + if kerneltype.IsNextGen() { + systemState, ok := statesByID[constant.SystemKeyspaceID] + re.True(ok) + re.True(systemState.IsKeyspaceLevelGC) + } else { + unifiedKeyspaceIDString := strconv.FormatUint(uint64(unifiedKeyspaceID), 10) + output, err = tests.ExecuteCommand( + ctl.GetRootCmd(), "-u", pdAddr, "gc-state", "keyspace", unifiedKeyspaceIDString, + ) + re.NoError(err) + var unifiedKeyspaceResponse gcStateCommandSingle + re.NoError(json.Unmarshal(output, &unifiedKeyspaceResponse), string(output)) + re.Equal(unifiedKeyspaceID, unifiedKeyspaceResponse.RequestedKeyspaceID) + re.Equal(constant.NullKeyspaceID, unifiedKeyspaceResponse.EffectiveKeyspaceID) + re.False(unifiedKeyspaceResponse.IsKeyspaceLevelGC) + re.Equal(uint64(100), unifiedKeyspaceResponse.TxnSafePoint) + re.Equal(uint64(90), unifiedKeyspaceResponse.GCSafePoint) + requireGCStateCommandBarriers(re, unifiedKeyspaceResponse.GCBarriers, []expectedGCStateCommandBarrier{ + {barrierID: "a-null", barrierTS: 110}, + {barrierID: "z-null", barrierTS: 120, expires: true}, + }) + requireGCStateCommandBarriers( + re, + unifiedKeyspaceResponse.GlobalGCBarriers, + []expectedGCStateCommandBarrier{ + {barrierID: "a-global", barrierTS: 310}, + {barrierID: "z-global", barrierTS: 320}, + }, + ) + + re.NotContains(statesByID, unifiedKeyspaceID) + } + + output, err = tests.ExecuteCommand(ctl.GetRootCmd(), "-u", pdAddr, "service-gc-safepoint") + re.NoError(err) + re.True(json.Valid(output), string(output)) +}