From 36cc83fd8a7025df534470f536e1a1386fc4edba Mon Sep 17 00:00:00 2001 From: lidezhu Date: Fri, 17 Jul 2026 09:54:45 +0800 Subject: [PATCH 01/26] logpuller: extract region request scheduler from subscription client --- .../logpuller/region_request_scheduler.go | 125 +++++++++++++++++- logservice/logpuller/requested_store.go | 83 ++++++++++++ logservice/logpuller/subscription_client.go | 68 ++++++++++ .../logpuller/subscription_client_test.go | 62 +++++++++ 4 files changed, 337 insertions(+), 1 deletion(-) create mode 100644 logservice/logpuller/requested_store.go diff --git a/logservice/logpuller/region_request_scheduler.go b/logservice/logpuller/region_request_scheduler.go index 4d4e5caadd..bff6f9ea7e 100644 --- a/logservice/logpuller/region_request_scheduler.go +++ b/logservice/logpuller/region_request_scheduler.go @@ -20,14 +20,20 @@ import ( "sync/atomic" "github.com/pingcap/log" +<<<<<<< HEAD "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/config" "github.com/pingcap/ticdc/pkg/errors" "github.com/pingcap/ticdc/pkg/metrics" +======= + "github.com/pingcap/ticdc/pkg/config" + "github.com/pingcap/ticdc/pkg/errors" +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) "github.com/pingcap/ticdc/utils/priorityqueue" kvclientv2 "github.com/tikv/client-go/v2/kv" "github.com/tikv/client-go/v2/tikv" "go.uber.org/zap" +<<<<<<< HEAD "go.uber.org/zap/zapcore" "golang.org/x/sync/errgroup" ) @@ -49,6 +55,24 @@ type regionRequestScheduler struct { sequence atomic.Uint64 // stores maps TiKV addresses to regionRequestStore. Stores are created only // by Run, but are also read by metrics and deregistration goroutines. +======= + "golang.org/x/sync/errgroup" +) + +// regionRequestScheduler routes locked Region requests through the global +// priority queue to a worker connected to the Region's TiKV store. Range +// resolution and retry policy remain owned by subscriptionClient and +// regionFailureHandler respectively. +type regionRequestScheduler struct { + client *subscriptionClient + + // taskQueue orders all Regions before they are assigned to a TiKV store. + taskQueue *priorityqueue.PriorityQueue[*regionPriorityTask] + // sequence is the FIFO tie-breaker for Regions in the same priority class. + sequence atomic.Uint64 + // stores maps TiKV addresses to requestedStore. Stores are created only by + // run, but are also read by metrics and deregistration goroutines. +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) stores sync.Map // workerCount is the configured number of request workers per store. @@ -59,6 +83,7 @@ type regionRequestScheduler struct { maxWindowMultiplier int } +<<<<<<< HEAD func newRegionRequestScheduler( upstream *upstreamHandle, eventSink *regionEventSink, @@ -71,6 +96,17 @@ func newRegionRequestScheduler( upstream: upstream, eventSink: eventSink, failureHandler: failureHandler, +======= +func newRegionRequestScheduler(client *subscriptionClient) *regionRequestScheduler { + pullerConfig := config.GetGlobalServerConfig().Debug.Puller + workerCount := int(client.config.RegionRequestWorkerPerStore) + if workerCount <= 0 { + workerCount = 1 + } + workerWindow := (pullerConfig.PendingRegionRequestQueueSize + workerCount - 1) / workerCount + return ®ionRequestScheduler{ + client: client, +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) taskQueue: priorityqueue.New[*regionPriorityTask](), workerCount: workerCount, workerWindow: workerWindow, @@ -78,6 +114,7 @@ func newRegionRequestScheduler( } } +<<<<<<< HEAD func (s *regionRequestScheduler) Submit(region regionInfo) { if log.GetLevel() <= zapcore.DebugLevel { log.Debug("cdc region scan task enqueued", @@ -102,6 +139,15 @@ func (s *regionRequestScheduler) Run(ctx context.Context, workerGroup *errgroup. }) }() +======= +func (s *regionRequestScheduler) submit(region regionInfo) { + s.taskQueue.Push(NewRegionPriorityTask( + region, s.client.pdClock.CurrentTS(), s.sequence.Add(1))) +} + +func (s *regionRequestScheduler) run(ctx context.Context, group *errgroup.Group) error { + defer s.closeStores() +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) for { select { case <-ctx.Done(): @@ -117,6 +163,7 @@ func (s *regionRequestScheduler) Run(ctx context.Context, workerGroup *errgroup. return err } +<<<<<<< HEAD region, err := s.attachRPCContext(ctx, task.regionInfo) if err != nil { s.failureHandler.Report(newRegionErrorInfo(region, err)) @@ -142,12 +189,30 @@ func (s *regionRequestScheduler) Run(ctx context.Context, workerGroup *errgroup. zap.Uint64("regionID", region.verID.GetID()), zap.String("addr", region.rpcCtx.Addr)) } +======= + region, ok := s.attachRPCContext(ctx, task.GetRegionInfo()) + if !ok { + continue + } + + store := s.getOrCreateStore(ctx, group, region.rpcCtx.Addr) + task.updateRegion(region, s.client.pdClock.CurrentTS()) + if !store.submit(task) { + return context.Canceled + } + + log.Debug("subscription client will request a region", + zap.Uint64("subscriptionID", uint64(region.subscribedSpan.subID)), + zap.Uint64("regionID", region.verID.GetID()), + zap.String("addr", store.storeAddr)) +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) } } func (s *regionRequestScheduler) attachRPCContext( ctx context.Context, region regionInfo, +<<<<<<< HEAD ) (regionInfo, error) { bo := tikv.NewBackoffer(ctx, tikvRequestMaxBackoff) rpcCtx, err := s.upstream.regionCache.GetTiKVRPCContext( @@ -158,15 +223,33 @@ func (s *regionRequestScheduler) attachRPCContext( } if err != nil { log.Debug("region request scheduler failed to get RPC context", +======= +) (regionInfo, bool) { + bo := tikv.NewBackoffer(ctx, tikvRequestMaxBackoff) + rpcCtx, err := s.client.regionCache.GetTiKVRPCContext( + bo, region.verID, kvclientv2.ReplicaReadLeader, 0) + if rpcCtx != nil { + region.rpcCtx = rpcCtx + return region, true + } + if err != nil { + log.Debug("subscription client get rpc context fail", +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) zap.Uint64("subscriptionID", uint64(region.subscribedSpan.subID)), zap.Uint64("regionID", region.verID.GetID()), zap.Error(err)) } +<<<<<<< HEAD return region, &rpcCtxUnavailableErr{verID: region.verID} +======= + s.client.onRegionFail(newRegionErrorInfo(region, &rpcCtxUnavailableErr{verID: region.verID})) + return region, false +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) } func (s *regionRequestScheduler) getOrCreateStore( ctx context.Context, +<<<<<<< HEAD workerGroup *errgroup.Group, storeAddr string, ) *regionRequestStore { @@ -191,11 +274,34 @@ func (s *regionRequestScheduler) getOrCreateStore( } func (s *regionRequestScheduler) BroadcastDeregister( +======= + group *errgroup.Group, + storeAddr string, +) *requestedStore { + if value, ok := s.stores.Load(storeAddr); ok { + return value.(*requestedStore) + } + + store := newRequestedStore( + s.client, storeAddr, s.workerCount, s.workerWindow, s.maxWindowMultiplier) + // run is the only writer. Publish the store after its immutable worker list + // is complete, then start its workers. + s.stores.Store(storeAddr, store) + store.run(ctx, group) + return store +} + +func (s *regionRequestScheduler) broadcastDeregister( +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) subID SubscriptionID, filterLoop bool, ) { s.stores.Range(func(_, value any) bool { +<<<<<<< HEAD value.(*regionRequestStore).broadcastDeregister(subID, filterLoop) +======= + value.(*requestedStore).broadcastDeregister(subID, filterLoop) +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) return true }) } @@ -203,12 +309,17 @@ func (s *regionRequestScheduler) BroadcastDeregister( func (s *regionRequestScheduler) requestedRegionCount() int { count := 0 s.stores.Range(func(_, value any) bool { - count += value.(*regionRequestStore).requestedRegionCount() + <<<<<<< HEAD + count += value.(*regionRequestStore).inflightCount() + ======= + count += value.(*requestedStore).inflightCount() + >>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) return true }) return count } +<<<<<<< HEAD func (s *regionRequestScheduler) UpdateMetrics() { metrics.SubscriptionClientRequestedRegionCount.WithLabelValues("pending"). Set(float64(s.requestedRegionCount())) @@ -217,3 +328,15 @@ func (s *regionRequestScheduler) UpdateMetrics() { func (s *regionRequestScheduler) Close() { s.taskQueue.Close() } +======= +func (s *regionRequestScheduler) close() { + s.taskQueue.Close() +} + +func (s *regionRequestScheduler) closeStores() { + s.stores.Range(func(_, value any) bool { + value.(*requestedStore).close() + return true + }) +} +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) diff --git a/logservice/logpuller/requested_store.go b/logservice/logpuller/requested_store.go new file mode 100644 index 0000000000..f98b59b679 --- /dev/null +++ b/logservice/logpuller/requested_store.go @@ -0,0 +1,83 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package logpuller + +import ( + "context" + "sync/atomic" + + "golang.org/x/sync/errgroup" +) + +// requestedStore owns the request workers connected to one TiKV store. The +// worker slice is complete before the store is published and is immutable +// afterwards, so task submission only needs an atomic round-robin counter. +type requestedStore struct { + storeAddr string + workers []*regionRequestWorker + nextWorker atomic.Uint64 +} + +func newRequestedStore( + client *subscriptionClient, + storeAddr string, + workerCount int, + workerWindow int, + maxWindowMultiplier int, +) *requestedStore { + store := &requestedStore{ + storeAddr: storeAddr, + workers: make([]*regionRequestWorker, 0, workerCount), + } + for i := 0; i < workerCount; i++ { + store.workers = append(store.workers, newRegionRequestWorker( + client, store, workerWindow, maxWindowMultiplier)) + } + return store +} + +func (s *requestedStore) run(ctx context.Context, group *errgroup.Group) { + for _, worker := range s.workers { + group.Go(func() error { return worker.Run(ctx) }) + } +} + +func (s *requestedStore) submit(task *regionPriorityTask) bool { + if len(s.workers) == 0 { + return false + } + index := (s.nextWorker.Add(1) - 1) % uint64(len(s.workers)) + return s.workers[index].admission.submit(task) +} + +func (s *requestedStore) broadcastDeregister(subID SubscriptionID, filterLoop bool) { + for _, worker := range s.workers { + worker.controlQueue.push(deregisterRequest{subID: subID, filterLoop: filterLoop}) + } +} + +func (s *requestedStore) close() { + for _, worker := range s.workers { + worker.admission.close() + } +} + +func (s *requestedStore) inflightCount() int { + count := 0 + for _, worker := range s.workers { + count += worker.admission.stats().inflight + } + return count +} diff --git a/logservice/logpuller/subscription_client.go b/logservice/logpuller/subscription_client.go index 8fa9d17238..7614ab01dc 100644 --- a/logservice/logpuller/subscription_client.go +++ b/logservice/logpuller/subscription_client.go @@ -26,12 +26,19 @@ import ( "github.com/pingcap/ticdc/logservice/txnutil" "github.com/pingcap/ticdc/pkg/common" appcontext "github.com/pingcap/ticdc/pkg/common/context" +<<<<<<< HEAD "github.com/pingcap/ticdc/pkg/config" +======= +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) "github.com/pingcap/ticdc/pkg/metrics" "github.com/pingcap/ticdc/pkg/pdutil" "github.com/pingcap/ticdc/pkg/security" "github.com/pingcap/ticdc/pkg/spanz" "github.com/pingcap/ticdc/pkg/util" +<<<<<<< HEAD +======= + "github.com/prometheus/client_golang/prometheus" +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) "github.com/tikv/client-go/v2/tikv" pd "github.com/tikv/pd/client" "go.uber.org/zap" @@ -129,13 +136,23 @@ type subscriptionClient struct { lockResolver txnutil.LockResolver +<<<<<<< HEAD +======= + // the credential to connect tikv + credential *security.Credential + +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) // failureHandler handles failed regions and owns reschedule/retry decisions. failureHandler *regionFailureHandler // eventSink delivers region events and owns dynstream interaction. eventSink *regionEventSink // spanRegistry tracks subscribed spans and owns span-level background tasks. spanRegistry *spanRegistry +<<<<<<< HEAD // regionScheduler assigns locked region requests to per-store workers. +======= + // regionScheduler assigns locked Region requests to per-store workers. +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) regionScheduler *regionRequestScheduler // rangeTaskCh is used to receive range tasks. @@ -154,12 +171,20 @@ func NewSubscriptionClient( credential *security.Credential, ) SubscriptionClient { subClient := &subscriptionClient{ +<<<<<<< HEAD upstream: &upstreamHandle{ pd: pd, regionCache: appcontext.GetService[*tikv.RegionCache](appcontext.RegionCache), pdClock: appcontext.GetService[pdutil.Clock](appcontext.DefaultPDClock), credential: credential, }, +======= + config: config, + + pd: pd, + regionCache: appcontext.GetService[*tikv.RegionCache](appcontext.RegionCache), + pdClock: appcontext.GetService[pdutil.Clock](appcontext.DefaultPDClock), +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) lockResolver: lockResolver, rangeTaskCh: make(chan rangeTask, 1024), @@ -167,6 +192,7 @@ func NewSubscriptionClient( resolveLockRateLimiter: newResolveLockRateLimiter(), } subClient.ctx, subClient.cancel = context.WithCancel(context.Background()) +<<<<<<< HEAD subClient.failureHandler = newRegionFailureHandler( subClient.upstream.regionCache, subClient.onTableDrained, @@ -180,6 +206,14 @@ func NewSubscriptionClient( subClient.eventSink, subClient.failureHandler, ) +======= + subClient.failureHandler = newRegionFailureHandler(subClient) + subClient.eventSink = newRegionEventSink(subClient.ctx, subClient.failureHandler) + subClient.spanRegistry = newSpanRegistry(subClient.pd, subClient.pdClock) + subClient.regionScheduler = newRegionRequestScheduler(subClient) + + subClient.initMetrics() +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) return subClient } @@ -200,7 +234,13 @@ func (s *subscriptionClient) updateMetrics(ctx context.Context) error { case <-ctx.Done(): return ctx.Err() case <-ticker.C: +<<<<<<< HEAD s.regionScheduler.UpdateMetrics() +======= + pendingRegionReqCount := s.regionScheduler.inflightCount() + + metrics.SubscriptionClientRequestedRegionCount.WithLabelValues("pending").Set(float64(pendingRegionReqCount)) +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) s.eventSink.UpdateMetrics() s.spanRegistry.UpdateMetrics() } @@ -282,8 +322,12 @@ func (s *subscriptionClient) Run(ctx context.Context) error { // The goroutines are listed by data flow; errgroup does not guarantee their // actual startup order. g.Go(func() error { return s.handleRangeTasks(ctx) }) +<<<<<<< HEAD g.Go(func() error { return s.regionScheduler.Run(ctx, g) }) g.Go(func() error { return s.eventSink.Run(ctx) }) +======= + g.Go(func() error { return s.regionScheduler.run(ctx, g) }) +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) g.Go(func() error { return s.failureHandler.Run(ctx) }) g.Go(func() error { return s.spanRegistry.Run(ctx) }) g.Go(func() error { return s.handleResolveLockTasks(ctx) }) @@ -298,7 +342,11 @@ func (s *subscriptionClient) Run(ctx context.Context) error { func (s *subscriptionClient) Close(ctx context.Context) error { s.cancel() s.eventSink.Close() +<<<<<<< HEAD s.regionScheduler.Close() +======= + s.regionScheduler.close() +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) return nil } @@ -309,7 +357,11 @@ func (s *subscriptionClient) setTableStopped(rt *subscribedSpan) { // Set stopped to true so we can stop handling region events from the table, // then notify every existing worker to deregister the subscription. if rt.stopped.CompareAndSwap(false, true) { +<<<<<<< HEAD s.regionScheduler.BroadcastDeregister(rt.subID, rt.filterLoop) +======= + s.regionScheduler.broadcastDeregister(rt.subID, rt.filterLoop) +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) if rt.rangeLock.Stop() { s.onTableDrained(rt) } @@ -329,6 +381,14 @@ func (s *subscriptionClient) onTableDrained(rt *subscribedSpan) { s.spanRegistry.Remove(rt.subID) } +<<<<<<< HEAD +======= +// Note: don't block the caller, otherwise there may be deadlock +func (s *subscriptionClient) onRegionFail(errInfo regionErrorInfo) { + s.failureHandler.Report(errInfo) +} + +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) func (s *subscriptionClient) handleRangeTasks(ctx context.Context) error { g, ctx := errgroup.WithContext(ctx) // Limit the concurrent number of goroutines to convert range tasks to region tasks. @@ -433,7 +493,11 @@ func (s *subscriptionClient) divideSpanAndScheduleRegionRequests( } } +<<<<<<< HEAD // scheduleRegionRequest locks the region's range before submitting it to the +======= +// scheduleRegionRequest locks the Region's range before submitting it to the +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) // request scheduler. func (s *subscriptionClient) scheduleRegionRequest(ctx context.Context, region regionInfo) { lockRangeResult := region.subscribedSpan.rangeLock.LockRange( @@ -446,12 +510,16 @@ func (s *subscriptionClient) scheduleRegionRequest(ctx context.Context, region r switch lockRangeResult.Status { case regionlock.LockRangeStatusSuccess: region.lockedRangeState = lockRangeResult.LockedRangeState +<<<<<<< HEAD region.scanPriority = region.subscribedSpan.priorityPolicy.resolve( region.scanPriority, region.resolvedTs(), s.upstream.pdClock.CurrentTime(), ) s.regionScheduler.Submit(region) +======= + s.regionScheduler.submit(region) +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) case regionlock.LockRangeStatusStale: for _, r := range lockRangeResult.RetryRanges { s.scheduleRangeRequest(ctx, rangeTask{ diff --git a/logservice/logpuller/subscription_client_test.go b/logservice/logpuller/subscription_client_test.go index 818a745e54..8326899ac7 100644 --- a/logservice/logpuller/subscription_client_test.go +++ b/logservice/logpuller/subscription_client_test.go @@ -334,11 +334,18 @@ func TestStopTaskUsesSubscribedSpanFilterLoop(t *testing.T) { res := span.rangeLock.LockRange(context.Background(), rawSpan.StartKey, rawSpan.EndKey, 1, 1) require.Equal(t, regionlock.LockRangeStatusSuccess, res.Status) +<<<<<<< HEAD const storeAddr = "store-1" worker := ®ionRequestWorker{storeAddr: storeAddr, controlQueue: newControlQueue()} store := ®ionRequestStore{workers: []*regionRequestWorker{worker}} client.regionScheduler = ®ionRequestScheduler{} client.regionScheduler.stores.Store(storeAddr, store) +======= + worker := ®ionRequestWorker{controlQueue: newControlQueue()} + store := &requestedStore{storeAddr: "store-1", workers: []*regionRequestWorker{worker}} + client.regionScheduler = ®ionRequestScheduler{client: client} + client.regionScheduler.stores.Store(store.storeAddr, store) +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) client.setTableStopped(span) @@ -430,6 +437,10 @@ func TestRegionEventSinkPushUnblocksOnClientClose(t *testing.T) { sink.cond = sync.NewCond(&sink.mu) client := &subscriptionClient{eventSink: sink} client.regionScheduler = ®ionRequestScheduler{ +<<<<<<< HEAD +======= + client: client, +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) taskQueue: priorityqueue.New[*regionPriorityTask](), } client.ctx, client.cancel = context.WithCancel(context.Background()) @@ -457,6 +468,57 @@ func TestRegionEventSinkPushUnblocksOnClientClose(t *testing.T) { } } +<<<<<<< HEAD +======= +func TestBroadcastDeregisterUsesWorkerControlQueue(t *testing.T) { + client := &subscriptionClient{} + scheduler := ®ionRequestScheduler{client: client} + admission := newRegionAdmissionController(1, 1) + + worker := ®ionRequestWorker{ + admission: admission, + controlQueue: newControlQueue(), + } + store := &requestedStore{storeAddr: "store-1", workers: []*regionRequestWorker{worker}} + scheduler.stores.Store(store.storeAddr, store) + + dummyRegion := regionInfo{ + subscribedSpan: &subscribedSpan{subID: SubscriptionID(2)}, + lockedRangeState: ®ionlock.LockedRangeState{}, + } + require.True(t, admission.submit(NewRegionPriorityTask(dummyRegion, 1, 1))) + + scheduler.broadcastDeregister(SubscriptionID(1), true) + require.Equal(t, 1, worker.controlQueue.len()) + req, ok := worker.controlQueue.tryPop() + require.True(t, ok) + require.Equal(t, SubscriptionID(1), req.subID) + require.True(t, req.filterLoop) + require.Equal(t, 1, admission.stats().pending) +} + +func TestRequestedStoreDistributesRegionsAcrossWorkerBuffers(t *testing.T) { + worker1 := ®ionRequestWorker{admission: newRegionAdmissionController(1, 1)} + worker2 := ®ionRequestWorker{admission: newRegionAdmissionController(1, 1)} + store := &requestedStore{ + storeAddr: "store-1", + workers: []*regionRequestWorker{worker1, worker2}, + } + + for i := uint64(1); i <= 4; i++ { + region := regionInfo{ + verID: tikv.NewRegionVerID(i, 1, 1), + subscribedSpan: &subscribedSpan{subID: 1}, + lockedRangeState: ®ionlock.LockedRangeState{}, + } + require.True(t, store.submit(NewRegionPriorityTask(region, 1, i))) + } + + require.Equal(t, 2, worker1.admission.stats().pending) + require.Equal(t, 2, worker2.admission.stats().pending) +} + +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) func TestSubscriptionWithFailedTiKV(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) mockPDClock := pdutil.NewClock4Test() From 1dc8f6a5f69fd91ecbc9f92fb11e6438f5770e18 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Fri, 17 Jul 2026 11:07:32 +0800 Subject: [PATCH 02/26] refactor --- logservice/logpuller/requested_store.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/logservice/logpuller/requested_store.go b/logservice/logpuller/requested_store.go index f98b59b679..97294a8e33 100644 --- a/logservice/logpuller/requested_store.go +++ b/logservice/logpuller/requested_store.go @@ -31,7 +31,9 @@ type requestedStore struct { } func newRequestedStore( - client *subscriptionClient, + upstream *upstreamHandle, + eventSink *regionEventSink, + failureHandler *regionFailureHandler, storeAddr string, workerCount int, workerWindow int, @@ -43,7 +45,7 @@ func newRequestedStore( } for i := 0; i < workerCount; i++ { store.workers = append(store.workers, newRegionRequestWorker( - client, store, workerWindow, maxWindowMultiplier)) + upstream, eventSink, failureHandler, store, workerWindow, maxWindowMultiplier)) } return store } From 7d6b59e862ef4f792f2c22e1335875a157b57888 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Fri, 17 Jul 2026 16:41:06 +0800 Subject: [PATCH 03/26] refactor --- logservice/logpuller/priority_task.go | 21 ++++++++ logservice/logpuller/priority_task_test.go | 49 +++++++++++++++++++ .../region_admission_controller_test.go | 9 +++- .../logpuller/region_event_handler_test.go | 1 - .../logpuller/region_failure_handler.go | 24 +++++++++ .../logpuller/region_request_scheduler.go | 8 +++ logservice/logpuller/region_request_worker.go | 29 +++++++++++ .../logpuller/region_request_worker_test.go | 24 +++++++++ logservice/logpuller/region_state.go | 5 +- logservice/logpuller/region_tracker.go | 12 +++++ logservice/logpuller/subscription_client.go | 20 ++++++++ .../logpuller/subscription_client_test.go | 22 ++++++++- 12 files changed, 219 insertions(+), 5 deletions(-) diff --git a/logservice/logpuller/priority_task.go b/logservice/logpuller/priority_task.go index 38cd813c8b..89d43dc03a 100644 --- a/logservice/logpuller/priority_task.go +++ b/logservice/logpuller/priority_task.go @@ -32,6 +32,7 @@ type regionPriorityTask struct { heapIndex int // for heap.Item interface } +<<<<<<< HEAD // newRegionPriorityTask creates a new priority task for region. func newRegionPriorityTask(regionInfo regionInfo, sequence uint64) *regionPriorityTask { regionInfo.scanPriority = normalizeScanPriority(regionInfo.scanPriority) @@ -39,11 +40,31 @@ func newRegionPriorityTask(regionInfo regionInfo, sequence uint64) *regionPriori regionInfo: regionInfo, sequence: sequence, heapIndex: 0, // 0 means not in heap +======= +func newRegionPriorityTask(regionInfo regionInfo, currentTs, sequence uint64) *regionPriorityTask { + task := ®ionPriorityTask{ + sequence: sequence, + heapIndex: 0, // 0 means not in heap +>>>>>>> 9903a1be7 (refactor) } } +<<<<<<< HEAD func (pt *regionPriorityTask) priority() cdcpb.ScanPriority { return normalizeScanPriority(pt.regionInfo.scanPriority) +======= +// updateRegion refreshes both the request data and its priority before the task +// enters another scheduling stage. +func (pt *regionPriorityTask) updateRegion(regionInfo regionInfo, currentTs uint64) { + priority := normalRegionPriority + if regionInfo.wasInitialized { + priority = initializedRegionPriority + } else if regionScanLag(currentTs, regionInfo.resolvedTs()) < lowLagRegionThreshold { + priority = lowLagRegionPriority + } + pt.regionInfo = regionInfo + pt.priority = priority +>>>>>>> 9903a1be7 (refactor) } func (pt *regionPriorityTask) canUseMaxWindow() bool { diff --git a/logservice/logpuller/priority_task_test.go b/logservice/logpuller/priority_task_test.go index 41501181fb..35b8209c2b 100644 --- a/logservice/logpuller/priority_task_test.go +++ b/logservice/logpuller/priority_task_test.go @@ -60,6 +60,7 @@ func TestRegionPriorityTaskQueueOrder(t *testing.T) { queue := priorityqueue.New[*regionPriorityTask]() currentTime := time.Now() +<<<<<<< HEAD lowTask := newRegionPriorityTask( withScanPriority( newPriorityTestRegion(1, oracle.GoTimeToTS(currentTime.Add(-time.Hour))), @@ -80,6 +81,19 @@ func TestRegionPriorityTaskQueueOrder(t *testing.T) { cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, ), 1, +======= + normalTask := newRegionPriorityTask( + newPriorityTestRegion(1, oracle.GoTimeToTS(currentTime.Add(-time.Hour)), false), + currentTs, 3, + ) + lowLagTask := newRegionPriorityTask( + newPriorityTestRegion(2, oracle.GoTimeToTS(currentTime.Add(-10*time.Minute)), false), + currentTs, 2, + ) + initializedTask := newRegionPriorityTask( + newPriorityTestRegion(3, oracle.GoTimeToTS(currentTime.Add(-time.Hour)), true), + currentTs, 1, +>>>>>>> 9903a1be7 (refactor) ) require.True(t, queue.Push(lowTask)) @@ -98,10 +112,15 @@ func TestRegionPriorityTaskFIFOWithinPriority(t *testing.T) { currentTime := time.Now() checkpointTs := oracle.GoTimeToTS(currentTime.Add(-time.Hour)) +<<<<<<< HEAD first := newRegionPriorityTask( withScanPriority(newPriorityTestRegion(1, checkpointTs), cdcpb.ScanPriority_SCAN_PRIORITY_HIGH), 1) second := newRegionPriorityTask( withScanPriority(newPriorityTestRegion(2, checkpointTs), cdcpb.ScanPriority_SCAN_PRIORITY_HIGH), 2) +======= + first := newRegionPriorityTask(newPriorityTestRegion(1, checkpointTs, false), currentTs, 1) + second := newRegionPriorityTask(newPriorityTestRegion(2, checkpointTs, false), currentTs, 2) +>>>>>>> 9903a1be7 (refactor) require.True(t, queue.Push(second)) require.True(t, queue.Push(first)) @@ -120,6 +139,7 @@ func TestRegionPriorityTaskUsesHighPriorityWindow(t *testing.T) { lowTask := newRegionPriorityTask( withScanPriority(newPriorityTestRegion(2, 1), cdcpb.ScanPriority_SCAN_PRIORITY_LOW), 2) +<<<<<<< HEAD require.True(t, highTask.canUseMaxWindow()) require.False(t, lowTask.canUseMaxWindow()) } @@ -128,6 +148,35 @@ func TestRegionPriorityTaskRefreshesRegionInfoBetweenStages(t *testing.T) { region := withScanPriority(newPriorityTestRegion(1, 1), cdcpb.ScanPriority_SCAN_PRIORITY_LOW) task := newRegionPriorityTask(region, 1) require.Equal(t, cdcpb.ScanPriority_SCAN_PRIORITY_LOW, task.priority()) +======= + belowThreshold := newRegionPriorityTask(newPriorityTestRegion( + 1, + oracle.GoTimeToTS(currentTime.Add(-lowLagRegionThreshold+time.Millisecond)), + false, + ), currentTs, 1) + atThreshold := newRegionPriorityTask(newPriorityTestRegion( + 2, + oracle.GoTimeToTS(currentTime.Add(-lowLagRegionThreshold)), + false, + ), currentTs, 2) + futureCheckpoint := newRegionPriorityTask(newPriorityTestRegion( + 3, + oracle.GoTimeToTS(currentTime.Add(time.Second)), + false, + ), currentTs, 3) + + require.Equal(t, lowLagRegionPriority, belowThreshold.priority) + require.Equal(t, normalRegionPriority, atThreshold.priority) + require.Equal(t, lowLagRegionPriority, futureCheckpoint.priority) +} + +func TestRegionPriorityTaskRefreshesPriorityBetweenStages(t *testing.T) { + checkpointTime := time.Now() + checkpointTs := oracle.GoTimeToTS(checkpointTime) + region := newPriorityTestRegion(1, checkpointTs, false) + task := newRegionPriorityTask(region, oracle.GoTimeToTS(checkpointTime.Add(time.Minute)), 1) + require.Equal(t, lowLagRegionPriority, task.priority) +>>>>>>> 9903a1be7 (refactor) region.scanPriority = cdcpb.ScanPriority_SCAN_PRIORITY_HIGH task.regionInfo = region diff --git a/logservice/logpuller/region_admission_controller_test.go b/logservice/logpuller/region_admission_controller_test.go index 13fc2ed2bc..706e140501 100644 --- a/logservice/logpuller/region_admission_controller_test.go +++ b/logservice/logpuller/region_admission_controller_test.go @@ -39,7 +39,6 @@ func createTestRegionInfo(subID SubscriptionID, regionID uint64) regionInfo { span, nil, &subscribedSpan{subID: subID, startTs: 100, span: span}, - false, ) } @@ -56,7 +55,11 @@ func submitRegionForAdmission( currentTs uint64, ) { t.Helper() +<<<<<<< HEAD task := newRegionPriorityTask(region, region.verID.GetID()) +======= + task := newRegionPriorityTask(region, currentTs, region.verID.GetID()) +>>>>>>> 9903a1be7 (refactor) require.True(t, controller.submit(task)) } @@ -191,7 +194,11 @@ func TestRegionAdmissionControllerClose(t *testing.T) { controller := newRegionAdmissionController(1, 1) controller.close() region := prepareRegionForAdmission(createTestRegionInfo(1, 1), 1) +<<<<<<< HEAD require.False(t, controller.submit(newRegionPriorityTask(region, 1))) +======= + require.False(t, controller.submit(newRegionPriorityTask(region, 1, 1))) +>>>>>>> 9903a1be7 (refactor) _, err := controller.pop(context.Background(), nil) require.ErrorIs(t, err, context.Canceled) diff --git a/logservice/logpuller/region_event_handler_test.go b/logservice/logpuller/region_event_handler_test.go index fa6e76eaad..f2e1885197 100644 --- a/logservice/logpuller/region_event_handler_test.go +++ b/logservice/logpuller/region_event_handler_test.go @@ -87,7 +87,6 @@ func TestHandleEventEntryEventOutOfOrder(t *testing.T) { span, &tikv.RPCContext{}, subSpan, - false, ) lockResult := subSpan.rangeLock.LockRange( context.Background(), span.StartKey, span.EndKey, 1, 1) diff --git a/logservice/logpuller/region_failure_handler.go b/logservice/logpuller/region_failure_handler.go index 3622d927d9..5ca0f03c96 100644 --- a/logservice/logpuller/region_failure_handler.go +++ b/logservice/logpuller/region_failure_handler.go @@ -64,6 +64,14 @@ func newRegionFailureHandler( } } +func (r *regionFailureHandler) retryRange(ctx context.Context, errInfo regionErrorInfo) { + r.scheduleRangeRequest(ctx, rangeTask{ + span: errInfo.span, + subscribedSpan: errInfo.subscribedSpan, + wasInitialized: errInfo.wasInitialized, + }) +} + // Report admits a region failure into the recovery pipeline. It releases the // corresponding range lock before enqueueing the failure so new range tasks are // not blocked by stale region ownership. @@ -159,12 +167,20 @@ func (r *regionFailureHandler) handleError(ctx context.Context, errInfo regionEr } if innerErr.GetEpochNotMatch() != nil { metricFeedEpochNotMatchCounter.Inc() +<<<<<<< HEAD rescheduleRange() +======= + r.retryRange(ctx, errInfo) +>>>>>>> 9903a1be7 (refactor) return nil } if innerErr.GetRegionNotFound() != nil { metricFeedRegionNotFoundCounter.Inc() +<<<<<<< HEAD rescheduleRange() +======= + r.retryRange(ctx, errInfo) +>>>>>>> 9903a1be7 (refactor) return nil } if innerErr.GetCongested() != nil { @@ -197,14 +213,22 @@ func (r *regionFailureHandler) handleError(ctx context.Context, errInfo regionEr return nil case *rpcCtxUnavailableErr: metricFeedRPCCtxUnavailable.Inc() +<<<<<<< HEAD rescheduleRange() +======= + r.retryRange(ctx, errInfo) +>>>>>>> 9903a1be7 (refactor) return nil case *getStoreErr: metricGetStoreErr.Inc() bo := tikv.NewBackoffer(ctx, tikvRequestMaxBackoff) // cannot get the store the region belongs to, so we need to reload the region. r.regionCache.OnSendFail(bo, errInfo.rpcCtx, true, err) +<<<<<<< HEAD rescheduleRange() +======= + r.retryRange(ctx, errInfo) +>>>>>>> 9903a1be7 (refactor) return nil case *storeStreamErr: metricStoreSendRequestErr.Inc() diff --git a/logservice/logpuller/region_request_scheduler.go b/logservice/logpuller/region_request_scheduler.go index bff6f9ea7e..135b700e28 100644 --- a/logservice/logpuller/region_request_scheduler.go +++ b/logservice/logpuller/region_request_scheduler.go @@ -116,6 +116,7 @@ func newRegionRequestScheduler(client *subscriptionClient) *regionRequestSchedul <<<<<<< HEAD func (s *regionRequestScheduler) Submit(region regionInfo) { +<<<<<<< HEAD if log.GetLevel() <= zapcore.DebugLevel { log.Debug("cdc region scan task enqueued", zap.Uint64("subscriptionID", uint64(region.subscribedSpan.subID)), @@ -129,6 +130,10 @@ func (s *regionRequestScheduler) Submit(region regionInfo) { zap.String("span", common.FormatTableSpan(®ion.span))) } s.taskQueue.Push(newRegionPriorityTask(region, s.sequence.Add(1))) +======= + s.taskQueue.Push(newRegionPriorityTask( + region, s.upstream.pdClock.CurrentTS(), s.sequence.Add(1))) +>>>>>>> 9903a1be7 (refactor) } func (s *regionRequestScheduler) Run(ctx context.Context, workerGroup *errgroup.Group) error { @@ -164,6 +169,9 @@ func (s *regionRequestScheduler) run(ctx context.Context, group *errgroup.Group) } <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 9903a1be7 (refactor) region, err := s.attachRPCContext(ctx, task.regionInfo) if err != nil { s.failureHandler.Report(newRegionErrorInfo(region, err)) diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index 3d3e0dca4b..8e519db935 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -174,6 +174,35 @@ func (s *regionRequestWorker) Run(ctx context.Context) error { } } +<<<<<<< HEAD +======= +// failStreamRegions transfers every request sent by a failed stream to the +// recovery pipeline. +func (s *regionRequestWorker) failStreamRegions(err error) { + for _, state := range s.tracker.Drain() { + s.notifyRegionError(state, err) + } + // The failed stream no longer owns remote registrations. + s.controlQueue.drain() +} + +// failPendingRegions transfers requests owned by this worker but not yet sent +// to the recovery pipeline, so they can be resolved and routed again. +func (s *regionRequestWorker) failPendingRegions(err error) { + for _, task := range s.admission.drain() { + s.failureHandler.Report(newRegionErrorInfo(task.regionInfo, err)) + } +} + +func (s *regionRequestWorker) notifyRegionError(state *regionFeedState, err error) { + state.markStopped(err) + s.eventSink.Push( + SubscriptionID(state.requestID), + regionEvent{states: []*regionFeedState{state}}, + ) +} + +>>>>>>> 9903a1be7 (refactor) func (s *regionRequestWorker) waitForRegionRequest(ctx context.Context) (*regionReq, error) { // Without a stream there are no remote registrations to deregister. s.controlQueue.drain() diff --git a/logservice/logpuller/region_request_worker_test.go b/logservice/logpuller/region_request_worker_test.go index 32228d8e7c..daef4db768 100644 --- a/logservice/logpuller/region_request_worker_test.go +++ b/logservice/logpuller/region_request_worker_test.go @@ -584,6 +584,7 @@ func TestRunStreamFailureReportsPendingRegionsToFailureHandler(t *testing.T) { pdClient := newFailureRecoveryTestPDClient(t) defer pdClient.Close() +<<<<<<< HEAD handler := newRegionFailureHandler(nil, func(*subscribedSpan) {}, nil, nil) worker := ®ionRequestWorker{ upstream: &upstreamHandle{pd: pdClient, credential: &security.Credential{}}, @@ -593,6 +594,29 @@ func TestRunStreamFailureReportsPendingRegionsToFailureHandler(t *testing.T) { controlQueue: newControlQueue(), tracker: newRegionTracker(), storeAddr: "127.0.0.1:1", +======= + admission := newRegionAdmissionController(1, 1) + failureHandler := ®ionFailureHandler{cache: newErrCache()} + worker := ®ionRequestWorker{failureHandler: failureHandler, admission: admission} + regions := []regionInfo{ + { + verID: tikv.NewRegionVerID(1, 1, 1), + span: heartbeatpb.TableSpan{ + TableID: 1, StartKey: []byte("a"), EndKey: []byte("m"), + }, + subscribedSpan: span, lockedRangeState: lock1.LockedRangeState, + }, + { + verID: tikv.NewRegionVerID(2, 1, 1), + span: heartbeatpb.TableSpan{ + TableID: 1, StartKey: []byte("m"), EndKey: []byte("z"), + }, + subscribedSpan: span, lockedRangeState: lock2.LockedRangeState, + }, + } + for i, region := range regions { + require.True(t, admission.submit(newRegionPriorityTask(region, 1, uint64(i+1)))) +>>>>>>> 9903a1be7 (refactor) } firstRegion := createFailureRecoveryTestRegion(t, 1, 1) diff --git a/logservice/logpuller/region_state.go b/logservice/logpuller/region_state.go index 0f2f84ce3e..6a2837564f 100644 --- a/logservice/logpuller/region_state.go +++ b/logservice/logpuller/region_state.go @@ -58,15 +58,18 @@ func newRegionInfo( span heartbeatpb.TableSpan, rpcCtx *tikv.RPCContext, subscribedSpan *subscribedSpan, - filterLoop bool, ) regionInfo { return regionInfo{ verID: verID, span: span, rpcCtx: rpcCtx, subscribedSpan: subscribedSpan, +<<<<<<< HEAD filterLoop: filterLoop, scanPriority: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, +======= + filterLoop: subscribedSpan.filterLoop, +>>>>>>> 9903a1be7 (refactor) } } diff --git a/logservice/logpuller/region_tracker.go b/logservice/logpuller/region_tracker.go index 81a27dc86c..4d69bf8bb3 100644 --- a/logservice/logpuller/region_tracker.go +++ b/logservice/logpuller/region_tracker.go @@ -112,6 +112,7 @@ func (t *regionTracker) Drain() []*regionFeedState { t.statesBySubscription = make(map[SubscriptionID]regionStatesByID) t.mu.Unlock() +<<<<<<< HEAD totalStates := 0 for _, states := range statesBySubscription { totalStates += len(states) @@ -119,6 +120,17 @@ func (t *regionTracker) Drain() []*regionFeedState { drainedStates := make([]*regionFeedState, 0, totalStates) for _, states := range statesBySubscription { drainedStates = append(drainedStates, slices.Collect(maps.Values(states))...) +======= + stateCount := 0 + for _, states := range statesBySubscription { + stateCount += len(states) + } + drainedStates := make([]*regionFeedState, 0, stateCount) + for _, states := range statesBySubscription { + for _, state := range states { + drainedStates = append(drainedStates, state) + } +>>>>>>> 9903a1be7 (refactor) } return drainedStates } diff --git a/logservice/logpuller/subscription_client.go b/logservice/logpuller/subscription_client.go index 7614ab01dc..d77de7dc5b 100644 --- a/logservice/logpuller/subscription_client.go +++ b/logservice/logpuller/subscription_client.go @@ -89,8 +89,12 @@ type resolveLockTask struct { type rangeTask struct { span heartbeatpb.TableSpan subscribedSpan *subscribedSpan +<<<<<<< HEAD filterLoop bool priority cdcpb.ScanPriority +======= + wasInitialized bool +>>>>>>> 9903a1be7 (refactor) } // upstreamHandle contains the stable TiKV and PD dependencies shared by the @@ -286,12 +290,16 @@ func (s *subscriptionClient) Subscribe( select { case <-s.ctx.Done(): log.Warn("subscribes span failed, the subscription client has closed") +<<<<<<< HEAD case s.rangeTaskCh <- rangeTask{ span: span, subscribedSpan: rt, filterLoop: rt.filterLoop, priority: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, }: +======= + case s.rangeTaskCh <- rangeTask{span: span, subscribedSpan: rt}: +>>>>>>> 9903a1be7 (refactor) log.Info("subscribes span done", zap.Uint64("subscriptionID", uint64(subID)), zap.Int64("tableID", span.TableID), zap.Uint64("startTs", startTs), zap.String("startKey", spanz.HexKey(span.StartKey)), zap.String("endKey", spanz.HexKey(span.EndKey))) @@ -325,9 +333,12 @@ func (s *subscriptionClient) Run(ctx context.Context) error { <<<<<<< HEAD g.Go(func() error { return s.regionScheduler.Run(ctx, g) }) g.Go(func() error { return s.eventSink.Run(ctx) }) +<<<<<<< HEAD ======= g.Go(func() error { return s.regionScheduler.run(ctx, g) }) >>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) +======= +>>>>>>> 9903a1be7 (refactor) g.Go(func() error { return s.failureHandler.Run(ctx) }) g.Go(func() error { return s.spanRegistry.Run(ctx) }) g.Go(func() error { return s.handleResolveLockTasks(ctx) }) @@ -477,8 +488,13 @@ func (s *subscriptionClient) divideSpanAndScheduleRegionRequests( } verID := tikv.NewRegionVerID(regionMeta.Id, regionMeta.RegionEpoch.ConfVer, regionMeta.RegionEpoch.Version) +<<<<<<< HEAD regionInfo := newRegionInfo(verID, intersectSpan, nil, subscribedSpan, task.filterLoop) regionInfo.scanPriority = normalizeScanPriority(task.priority) +======= + regionInfo := newRegionInfo(verID, intersectSpan, nil, subscribedSpan) + regionInfo.wasInitialized = task.wasInitialized +>>>>>>> 9903a1be7 (refactor) // Schedule a region request to subscribe the region. s.scheduleRegionRequest(ctx, regionInfo) @@ -525,8 +541,12 @@ func (s *subscriptionClient) scheduleRegionRequest(ctx context.Context, region r s.scheduleRangeRequest(ctx, rangeTask{ span: r, subscribedSpan: region.subscribedSpan, +<<<<<<< HEAD filterLoop: region.filterLoop, priority: region.scanPriority, +======= + wasInitialized: region.wasInitialized, +>>>>>>> 9903a1be7 (refactor) }) } default: diff --git a/logservice/logpuller/subscription_client_test.go b/logservice/logpuller/subscription_client_test.go index 8326899ac7..048517b011 100644 --- a/logservice/logpuller/subscription_client_test.go +++ b/logservice/logpuller/subscription_client_test.go @@ -335,17 +335,23 @@ func TestStopTaskUsesSubscribedSpanFilterLoop(t *testing.T) { res := span.rangeLock.LockRange(context.Background(), rawSpan.StartKey, rawSpan.EndKey, 1, 1) require.Equal(t, regionlock.LockRangeStatusSuccess, res.Status) <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 9903a1be7 (refactor) const storeAddr = "store-1" worker := ®ionRequestWorker{storeAddr: storeAddr, controlQueue: newControlQueue()} store := ®ionRequestStore{workers: []*regionRequestWorker{worker}} client.regionScheduler = ®ionRequestScheduler{} client.regionScheduler.stores.Store(storeAddr, store) +<<<<<<< HEAD ======= worker := ®ionRequestWorker{controlQueue: newControlQueue()} store := &requestedStore{storeAddr: "store-1", workers: []*regionRequestWorker{worker}} client.regionScheduler = ®ionRequestScheduler{client: client} client.regionScheduler.stores.Store(store.storeAddr, store) >>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) +======= +>>>>>>> 9903a1be7 (refactor) client.setTableStopped(span) @@ -475,18 +481,25 @@ func TestBroadcastDeregisterUsesWorkerControlQueue(t *testing.T) { scheduler := ®ionRequestScheduler{client: client} admission := newRegionAdmissionController(1, 1) + const storeAddr = "store-1" worker := ®ionRequestWorker{ + storeAddr: storeAddr, admission: admission, controlQueue: newControlQueue(), } +<<<<<<< HEAD store := &requestedStore{storeAddr: "store-1", workers: []*regionRequestWorker{worker}} scheduler.stores.Store(store.storeAddr, store) +======= + store := ®ionRequestStore{workers: []*regionRequestWorker{worker}} + scheduler.stores.Store(storeAddr, store) +>>>>>>> 9903a1be7 (refactor) dummyRegion := regionInfo{ subscribedSpan: &subscribedSpan{subID: SubscriptionID(2)}, lockedRangeState: ®ionlock.LockedRangeState{}, } - require.True(t, admission.submit(NewRegionPriorityTask(dummyRegion, 1, 1))) + require.True(t, admission.submit(newRegionPriorityTask(dummyRegion, 1, 1))) scheduler.broadcastDeregister(SubscriptionID(1), true) require.Equal(t, 1, worker.controlQueue.len()) @@ -500,9 +513,14 @@ func TestBroadcastDeregisterUsesWorkerControlQueue(t *testing.T) { func TestRequestedStoreDistributesRegionsAcrossWorkerBuffers(t *testing.T) { worker1 := ®ionRequestWorker{admission: newRegionAdmissionController(1, 1)} worker2 := ®ionRequestWorker{admission: newRegionAdmissionController(1, 1)} +<<<<<<< HEAD store := &requestedStore{ storeAddr: "store-1", workers: []*regionRequestWorker{worker1, worker2}, +======= + store := ®ionRequestStore{ + workers: []*regionRequestWorker{worker1, worker2}, +>>>>>>> 9903a1be7 (refactor) } for i := uint64(1); i <= 4; i++ { @@ -511,7 +529,7 @@ func TestRequestedStoreDistributesRegionsAcrossWorkerBuffers(t *testing.T) { subscribedSpan: &subscribedSpan{subID: 1}, lockedRangeState: ®ionlock.LockedRangeState{}, } - require.True(t, store.submit(NewRegionPriorityTask(region, 1, i))) + require.True(t, store.submit(newRegionPriorityTask(region, 1, i))) } require.Equal(t, 2, worker1.admission.stats().pending) From cb55332a4699e7670a4194d31bfe24b25c5206ac Mon Sep 17 00:00:00 2001 From: lidezhu Date: Fri, 17 Jul 2026 19:31:15 +0800 Subject: [PATCH 04/26] refactor --- logservice/eventstore/pebble.go | 2 +- logservice/logpuller/memory_quota.go | 415 ++++++++++++++++++ logservice/logpuller/memory_quota_test.go | 296 +++++++++++++ logservice/logpuller/priority_task.go | 21 - logservice/logpuller/priority_task_test.go | 49 --- .../logpuller/region_admission_controller.go | 49 ++- .../region_admission_controller_test.go | 42 +- logservice/logpuller/region_event_handler.go | 52 ++- .../logpuller/region_event_handler_test.go | 183 ++++---- logservice/logpuller/region_event_sink.go | 115 +++-- .../logpuller/region_event_sink_test.go | 120 +++-- .../logpuller/region_failure_handler.go | 24 - .../logpuller/region_request_scheduler.go | 140 +----- logservice/logpuller/region_request_store.go | 16 +- logservice/logpuller/region_request_worker.go | 41 +- .../logpuller/region_request_worker_test.go | 24 - logservice/logpuller/region_state.go | 5 +- logservice/logpuller/region_tracker.go | 12 - logservice/logpuller/span_registry.go | 12 + logservice/logpuller/subscription_client.go | 103 +---- .../logpuller/subscription_client_test.go | 80 ---- metrics/grafana/ticdc_new_arch.json | 90 ++++ .../ticdc_new_arch_next_gen.json | 90 ++++ pkg/config/debug.go | 18 + pkg/config/debug_test.go | 15 +- pkg/metrics/log_puller.go | 8 + 26 files changed, 1294 insertions(+), 728 deletions(-) create mode 100644 logservice/logpuller/memory_quota.go create mode 100644 logservice/logpuller/memory_quota_test.go diff --git a/logservice/eventstore/pebble.go b/logservice/eventstore/pebble.go index f5f48bf71b..0d950c99d0 100644 --- a/logservice/eventstore/pebble.go +++ b/logservice/eventstore/pebble.go @@ -42,7 +42,7 @@ func newPebbleOptions(dbNum int) *pebble.Options { MaxOpenFiles: maxOpenFilesPerDB, - MaxConcurrentCompactions: func() int { return 6 }, + MaxConcurrentCompactions: func() int { return 2 }, // Decrease compaction frequency L0CompactionThreshold: 20, diff --git a/logservice/logpuller/memory_quota.go b/logservice/logpuller/memory_quota.go new file mode 100644 index 0000000000..6596fc0f29 --- /dev/null +++ b/logservice/logpuller/memory_quota.go @@ -0,0 +1,415 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package logpuller + +import ( + "context" + "math" + "sync" + "sync/atomic" + "time" +) + +const ( + defaultLogPullerMemoryQuota uint64 = 1024 * 1024 * 1024 + + defaultPauseWarmingRatio = 0.15 + defaultResumeWarmingRatio = 0.05 + defaultFreezeAllRatio = 0.8 + defaultResumeAllRatio = 0.6 + defaultHardLimitRatio = 5.0 + + defaultScanBaseSize uint64 = 8 * 1024 * 1024 + defaultScanLagUnit = 10 * time.Minute + defaultScanLagWeight = 0.22 + defaultMaxScanLagFactor = 16 +) + +type admissionLevel uint8 + +const ( + admissionNormal admissionLevel = iota + admissionPauseWarming + admissionFreezeAllNewScans +) + +type memoryQuotaLease struct { + once sync.Once + release func() +} + +func (l *memoryQuotaLease) Release() { + if l == nil { + return + } + l.once.Do(func() { + if l.release != nil { + l.release() + } + }) +} + +type subscriptionQuotaState struct { + eventLeases map[*memoryQuotaLease]struct{} + scanLeases map[*memoryQuotaLease]struct{} +} + +func newSubscriptionQuotaState() *subscriptionQuotaState { + return &subscriptionQuotaState{ + eventLeases: make(map[*memoryQuotaLease]struct{}), + scanLeases: make(map[*memoryQuotaLease]struct{}), + } +} + +// memoryQuotaController tracks event memory retained by downstream callbacks +// and estimated memory for admitted initial scans. Event memory is allowed to +// exceed the soft capacity, but the receive path waits at the hard limit. Scan +// admission first pauses uninitialized high-lag spans and freezes all new scans +// only under heavier pressure; both transitions use hysteresis when resuming. +type memoryQuotaController struct { + mu sync.Mutex + cond *sync.Cond + + capacity uint64 + // used tracks event bytes retained until downstream finishes consuming them. + used uint64 + + // scanUsed tracks the estimated memory of all admitted initial scans. + // warmingScanUsed is the subset used by uninitialized, high-lag spans. + scanUsed uint64 + warmingScanUsed uint64 + level admissionLevel + + pauseWarmingRatio float64 + resumeWarmingRatio float64 + freezeAllRatio float64 + resumeAllRatio float64 + hardLimitRatio float64 + + scanEstimate uint64 + + subscriptions map[SubscriptionID]*subscriptionQuotaState + onAvailable atomic.Value // func() +} + +func newMemoryQuotaController(capacity, scanBaseSize uint64) *memoryQuotaController { + if capacity == 0 { + capacity = defaultLogPullerMemoryQuota + } + if scanBaseSize == 0 { + scanBaseSize = defaultScanBaseSize + } + c := &memoryQuotaController{ + capacity: capacity, + level: admissionNormal, + pauseWarmingRatio: defaultPauseWarmingRatio, + resumeWarmingRatio: defaultResumeWarmingRatio, + freezeAllRatio: defaultFreezeAllRatio, + resumeAllRatio: defaultResumeAllRatio, + hardLimitRatio: defaultHardLimitRatio, + scanEstimate: scanBaseSize, + subscriptions: make(map[SubscriptionID]*subscriptionQuotaState), + } + c.cond = sync.NewCond(&c.mu) + return c +} + +func (c *memoryQuotaController) setOnAvailable(fn func()) { + c.onAvailable.Store(fn) +} + +func (c *memoryQuotaController) notifyAvailable() { + if fn, ok := c.onAvailable.Load().(func()); ok && fn != nil { + fn() + } +} + +func (c *memoryQuotaController) wakeAll() { + c.mu.Lock() + c.cond.Broadcast() + c.mu.Unlock() +} + +func (c *memoryQuotaController) snapshot() (used, capacity uint64, level admissionLevel) { + c.mu.Lock() + defer c.mu.Unlock() + return c.used, c.capacity, c.level +} + +func (c *memoryQuotaController) scanSnapshot() ( + scanUsed uint64, + warmingScanUsed uint64, + warmingScanBudget uint64, + scanEstimate uint64, + hardLimit uint64, +) { + c.mu.Lock() + defer c.mu.Unlock() + return c.scanUsed, c.warmingScanUsed, c.warmingScanBudgetLocked(), + c.scanEstimate, c.hardLimitLocked() +} + +func (c *memoryQuotaController) addSubscription(span *subscribedSpan) { + if span == nil { + return + } + c.mu.Lock() + c.subscriptions[span.subID] = newSubscriptionQuotaState() + c.mu.Unlock() +} + +func (c *memoryQuotaController) removeSubscription(span *subscribedSpan) { + if span == nil { + return + } + + c.mu.Lock() + state, ok := c.subscriptions[span.subID] + if !ok { + c.mu.Unlock() + return + } + delete(c.subscriptions, span.subID) + leases := make([]*memoryQuotaLease, 0, len(state.eventLeases)+len(state.scanLeases)) + for lease := range state.eventLeases { + leases = append(leases, lease) + } + for lease := range state.scanLeases { + leases = append(leases, lease) + } + // Wake event receivers so they can observe that the subscription was removed. + c.cond.Broadcast() + c.mu.Unlock() + + for _, lease := range leases { + lease.Release() + } + // Removing a subscription can make a pending request eligible even when the + // subscription itself did not own any lease. + c.notifyAvailable() +} + +func (c *memoryQuotaController) markSubscriptionInitialized() { + c.notifyAvailable() +} + +func (c *memoryQuotaController) acquireScan( + region regionInfo, + currentTs uint64, +) (*memoryQuotaLease, bool) { + span := region.subscribedSpan + if span == nil { + return nil, true + } + + c.mu.Lock() + state, ok := c.subscriptions[span.subID] + if !ok { + // The subscription has already been removed. Let the request continue to + // the worker, where the normal stopped-subscription path will discard it. + c.mu.Unlock() + return nil, true + } + c.refreshLevelLocked() + if c.level == admissionFreezeAllNewScans { + c.mu.Unlock() + return nil, false + } + + bytes := c.estimateScanSizeLocked(region, currentTs) + warming := isWarmingScan(region, currentTs) + if c.isWarmingScanBlockedLocked(warming, bytes) { + c.mu.Unlock() + return nil, false + } + + lease := &memoryQuotaLease{} + lease.release = func() { + c.mu.Lock() + previousLevel := c.level + c.scanUsed = subtractFloor(c.scanUsed, bytes) + if warming { + c.warmingScanUsed = subtractFloor(c.warmingScanUsed, bytes) + } + delete(state.scanLeases, lease) + c.refreshLevelLocked() + shouldNotifyAdmission := c.level < previousLevel || + (warming && c.level == admissionNormal) + c.mu.Unlock() + + if shouldNotifyAdmission { + c.notifyAvailable() + } + } + c.scanUsed += bytes + if warming { + c.warmingScanUsed += bytes + } + state.scanLeases[lease] = struct{}{} + c.refreshLevelLocked() + c.mu.Unlock() + return lease, true +} + +func (c *memoryQuotaController) trackEvent( + ctx context.Context, + span *subscribedSpan, + bytes uint64, +) *memoryQuotaLease { + if span == nil || bytes == 0 { + return nil + } + + c.mu.Lock() + if ctx.Err() != nil { + c.mu.Unlock() + return nil + } + state := c.subscriptions[span.subID] + if state == nil { + c.mu.Unlock() + return nil + } + + for c.used > 0 && wouldExceed(c.used, bytes, c.hardLimitLocked()) { + c.cond.Wait() + if ctx.Err() != nil { + c.mu.Unlock() + return nil + } + if c.subscriptions[span.subID] != state { + c.mu.Unlock() + return nil + } + } + + c.used += bytes + c.refreshLevelLocked() + lease := &memoryQuotaLease{} + lease.release = func() { + c.mu.Lock() + previousLevel := c.level + c.used = subtractFloor(c.used, bytes) + delete(state.eventLeases, lease) + c.refreshLevelLocked() + shouldNotifyAdmission := c.level < previousLevel + c.cond.Broadcast() + c.mu.Unlock() + + if shouldNotifyAdmission { + c.notifyAvailable() + } + } + state.eventLeases[lease] = struct{}{} + c.mu.Unlock() + return lease +} + +func (c *memoryQuotaController) estimateScanSizeLocked(region regionInfo, currentTs uint64) uint64 { + raw := float64(c.scanEstimate) * scanLagFactor(region.resolvedTs(), currentTs) + estimate := uint64(raw) + if estimate < c.scanEstimate { + estimate = c.scanEstimate + } + maxEstimate := uint64(math.MaxUint64) + if c.scanEstimate <= math.MaxUint64/defaultMaxScanLagFactor { + maxEstimate = c.scanEstimate * defaultMaxScanLagFactor + } + if estimate > maxEstimate { + estimate = maxEstimate + } + if estimate == 0 { + estimate = c.scanEstimate + } + return estimate +} + +func scanLagFactor(startTs, currentTs uint64) float64 { + lag := regionScanLag(currentTs, startTs) + if lag <= 0 { + return 1 + } + return min(defaultMaxScanLagFactor, + 1+defaultScanLagWeight*math.Log2(1+float64(lag)/float64(defaultScanLagUnit))) +} + +func isWarmingScan(region regionInfo, currentTs uint64) bool { + span := region.subscribedSpan + if span == nil || span.initialized.Load() { + return false + } + return regionScanLag(currentTs, region.resolvedTs()) >= lowLagRegionThreshold +} + +func (c *memoryQuotaController) isWarmingScanBlockedLocked(warming bool, bytes uint64) bool { + if !warming { + return false + } + if c.level == admissionPauseWarming { + return true + } + return wouldExceed(c.warmingScanUsed, bytes, c.warmingScanBudgetLocked()) +} + +func (c *memoryQuotaController) warmingScanBudgetLocked() uint64 { + budget := uint64(float64(c.capacity) * c.pauseWarmingRatio) + return max(budget, c.scanEstimate) +} + +func (c *memoryQuotaController) hardLimitLocked() uint64 { + return uint64(float64(c.capacity) * c.hardLimitRatio) +} + +func (c *memoryQuotaController) refreshLevelLocked() { + // scanUsed predicts the event memory an initial scan may produce, so adding + // it to actual event bytes would count the same pressure twice. + pressure := max(c.used, c.scanUsed) + usage := float64(pressure) / float64(c.capacity) + switch c.level { + case admissionFreezeAllNewScans: + if usage <= c.resumeAllRatio { + if usage >= c.pauseWarmingRatio { + c.level = admissionPauseWarming + } else { + c.level = admissionNormal + } + } + case admissionPauseWarming: + switch { + case usage >= c.freezeAllRatio: + c.level = admissionFreezeAllNewScans + case usage <= c.resumeWarmingRatio: + c.level = admissionNormal + } + default: + switch { + case usage >= c.freezeAllRatio: + c.level = admissionFreezeAllNewScans + case usage >= c.pauseWarmingRatio: + c.level = admissionPauseWarming + } + } +} + +func wouldExceed(used, bytes, limit uint64) bool { + return bytes > limit || used > limit-bytes +} + +func subtractFloor(value, delta uint64) uint64 { + if value < delta { + return 0 + } + return value - delta +} diff --git a/logservice/logpuller/memory_quota_test.go b/logservice/logpuller/memory_quota_test.go new file mode 100644 index 0000000000..be4d88d89d --- /dev/null +++ b/logservice/logpuller/memory_quota_test.go @@ -0,0 +1,296 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package logpuller + +import ( + "context" + "testing" + "time" + + "github.com/pingcap/ticdc/logservice/logpuller/regionlock" + "github.com/stretchr/testify/require" + "github.com/tikv/client-go/v2/oracle" +) + +func newTestQuotaSpan(subID SubscriptionID) *subscribedSpan { + span := &subscribedSpan{subID: subID} + span.resolvedTs.Store(oracle.GoTimeToTS(time.Now())) + return span +} + +func newTestQuotaRegion(span *subscribedSpan) regionInfo { + state := ®ionlock.LockedRangeState{} + state.ResolvedTs.Store(span.resolvedTs.Load()) + return regionInfo{ + subscribedSpan: span, + lockedRangeState: state, + } +} + +func setTestQuotaSpanLag(span *subscribedSpan, lag time.Duration) uint64 { + now := time.Now() + span.resolvedTs.Store(oracle.GoTimeToTS(now.Add(-lag))) + return oracle.GoTimeToTS(now) +} + +func TestMemoryQuotaAdmissionLevels(t *testing.T) { + quota := newMemoryQuotaController(100, 10) + warmingSpan := newTestQuotaSpan(1) + initializedSpan := newTestQuotaSpan(2) + initializedSpan.initialized.Store(true) + warmingTs := setTestQuotaSpanLag(warmingSpan, lowLagRegionThreshold+time.Minute) + initializedTs := setTestQuotaSpanLag(initializedSpan, lowLagRegionThreshold+time.Minute) + quota.addSubscription(warmingSpan) + quota.addSubscription(initializedSpan) + + lowLease := quota.trackEvent(context.Background(), initializedSpan, 5) + pauseLease := quota.trackEvent(context.Background(), initializedSpan, 10) + require.NotNil(t, lowLease) + require.NotNil(t, pauseLease) + scanLease, admitted := quota.acquireScan(newTestQuotaRegion(warmingSpan), warmingTs) + require.False(t, admitted) + require.Nil(t, scanLease) + + scanLease, admitted = quota.acquireScan( + newTestQuotaRegion(initializedSpan), initializedTs) + require.True(t, admitted) + scanLease.Release() + + middleLease := quota.trackEvent(context.Background(), initializedSpan, 45) + freezeLease := quota.trackEvent(context.Background(), initializedSpan, 20) + require.NotNil(t, middleLease) + require.NotNil(t, freezeLease) + scanLease, admitted = quota.acquireScan( + newTestQuotaRegion(initializedSpan), initializedTs) + require.False(t, admitted) + require.Nil(t, scanLease) + + freezeLease.Release() + _, _, level := quota.snapshot() + require.Equal(t, admissionPauseWarming, level) + middleLease.Release() + _, _, level = quota.snapshot() + require.Equal(t, admissionPauseWarming, level) + pauseLease.Release() + _, _, level = quota.snapshot() + require.Equal(t, admissionNormal, level) + lowLease.Release() +} + +func TestMemoryQuotaRemoveSubscriptionReleasesOwnedMemory(t *testing.T) { + quota := newMemoryQuotaController(100, 10) + span1 := newTestQuotaSpan(1) + span2 := newTestQuotaSpan(2) + quota.addSubscription(span1) + quota.addSubscription(span2) + + lease1 := quota.trackEvent(context.Background(), span1, 30) + lease2 := quota.trackEvent(context.Background(), span2, 40) + require.NotNil(t, lease1) + require.NotNil(t, lease2) + scanLease, admitted := quota.acquireScan(newTestQuotaRegion(span1), span1.resolvedTs.Load()) + require.True(t, admitted) + require.NotNil(t, scanLease) + + quota.removeSubscription(span1) + used, _, _ := quota.snapshot() + require.Equal(t, uint64(40), used) + scanUsed, _, _, _, _ := quota.scanSnapshot() + require.Zero(t, scanUsed) + require.NotContains(t, quota.subscriptions, span1.subID) + quota.removeSubscription(span1) + + lease1.Release() + scanLease.Release() + used, _, _ = quota.snapshot() + require.Equal(t, uint64(40), used) + + // Late region tasks are allowed to reach the stopped-subscription cleanup + // path without recreating quota state. + scanLease, admitted = quota.acquireScan(newTestQuotaRegion(span1), span1.resolvedTs.Load()) + require.True(t, admitted) + require.Nil(t, scanLease) + require.NotContains(t, quota.subscriptions, span1.subID) + + lease2.Release() + used, _, _ = quota.snapshot() + require.Zero(t, used) +} + +func TestMemoryQuotaBlockedEventStopsWhenSubscriptionIsRemoved(t *testing.T) { + quota := newMemoryQuotaController(100, 10) + quota.hardLimitRatio = 1 + span := newTestQuotaSpan(1) + quota.addSubscription(span) + + lease := quota.trackEvent(context.Background(), span, 100) + require.NotNil(t, lease) + acquired := make(chan *memoryQuotaLease, 1) + go func() { + acquired <- quota.trackEvent(context.Background(), span, 1) + }() + + select { + case <-acquired: + t.Fatal("event memory should wait at the hard limit") + case <-time.After(100 * time.Millisecond): + } + + quota.removeSubscription(span) + select { + case blockedLease := <-acquired: + require.Nil(t, blockedLease) + case <-time.After(time.Second): + t.Fatal("removing the subscription did not wake the blocked event") + } + require.NotContains(t, quota.subscriptions, span.subID) + lease.Release() +} + +func TestMemoryQuotaBlockedEventResumesAfterRelease(t *testing.T) { + quota := newMemoryQuotaController(100, 10) + quota.hardLimitRatio = 1 + span := newTestQuotaSpan(1) + quota.addSubscription(span) + + lease := quota.trackEvent(context.Background(), span, 100) + require.NotNil(t, lease) + acquired := make(chan *memoryQuotaLease, 1) + go func() { + acquired <- quota.trackEvent(context.Background(), span, 1) + }() + + select { + case <-acquired: + t.Fatal("event memory should wait at the hard limit") + case <-time.After(100 * time.Millisecond): + } + + lease.Release() + select { + case nextLease := <-acquired: + require.NotNil(t, nextLease) + nextLease.Release() + case <-time.After(time.Second): + t.Fatal("event memory did not resume after memory was released") + } +} + +func TestMemoryQuotaWarmingScanBudget(t *testing.T) { + quota := newMemoryQuotaController(200, 10) + span := newTestQuotaSpan(1) + currentTs := setTestQuotaSpanLag(span, lowLagRegionThreshold+time.Minute) + quota.addSubscription(span) + region := newTestQuotaRegion(span) + + lease1, admitted := quota.acquireScan(region, currentTs) + require.True(t, admitted) + lease2, admitted := quota.acquireScan(region, currentTs) + require.True(t, admitted) + + lease3, admitted := quota.acquireScan(region, currentTs) + require.False(t, admitted) + require.Nil(t, lease3) + + lease1.Release() + lease3, admitted = quota.acquireScan(region, currentTs) + require.True(t, admitted) + lease2.Release() + lease3.Release() +} + +func TestMemoryQuotaLowLagScanBypassesWarmingGate(t *testing.T) { + quota := newMemoryQuotaController(100, 10) + span := newTestQuotaSpan(1) + currentTs := setTestQuotaSpanLag(span, lowLagRegionThreshold-time.Second) + quota.addSubscription(span) + + pressureLease := quota.trackEvent(context.Background(), span, 20) + require.NotNil(t, pressureLease) + scanLease, admitted := quota.acquireScan(newTestQuotaRegion(span), currentTs) + require.True(t, admitted) + require.NotNil(t, scanLease) + _, warmingScanUsed, _, _, _ := quota.scanSnapshot() + require.Zero(t, warmingScanUsed) + + scanLease.Release() + pressureLease.Release() +} + +func TestMemoryQuotaRemovalNotifiesAdmissionWithoutLeases(t *testing.T) { + quota := newMemoryQuotaController(100, 10) + span := newTestQuotaSpan(1) + quota.addSubscription(span) + + notified := make(chan struct{}, 1) + quota.setOnAvailable(func() { + select { + case notified <- struct{}{}: + default: + } + }) + quota.removeSubscription(span) + select { + case <-notified: + case <-time.After(time.Second): + t.Fatal("subscription removal did not notify admission") + } +} + +func TestAdmissionWaitsForMemoryAndReleasesScanLease(t *testing.T) { + quota := newMemoryQuotaController(100, 10) + span := newTestQuotaSpan(1) + currentTs := setTestQuotaSpanLag(span, lowLagRegionThreshold+time.Minute) + quota.addSubscription(span) + controller := newRegionAdmissionController(1, 1, quota, func() uint64 { + return currentTs + }) + quota.setOnAvailable(controller.notifyAvailable) + + pressureLease := quota.trackEvent(context.Background(), span, 20) + require.NotNil(t, pressureLease) + region := newTestQuotaRegion(span) + require.True(t, controller.submit(newRegionPriorityTask(region, currentTs, 1))) + + type popResult struct { + req *regionReq + err error + } + result := make(chan popResult, 1) + go func() { + req, err := controller.pop(context.Background(), nil) + result <- popResult{req: req, err: err} + }() + select { + case <-result: + t.Fatal("warming scan should wait while memory is under pressure") + case <-time.After(100 * time.Millisecond): + } + + pressureLease.Release() + var resultValue popResult + select { + case resultValue = <-result: + case <-time.After(time.Second): + t.Fatal("scan admission was not notified after memory became available") + } + require.NoError(t, resultValue.err) + req := resultValue.req + scanUsed, _, _, _, _ := quota.scanSnapshot() + require.NotZero(t, scanUsed) + require.True(t, req.abort()) + scanUsed, _, _, _, _ = quota.scanSnapshot() + require.Zero(t, scanUsed) +} diff --git a/logservice/logpuller/priority_task.go b/logservice/logpuller/priority_task.go index 89d43dc03a..38cd813c8b 100644 --- a/logservice/logpuller/priority_task.go +++ b/logservice/logpuller/priority_task.go @@ -32,7 +32,6 @@ type regionPriorityTask struct { heapIndex int // for heap.Item interface } -<<<<<<< HEAD // newRegionPriorityTask creates a new priority task for region. func newRegionPriorityTask(regionInfo regionInfo, sequence uint64) *regionPriorityTask { regionInfo.scanPriority = normalizeScanPriority(regionInfo.scanPriority) @@ -40,31 +39,11 @@ func newRegionPriorityTask(regionInfo regionInfo, sequence uint64) *regionPriori regionInfo: regionInfo, sequence: sequence, heapIndex: 0, // 0 means not in heap -======= -func newRegionPriorityTask(regionInfo regionInfo, currentTs, sequence uint64) *regionPriorityTask { - task := ®ionPriorityTask{ - sequence: sequence, - heapIndex: 0, // 0 means not in heap ->>>>>>> 9903a1be7 (refactor) } } -<<<<<<< HEAD func (pt *regionPriorityTask) priority() cdcpb.ScanPriority { return normalizeScanPriority(pt.regionInfo.scanPriority) -======= -// updateRegion refreshes both the request data and its priority before the task -// enters another scheduling stage. -func (pt *regionPriorityTask) updateRegion(regionInfo regionInfo, currentTs uint64) { - priority := normalRegionPriority - if regionInfo.wasInitialized { - priority = initializedRegionPriority - } else if regionScanLag(currentTs, regionInfo.resolvedTs()) < lowLagRegionThreshold { - priority = lowLagRegionPriority - } - pt.regionInfo = regionInfo - pt.priority = priority ->>>>>>> 9903a1be7 (refactor) } func (pt *regionPriorityTask) canUseMaxWindow() bool { diff --git a/logservice/logpuller/priority_task_test.go b/logservice/logpuller/priority_task_test.go index 35b8209c2b..41501181fb 100644 --- a/logservice/logpuller/priority_task_test.go +++ b/logservice/logpuller/priority_task_test.go @@ -60,7 +60,6 @@ func TestRegionPriorityTaskQueueOrder(t *testing.T) { queue := priorityqueue.New[*regionPriorityTask]() currentTime := time.Now() -<<<<<<< HEAD lowTask := newRegionPriorityTask( withScanPriority( newPriorityTestRegion(1, oracle.GoTimeToTS(currentTime.Add(-time.Hour))), @@ -81,19 +80,6 @@ func TestRegionPriorityTaskQueueOrder(t *testing.T) { cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, ), 1, -======= - normalTask := newRegionPriorityTask( - newPriorityTestRegion(1, oracle.GoTimeToTS(currentTime.Add(-time.Hour)), false), - currentTs, 3, - ) - lowLagTask := newRegionPriorityTask( - newPriorityTestRegion(2, oracle.GoTimeToTS(currentTime.Add(-10*time.Minute)), false), - currentTs, 2, - ) - initializedTask := newRegionPriorityTask( - newPriorityTestRegion(3, oracle.GoTimeToTS(currentTime.Add(-time.Hour)), true), - currentTs, 1, ->>>>>>> 9903a1be7 (refactor) ) require.True(t, queue.Push(lowTask)) @@ -112,15 +98,10 @@ func TestRegionPriorityTaskFIFOWithinPriority(t *testing.T) { currentTime := time.Now() checkpointTs := oracle.GoTimeToTS(currentTime.Add(-time.Hour)) -<<<<<<< HEAD first := newRegionPriorityTask( withScanPriority(newPriorityTestRegion(1, checkpointTs), cdcpb.ScanPriority_SCAN_PRIORITY_HIGH), 1) second := newRegionPriorityTask( withScanPriority(newPriorityTestRegion(2, checkpointTs), cdcpb.ScanPriority_SCAN_PRIORITY_HIGH), 2) -======= - first := newRegionPriorityTask(newPriorityTestRegion(1, checkpointTs, false), currentTs, 1) - second := newRegionPriorityTask(newPriorityTestRegion(2, checkpointTs, false), currentTs, 2) ->>>>>>> 9903a1be7 (refactor) require.True(t, queue.Push(second)) require.True(t, queue.Push(first)) @@ -139,7 +120,6 @@ func TestRegionPriorityTaskUsesHighPriorityWindow(t *testing.T) { lowTask := newRegionPriorityTask( withScanPriority(newPriorityTestRegion(2, 1), cdcpb.ScanPriority_SCAN_PRIORITY_LOW), 2) -<<<<<<< HEAD require.True(t, highTask.canUseMaxWindow()) require.False(t, lowTask.canUseMaxWindow()) } @@ -148,35 +128,6 @@ func TestRegionPriorityTaskRefreshesRegionInfoBetweenStages(t *testing.T) { region := withScanPriority(newPriorityTestRegion(1, 1), cdcpb.ScanPriority_SCAN_PRIORITY_LOW) task := newRegionPriorityTask(region, 1) require.Equal(t, cdcpb.ScanPriority_SCAN_PRIORITY_LOW, task.priority()) -======= - belowThreshold := newRegionPriorityTask(newPriorityTestRegion( - 1, - oracle.GoTimeToTS(currentTime.Add(-lowLagRegionThreshold+time.Millisecond)), - false, - ), currentTs, 1) - atThreshold := newRegionPriorityTask(newPriorityTestRegion( - 2, - oracle.GoTimeToTS(currentTime.Add(-lowLagRegionThreshold)), - false, - ), currentTs, 2) - futureCheckpoint := newRegionPriorityTask(newPriorityTestRegion( - 3, - oracle.GoTimeToTS(currentTime.Add(time.Second)), - false, - ), currentTs, 3) - - require.Equal(t, lowLagRegionPriority, belowThreshold.priority) - require.Equal(t, normalRegionPriority, atThreshold.priority) - require.Equal(t, lowLagRegionPriority, futureCheckpoint.priority) -} - -func TestRegionPriorityTaskRefreshesPriorityBetweenStages(t *testing.T) { - checkpointTime := time.Now() - checkpointTs := oracle.GoTimeToTS(checkpointTime) - region := newPriorityTestRegion(1, checkpointTs, false) - task := newRegionPriorityTask(region, oracle.GoTimeToTS(checkpointTime.Add(time.Minute)), 1) - require.Equal(t, lowLagRegionPriority, task.priority) ->>>>>>> 9903a1be7 (refactor) region.scanPriority = cdcpb.ScanPriority_SCAN_PRIORITY_HIGH task.regionInfo = region diff --git a/logservice/logpuller/region_admission_controller.go b/logservice/logpuller/region_admission_controller.go index 1564aaed74..b7ee52f3da 100644 --- a/logservice/logpuller/region_admission_controller.go +++ b/logservice/logpuller/region_admission_controller.go @@ -36,6 +36,7 @@ type regionReq struct { regionInfo regionInfo createTime time.Time controller *regionAdmissionController + scanQuota *memoryQuotaLease released atomic.Bool } @@ -72,6 +73,7 @@ func (r *regionReq) release() bool { if !r.released.CompareAndSwap(false, true) { return false } + r.scanQuota.Release() r.controller.release() return true } @@ -96,6 +98,11 @@ type regionAdmissionController struct { // closed prevents new submissions and makes waiting workers exit. closed bool } + + // memoryQuota gates initial scans using the log puller's global memory + // pressure. currentTs is sampled when a request is admitted. + memoryQuota *memoryQuotaController + currentTs func() uint64 // notify wakes workers when a request is submitted or an admission slot is // released. The one-element buffer prevents a wakeup from being lost between // checking the admission condition and waiting on this channel. Notifications @@ -109,7 +116,12 @@ type regionAdmissionStats struct { inflight int } -func newRegionAdmissionController(currentWindow, maxWindowMultiplier int) *regionAdmissionController { +func newRegionAdmissionController( + currentWindow int, + maxWindowMultiplier int, + memoryQuota *memoryQuotaController, + currentTs func() uint64, +) *regionAdmissionController { if currentWindow <= 0 { currentWindow = 1 } @@ -123,6 +135,8 @@ func newRegionAdmissionController(currentWindow, maxWindowMultiplier int) *regio controller := ®ionAdmissionController{ currentWindow: currentWindow, maxWindow: maxWindow, + memoryQuota: memoryQuota, + currentTs: currentTs, notify: make(chan struct{}, 1), } controller.state.pending = heap.NewHeap[*regionPriorityTask]() @@ -153,7 +167,7 @@ func (c *regionAdmissionController) pop( c.state.Unlock() return nil, context.Canceled } - request := c.popEligibleLocked() + request, scanQuota := c.popEligibleLocked() if request != nil { c.state.inflight++ c.state.Unlock() @@ -161,6 +175,7 @@ func (c *regionAdmissionController) pop( regionInfo: request.regionInfo, createTime: time.Now(), controller: c, + scanQuota: scanQuota, }, nil } c.state.Unlock() @@ -175,16 +190,32 @@ func (c *regionAdmissionController) pop( } } -func (c *regionAdmissionController) popEligibleLocked() *regionPriorityTask { +func (c *regionAdmissionController) popEligibleLocked() ( + *regionPriorityTask, + *memoryQuotaLease, +) { request, ok := c.state.pending.PeekTop() if !ok { - return nil + return nil, nil } if c.state.inflight >= c.windowFor(request) { - return nil + return nil, nil + } + + var scanQuota *memoryQuotaLease + if c.memoryQuota != nil { + currentTs := uint64(0) + if c.currentTs != nil { + currentTs = c.currentTs() + } + var admitted bool + scanQuota, admitted = c.memoryQuota.acquireScan(request.regionInfo, currentTs) + if !admitted { + return nil, nil + } } request, _ = c.state.pending.PopTop() - return request + return request, scanQuota } func (c *regionAdmissionController) windowFor(request *regionPriorityTask) int { @@ -203,6 +234,12 @@ func (c *regionAdmissionController) release() { c.state.Unlock() } +func (c *regionAdmissionController) notifyAvailable() { + c.state.Lock() + c.notifyOneLocked() + c.state.Unlock() +} + func (c *regionAdmissionController) close() { c.state.Lock() if !c.state.closed { diff --git a/logservice/logpuller/region_admission_controller_test.go b/logservice/logpuller/region_admission_controller_test.go index 706e140501..ed68f14a2f 100644 --- a/logservice/logpuller/region_admission_controller_test.go +++ b/logservice/logpuller/region_admission_controller_test.go @@ -20,7 +20,6 @@ import ( "testing" "time" - "github.com/pingcap/kvproto/pkg/cdcpb" "github.com/pingcap/ticdc/heartbeatpb" "github.com/pingcap/ticdc/logservice/logpuller/regionlock" "github.com/stretchr/testify/require" @@ -55,16 +54,12 @@ func submitRegionForAdmission( currentTs uint64, ) { t.Helper() -<<<<<<< HEAD - task := newRegionPriorityTask(region, region.verID.GetID()) -======= task := newRegionPriorityTask(region, currentTs, region.verID.GetID()) ->>>>>>> 9903a1be7 (refactor) require.True(t, controller.submit(task)) } func TestRegionAdmissionControllerNormalWindow(t *testing.T) { - controller := newRegionAdmissionController(1, 2) + controller := newRegionAdmissionController(1, 2, nil, nil) currentTs := oracle.GoTimeToTS(time.Now()) checkpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Hour)) region1 := prepareRegionForAdmission(createTestRegionInfo(1, 1), checkpointTs) @@ -88,10 +83,11 @@ func TestRegionAdmissionControllerNormalWindow(t *testing.T) { require.True(t, req2.abort()) } -func TestRegionAdmissionControllerHighPriorityUsesMaxWindow(t *testing.T) { - controller := newRegionAdmissionController(1, 2) +func TestRegionAdmissionControllerLowLagUsesMaxWindow(t *testing.T) { + controller := newRegionAdmissionController(1, 2, nil, nil) currentTs := oracle.GoTimeToTS(time.Now()) slowCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Hour)) + lowLagCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Minute)) submitRegionForAdmission(t, controller, prepareRegionForAdmission(createTestRegionInfo(1, 1), slowCheckpointTs), @@ -102,9 +98,9 @@ func TestRegionAdmissionControllerHighPriorityUsesMaxWindow(t *testing.T) { submitRegionForAdmission(t, controller, prepareRegionForAdmission(createTestRegionInfo(1, 2), slowCheckpointTs), currentTs) - highPriorityRegion := prepareRegionForAdmission(createTestRegionInfo(1, 3), slowCheckpointTs) - highPriorityRegion.scanPriority = cdcpb.ScanPriority_SCAN_PRIORITY_HIGH - submitRegionForAdmission(t, controller, highPriorityRegion, currentTs) + submitRegionForAdmission(t, controller, + prepareRegionForAdmission(createTestRegionInfo(1, 3), lowLagCheckpointTs), + currentTs) req2, err := controller.pop(t.Context(), nil) require.NoError(t, err) @@ -124,10 +120,11 @@ func TestRegionAdmissionControllerHighPriorityUsesMaxWindow(t *testing.T) { require.True(t, req3.abort()) } -func TestRegionAdmissionControllerPrioritizesHighPriorityRegion(t *testing.T) { - controller := newRegionAdmissionController(1, 2) +func TestRegionAdmissionControllerPrioritizesInitializedRegion(t *testing.T) { + controller := newRegionAdmissionController(1, 2, nil, nil) currentTs := oracle.GoTimeToTS(time.Now()) slowCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Hour)) + lowLagCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Minute)) submitRegionForAdmission(t, controller, prepareRegionForAdmission(createTestRegionInfo(1, 1), slowCheckpointTs), @@ -136,11 +133,12 @@ func TestRegionAdmissionControllerPrioritizesHighPriorityRegion(t *testing.T) { require.NoError(t, err) submitRegionForAdmission(t, controller, - prepareRegionForAdmission(createTestRegionInfo(1, 2), slowCheckpointTs), + prepareRegionForAdmission(createTestRegionInfo(1, 2), lowLagCheckpointTs), currentTs) - highPriorityRegion := prepareRegionForAdmission(createTestRegionInfo(1, 3), slowCheckpointTs) - highPriorityRegion.scanPriority = cdcpb.ScanPriority_SCAN_PRIORITY_HIGH - submitRegionForAdmission(t, controller, highPriorityRegion, currentTs) + initializedRegion := prepareRegionForAdmission(createTestRegionInfo(1, 3), slowCheckpointTs) + initializedRegion.wasInitialized = true + submitRegionForAdmission(t, controller, + initializedRegion, currentTs) req2, err := controller.pop(t.Context(), nil) require.NoError(t, err) @@ -155,7 +153,7 @@ func TestRegionAdmissionControllerPrioritizesHighPriorityRegion(t *testing.T) { } func TestRegionAdmissionLeaseReleasedOnce(t *testing.T) { - controller := newRegionAdmissionController(1, 1) + controller := newRegionAdmissionController(1, 1, nil, nil) currentTs := oracle.GoTimeToTS(time.Now()) region := prepareRegionForAdmission(createTestRegionInfo(1, 1), currentTs) submitRegionForAdmission(t, controller, region, currentTs) @@ -191,21 +189,17 @@ func TestRegionAdmissionLeaseReleasedOnce(t *testing.T) { } func TestRegionAdmissionControllerClose(t *testing.T) { - controller := newRegionAdmissionController(1, 1) + controller := newRegionAdmissionController(1, 1, nil, nil) controller.close() region := prepareRegionForAdmission(createTestRegionInfo(1, 1), 1) -<<<<<<< HEAD - require.False(t, controller.submit(newRegionPriorityTask(region, 1))) -======= require.False(t, controller.submit(newRegionPriorityTask(region, 1, 1))) ->>>>>>> 9903a1be7 (refactor) _, err := controller.pop(context.Background(), nil) require.ErrorIs(t, err, context.Canceled) } func TestRegionAdmissionControllerDrainPending(t *testing.T) { - controller := newRegionAdmissionController(1, 1) + controller := newRegionAdmissionController(1, 1, nil, nil) region1 := prepareRegionForAdmission(createTestRegionInfo(1, 1), 1) region2 := prepareRegionForAdmission(createTestRegionInfo(1, 2), 1) submitRegionForAdmission(t, controller, region1, 1) diff --git a/logservice/logpuller/region_event_handler.go b/logservice/logpuller/region_event_handler.go index e13b91cbd0..c23184ddab 100644 --- a/logservice/logpuller/region_event_handler.go +++ b/logservice/logpuller/region_event_handler.go @@ -29,7 +29,8 @@ import ( ) var ( - metricsEventCount = metrics.PullerEventCounter.WithLabelValues("event") + metricsResolvedTsCount = metrics.PullerEventCounter.WithLabelValues("resolved_ts") + metricsEventCount = metrics.PullerEventCounter.WithLabelValues("event") metricRegionEventHandleDurationEntries = metrics.SubscriptionClientRegionEventHandleDuration.WithLabelValues("entries") metricRegionEventHandleDurationResolved = metrics.SubscriptionClientRegionEventHandleDuration.WithLabelValues("resolved") @@ -53,8 +54,18 @@ type regionEvent struct { // Resolved-ts events: `resolvedTs` is set and `states` contains all related regions. states []*regionFeedState - entries *cdcpb.Event_Entries_ - resolvedTs uint64 + entries *cdcpb.Event_Entries_ + resolvedTs uint64 + memoryQuota *memoryQuotaLease +} + +func (event *regionEvent) needMemoryQuota() bool { + return event.entries != nil +} + +func (event *regionEvent) releaseMemoryQuota() { + event.memoryQuota.Release() + event.memoryQuota = nil } func (event *regionEvent) getSize() int { @@ -121,7 +132,12 @@ func (h *regionEventHandler) Handle(span *subscribedSpan, events ...regionEvent) } newResolvedTs := uint64(0) + wasInitialized := span.initialized.Load() + quotaLeases := make([]*memoryQuotaLease, 0, len(events)) for _, event := range events { + if event.memoryQuota != nil { + quotaLeases = append(quotaLeases, event.memoryQuota) + } if len(event.states) == 1 && event.states[0].isStale() { hasError = true h.handleRegionError(event.states[0]) @@ -142,14 +158,24 @@ func (h *regionEventHandler) Handle(span *subscribedSpan, events ...regionEvent) log.Panic("should not reach", zap.Any("event", event), zap.Any("events", events)) } } + if !wasInitialized && span.initialized.Load() && + h.eventSink != nil && h.eventSink.memoryQuota != nil { + h.eventSink.memoryQuota.markSubscriptionInitialized() + } tryAdvanceResolvedTs := func() { if newResolvedTs != 0 { span.advanceResolvedTs(newResolvedTs) } } + releaseMemoryQuota := func() { + for _, lease := range quotaLeases { + lease.Release() + } + } if len(span.kvEventsCache) > 0 { metricsEventCount.Add(float64(len(span.kvEventsCache))) await := span.consumeKVEvents(span.kvEventsCache, func() { + defer releaseMemoryQuota() start := time.Now() span.clearKVEventsCache() metricConsumeKVEventsCallbackDurationClearCache.Observe(time.Since(start).Seconds()) @@ -166,10 +192,12 @@ func (h *regionEventHandler) Handle(span *subscribedSpan, events ...regionEvent) if !await { span.clearKVEventsCache() tryAdvanceResolvedTs() + releaseMemoryQuota() } return await } else { tryAdvanceResolvedTs() + releaseMemoryQuota() } return false } @@ -226,6 +254,7 @@ func (h *regionEventHandler) GetType(event regionEvent) dynstream.EventType { } func (h *regionEventHandler) OnDrop(event regionEvent) interface{} { + event.releaseMemoryQuota() // TODO: Distinguish between drop events caused by "path not found" errors and memory control. state := event.mustFirstState() fields := []zap.Field{ @@ -285,7 +314,7 @@ func handleEventEntries(span *subscribedSpan, state *regionFeedState, entries *c for _, entry := range entries.Entries.GetEntries() { switch entry.Type { case cdcpb.Event_INITIALIZED: - span.markRegionInitialized(state) + state.setInitialized() log.Debug("region is initialized", zap.Int64("tableID", span.span.TableID), zap.Uint64("regionID", regionID), @@ -392,6 +421,7 @@ func handleResolvedTs(span *subscribedSpan, state *regionFeedState, resolvedTs u } if shouldAdvance { + span.tryMarkInitialized(regionID, ts) lastResolvedTs := span.resolvedTs.Load() nextResolvedPhyTs := oracle.ExtractPhysical(ts) // Generally, we don't want to send duplicate resolved ts, @@ -399,16 +429,7 @@ func handleResolvedTs(span *subscribedSpan, state *regionFeedState, resolvedTs u // but when `ts` == `lastResolvedTs` == `span.startTs`, // the span may just be initialized and have not receive any resolved ts before, // so we also send ts in this case for quick notification to downstream. - if ts > lastResolvedTs || - (span.initialized.Load() && ts == lastResolvedTs && lastResolvedTs == span.startTs) { - if lastResolvedTs == span.startTs && ts > span.startTs && !span.initialized.Load() { - log.Warn("should not happen: resolved ts advances before span is initialized", - zap.Uint64("subscriptionID", uint64(span.subID)), - zap.Int64("tableID", span.span.TableID), - zap.Uint64("regionID", regionID), - zap.Uint64("startTs", span.startTs), - zap.Uint64("resolvedTs", ts)) - } + if ts > lastResolvedTs || (ts == lastResolvedTs && lastResolvedTs == span.startTs) { resolvedPhyTs := oracle.ExtractPhysical(lastResolvedTs) decreaseLag := float64(nextResolvedPhyTs-resolvedPhyTs) / 1e3 const largeResolvedTsAdvanceStepInSecs = 30 @@ -421,7 +442,8 @@ func handleResolvedTs(span *subscribedSpan, state *regionFeedState, resolvedTs u zap.Uint64("lastResolvedTs", lastResolvedTs), zap.Float64("decreaseLag(s)", decreaseLag)) } - span.recordResolvedTs(ts) + span.resolvedTs.Store(ts) + span.resolvedTsUpdated.Store(time.Now().Unix()) return ts } } diff --git a/logservice/logpuller/region_event_handler_test.go b/logservice/logpuller/region_event_handler_test.go index f2e1885197..92fc3a8e03 100644 --- a/logservice/logpuller/region_event_handler_test.go +++ b/logservice/logpuller/region_event_handler_test.go @@ -23,7 +23,6 @@ import ( "github.com/pingcap/ticdc/heartbeatpb" "github.com/pingcap/ticdc/logservice/logpuller/regionlock" "github.com/pingcap/ticdc/pkg/common" - "github.com/pingcap/ticdc/pkg/pdutil" "github.com/pingcap/ticdc/utils/dynstream" "github.com/stretchr/testify/require" "github.com/tikv/client-go/v2/tikv" @@ -72,7 +71,6 @@ func TestHandleEventEntryEventOutOfOrder(t *testing.T) { subID: subID, span: span, startTs: 1000, // not used - rangeLock: regionlock.NewRangeLock(uint64(subID), span.StartKey, span.EndKey, 1000), consumeKVEvents: consumeKVEvents, advanceResolvedTs: advanceResolvedTs, advanceInterval: 0, @@ -83,15 +81,12 @@ func TestHandleEventEntryEventOutOfOrder(t *testing.T) { tracker: newRegionTracker(), } region := newRegionInfo( - tikv.NewRegionVerID(1, 1, 1), + tikv.RegionVerID{}, span, &tikv.RPCContext{}, subSpan, ) - lockResult := subSpan.rangeLock.LockRange( - context.Background(), span.StartKey, span.EndKey, 1, 1) - require.Equal(t, regionlock.LockRangeStatusSuccess, lockResult.Status) - region.lockedRangeState = lockResult.LockedRangeState + region.lockedRangeState = ®ionlock.LockedRangeState{} state := newRegionFeedState(region, 1, worker, nil) // Receive prewrite2 with empty value. @@ -209,8 +204,6 @@ func TestHandleEventEntryEventOutOfOrder(t *testing.T) { func TestHandleResolvedTs(t *testing.T) { // initialize option := dynstream.NewOption() - pdClock := pdutil.NewClock4Test() - pdClock.(*pdutil.Clock4Test).SetTS(10) ds := dynstream.NewParallelDynamicStream("test", ®ionEventHandler{}, option) ds.Start() @@ -225,28 +218,23 @@ func TestHandleResolvedTs(t *testing.T) { tracker: newRegionTracker(), } state1 := newRegionFeedState(regionInfo{verID: tikv.NewRegionVerID(1, 1, 1)}, uint64(subID1), worker, nil) - var subSpan1 *subscribedSpan { span := heartbeatpb.TableSpan{ TableID: 100, StartKey: common.ToComparableKey([]byte{}), // TODO: remove spanz dependency EndKey: common.ToComparableKey(common.UpperBoundKey), } - subSpan1 = &subscribedSpan{ + subSpan := &subscribedSpan{ subID: subID1, span: heartbeatpb.TableSpan{}, rangeLock: regionlock.NewRangeLock(uint64(subID1), span.StartKey, span.EndKey, 1), consumeKVEvents: consumeKVEvents, advanceResolvedTs: advanceResolvedTs, advanceInterval: 0, - priorityPolicy: newScanPriorityPolicy(pdClock, 30*time.Minute), } - ds.AddPath(subID1, subSpan1, dynstream.AreaSettings{}) - state1.region.subscribedSpan = subSpan1 - lockResult := subSpan1.rangeLock.LockRange( - context.Background(), span.StartKey, span.EndKey, 1, 1) - require.Equal(t, regionlock.LockRangeStatusSuccess, lockResult.Status) - state1.region.lockedRangeState = lockResult.LockedRangeState + ds.AddPath(subID1, subSpan, dynstream.AreaSettings{}) + state1.region.subscribedSpan = subSpan + state1.region.lockedRangeState = ®ionlock.LockedRangeState{} state1.setInitialized() state1.updateResolvedTs(9) } @@ -266,14 +254,10 @@ func TestHandleResolvedTs(t *testing.T) { consumeKVEvents: consumeKVEvents, advanceResolvedTs: advanceResolvedTs, advanceInterval: 0, - priorityPolicy: newScanPriorityPolicy(pdClock, 30*time.Minute), } ds.AddPath(subID2, subSpan, dynstream.AreaSettings{}) state2.region.subscribedSpan = subSpan - lockResult := subSpan.rangeLock.LockRange( - context.Background(), span.StartKey, span.EndKey, 2, 2) - require.Equal(t, regionlock.LockRangeStatusSuccess, lockResult.Status) - state2.region.lockedRangeState = lockResult.LockedRangeState + state2.region.lockedRangeState = ®ionlock.LockedRangeState{} state2.setInitialized() state2.updateResolvedTs(11) } @@ -293,14 +277,10 @@ func TestHandleResolvedTs(t *testing.T) { consumeKVEvents: consumeKVEvents, advanceResolvedTs: advanceResolvedTs, advanceInterval: 0, - priorityPolicy: newScanPriorityPolicy(pdClock, 30*time.Minute), } ds.AddPath(subID3, subSpan, dynstream.AreaSettings{}) state3.region.subscribedSpan = subSpan - lockResult := subSpan.rangeLock.LockRange( - context.Background(), span.StartKey, span.EndKey, 3, 3) - require.Equal(t, regionlock.LockRangeStatusSuccess, lockResult.Status) - state3.region.lockedRangeState = lockResult.LockedRangeState + state3.region.lockedRangeState = ®ionlock.LockedRangeState{} state3.updateResolvedTs(8) } @@ -345,7 +325,6 @@ func TestHandleResolvedTs(t *testing.T) { require.Equal(t, uint64(10), state1.getLastResolvedTs()) require.Equal(t, uint64(11), state2.getLastResolvedTs()) require.Equal(t, uint64(8), state3.getLastResolvedTs()) - require.True(t, subSpan1.priorityPolicy.everCaughtUp.Load()) } func TestHandleResolvedTsThrottled(t *testing.T) { @@ -375,7 +354,6 @@ func TestHandleResolvedTsThrottled(t *testing.T) { subID: SubscriptionID(1), rangeLock: l, advanceInterval: 100, - priorityPolicy: newScanPriorityPolicy(pdutil.NewClock4Test(), 30*time.Minute), } span.lastAdvanceTime.Store(0) worker := ®ionRequestWorker{tracker: newRegionTracker()} @@ -393,63 +371,104 @@ func TestHandleResolvedTsThrottled(t *testing.T) { require.Equal(t, uint64(200), handleResolvedTs(span, state, 300)) } -func TestSpanInitializedAfterAllRangesInitialized(t *testing.T) { - ctx := context.Background() - rangeLock := regionlock.NewRangeLock(1, []byte("a"), []byte("z"), 100) - firstLock := rangeLock.LockRange(ctx, []byte("a"), []byte("m"), 1, 1) - require.Equal(t, regionlock.LockRangeStatusSuccess, firstLock.Status) - secondLock := rangeLock.LockRange(ctx, []byte("m"), []byte("z"), 2, 1) - require.Equal(t, regionlock.LockRangeStatusSuccess, secondLock.Status) - - span := &subscribedSpan{ - subID: SubscriptionID(1), - startTs: 100, - span: heartbeatpb.TableSpan{StartKey: []byte("a"), EndKey: []byte("z")}, - rangeLock: rangeLock, - priorityPolicy: newScanPriorityPolicy(pdutil.NewClock4Test(), 30*time.Minute), +func TestHandleEntriesReleasesMemoryAfterDownstreamCallback(t *testing.T) { + quota := newMemoryQuotaController(1024, 8) + span := newTestQuotaSpan(1) + quota.addSubscription(span) + callbackCh := make(chan func(), 1) + span.consumeKVEvents = func(_ []common.RawKVEntry, callback func()) bool { + callbackCh <- callback + return true } - span.resolvedTs.Store(span.startTs) - worker := ®ionRequestWorker{tracker: newRegionTracker()} - newState := func( - regionID uint64, regionSpan heartbeatpb.TableSpan, - lockedRangeState *regionlock.LockedRangeState, - ) *regionFeedState { - state := newRegionFeedState( - regionInfo{ - verID: tikv.NewRegionVerID(regionID, 1, 1), - span: regionSpan, - rpcCtx: &tikv.RPCContext{}, - subscribedSpan: span, - lockedRangeState: lockedRangeState, - }, - uint64(span.subID), - worker, - nil, - ) - return state + span.advanceResolvedTs = func(uint64) {} + + lockedState := ®ionlock.LockedRangeState{} + lockedState.ResolvedTs.Store(100) + state := ®ionFeedState{ + region: regionInfo{ + verID: tikv.NewRegionVerID(1, 1, 1), + rpcCtx: &tikv.RPCContext{}, + subscribedSpan: span, + lockedRangeState: lockedState, + }, } - firstState := newState(1, - heartbeatpb.TableSpan{StartKey: []byte("a"), EndKey: []byte("m")}, - firstLock.LockedRangeState) - secondState := newState(2, - heartbeatpb.TableSpan{StartKey: []byte("m"), EndKey: []byte("z")}, - secondLock.LockedRangeState) - - handler := ®ionEventHandler{} - initializedEvent := func(state *regionFeedState) regionEvent { - return regionEvent{ - states: []*regionFeedState{state}, - entries: &cdcpb.Event_Entries_{Entries: &cdcpb.Event_Entries{ - Entries: []*cdcpb.Event_Row{{Type: cdcpb.Event_INITIALIZED}}, + lease := quota.trackEvent(context.Background(), span, 10) + require.NotNil(t, lease) + handler := ®ionEventHandler{eventSink: ®ionEventSink{ + ds: newMockRegionEventSinkStream(), + memoryQuota: quota, + }} + + await := handler.Handle(span, regionEvent{ + states: []*regionFeedState{state}, + memoryQuota: lease, + entries: &cdcpb.Event_Entries_{Entries: &cdcpb.Event_Entries{ + Entries: []*cdcpb.Event_Row{{ + Type: cdcpb.Event_COMMITTED, + OpType: cdcpb.Event_Row_PUT, + CommitTs: 101, }}, - } - } + }}, + }) + require.True(t, await) + used, _, _ := quota.snapshot() + require.Equal(t, uint64(10), used) + + callback := <-callbackCh + callback() + used, _, _ = quota.snapshot() + require.Zero(t, used) +} - require.False(t, handler.Handle(span, initializedEvent(firstState))) +func TestTryMarkSpanInitializedByResolvedTs(t *testing.T) { + span := &subscribedSpan{subID: 1, startTs: 100} + require.False(t, span.tryMarkInitialized(1, 100)) require.False(t, span.initialized.Load()) - require.Equal(t, uint64(0), handleResolvedTs(span, firstState, span.startTs)) + require.True(t, span.tryMarkInitialized(1, 101)) + require.True(t, span.initialized.Load()) + require.False(t, span.tryMarkInitialized(1, 102)) +} + +func TestSpanInitializationNotifiesMemoryAdmission(t *testing.T) { + quota := newMemoryQuotaController(1024, 8) + notified := make(chan struct{}, 1) + quota.setOnAvailable(func() { + select { + case notified <- struct{}{}: + default: + } + }) + + const startTs = 100 + rangeLock := regionlock.NewRangeLock(1, []byte("a"), []byte("z"), startTs) + lockResult := rangeLock.LockRange(t.Context(), []byte("a"), []byte("z"), 1, 1) + require.Equal(t, regionlock.LockRangeStatusSuccess, lockResult.Status) + lockResult.LockedRangeState.Initialized.Store(true) - require.False(t, handler.Handle(span, initializedEvent(secondState))) + span := &subscribedSpan{ + subID: 1, + startTs: startTs, + rangeLock: rangeLock, + consumeKVEvents: func([]common.RawKVEntry, func()) bool { return false }, + advanceResolvedTs: func(uint64) {}, + } + span.resolvedTs.Store(startTs) + quota.addSubscription(span) + state := newRegionFeedState(regionInfo{ + verID: tikv.NewRegionVerID(1, 1, 1), + subscribedSpan: span, + lockedRangeState: lockResult.LockedRangeState, + }, uint64(span.subID), ®ionRequestWorker{}, nil) + handler := ®ionEventHandler{eventSink: ®ionEventSink{memoryQuota: quota}} + + require.False(t, handler.Handle(span, regionEvent{ + states: []*regionFeedState{state}, + resolvedTs: startTs + 1, + })) require.True(t, span.initialized.Load()) - require.Equal(t, span.startTs, handleResolvedTs(span, secondState, span.startTs)) + select { + case <-notified: + case <-time.After(time.Second): + t.Fatal("span initialization did not notify memory admission") + } } diff --git a/logservice/logpuller/region_event_sink.go b/logservice/logpuller/region_event_sink.go index a89a0555bc..3c161fda1a 100644 --- a/logservice/logpuller/region_event_sink.go +++ b/logservice/logpuller/region_event_sink.go @@ -26,21 +26,22 @@ import ( // regionEventSink delivers region events to dynstream and owns push-side flow control. type regionEventSink struct { - // mu/cond coordinate the paused push path with pause/resume and shutdown signals. - mu sync.Mutex - cond *sync.Cond - // paused tracks whether region event pushing is temporarily held back by feedback. - paused atomic.Bool - // stopped marks the sink as shutting down so blocked pushers can exit instead of waiting for resume. - stopped atomic.Bool + ctx context.Context + ds dynstream.DynamicStream[int, SubscriptionID, regionEvent, *subscribedSpan, *regionEventHandler] - // ds owns the dynstream used to deliver region events and receive flow-control feedback. - ds dynstream.DynamicStream[int, SubscriptionID, regionEvent, *subscribedSpan, *regionEventHandler] + memoryQuota *memoryQuotaController + // the following three fields are used to manage feedback from ds and notify other goroutines + mu sync.Mutex + cond *sync.Cond + paused atomic.Bool } -func newRegionEventSink(failureHandler *regionFailureHandler) *regionEventSink { - sink := ®ionEventSink{} - sink.cond = sync.NewCond(&sink.mu) +func newRegionEventSink( + ctx context.Context, + failureHandler *regionFailureHandler, + memoryQuota *memoryQuotaController, +) *regionEventSink { + sink := ®ionEventSink{ctx: ctx, memoryQuota: memoryQuota} option := dynstream.NewOption() // Note: it is max batch size of the kv sent from tikv(not committed rows) @@ -48,7 +49,7 @@ func newRegionEventSink(failureHandler *regionFailureHandler) *regionEventSink { // TODO: Set `UseBuffer` to true until we refactor the `regionEventHandler.Handle` method so that it doesn't call any method of the dynamic stream. Currently, if `UseBuffer` is set to false, there will be a deadlock: // ds.handleLoop fetch events from `ch` -> regionEventHandler.Handle -> ds.RemovePath -> send event to `ch` option.UseBuffer = true - option.EnableMemoryControl = true + option.EnableMemoryControl = false ds := dynstream.NewParallelDynamicStream( "log-puller", ®ionEventHandler{eventSink: sink, failureHandler: failureHandler}, @@ -56,12 +57,12 @@ func newRegionEventSink(failureHandler *regionFailureHandler) *regionEventSink { ) ds.Start() sink.ds = ds + sink.cond = sync.NewCond(&sink.mu) return sink } func (s *regionEventSink) AddPath(rt *subscribedSpan) { - areaSetting := dynstream.NewAreaSettingsWithMaxPendingSize(1*1024*1024*1024, dynstream.MemoryControlForPuller, "logPuller") // 1GB - if err := s.ds.AddPath(rt.subID, rt, areaSetting); err != nil { + if err := s.ds.AddPath(rt.subID, rt); err != nil { log.Warn("subscription client add path failed", zap.Uint64("subscriptionID", uint64(rt.subID)), zap.Error(err)) @@ -77,8 +78,12 @@ func (s *regionEventSink) Wake(subID SubscriptionID) { } func (s *regionEventSink) Push(subID SubscriptionID, event regionEvent) { - if s.stopped.Load() { - return + if event.needMemoryQuota() && s.memoryQuota != nil { + span := event.mustFirstState().region.subscribedSpan + event.memoryQuota = s.memoryQuota.trackEvent(s.ctx, span, uint64(event.getSize())) + if event.memoryQuota == nil { + return + } } // fast path if !s.paused.Load() { @@ -88,15 +93,17 @@ func (s *regionEventSink) Push(subID SubscriptionID, event regionEvent) { // slow path: wait until paused is false s.mu.Lock() - for s.paused.Load() && !s.stopped.Load() { - s.cond.Wait() + for s.paused.Load() { + select { + case <-s.ctx.Done(): + s.mu.Unlock() + event.releaseMemoryQuota() + return + default: + s.cond.Wait() + } } - stopped := s.stopped.Load() s.mu.Unlock() - - if stopped { - return - } s.ds.Push(subID, event) } @@ -104,15 +111,19 @@ func (s *regionEventSink) Run(ctx context.Context) error { for { select { case <-ctx.Done(): - s.stop() return nil case feedback := <-s.ds.Feedback(): switch feedback.FeedbackType { case dynstream.PauseArea: - s.pause() + s.mu.Lock() + s.paused.Store(true) + s.mu.Unlock() log.Info("subscription client pause push region event") case dynstream.ResumeArea: - s.resume() + s.mu.Lock() + s.paused.Store(false) + s.cond.Broadcast() + s.mu.Unlock() log.Info("subscription client resume push region event") case dynstream.ReleasePath, dynstream.ResumePath: // Ignore it, because it is no need to pause and resume a path in puller. @@ -126,59 +137,41 @@ func (s *regionEventSink) UpdateMetrics() { metricSubscriptionClientDSChannelSize.Set(float64(dsMetrics.EventChanSize)) metricSubscriptionClientDSPendingQueueLen.Set(float64(dsMetrics.PendingQueueLen)) - if len(dsMetrics.MemoryControl.AreaMemoryMetrics) == 0 { - return - } - if len(dsMetrics.MemoryControl.AreaMemoryMetrics) != 1 { - log.Warn("subscription client should have exactly one area") + if s.memoryQuota == nil { return } - areaMetric := dsMetrics.MemoryControl.AreaMemoryMetrics[0] + used, capacity, _ := s.memoryQuota.snapshot() + scanUsed, warmingScanUsed, warmingScanBudget, scanEstimate, hardLimit := + s.memoryQuota.scanSnapshot() + metrics.LogPullerMemoryQuota.WithLabelValues("max").Set(float64(capacity)) + metrics.LogPullerMemoryQuota.WithLabelValues("used").Set(float64(used)) + metrics.LogPullerMemoryQuota.WithLabelValues("scan_used").Set(float64(scanUsed)) + metrics.LogPullerMemoryQuota.WithLabelValues("warming_scan_used").Set(float64(warmingScanUsed)) + metrics.LogPullerMemoryQuota.WithLabelValues("warming_scan_budget").Set(float64(warmingScanBudget)) + metrics.LogPullerMemoryQuota.WithLabelValues("scan_estimate").Set(float64(scanEstimate)) + metrics.LogPullerMemoryQuota.WithLabelValues("hard_limit").Set(float64(hardLimit)) metrics.DynamicStreamMemoryUsage.WithLabelValues( "log-puller", "max", "default", "default", - ).Set(float64(areaMetric.MaxMemory())) + ).Set(float64(capacity)) metrics.DynamicStreamMemoryUsage.WithLabelValues( "log-puller", "used", "default", "default", - ).Set(float64(areaMetric.MemoryUsage())) + ).Set(float64(used)) } func (s *regionEventSink) Close() { - s.stop() - s.ds.Close() -} - -func (s *regionEventSink) pause() { - s.mu.Lock() - defer s.mu.Unlock() - if s.stopped.Load() || s.paused.Load() { - return - } - s.paused.Store(true) -} - -func (s *regionEventSink) resume() { - s.mu.Lock() - defer s.mu.Unlock() - if !s.paused.Load() { - return - } - s.paused.Store(false) - s.cond.Broadcast() -} - -func (s *regionEventSink) stop() { - if !s.stopped.CompareAndSwap(false, true) { - return + if s.memoryQuota != nil { + s.memoryQuota.wakeAll() } s.mu.Lock() s.paused.Store(false) s.cond.Broadcast() s.mu.Unlock() + s.ds.Close() } diff --git a/logservice/logpuller/region_event_sink_test.go b/logservice/logpuller/region_event_sink_test.go index 0698d3fb4a..1f08aa51f9 100644 --- a/logservice/logpuller/region_event_sink_test.go +++ b/logservice/logpuller/region_event_sink_test.go @@ -20,6 +20,7 @@ import ( "testing" "time" + "github.com/pingcap/kvproto/pkg/cdcpb" "github.com/pingcap/ticdc/pkg/metrics" "github.com/pingcap/ticdc/utils/dynstream" "github.com/prometheus/client_golang/prometheus/testutil" @@ -30,6 +31,7 @@ type mockRegionEventSinkStream struct { feedbackCh chan dynstream.Feedback[int, SubscriptionID, *subscribedSpan] pushCount atomic.Int32 pushCh chan struct{} + eventCh chan regionEvent metrics dynstream.Metrics[int, SubscriptionID] } @@ -37,6 +39,7 @@ func newMockRegionEventSinkStream() *mockRegionEventSinkStream { return &mockRegionEventSinkStream{ feedbackCh: make(chan dynstream.Feedback[int, SubscriptionID, *subscribedSpan], 2), pushCh: make(chan struct{}, 1), + eventCh: make(chan regionEvent, 1), } } @@ -44,9 +47,10 @@ func (s *mockRegionEventSinkStream) Start() {} func (s *mockRegionEventSinkStream) Close() {} -func (s *mockRegionEventSinkStream) Push(_ SubscriptionID, _ regionEvent) { +func (s *mockRegionEventSinkStream) Push(_ SubscriptionID, event regionEvent) { s.pushCount.Add(1) s.pushCh <- struct{}{} + s.eventCh <- event } func (s *mockRegionEventSinkStream) Wake(_ SubscriptionID) {} @@ -71,20 +75,16 @@ func (s *mockRegionEventSinkStream) GetMetrics() dynstream.Metrics[int, Subscrip return s.metrics } -func newTestRegionEventSink( - ds dynstream.DynamicStream[int, SubscriptionID, regionEvent, *subscribedSpan, *regionEventHandler], -) *regionEventSink { - sink := ®ionEventSink{ds: ds} - sink.cond = sync.NewCond(&sink.mu) - return sink -} - func TestRegionEventSinkRunPausesAndResumesPush(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() ds := newMockRegionEventSinkStream() - sink := newTestRegionEventSink(ds) + sink := ®ionEventSink{ + ctx: ctx, + ds: ds, + } + sink.cond = sync.NewCond(&sink.mu) runErrCh := make(chan error, 1) go func() { @@ -135,7 +135,7 @@ func TestRegionEventSinkRunPausesAndResumesPush(t *testing.T) { } func TestRegionEventSinkUpdateMetrics(t *testing.T) { - t.Run("empty area metrics returns after queue gauges", func(t *testing.T) { + t.Run("without quota only updates queue gauges", func(t *testing.T) { ds := newMockRegionEventSinkStream() ds.metrics = dynstream.Metrics[int, SubscriptionID]{ EventChanSize: 11, @@ -156,8 +156,10 @@ func TestRegionEventSinkUpdateMetrics(t *testing.T) { ).Set(456) sink := ®ionEventSink{ - ds: ds, + ctx: context.Background(), + ds: ds, } + sink.cond = sync.NewCond(&sink.mu) sink.UpdateMetrics() require.Equal(t, float64(11), testutil.ToFloat64(metricSubscriptionClientDSChannelSize)) @@ -176,25 +178,25 @@ func TestRegionEventSinkUpdateMetrics(t *testing.T) { ))) }) - t.Run("single area metrics updates memory gauges", func(t *testing.T) { + t.Run("quota updates memory gauges", func(t *testing.T) { ds := newMockRegionEventSinkStream() ds.metrics = dynstream.Metrics[int, SubscriptionID]{ EventChanSize: 33, PendingQueueLen: 44, - MemoryControl: dynstream.MemoryMetric[int, SubscriptionID]{ - AreaMemoryMetrics: []dynstream.AreaMemoryMetric[int, SubscriptionID]{ - { - UsedMemoryValue: 55, - MaxMemoryValue: 66, - PathMaxMemoryValue: 66, - }, - }, - }, } + quota := newMemoryQuotaController(66, 8) + span := newTestQuotaSpan(1) + quota.addSubscription(span) + lease := quota.trackEvent(context.Background(), span, 55) + require.NotNil(t, lease) + t.Cleanup(lease.Release) sink := ®ionEventSink{ - ds: ds, + ctx: context.Background(), + ds: ds, + memoryQuota: quota, } + sink.cond = sync.NewCond(&sink.mu) sink.UpdateMetrics() require.Equal(t, float64(33), testutil.ToFloat64(metricSubscriptionClientDSChannelSize)) @@ -211,51 +213,41 @@ func TestRegionEventSinkUpdateMetrics(t *testing.T) { "default", "default", ))) + require.Equal(t, float64(66), testutil.ToFloat64( + metrics.LogPullerMemoryQuota.WithLabelValues("max"))) + require.Equal(t, float64(55), testutil.ToFloat64( + metrics.LogPullerMemoryQuota.WithLabelValues("used"))) }) } -func TestRegionEventSinkRunCancelUnblocksPush(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - - ds := newMockRegionEventSinkStream() - sink := newTestRegionEventSink(ds) - - runErrCh := make(chan error, 1) - go func() { - runErrCh <- sink.Run(ctx) - }() - - ds.feedbackCh <- dynstream.Feedback[int, SubscriptionID, *subscribedSpan]{ - FeedbackType: dynstream.PauseArea, - } - require.Eventually(t, sink.paused.Load, time.Second, 10*time.Millisecond) - - pushDone := make(chan struct{}) - go func() { - sink.Push(SubscriptionID(1), regionEvent{resolvedTs: 100}) - close(pushDone) - }() - - select { - case <-pushDone: - t.Fatal("Push should block while the sink is paused") - case <-time.After(100 * time.Millisecond): +func TestRegionEventSinkTracksEntriesUntilDrop(t *testing.T) { + quota := newMemoryQuotaController(1024, 8) + span := newTestQuotaSpan(1) + quota.addSubscription(span) + state := ®ionFeedState{ + region: regionInfo{subscribedSpan: span}, + worker: ®ionRequestWorker{}, } - require.Equal(t, int32(0), ds.pushCount.Load()) - - cancel() - - select { - case <-pushDone: - case <-time.After(time.Second): - t.Fatal("Push should be unblocked by Run context cancellation") + ds := newMockRegionEventSinkStream() + sink := ®ionEventSink{ + ctx: context.Background(), + ds: ds, + memoryQuota: quota, } - require.Equal(t, int32(0), ds.pushCount.Load()) + sink.cond = sync.NewCond(&sink.mu) - select { - case err := <-runErrCh: - require.NoError(t, err) - case <-time.After(time.Second): - t.Fatal("Run should exit after context cancellation") - } + sink.Push(span.subID, regionEvent{ + states: []*regionFeedState{state}, + entries: &cdcpb.Event_Entries_{Entries: &cdcpb.Event_Entries{ + Entries: []*cdcpb.Event_Row{{Key: []byte("key"), Value: []byte("value")}}, + }}, + }) + pushed := <-ds.eventCh + require.NotNil(t, pushed.memoryQuota) + used, _, _ := quota.snapshot() + require.NotZero(t, used) + + (®ionEventHandler{}).OnDrop(pushed) + used, _, _ = quota.snapshot() + require.Zero(t, used) } diff --git a/logservice/logpuller/region_failure_handler.go b/logservice/logpuller/region_failure_handler.go index 5ca0f03c96..3622d927d9 100644 --- a/logservice/logpuller/region_failure_handler.go +++ b/logservice/logpuller/region_failure_handler.go @@ -64,14 +64,6 @@ func newRegionFailureHandler( } } -func (r *regionFailureHandler) retryRange(ctx context.Context, errInfo regionErrorInfo) { - r.scheduleRangeRequest(ctx, rangeTask{ - span: errInfo.span, - subscribedSpan: errInfo.subscribedSpan, - wasInitialized: errInfo.wasInitialized, - }) -} - // Report admits a region failure into the recovery pipeline. It releases the // corresponding range lock before enqueueing the failure so new range tasks are // not blocked by stale region ownership. @@ -167,20 +159,12 @@ func (r *regionFailureHandler) handleError(ctx context.Context, errInfo regionEr } if innerErr.GetEpochNotMatch() != nil { metricFeedEpochNotMatchCounter.Inc() -<<<<<<< HEAD rescheduleRange() -======= - r.retryRange(ctx, errInfo) ->>>>>>> 9903a1be7 (refactor) return nil } if innerErr.GetRegionNotFound() != nil { metricFeedRegionNotFoundCounter.Inc() -<<<<<<< HEAD rescheduleRange() -======= - r.retryRange(ctx, errInfo) ->>>>>>> 9903a1be7 (refactor) return nil } if innerErr.GetCongested() != nil { @@ -213,22 +197,14 @@ func (r *regionFailureHandler) handleError(ctx context.Context, errInfo regionEr return nil case *rpcCtxUnavailableErr: metricFeedRPCCtxUnavailable.Inc() -<<<<<<< HEAD rescheduleRange() -======= - r.retryRange(ctx, errInfo) ->>>>>>> 9903a1be7 (refactor) return nil case *getStoreErr: metricGetStoreErr.Inc() bo := tikv.NewBackoffer(ctx, tikvRequestMaxBackoff) // cannot get the store the region belongs to, so we need to reload the region. r.regionCache.OnSendFail(bo, errInfo.rpcCtx, true, err) -<<<<<<< HEAD rescheduleRange() -======= - r.retryRange(ctx, errInfo) ->>>>>>> 9903a1be7 (refactor) return nil case *storeStreamErr: metricStoreSendRequestErr.Inc() diff --git a/logservice/logpuller/region_request_scheduler.go b/logservice/logpuller/region_request_scheduler.go index 135b700e28..d2e93081e3 100644 --- a/logservice/logpuller/region_request_scheduler.go +++ b/logservice/logpuller/region_request_scheduler.go @@ -20,20 +20,14 @@ import ( "sync/atomic" "github.com/pingcap/log" -<<<<<<< HEAD "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/config" "github.com/pingcap/ticdc/pkg/errors" "github.com/pingcap/ticdc/pkg/metrics" -======= - "github.com/pingcap/ticdc/pkg/config" - "github.com/pingcap/ticdc/pkg/errors" ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) "github.com/pingcap/ticdc/utils/priorityqueue" kvclientv2 "github.com/tikv/client-go/v2/kv" "github.com/tikv/client-go/v2/tikv" "go.uber.org/zap" -<<<<<<< HEAD "go.uber.org/zap/zapcore" "golang.org/x/sync/errgroup" ) @@ -48,6 +42,7 @@ type regionRequestScheduler struct { upstream *upstreamHandle eventSink *regionEventSink failureHandler *regionFailureHandler + memoryQuota *memoryQuotaController // taskQueue orders all regions before they are assigned to a TiKV store. taskQueue *priorityqueue.PriorityQueue[*regionPriorityTask] @@ -55,24 +50,6 @@ type regionRequestScheduler struct { sequence atomic.Uint64 // stores maps TiKV addresses to regionRequestStore. Stores are created only // by Run, but are also read by metrics and deregistration goroutines. -======= - "golang.org/x/sync/errgroup" -) - -// regionRequestScheduler routes locked Region requests through the global -// priority queue to a worker connected to the Region's TiKV store. Range -// resolution and retry policy remain owned by subscriptionClient and -// regionFailureHandler respectively. -type regionRequestScheduler struct { - client *subscriptionClient - - // taskQueue orders all Regions before they are assigned to a TiKV store. - taskQueue *priorityqueue.PriorityQueue[*regionPriorityTask] - // sequence is the FIFO tie-breaker for Regions in the same priority class. - sequence atomic.Uint64 - // stores maps TiKV addresses to requestedStore. Stores are created only by - // run, but are also read by metrics and deregistration goroutines. ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) stores sync.Map // workerCount is the configured number of request workers per store. @@ -83,11 +60,11 @@ type regionRequestScheduler struct { maxWindowMultiplier int } -<<<<<<< HEAD func newRegionRequestScheduler( upstream *upstreamHandle, eventSink *regionEventSink, failureHandler *regionFailureHandler, + memoryQuota *memoryQuotaController, ) *regionRequestScheduler { pullerConfig := config.GetGlobalServerConfig().Debug.Puller workerCount := regionRequestWorkerPerStore @@ -96,17 +73,7 @@ func newRegionRequestScheduler( upstream: upstream, eventSink: eventSink, failureHandler: failureHandler, -======= -func newRegionRequestScheduler(client *subscriptionClient) *regionRequestScheduler { - pullerConfig := config.GetGlobalServerConfig().Debug.Puller - workerCount := int(client.config.RegionRequestWorkerPerStore) - if workerCount <= 0 { - workerCount = 1 - } - workerWindow := (pullerConfig.PendingRegionRequestQueueSize + workerCount - 1) / workerCount - return ®ionRequestScheduler{ - client: client, ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) + memoryQuota: memoryQuota, taskQueue: priorityqueue.New[*regionPriorityTask](), workerCount: workerCount, workerWindow: workerWindow, @@ -114,9 +81,7 @@ func newRegionRequestScheduler(client *subscriptionClient) *regionRequestSchedul } } -<<<<<<< HEAD func (s *regionRequestScheduler) Submit(region regionInfo) { -<<<<<<< HEAD if log.GetLevel() <= zapcore.DebugLevel { log.Debug("cdc region scan task enqueued", zap.Uint64("subscriptionID", uint64(region.subscribedSpan.subID)), @@ -130,10 +95,6 @@ func (s *regionRequestScheduler) Submit(region regionInfo) { zap.String("span", common.FormatTableSpan(®ion.span))) } s.taskQueue.Push(newRegionPriorityTask(region, s.sequence.Add(1))) -======= - s.taskQueue.Push(newRegionPriorityTask( - region, s.upstream.pdClock.CurrentTS(), s.sequence.Add(1))) ->>>>>>> 9903a1be7 (refactor) } func (s *regionRequestScheduler) Run(ctx context.Context, workerGroup *errgroup.Group) error { @@ -144,15 +105,6 @@ func (s *regionRequestScheduler) Run(ctx context.Context, workerGroup *errgroup. }) }() -======= -func (s *regionRequestScheduler) submit(region regionInfo) { - s.taskQueue.Push(NewRegionPriorityTask( - region, s.client.pdClock.CurrentTS(), s.sequence.Add(1))) -} - -func (s *regionRequestScheduler) run(ctx context.Context, group *errgroup.Group) error { - defer s.closeStores() ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) for { select { case <-ctx.Done(): @@ -168,10 +120,6 @@ func (s *regionRequestScheduler) run(ctx context.Context, group *errgroup.Group) return err } -<<<<<<< HEAD -<<<<<<< HEAD -======= ->>>>>>> 9903a1be7 (refactor) region, err := s.attachRPCContext(ctx, task.regionInfo) if err != nil { s.failureHandler.Report(newRegionErrorInfo(region, err)) @@ -197,30 +145,12 @@ func (s *regionRequestScheduler) run(ctx context.Context, group *errgroup.Group) zap.Uint64("regionID", region.verID.GetID()), zap.String("addr", region.rpcCtx.Addr)) } -======= - region, ok := s.attachRPCContext(ctx, task.GetRegionInfo()) - if !ok { - continue - } - - store := s.getOrCreateStore(ctx, group, region.rpcCtx.Addr) - task.updateRegion(region, s.client.pdClock.CurrentTS()) - if !store.submit(task) { - return context.Canceled - } - - log.Debug("subscription client will request a region", - zap.Uint64("subscriptionID", uint64(region.subscribedSpan.subID)), - zap.Uint64("regionID", region.verID.GetID()), - zap.String("addr", store.storeAddr)) ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) } } func (s *regionRequestScheduler) attachRPCContext( ctx context.Context, region regionInfo, -<<<<<<< HEAD ) (regionInfo, error) { bo := tikv.NewBackoffer(ctx, tikvRequestMaxBackoff) rpcCtx, err := s.upstream.regionCache.GetTiKVRPCContext( @@ -231,33 +161,15 @@ func (s *regionRequestScheduler) attachRPCContext( } if err != nil { log.Debug("region request scheduler failed to get RPC context", -======= -) (regionInfo, bool) { - bo := tikv.NewBackoffer(ctx, tikvRequestMaxBackoff) - rpcCtx, err := s.client.regionCache.GetTiKVRPCContext( - bo, region.verID, kvclientv2.ReplicaReadLeader, 0) - if rpcCtx != nil { - region.rpcCtx = rpcCtx - return region, true - } - if err != nil { - log.Debug("subscription client get rpc context fail", ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) zap.Uint64("subscriptionID", uint64(region.subscribedSpan.subID)), zap.Uint64("regionID", region.verID.GetID()), zap.Error(err)) } -<<<<<<< HEAD return region, &rpcCtxUnavailableErr{verID: region.verID} -======= - s.client.onRegionFail(newRegionErrorInfo(region, &rpcCtxUnavailableErr{verID: region.verID})) - return region, false ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) } func (s *regionRequestScheduler) getOrCreateStore( ctx context.Context, -<<<<<<< HEAD workerGroup *errgroup.Group, storeAddr string, ) *regionRequestStore { @@ -273,6 +185,7 @@ func (s *regionRequestScheduler) getOrCreateStore( s.workerCount, s.workerWindow, s.maxWindowMultiplier, + s.memoryQuota, ) // The scheduler run loop is the only writer. Publish the store after its // immutable worker list is complete, then start its workers. @@ -282,34 +195,11 @@ func (s *regionRequestScheduler) getOrCreateStore( } func (s *regionRequestScheduler) BroadcastDeregister( -======= - group *errgroup.Group, - storeAddr string, -) *requestedStore { - if value, ok := s.stores.Load(storeAddr); ok { - return value.(*requestedStore) - } - - store := newRequestedStore( - s.client, storeAddr, s.workerCount, s.workerWindow, s.maxWindowMultiplier) - // run is the only writer. Publish the store after its immutable worker list - // is complete, then start its workers. - s.stores.Store(storeAddr, store) - store.run(ctx, group) - return store -} - -func (s *regionRequestScheduler) broadcastDeregister( ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) subID SubscriptionID, filterLoop bool, ) { s.stores.Range(func(_, value any) bool { -<<<<<<< HEAD value.(*regionRequestStore).broadcastDeregister(subID, filterLoop) -======= - value.(*requestedStore).broadcastDeregister(subID, filterLoop) ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) return true }) } @@ -317,34 +207,24 @@ func (s *regionRequestScheduler) broadcastDeregister( func (s *regionRequestScheduler) requestedRegionCount() int { count := 0 s.stores.Range(func(_, value any) bool { - <<<<<<< HEAD count += value.(*regionRequestStore).inflightCount() - ======= - count += value.(*requestedStore).inflightCount() - >>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) return true }) return count } -<<<<<<< HEAD func (s *regionRequestScheduler) UpdateMetrics() { metrics.SubscriptionClientRequestedRegionCount.WithLabelValues("pending"). Set(float64(s.requestedRegionCount())) } -func (s *regionRequestScheduler) Close() { - s.taskQueue.Close() -} -======= -func (s *regionRequestScheduler) close() { - s.taskQueue.Close() -} - -func (s *regionRequestScheduler) closeStores() { +func (s *regionRequestScheduler) notifyAvailable() { s.stores.Range(func(_, value any) bool { - value.(*requestedStore).close() + value.(*regionRequestStore).notifyAvailable() return true }) } ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) + +func (s *regionRequestScheduler) Close() { + s.taskQueue.Close() +} diff --git a/logservice/logpuller/region_request_store.go b/logservice/logpuller/region_request_store.go index db7de05006..53a0d7a284 100644 --- a/logservice/logpuller/region_request_store.go +++ b/logservice/logpuller/region_request_store.go @@ -37,13 +37,21 @@ func newRegionRequestStore( workerCount int, workerWindow int, maxWindowMultiplier int, + memoryQuota *memoryQuotaController, ) *regionRequestStore { store := ®ionRequestStore{ workers: make([]*regionRequestWorker, 0, workerCount), } for range workerCount { store.workers = append(store.workers, newRegionRequestWorker( - upstream, eventSink, failureHandler, storeAddr, workerWindow, maxWindowMultiplier)) + upstream, + eventSink, + failureHandler, + storeAddr, + workerWindow, + maxWindowMultiplier, + memoryQuota, + )) } return store } @@ -79,3 +87,9 @@ func (s *regionRequestStore) requestedRegionCount() int { } return count } + +func (s *regionRequestStore) notifyAvailable() { + for _, worker := range s.workers { + worker.admission.notifyAvailable() + } +} diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index 8e519db935..8e333cf187 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -112,6 +112,7 @@ func newRegionRequestWorker( storeAddr string, currentWindow int, maxWindowMultiplier int, + memoryQuota *memoryQuotaController, ) *regionRequestWorker { workerID := workerIDGen.Add(1) return ®ionRequestWorker{ @@ -120,9 +121,14 @@ func newRegionRequestWorker( eventSink: eventSink, failureHandler: failureHandler, storeAddr: storeAddr, - admission: newRegionAdmissionController(currentWindow, maxWindowMultiplier), - controlQueue: newControlQueue(), - tracker: newRegionTracker(), + admission: newRegionAdmissionController( + currentWindow, + maxWindowMultiplier, + memoryQuota, + upstream.pdClock.CurrentTS, + ), + controlQueue: newControlQueue(), + tracker: newRegionTracker(), } } @@ -174,35 +180,6 @@ func (s *regionRequestWorker) Run(ctx context.Context) error { } } -<<<<<<< HEAD -======= -// failStreamRegions transfers every request sent by a failed stream to the -// recovery pipeline. -func (s *regionRequestWorker) failStreamRegions(err error) { - for _, state := range s.tracker.Drain() { - s.notifyRegionError(state, err) - } - // The failed stream no longer owns remote registrations. - s.controlQueue.drain() -} - -// failPendingRegions transfers requests owned by this worker but not yet sent -// to the recovery pipeline, so they can be resolved and routed again. -func (s *regionRequestWorker) failPendingRegions(err error) { - for _, task := range s.admission.drain() { - s.failureHandler.Report(newRegionErrorInfo(task.regionInfo, err)) - } -} - -func (s *regionRequestWorker) notifyRegionError(state *regionFeedState, err error) { - state.markStopped(err) - s.eventSink.Push( - SubscriptionID(state.requestID), - regionEvent{states: []*regionFeedState{state}}, - ) -} - ->>>>>>> 9903a1be7 (refactor) func (s *regionRequestWorker) waitForRegionRequest(ctx context.Context) (*regionReq, error) { // Without a stream there are no remote registrations to deregister. s.controlQueue.drain() diff --git a/logservice/logpuller/region_request_worker_test.go b/logservice/logpuller/region_request_worker_test.go index daef4db768..32228d8e7c 100644 --- a/logservice/logpuller/region_request_worker_test.go +++ b/logservice/logpuller/region_request_worker_test.go @@ -584,7 +584,6 @@ func TestRunStreamFailureReportsPendingRegionsToFailureHandler(t *testing.T) { pdClient := newFailureRecoveryTestPDClient(t) defer pdClient.Close() -<<<<<<< HEAD handler := newRegionFailureHandler(nil, func(*subscribedSpan) {}, nil, nil) worker := ®ionRequestWorker{ upstream: &upstreamHandle{pd: pdClient, credential: &security.Credential{}}, @@ -594,29 +593,6 @@ func TestRunStreamFailureReportsPendingRegionsToFailureHandler(t *testing.T) { controlQueue: newControlQueue(), tracker: newRegionTracker(), storeAddr: "127.0.0.1:1", -======= - admission := newRegionAdmissionController(1, 1) - failureHandler := ®ionFailureHandler{cache: newErrCache()} - worker := ®ionRequestWorker{failureHandler: failureHandler, admission: admission} - regions := []regionInfo{ - { - verID: tikv.NewRegionVerID(1, 1, 1), - span: heartbeatpb.TableSpan{ - TableID: 1, StartKey: []byte("a"), EndKey: []byte("m"), - }, - subscribedSpan: span, lockedRangeState: lock1.LockedRangeState, - }, - { - verID: tikv.NewRegionVerID(2, 1, 1), - span: heartbeatpb.TableSpan{ - TableID: 1, StartKey: []byte("m"), EndKey: []byte("z"), - }, - subscribedSpan: span, lockedRangeState: lock2.LockedRangeState, - }, - } - for i, region := range regions { - require.True(t, admission.submit(newRegionPriorityTask(region, 1, uint64(i+1)))) ->>>>>>> 9903a1be7 (refactor) } firstRegion := createFailureRecoveryTestRegion(t, 1, 1) diff --git a/logservice/logpuller/region_state.go b/logservice/logpuller/region_state.go index 6a2837564f..0f2f84ce3e 100644 --- a/logservice/logpuller/region_state.go +++ b/logservice/logpuller/region_state.go @@ -58,18 +58,15 @@ func newRegionInfo( span heartbeatpb.TableSpan, rpcCtx *tikv.RPCContext, subscribedSpan *subscribedSpan, + filterLoop bool, ) regionInfo { return regionInfo{ verID: verID, span: span, rpcCtx: rpcCtx, subscribedSpan: subscribedSpan, -<<<<<<< HEAD filterLoop: filterLoop, scanPriority: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, -======= - filterLoop: subscribedSpan.filterLoop, ->>>>>>> 9903a1be7 (refactor) } } diff --git a/logservice/logpuller/region_tracker.go b/logservice/logpuller/region_tracker.go index 4d69bf8bb3..81a27dc86c 100644 --- a/logservice/logpuller/region_tracker.go +++ b/logservice/logpuller/region_tracker.go @@ -112,7 +112,6 @@ func (t *regionTracker) Drain() []*regionFeedState { t.statesBySubscription = make(map[SubscriptionID]regionStatesByID) t.mu.Unlock() -<<<<<<< HEAD totalStates := 0 for _, states := range statesBySubscription { totalStates += len(states) @@ -120,17 +119,6 @@ func (t *regionTracker) Drain() []*regionFeedState { drainedStates := make([]*regionFeedState, 0, totalStates) for _, states := range statesBySubscription { drainedStates = append(drainedStates, slices.Collect(maps.Values(states))...) -======= - stateCount := 0 - for _, states := range statesBySubscription { - stateCount += len(states) - } - drainedStates := make([]*regionFeedState, 0, stateCount) - for _, states := range statesBySubscription { - for _, state := range states { - drainedStates = append(drainedStates, state) - } ->>>>>>> 9903a1be7 (refactor) } return drainedStates } diff --git a/logservice/logpuller/span_registry.go b/logservice/logpuller/span_registry.go index 797e42d833..5c8f70fba8 100644 --- a/logservice/logpuller/span_registry.go +++ b/logservice/logpuller/span_registry.go @@ -190,6 +190,18 @@ func (span *subscribedSpan) resolveStaleLocks(targetTs uint64) { zap.Any("ranges", res)) } +func (span *subscribedSpan) tryMarkInitialized(regionID, resolvedTs uint64) bool { + if resolvedTs <= span.startTs || !span.initialized.CompareAndSwap(false, true) { + return false + } + log.Info("subscription client is initialized", + zap.Uint64("subscriptionID", uint64(span.subID)), + zap.Uint64("regionID", regionID), + zap.Uint64("resolvedTs", resolvedTs), + zap.Uint64("startTs", span.startTs)) + return true +} + func newSpanRegistry(pd pd.Client, pdClock pdutil.Clock) *spanRegistry { return &spanRegistry{ spans: make(map[SubscriptionID]*subscribedSpan), diff --git a/logservice/logpuller/subscription_client.go b/logservice/logpuller/subscription_client.go index d77de7dc5b..28b26944c2 100644 --- a/logservice/logpuller/subscription_client.go +++ b/logservice/logpuller/subscription_client.go @@ -26,19 +26,12 @@ import ( "github.com/pingcap/ticdc/logservice/txnutil" "github.com/pingcap/ticdc/pkg/common" appcontext "github.com/pingcap/ticdc/pkg/common/context" -<<<<<<< HEAD "github.com/pingcap/ticdc/pkg/config" -======= ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) "github.com/pingcap/ticdc/pkg/metrics" "github.com/pingcap/ticdc/pkg/pdutil" "github.com/pingcap/ticdc/pkg/security" "github.com/pingcap/ticdc/pkg/spanz" "github.com/pingcap/ticdc/pkg/util" -<<<<<<< HEAD -======= - "github.com/prometheus/client_golang/prometheus" ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) "github.com/tikv/client-go/v2/tikv" pd "github.com/tikv/pd/client" "go.uber.org/zap" @@ -89,12 +82,8 @@ type resolveLockTask struct { type rangeTask struct { span heartbeatpb.TableSpan subscribedSpan *subscribedSpan -<<<<<<< HEAD filterLoop bool priority cdcpb.ScanPriority -======= - wasInitialized bool ->>>>>>> 9903a1be7 (refactor) } // upstreamHandle contains the stable TiKV and PD dependencies shared by the @@ -140,24 +129,16 @@ type subscriptionClient struct { lockResolver txnutil.LockResolver -<<<<<<< HEAD -======= - // the credential to connect tikv - credential *security.Credential - ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) // failureHandler handles failed regions and owns reschedule/retry decisions. failureHandler *regionFailureHandler // eventSink delivers region events and owns dynstream interaction. eventSink *regionEventSink // spanRegistry tracks subscribed spans and owns span-level background tasks. spanRegistry *spanRegistry -<<<<<<< HEAD // regionScheduler assigns locked region requests to per-store workers. -======= - // regionScheduler assigns locked Region requests to per-store workers. ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) regionScheduler *regionRequestScheduler + // memoryQuota owns event-memory accounting and initial-scan admission. + memoryQuota *memoryQuotaController // rangeTaskCh is used to receive range tasks. // The tasks will be handled in `handleRangeTask` goroutine. @@ -175,20 +156,12 @@ func NewSubscriptionClient( credential *security.Credential, ) SubscriptionClient { subClient := &subscriptionClient{ -<<<<<<< HEAD upstream: &upstreamHandle{ pd: pd, regionCache: appcontext.GetService[*tikv.RegionCache](appcontext.RegionCache), pdClock: appcontext.GetService[pdutil.Clock](appcontext.DefaultPDClock), credential: credential, }, -======= - config: config, - - pd: pd, - regionCache: appcontext.GetService[*tikv.RegionCache](appcontext.RegionCache), - pdClock: appcontext.GetService[pdutil.Clock](appcontext.DefaultPDClock), ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) lockResolver: lockResolver, rangeTaskCh: make(chan rangeTask, 1024), @@ -196,28 +169,28 @@ func NewSubscriptionClient( resolveLockRateLimiter: newResolveLockRateLimiter(), } subClient.ctx, subClient.cancel = context.WithCancel(context.Background()) -<<<<<<< HEAD + pullerConfig := config.GetGlobalServerConfig().Debug.Puller + subClient.memoryQuota = newMemoryQuotaController( + pullerConfig.MemoryQuota, pullerConfig.ScanBaseSize) subClient.failureHandler = newRegionFailureHandler( subClient.upstream.regionCache, subClient.onTableDrained, subClient.scheduleRegionRequest, subClient.scheduleRangeRequest, ) - subClient.eventSink = newRegionEventSink(subClient.failureHandler) + subClient.eventSink = newRegionEventSink( + subClient.ctx, + subClient.failureHandler, + subClient.memoryQuota, + ) subClient.spanRegistry = newSpanRegistry(subClient.upstream.pd, subClient.upstream.pdClock) subClient.regionScheduler = newRegionRequestScheduler( subClient.upstream, subClient.eventSink, subClient.failureHandler, + subClient.memoryQuota, ) -======= - subClient.failureHandler = newRegionFailureHandler(subClient) - subClient.eventSink = newRegionEventSink(subClient.ctx, subClient.failureHandler) - subClient.spanRegistry = newSpanRegistry(subClient.pd, subClient.pdClock) - subClient.regionScheduler = newRegionRequestScheduler(subClient) - - subClient.initMetrics() ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) + subClient.memoryQuota.setOnAvailable(subClient.regionScheduler.notifyAvailable) return subClient } @@ -238,13 +211,7 @@ func (s *subscriptionClient) updateMetrics(ctx context.Context) error { case <-ctx.Done(): return ctx.Err() case <-ticker.C: -<<<<<<< HEAD s.regionScheduler.UpdateMetrics() -======= - pendingRegionReqCount := s.regionScheduler.inflightCount() - - metrics.SubscriptionClientRequestedRegionCount.WithLabelValues("pending").Set(float64(pendingRegionReqCount)) ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) s.eventSink.UpdateMetrics() s.spanRegistry.UpdateMetrics() } @@ -285,21 +252,18 @@ func (s *subscriptionClient) Subscribe( time.Duration(config.GetGlobalServerConfig().Debug.Puller.OldStartTsScanLowPriorityThreshold), ) s.spanRegistry.Add(rt) + s.memoryQuota.addSubscription(rt) s.eventSink.AddPath(rt) select { case <-s.ctx.Done(): log.Warn("subscribes span failed, the subscription client has closed") -<<<<<<< HEAD case s.rangeTaskCh <- rangeTask{ span: span, subscribedSpan: rt, filterLoop: rt.filterLoop, priority: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, }: -======= - case s.rangeTaskCh <- rangeTask{span: span, subscribedSpan: rt}: ->>>>>>> 9903a1be7 (refactor) log.Info("subscribes span done", zap.Uint64("subscriptionID", uint64(subID)), zap.Int64("tableID", span.TableID), zap.Uint64("startTs", startTs), zap.String("startKey", spanz.HexKey(span.StartKey)), zap.String("endKey", spanz.HexKey(span.EndKey))) @@ -330,15 +294,8 @@ func (s *subscriptionClient) Run(ctx context.Context) error { // The goroutines are listed by data flow; errgroup does not guarantee their // actual startup order. g.Go(func() error { return s.handleRangeTasks(ctx) }) -<<<<<<< HEAD g.Go(func() error { return s.regionScheduler.Run(ctx, g) }) g.Go(func() error { return s.eventSink.Run(ctx) }) -<<<<<<< HEAD -======= - g.Go(func() error { return s.regionScheduler.run(ctx, g) }) ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) -======= ->>>>>>> 9903a1be7 (refactor) g.Go(func() error { return s.failureHandler.Run(ctx) }) g.Go(func() error { return s.spanRegistry.Run(ctx) }) g.Go(func() error { return s.handleResolveLockTasks(ctx) }) @@ -353,11 +310,7 @@ func (s *subscriptionClient) Run(ctx context.Context) error { func (s *subscriptionClient) Close(ctx context.Context) error { s.cancel() s.eventSink.Close() -<<<<<<< HEAD s.regionScheduler.Close() -======= - s.regionScheduler.close() ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) return nil } @@ -368,11 +321,7 @@ func (s *subscriptionClient) setTableStopped(rt *subscribedSpan) { // Set stopped to true so we can stop handling region events from the table, // then notify every existing worker to deregister the subscription. if rt.stopped.CompareAndSwap(false, true) { -<<<<<<< HEAD s.regionScheduler.BroadcastDeregister(rt.subID, rt.filterLoop) -======= - s.regionScheduler.broadcastDeregister(rt.subID, rt.filterLoop) ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) if rt.rangeLock.Stop() { s.onTableDrained(rt) } @@ -389,17 +338,10 @@ func (s *subscriptionClient) onTableDrained(rt *subscribedSpan) { zap.Uint64("subscriptionID", uint64(rt.subID)), zap.Error(err)) } + s.memoryQuota.removeSubscription(rt) s.spanRegistry.Remove(rt.subID) } -<<<<<<< HEAD -======= -// Note: don't block the caller, otherwise there may be deadlock -func (s *subscriptionClient) onRegionFail(errInfo regionErrorInfo) { - s.failureHandler.Report(errInfo) -} - ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) func (s *subscriptionClient) handleRangeTasks(ctx context.Context) error { g, ctx := errgroup.WithContext(ctx) // Limit the concurrent number of goroutines to convert range tasks to region tasks. @@ -488,13 +430,8 @@ func (s *subscriptionClient) divideSpanAndScheduleRegionRequests( } verID := tikv.NewRegionVerID(regionMeta.Id, regionMeta.RegionEpoch.ConfVer, regionMeta.RegionEpoch.Version) -<<<<<<< HEAD regionInfo := newRegionInfo(verID, intersectSpan, nil, subscribedSpan, task.filterLoop) regionInfo.scanPriority = normalizeScanPriority(task.priority) -======= - regionInfo := newRegionInfo(verID, intersectSpan, nil, subscribedSpan) - regionInfo.wasInitialized = task.wasInitialized ->>>>>>> 9903a1be7 (refactor) // Schedule a region request to subscribe the region. s.scheduleRegionRequest(ctx, regionInfo) @@ -509,11 +446,7 @@ func (s *subscriptionClient) divideSpanAndScheduleRegionRequests( } } -<<<<<<< HEAD // scheduleRegionRequest locks the region's range before submitting it to the -======= -// scheduleRegionRequest locks the Region's range before submitting it to the ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) // request scheduler. func (s *subscriptionClient) scheduleRegionRequest(ctx context.Context, region regionInfo) { lockRangeResult := region.subscribedSpan.rangeLock.LockRange( @@ -526,27 +459,19 @@ func (s *subscriptionClient) scheduleRegionRequest(ctx context.Context, region r switch lockRangeResult.Status { case regionlock.LockRangeStatusSuccess: region.lockedRangeState = lockRangeResult.LockedRangeState -<<<<<<< HEAD region.scanPriority = region.subscribedSpan.priorityPolicy.resolve( region.scanPriority, region.resolvedTs(), s.upstream.pdClock.CurrentTime(), ) s.regionScheduler.Submit(region) -======= - s.regionScheduler.submit(region) ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) case regionlock.LockRangeStatusStale: for _, r := range lockRangeResult.RetryRanges { s.scheduleRangeRequest(ctx, rangeTask{ span: r, subscribedSpan: region.subscribedSpan, -<<<<<<< HEAD filterLoop: region.filterLoop, priority: region.scanPriority, -======= - wasInitialized: region.wasInitialized, ->>>>>>> 9903a1be7 (refactor) }) } default: diff --git a/logservice/logpuller/subscription_client_test.go b/logservice/logpuller/subscription_client_test.go index 048517b011..818a745e54 100644 --- a/logservice/logpuller/subscription_client_test.go +++ b/logservice/logpuller/subscription_client_test.go @@ -334,24 +334,11 @@ func TestStopTaskUsesSubscribedSpanFilterLoop(t *testing.T) { res := span.rangeLock.LockRange(context.Background(), rawSpan.StartKey, rawSpan.EndKey, 1, 1) require.Equal(t, regionlock.LockRangeStatusSuccess, res.Status) -<<<<<<< HEAD -<<<<<<< HEAD -======= ->>>>>>> 9903a1be7 (refactor) const storeAddr = "store-1" worker := ®ionRequestWorker{storeAddr: storeAddr, controlQueue: newControlQueue()} store := ®ionRequestStore{workers: []*regionRequestWorker{worker}} client.regionScheduler = ®ionRequestScheduler{} client.regionScheduler.stores.Store(storeAddr, store) -<<<<<<< HEAD -======= - worker := ®ionRequestWorker{controlQueue: newControlQueue()} - store := &requestedStore{storeAddr: "store-1", workers: []*regionRequestWorker{worker}} - client.regionScheduler = ®ionRequestScheduler{client: client} - client.regionScheduler.stores.Store(store.storeAddr, store) ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) -======= ->>>>>>> 9903a1be7 (refactor) client.setTableStopped(span) @@ -443,10 +430,6 @@ func TestRegionEventSinkPushUnblocksOnClientClose(t *testing.T) { sink.cond = sync.NewCond(&sink.mu) client := &subscriptionClient{eventSink: sink} client.regionScheduler = ®ionRequestScheduler{ -<<<<<<< HEAD -======= - client: client, ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) taskQueue: priorityqueue.New[*regionPriorityTask](), } client.ctx, client.cancel = context.WithCancel(context.Background()) @@ -474,69 +457,6 @@ func TestRegionEventSinkPushUnblocksOnClientClose(t *testing.T) { } } -<<<<<<< HEAD -======= -func TestBroadcastDeregisterUsesWorkerControlQueue(t *testing.T) { - client := &subscriptionClient{} - scheduler := ®ionRequestScheduler{client: client} - admission := newRegionAdmissionController(1, 1) - - const storeAddr = "store-1" - worker := ®ionRequestWorker{ - storeAddr: storeAddr, - admission: admission, - controlQueue: newControlQueue(), - } -<<<<<<< HEAD - store := &requestedStore{storeAddr: "store-1", workers: []*regionRequestWorker{worker}} - scheduler.stores.Store(store.storeAddr, store) -======= - store := ®ionRequestStore{workers: []*regionRequestWorker{worker}} - scheduler.stores.Store(storeAddr, store) ->>>>>>> 9903a1be7 (refactor) - - dummyRegion := regionInfo{ - subscribedSpan: &subscribedSpan{subID: SubscriptionID(2)}, - lockedRangeState: ®ionlock.LockedRangeState{}, - } - require.True(t, admission.submit(newRegionPriorityTask(dummyRegion, 1, 1))) - - scheduler.broadcastDeregister(SubscriptionID(1), true) - require.Equal(t, 1, worker.controlQueue.len()) - req, ok := worker.controlQueue.tryPop() - require.True(t, ok) - require.Equal(t, SubscriptionID(1), req.subID) - require.True(t, req.filterLoop) - require.Equal(t, 1, admission.stats().pending) -} - -func TestRequestedStoreDistributesRegionsAcrossWorkerBuffers(t *testing.T) { - worker1 := ®ionRequestWorker{admission: newRegionAdmissionController(1, 1)} - worker2 := ®ionRequestWorker{admission: newRegionAdmissionController(1, 1)} -<<<<<<< HEAD - store := &requestedStore{ - storeAddr: "store-1", - workers: []*regionRequestWorker{worker1, worker2}, -======= - store := ®ionRequestStore{ - workers: []*regionRequestWorker{worker1, worker2}, ->>>>>>> 9903a1be7 (refactor) - } - - for i := uint64(1); i <= 4; i++ { - region := regionInfo{ - verID: tikv.NewRegionVerID(i, 1, 1), - subscribedSpan: &subscribedSpan{subID: 1}, - lockedRangeState: ®ionlock.LockedRangeState{}, - } - require.True(t, store.submit(newRegionPriorityTask(region, 1, i))) - } - - require.Equal(t, 2, worker1.admission.stats().pending) - require.Equal(t, 2, worker2.admission.stats().pending) -} - ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) func TestSubscriptionWithFailedTiKV(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) mockPDClock := pdutil.NewClock4Test() diff --git a/metrics/grafana/ticdc_new_arch.json b/metrics/grafana/ticdc_new_arch.json index 0a3f4bbd37..29123f9715 100644 --- a/metrics/grafana/ticdc_new_arch.json +++ b/metrics/grafana/ticdc_new_arch.json @@ -8815,6 +8815,96 @@ "align": false, "alignLevel": null } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Log puller memory quota tracked by the local memory quota controller.", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 57 + }, + "hiddenSeries": false, + "id": 26001, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(ticdc_log_puller_memory_quota{k8s_cluster=~\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance, type)", + "interval": "", + "legendFormat": "{{instance}}-{{type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Quota Controller", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } } ], "title": "Log Puller", diff --git a/metrics/nextgengrafana/ticdc_new_arch_next_gen.json b/metrics/nextgengrafana/ticdc_new_arch_next_gen.json index f920ac2ccd..2bfe4b8015 100644 --- a/metrics/nextgengrafana/ticdc_new_arch_next_gen.json +++ b/metrics/nextgengrafana/ticdc_new_arch_next_gen.json @@ -8815,6 +8815,96 @@ "align": false, "alignLevel": null } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Log puller memory quota tracked by the local memory quota controller.", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 57 + }, + "hiddenSeries": false, + "id": 26001, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(ticdc_log_puller_memory_quota{k8s_cluster=~\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance, type)", + "interval": "", + "legendFormat": "{{instance}}-{{type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Quota Controller", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } } ], "title": "Log Puller", diff --git a/pkg/config/debug.go b/pkg/config/debug.go index 54a5ba868c..df2fb31d27 100644 --- a/pkg/config/debug.go +++ b/pkg/config/debug.go @@ -25,6 +25,8 @@ const ( // DefaultOldStartTsScanLowPriorityThreshold is the default lag threshold for // classifying scan tasks as low priority. DefaultOldStartTsScanLowPriorityThreshold = 10 * time.Minute + defaultLogPullerMemoryQuota uint64 = 1024 * 1024 * 1024 + defaultLogPullerScanBaseSize uint64 = 8 * 1024 * 1024 ) // DebugConfig represents config for ticdc unexposed feature configurations @@ -84,6 +86,10 @@ type PullerConfig struct { // Scans within this threshold are scheduled as high priority. Older scans // remain low priority until their span catches up once. OldStartTsScanLowPriorityThreshold TomlDuration `toml:"old-start-ts-scan-low-priority-threshold" json:"old_start_ts_scan_low_priority_threshold"` + // MemoryQuota is the log puller's local soft memory limit in bytes. + MemoryQuota uint64 `toml:"memory-quota" json:"memory_quota"` + // ScanBaseSize is the base memory estimate for one admitted initial scan. + ScanBaseSize uint64 `toml:"scan-base-size" json:"scan_base_size"` } // NewDefaultPullerConfig return the default puller configuration @@ -96,6 +102,8 @@ func NewDefaultPullerConfig() *PullerConfig { RegionRequestMaxWindowMultiplier: 4, // Allows high-priority scans to use up to 4 * PendingRegionRequestQueueSize. OldStartTsScanLowPriorityThreshold: TomlDuration( DefaultOldStartTsScanLowPriorityThreshold), + MemoryQuota: defaultLogPullerMemoryQuota, + ScanBaseSize: defaultLogPullerScanBaseSize, } } @@ -117,6 +125,16 @@ func (c *PullerConfig) ValidateAndAdjust() { if c.OldStartTsScanLowPriorityThreshold <= 0 { c.OldStartTsScanLowPriorityThreshold = TomlDuration(DefaultOldStartTsScanLowPriorityThreshold) } + if c.MemoryQuota == 0 { + log.Warn("log puller memory quota must be positive, use default value", + zap.Uint64("default", defaultCfg.MemoryQuota)) + c.MemoryQuota = defaultCfg.MemoryQuota + } + if c.ScanBaseSize == 0 { + log.Warn("log puller scan base size must be positive, use default value", + zap.Uint64("default", defaultCfg.ScanBaseSize)) + c.ScanBaseSize = defaultCfg.ScanBaseSize + } } type EventStoreConfig struct { diff --git a/pkg/config/debug_test.go b/pkg/config/debug_test.go index ced2d50bff..82904dcabf 100644 --- a/pkg/config/debug_test.go +++ b/pkg/config/debug_test.go @@ -24,19 +24,16 @@ func TestPullerConfigValidateAndAdjustRegionRequestWindow(t *testing.T) { defaultCfg := NewDefaultPullerConfig() require.Equal(t, 32, defaultCfg.PendingRegionRequestQueueSize) require.Equal(t, 4, defaultCfg.RegionRequestMaxWindowMultiplier) - require.Equal( - t, - TomlDuration(DefaultOldStartTsScanLowPriorityThreshold), - defaultCfg.OldStartTsScanLowPriorityThreshold, - ) + require.Equal(t, uint64(1024*1024*1024), defaultCfg.MemoryQuota) + require.Equal(t, uint64(8*1024*1024), defaultCfg.ScanBaseSize) cfg := &PullerConfig{ - PendingRegionRequestQueueSize: -1, - RegionRequestMaxWindowMultiplier: 0, - OldStartTsScanLowPriorityThreshold: 0, + PendingRegionRequestQueueSize: -1, + RegionRequestMaxWindowMultiplier: 0, } cfg.ValidateAndAdjust() require.Equal(t, defaultCfg.PendingRegionRequestQueueSize, cfg.PendingRegionRequestQueueSize) require.Equal(t, defaultCfg.RegionRequestMaxWindowMultiplier, cfg.RegionRequestMaxWindowMultiplier) - require.Equal(t, defaultCfg.OldStartTsScanLowPriorityThreshold, cfg.OldStartTsScanLowPriorityThreshold) + require.Equal(t, defaultCfg.MemoryQuota, cfg.MemoryQuota) + require.Equal(t, defaultCfg.ScanBaseSize, cfg.ScanBaseSize) } diff --git a/pkg/metrics/log_puller.go b/pkg/metrics/log_puller.go index d50c5a8526..be932f4a8c 100644 --- a/pkg/metrics/log_puller.go +++ b/pkg/metrics/log_puller.go @@ -64,6 +64,13 @@ var ( Name: "resolved_ts_lag", Help: "The lag of resolved ts", }) + LogPullerMemoryQuota = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "ticdc", + Subsystem: "log_puller", + Name: "memory_quota", + Help: "The log puller local memory quota usage.", + }, []string{"type"}) SubscriptionClientResolvedTsLagGauge = prometheus.NewGauge( prometheus.GaugeOpts{ @@ -156,6 +163,7 @@ func initLogPullerMetrics(registry *prometheus.Registry) { registry.MustRegister(LogPullerPrewriteCacheRowNum) registry.MustRegister(LogPullerMatcherCount) registry.MustRegister(LogPullerResolvedTsLag) + registry.MustRegister(LogPullerMemoryQuota) registry.MustRegister(SubscriptionClientRequestedRegionCount) registry.MustRegister(RegionRequestFinishScanDuration) registry.MustRegister(SubscriptionClientSubscribedRegionCount) From 428b2e3e57dfcd4b9a055ee1c2bf5404435cdb12 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Fri, 17 Jul 2026 20:02:36 +0800 Subject: [PATCH 05/26] small fix --- logservice/logpuller/memory_quota.go | 2 +- logservice/logpuller/memory_quota_test.go | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/logservice/logpuller/memory_quota.go b/logservice/logpuller/memory_quota.go index 6596fc0f29..85657987cd 100644 --- a/logservice/logpuller/memory_quota.go +++ b/logservice/logpuller/memory_quota.go @@ -29,7 +29,7 @@ const ( defaultResumeWarmingRatio = 0.05 defaultFreezeAllRatio = 0.8 defaultResumeAllRatio = 0.6 - defaultHardLimitRatio = 5.0 + defaultHardLimitRatio = 2.0 defaultScanBaseSize uint64 = 8 * 1024 * 1024 defaultScanLagUnit = 10 * time.Minute diff --git a/logservice/logpuller/memory_quota_test.go b/logservice/logpuller/memory_quota_test.go index be4d88d89d..4ffe57078f 100644 --- a/logservice/logpuller/memory_quota_test.go +++ b/logservice/logpuller/memory_quota_test.go @@ -161,11 +161,10 @@ func TestMemoryQuotaBlockedEventStopsWhenSubscriptionIsRemoved(t *testing.T) { func TestMemoryQuotaBlockedEventResumesAfterRelease(t *testing.T) { quota := newMemoryQuotaController(100, 10) - quota.hardLimitRatio = 1 span := newTestQuotaSpan(1) quota.addSubscription(span) - lease := quota.trackEvent(context.Background(), span, 100) + lease := quota.trackEvent(context.Background(), span, 200) require.NotNil(t, lease) acquired := make(chan *memoryQuotaLease, 1) go func() { From a084736478dc86a9d07a38553bc975d01e873c2d Mon Sep 17 00:00:00 2001 From: lidezhu Date: Fri, 17 Jul 2026 22:22:43 +0800 Subject: [PATCH 06/26] refactor --- logservice/logpuller/memory_quota.go | 81 ++++++++++--------- logservice/logpuller/memory_quota_test.go | 3 +- .../logpuller/region_admission_controller.go | 15 +--- .../region_admission_controller_test.go | 24 ++++-- logservice/logpuller/region_event_handler.go | 11 ++- .../logpuller/region_event_handler_test.go | 30 ++++--- logservice/logpuller/region_event_sink.go | 10 +-- .../logpuller/region_event_sink_test.go | 48 +---------- 8 files changed, 98 insertions(+), 124 deletions(-) diff --git a/logservice/logpuller/memory_quota.go b/logservice/logpuller/memory_quota.go index 85657987cd..c5d90837b1 100644 --- a/logservice/logpuller/memory_quota.go +++ b/logservice/logpuller/memory_quota.go @@ -23,18 +23,44 @@ import ( ) const ( + // defaultLogPullerMemoryQuota is the soft memory capacity shared by event + // accounting and initial-scan admission. defaultLogPullerMemoryQuota uint64 = 1024 * 1024 * 1024 - defaultPauseWarmingRatio = 0.15 + // Admission ratios compare max(accounted event memory, estimated scan + // memory) with the soft capacity. + + // defaultPauseWarmingRatio pauses new high-lag scans when memory pressure + // reaches 15% of the soft capacity. + defaultPauseWarmingRatio = 0.15 + + // defaultResumeWarmingRatio resumes high-lag scans after memory pressure + // falls to 5% of the soft capacity. defaultResumeWarmingRatio = 0.05 - defaultFreezeAllRatio = 0.8 - defaultResumeAllRatio = 0.6 - defaultHardLimitRatio = 2.0 - - defaultScanBaseSize uint64 = 8 * 1024 * 1024 - defaultScanLagUnit = 10 * time.Minute - defaultScanLagWeight = 0.22 - defaultMaxScanLagFactor = 16 + + // defaultFreezeAllRatio pauses every new scan when memory pressure reaches + // 80% of the soft capacity. + defaultFreezeAllRatio = 0.8 + + // defaultResumeAllRatio allows new scans again after memory pressure falls + // to 60% of the soft capacity. + defaultResumeAllRatio = 0.6 + + // defaultHardLimitRatio blocks receiving more events when accounted event + // memory reaches twice the soft capacity. + defaultHardLimitRatio = 2.0 + + // defaultScanBaseSize is the minimum memory estimate for one admitted scan. + defaultScanBaseSize uint64 = 8 * 1024 * 1024 + + // defaultScanLagUnit is the lag unit used by the logarithmic scan estimate. + defaultScanLagUnit = 10 * time.Minute + + // defaultScanLagWeight controls how quickly the scan estimate grows with lag. + defaultScanLagWeight = 0.22 + + // defaultMaxScanLagFactor caps one scan estimate at this multiple of the base. + defaultMaxScanLagFactor = 16 ) type admissionLevel uint8 @@ -50,15 +76,12 @@ type memoryQuotaLease struct { release func() } +func newMemoryQuotaLease(release func()) *memoryQuotaLease { + return &memoryQuotaLease{release: release} +} + func (l *memoryQuotaLease) Release() { - if l == nil { - return - } - l.once.Do(func() { - if l.release != nil { - l.release() - } - }) + l.once.Do(l.release) } type subscriptionQuotaState struct { @@ -123,6 +146,7 @@ func newMemoryQuotaController(capacity, scanBaseSize uint64) *memoryQuotaControl subscriptions: make(map[SubscriptionID]*subscriptionQuotaState), } c.cond = sync.NewCond(&c.mu) + c.onAvailable.Store(func() {}) return c } @@ -131,9 +155,7 @@ func (c *memoryQuotaController) setOnAvailable(fn func()) { } func (c *memoryQuotaController) notifyAvailable() { - if fn, ok := c.onAvailable.Load().(func()); ok && fn != nil { - fn() - } + c.onAvailable.Load().(func())() } func (c *memoryQuotaController) wakeAll() { @@ -162,19 +184,12 @@ func (c *memoryQuotaController) scanSnapshot() ( } func (c *memoryQuotaController) addSubscription(span *subscribedSpan) { - if span == nil { - return - } c.mu.Lock() c.subscriptions[span.subID] = newSubscriptionQuotaState() c.mu.Unlock() } func (c *memoryQuotaController) removeSubscription(span *subscribedSpan) { - if span == nil { - return - } - c.mu.Lock() state, ok := c.subscriptions[span.subID] if !ok { @@ -210,17 +225,13 @@ func (c *memoryQuotaController) acquireScan( currentTs uint64, ) (*memoryQuotaLease, bool) { span := region.subscribedSpan - if span == nil { - return nil, true - } - c.mu.Lock() state, ok := c.subscriptions[span.subID] if !ok { // The subscription has already been removed. Let the request continue to // the worker, where the normal stopped-subscription path will discard it. c.mu.Unlock() - return nil, true + return newMemoryQuotaLease(func() {}), true } c.refreshLevelLocked() if c.level == admissionFreezeAllNewScans { @@ -268,10 +279,6 @@ func (c *memoryQuotaController) trackEvent( span *subscribedSpan, bytes uint64, ) *memoryQuotaLease { - if span == nil || bytes == 0 { - return nil - } - c.mu.Lock() if ctx.Err() != nil { c.mu.Unlock() @@ -347,7 +354,7 @@ func scanLagFactor(startTs, currentTs uint64) float64 { func isWarmingScan(region regionInfo, currentTs uint64) bool { span := region.subscribedSpan - if span == nil || span.initialized.Load() { + if span.initialized.Load() { return false } return regionScanLag(currentTs, region.resolvedTs()) >= lowLagRegionThreshold diff --git a/logservice/logpuller/memory_quota_test.go b/logservice/logpuller/memory_quota_test.go index 4ffe57078f..5f45602f43 100644 --- a/logservice/logpuller/memory_quota_test.go +++ b/logservice/logpuller/memory_quota_test.go @@ -121,7 +121,8 @@ func TestMemoryQuotaRemoveSubscriptionReleasesOwnedMemory(t *testing.T) { // path without recreating quota state. scanLease, admitted = quota.acquireScan(newTestQuotaRegion(span1), span1.resolvedTs.Load()) require.True(t, admitted) - require.Nil(t, scanLease) + require.NotNil(t, scanLease) + scanLease.Release() require.NotContains(t, quota.subscriptions, span1.subID) lease2.Release() diff --git a/logservice/logpuller/region_admission_controller.go b/logservice/logpuller/region_admission_controller.go index b7ee52f3da..3ab3804394 100644 --- a/logservice/logpuller/region_admission_controller.go +++ b/logservice/logpuller/region_admission_controller.go @@ -202,17 +202,10 @@ func (c *regionAdmissionController) popEligibleLocked() ( return nil, nil } - var scanQuota *memoryQuotaLease - if c.memoryQuota != nil { - currentTs := uint64(0) - if c.currentTs != nil { - currentTs = c.currentTs() - } - var admitted bool - scanQuota, admitted = c.memoryQuota.acquireScan(request.regionInfo, currentTs) - if !admitted { - return nil, nil - } + scanQuota, admitted := c.memoryQuota.acquireScan( + request.regionInfo, c.currentTs()) + if !admitted { + return nil, nil } request, _ = c.state.pending.PopTop() return request, scanQuota diff --git a/logservice/logpuller/region_admission_controller_test.go b/logservice/logpuller/region_admission_controller_test.go index ed68f14a2f..a6277f80de 100644 --- a/logservice/logpuller/region_admission_controller_test.go +++ b/logservice/logpuller/region_admission_controller_test.go @@ -58,8 +58,20 @@ func submitRegionForAdmission( require.True(t, controller.submit(task)) } +func newTestRegionAdmissionController( + currentWindow int, + maxWindowMultiplier int, +) *regionAdmissionController { + return newRegionAdmissionController( + currentWindow, + maxWindowMultiplier, + newMemoryQuotaController(0, 0), + func() uint64 { return 0 }, + ) +} + func TestRegionAdmissionControllerNormalWindow(t *testing.T) { - controller := newRegionAdmissionController(1, 2, nil, nil) + controller := newTestRegionAdmissionController(1, 2) currentTs := oracle.GoTimeToTS(time.Now()) checkpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Hour)) region1 := prepareRegionForAdmission(createTestRegionInfo(1, 1), checkpointTs) @@ -84,7 +96,7 @@ func TestRegionAdmissionControllerNormalWindow(t *testing.T) { } func TestRegionAdmissionControllerLowLagUsesMaxWindow(t *testing.T) { - controller := newRegionAdmissionController(1, 2, nil, nil) + controller := newTestRegionAdmissionController(1, 2) currentTs := oracle.GoTimeToTS(time.Now()) slowCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Hour)) lowLagCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Minute)) @@ -121,7 +133,7 @@ func TestRegionAdmissionControllerLowLagUsesMaxWindow(t *testing.T) { } func TestRegionAdmissionControllerPrioritizesInitializedRegion(t *testing.T) { - controller := newRegionAdmissionController(1, 2, nil, nil) + controller := newTestRegionAdmissionController(1, 2) currentTs := oracle.GoTimeToTS(time.Now()) slowCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Hour)) lowLagCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Minute)) @@ -153,7 +165,7 @@ func TestRegionAdmissionControllerPrioritizesInitializedRegion(t *testing.T) { } func TestRegionAdmissionLeaseReleasedOnce(t *testing.T) { - controller := newRegionAdmissionController(1, 1, nil, nil) + controller := newTestRegionAdmissionController(1, 1) currentTs := oracle.GoTimeToTS(time.Now()) region := prepareRegionForAdmission(createTestRegionInfo(1, 1), currentTs) submitRegionForAdmission(t, controller, region, currentTs) @@ -189,7 +201,7 @@ func TestRegionAdmissionLeaseReleasedOnce(t *testing.T) { } func TestRegionAdmissionControllerClose(t *testing.T) { - controller := newRegionAdmissionController(1, 1, nil, nil) + controller := newTestRegionAdmissionController(1, 1) controller.close() region := prepareRegionForAdmission(createTestRegionInfo(1, 1), 1) require.False(t, controller.submit(newRegionPriorityTask(region, 1, 1))) @@ -199,7 +211,7 @@ func TestRegionAdmissionControllerClose(t *testing.T) { } func TestRegionAdmissionControllerDrainPending(t *testing.T) { - controller := newRegionAdmissionController(1, 1, nil, nil) + controller := newTestRegionAdmissionController(1, 1) region1 := prepareRegionForAdmission(createTestRegionInfo(1, 1), 1) region2 := prepareRegionForAdmission(createTestRegionInfo(1, 2), 1) submitRegionForAdmission(t, controller, region1, 1) diff --git a/logservice/logpuller/region_event_handler.go b/logservice/logpuller/region_event_handler.go index c23184ddab..92a649349a 100644 --- a/logservice/logpuller/region_event_handler.go +++ b/logservice/logpuller/region_event_handler.go @@ -64,14 +64,14 @@ func (event *regionEvent) needMemoryQuota() bool { } func (event *regionEvent) releaseMemoryQuota() { + if event.memoryQuota == nil { + return + } event.memoryQuota.Release() event.memoryQuota = nil } func (event *regionEvent) getSize() int { - if event == nil { - return 0 - } size := int(unsafe.Sizeof(*event)) if event.entries != nil { size += int(unsafe.Sizeof(*event.entries)) @@ -135,7 +135,7 @@ func (h *regionEventHandler) Handle(span *subscribedSpan, events ...regionEvent) wasInitialized := span.initialized.Load() quotaLeases := make([]*memoryQuotaLease, 0, len(events)) for _, event := range events { - if event.memoryQuota != nil { + if event.needMemoryQuota() { quotaLeases = append(quotaLeases, event.memoryQuota) } if len(event.states) == 1 && event.states[0].isStale() { @@ -158,8 +158,7 @@ func (h *regionEventHandler) Handle(span *subscribedSpan, events ...regionEvent) log.Panic("should not reach", zap.Any("event", event), zap.Any("events", events)) } } - if !wasInitialized && span.initialized.Load() && - h.eventSink != nil && h.eventSink.memoryQuota != nil { + if !wasInitialized && span.initialized.Load() { h.eventSink.memoryQuota.markSubscriptionInitialized() } tryAdvanceResolvedTs := func() { diff --git a/logservice/logpuller/region_event_handler_test.go b/logservice/logpuller/region_event_handler_test.go index 92fc3a8e03..dd4fc1d77b 100644 --- a/logservice/logpuller/region_event_handler_test.go +++ b/logservice/logpuller/region_event_handler_test.go @@ -48,7 +48,10 @@ import ( func TestHandleEventEntryEventOutOfOrder(t *testing.T) { // initialize option := dynstream.NewOption() - ds := dynstream.NewParallelDynamicStream("test", ®ionEventHandler{}, option) + handler := ®ionEventHandler{eventSink: ®ionEventSink{ + memoryQuota: newMemoryQuotaController(0, 0), + }} + ds := dynstream.NewParallelDynamicStream("test", handler, option) ds.Start() span := heartbeatpb.TableSpan{ @@ -104,8 +107,9 @@ func TestHandleEventEntryEventOutOfOrder(t *testing.T) { }, } regionEvent := regionEvent{ - states: []*regionFeedState{state}, - entries: events, + states: []*regionFeedState{state}, + entries: events, + memoryQuota: newMemoryQuotaLease(func() {}), } ds.Push(subID, regionEvent) } @@ -124,8 +128,9 @@ func TestHandleEventEntryEventOutOfOrder(t *testing.T) { }, } regionEvent := regionEvent{ - states: []*regionFeedState{state}, - entries: events, + states: []*regionFeedState{state}, + entries: events, + memoryQuota: newMemoryQuotaLease(func() {}), } ds.Push(subID, regionEvent) } @@ -154,8 +159,9 @@ func TestHandleEventEntryEventOutOfOrder(t *testing.T) { }, } regionEvent := regionEvent{ - states: []*regionFeedState{state}, - entries: events, + states: []*regionFeedState{state}, + entries: events, + memoryQuota: newMemoryQuotaLease(func() {}), } ds.Push(subID, regionEvent) } @@ -181,8 +187,9 @@ func TestHandleEventEntryEventOutOfOrder(t *testing.T) { }, } regionEvent := regionEvent{ - states: []*regionFeedState{state}, - entries: events, + states: []*regionFeedState{state}, + entries: events, + memoryQuota: newMemoryQuotaLease(func() {}), } ds.Push(subID, regionEvent) } @@ -204,7 +211,10 @@ func TestHandleEventEntryEventOutOfOrder(t *testing.T) { func TestHandleResolvedTs(t *testing.T) { // initialize option := dynstream.NewOption() - ds := dynstream.NewParallelDynamicStream("test", ®ionEventHandler{}, option) + handler := ®ionEventHandler{eventSink: ®ionEventSink{ + memoryQuota: newMemoryQuotaController(0, 0), + }} + ds := dynstream.NewParallelDynamicStream("test", handler, option) ds.Start() consumeKVEvents := func(events []common.RawKVEntry, _ func()) bool { return false } // not used diff --git a/logservice/logpuller/region_event_sink.go b/logservice/logpuller/region_event_sink.go index 3c161fda1a..25987ff997 100644 --- a/logservice/logpuller/region_event_sink.go +++ b/logservice/logpuller/region_event_sink.go @@ -78,7 +78,7 @@ func (s *regionEventSink) Wake(subID SubscriptionID) { } func (s *regionEventSink) Push(subID SubscriptionID, event regionEvent) { - if event.needMemoryQuota() && s.memoryQuota != nil { + if event.needMemoryQuota() { span := event.mustFirstState().region.subscribedSpan event.memoryQuota = s.memoryQuota.trackEvent(s.ctx, span, uint64(event.getSize())) if event.memoryQuota == nil { @@ -137,10 +137,6 @@ func (s *regionEventSink) UpdateMetrics() { metricSubscriptionClientDSChannelSize.Set(float64(dsMetrics.EventChanSize)) metricSubscriptionClientDSPendingQueueLen.Set(float64(dsMetrics.PendingQueueLen)) - if s.memoryQuota == nil { - return - } - used, capacity, _ := s.memoryQuota.snapshot() scanUsed, warmingScanUsed, warmingScanBudget, scanEstimate, hardLimit := s.memoryQuota.scanSnapshot() @@ -166,9 +162,7 @@ func (s *regionEventSink) UpdateMetrics() { } func (s *regionEventSink) Close() { - if s.memoryQuota != nil { - s.memoryQuota.wakeAll() - } + s.memoryQuota.wakeAll() s.mu.Lock() s.paused.Store(false) s.cond.Broadcast() diff --git a/logservice/logpuller/region_event_sink_test.go b/logservice/logpuller/region_event_sink_test.go index 1f08aa51f9..43fd6f088d 100644 --- a/logservice/logpuller/region_event_sink_test.go +++ b/logservice/logpuller/region_event_sink_test.go @@ -81,8 +81,9 @@ func TestRegionEventSinkRunPausesAndResumesPush(t *testing.T) { ds := newMockRegionEventSinkStream() sink := ®ionEventSink{ - ctx: ctx, - ds: ds, + ctx: ctx, + ds: ds, + memoryQuota: newMemoryQuotaController(0, 0), } sink.cond = sync.NewCond(&sink.mu) @@ -135,49 +136,6 @@ func TestRegionEventSinkRunPausesAndResumesPush(t *testing.T) { } func TestRegionEventSinkUpdateMetrics(t *testing.T) { - t.Run("without quota only updates queue gauges", func(t *testing.T) { - ds := newMockRegionEventSinkStream() - ds.metrics = dynstream.Metrics[int, SubscriptionID]{ - EventChanSize: 11, - PendingQueueLen: 22, - } - - metrics.DynamicStreamMemoryUsage.WithLabelValues( - "log-puller", - "max", - "default", - "default", - ).Set(123) - metrics.DynamicStreamMemoryUsage.WithLabelValues( - "log-puller", - "used", - "default", - "default", - ).Set(456) - - sink := ®ionEventSink{ - ctx: context.Background(), - ds: ds, - } - sink.cond = sync.NewCond(&sink.mu) - sink.UpdateMetrics() - - require.Equal(t, float64(11), testutil.ToFloat64(metricSubscriptionClientDSChannelSize)) - require.Equal(t, float64(22), testutil.ToFloat64(metricSubscriptionClientDSPendingQueueLen)) - require.Equal(t, float64(123), testutil.ToFloat64(metrics.DynamicStreamMemoryUsage.WithLabelValues( - "log-puller", - "max", - "default", - "default", - ))) - require.Equal(t, float64(456), testutil.ToFloat64(metrics.DynamicStreamMemoryUsage.WithLabelValues( - "log-puller", - "used", - "default", - "default", - ))) - }) - t.Run("quota updates memory gauges", func(t *testing.T) { ds := newMockRegionEventSinkStream() ds.metrics = dynstream.Metrics[int, SubscriptionID]{ From 99d400f7bb3b6ad950d3935d31f32e383f2c464b Mon Sep 17 00:00:00 2001 From: lidezhu Date: Sun, 19 Jul 2026 18:54:38 +0800 Subject: [PATCH 07/26] remove dependency on ds memory control --- logservice/logpuller/memory_quota.go | 383 +++++++++--------- logservice/logpuller/memory_quota_test.go | 309 ++++++++------ .../logpuller/region_admission_controller.go | 36 +- .../region_admission_controller_test.go | 12 +- logservice/logpuller/region_event_handler.go | 32 +- .../logpuller/region_event_handler_test.go | 37 +- logservice/logpuller/region_event_sink.go | 81 +--- .../logpuller/region_event_sink_test.go | 103 +---- .../logpuller/region_request_scheduler.go | 7 - logservice/logpuller/region_request_store.go | 6 - logservice/logpuller/subscription_client.go | 6 +- 11 files changed, 438 insertions(+), 574 deletions(-) diff --git a/logservice/logpuller/memory_quota.go b/logservice/logpuller/memory_quota.go index c5d90837b1..aa5396171e 100644 --- a/logservice/logpuller/memory_quota.go +++ b/logservice/logpuller/memory_quota.go @@ -71,29 +71,68 @@ const ( admissionFreezeAllNewScans ) -type memoryQuotaLease struct { - once sync.Once - release func() -} - -func newMemoryQuotaLease(release func()) *memoryQuotaLease { - return &memoryQuotaLease{release: release} +// eventMemoryNotifier wakes event receivers that are waiting for memory. Each +// notification closes the current ready channel to wake all current waiters, +// then creates a new channel for future waiters. +// +// To avoid missing a notification, a receiver waits in this order: +// +// 1. Register the waiter. +// 2. Read the current ready channel under mu. +// 3. Recheck memory and the span state before blocking on that channel. +// +// If a notification happens just before registration, the final recheck sees +// the released memory or stopped span, so the receiver does not block. +type eventMemoryNotifier struct { + mu sync.Mutex + ready chan struct{} + waiters atomic.Int64 } -func (l *memoryQuotaLease) Release() { - l.once.Do(l.release) +func newEventMemoryNotifier() *eventMemoryNotifier { + return &eventMemoryNotifier{ready: make(chan struct{})} } -type subscriptionQuotaState struct { - eventLeases map[*memoryQuotaLease]struct{} - scanLeases map[*memoryQuotaLease]struct{} +func (n *eventMemoryNotifier) wait( + ctx context.Context, + span *subscribedSpan, + tryAcquire func() bool, +) bool { + n.waiters.Add(1) + defer n.waiters.Add(-1) + for { + n.mu.Lock() + ready := n.ready + n.mu.Unlock() + + // This check must stay after waiter registration and loading ready. It + // closes both windows in which notify could otherwise be lost. + if tryAcquire() { + return true + } + if span.stopped.Load() { + return false + } + select { + case <-ready: + case <-ctx.Done(): + return false + } + } } -func newSubscriptionQuotaState() *subscriptionQuotaState { - return &subscriptionQuotaState{ - eventLeases: make(map[*memoryQuotaLease]struct{}), - scanLeases: make(map[*memoryQuotaLease]struct{}), +func (n *eventMemoryNotifier) notify() { + // waiters is only a fast-path hint. A waiter that registers after this load + // rechecks memory and the span state before blocking, so observing a stale + // zero cannot lose a wakeup. Observing a stale nonzero only causes a harmless + // extra broadcast. + if n.waiters.Load() == 0 { + return } + n.mu.Lock() + close(n.ready) + n.ready = make(chan struct{}) + n.mu.Unlock() } // memoryQuotaController tracks event memory retained by downstream callbacks @@ -102,29 +141,34 @@ func newSubscriptionQuotaState() *subscriptionQuotaState { // admission first pauses uninitialized high-lag spans and freezes all new scans // only under heavier pressure; both transitions use hysteresis when resuming. type memoryQuotaController struct { - mu sync.Mutex - cond *sync.Cond - capacity uint64 // used tracks event bytes retained until downstream finishes consuming them. - used uint64 + // Event accounting is on the receive hot path, so acquiring and releasing + // memory only use atomic operations while usage is below the hard limit. + used atomic.Uint64 - // scanUsed tracks the estimated memory of all admitted initial scans. - // warmingScanUsed is the subset used by uninitialized, high-lag spans. - scanUsed uint64 - warmingScanUsed uint64 - level admissionLevel + // eventNotifier owns the wait protocol used after the hard limit is reached. + eventNotifier *eventMemoryNotifier - pauseWarmingRatio float64 - resumeWarmingRatio float64 - freezeAllRatio float64 - resumeAllRatio float64 - hardLimitRatio float64 + // scanMu guards scan admission state and scanReady. Scan admission happens + // once per region rather than once per event batch, so it is intentionally + // kept simple instead of adding atomics to every field. + scanMu sync.Mutex + // scanUsed tracks the estimated memory of all admitted initial scans. + scanUsed uint64 + level admissionLevel + // scanReady is replaced and closed when a memory transition can make a + // rejected scan eligible. Workers wait on this channel directly, avoiding a + // synchronous broadcast to every store and request worker. + scanReady chan struct{} + + pauseWarmingLimit uint64 + resumeWarmingLimit uint64 + freezeAllLimit uint64 + resumeAllLimit uint64 + hardLimit uint64 scanEstimate uint64 - - subscriptions map[SubscriptionID]*subscriptionQuotaState - onAvailable atomic.Value // func() } func newMemoryQuotaController(capacity, scanBaseSize uint64) *memoryQuotaController { @@ -137,191 +181,150 @@ func newMemoryQuotaController(capacity, scanBaseSize uint64) *memoryQuotaControl c := &memoryQuotaController{ capacity: capacity, level: admissionNormal, - pauseWarmingRatio: defaultPauseWarmingRatio, - resumeWarmingRatio: defaultResumeWarmingRatio, - freezeAllRatio: defaultFreezeAllRatio, - resumeAllRatio: defaultResumeAllRatio, - hardLimitRatio: defaultHardLimitRatio, + pauseWarmingLimit: uint64(math.Ceil(float64(capacity) * defaultPauseWarmingRatio)), + resumeWarmingLimit: uint64(float64(capacity) * defaultResumeWarmingRatio), + freezeAllLimit: uint64(math.Ceil(float64(capacity) * defaultFreezeAllRatio)), + resumeAllLimit: uint64(float64(capacity) * defaultResumeAllRatio), + hardLimit: uint64(float64(capacity) * defaultHardLimitRatio), scanEstimate: scanBaseSize, - subscriptions: make(map[SubscriptionID]*subscriptionQuotaState), + eventNotifier: newEventMemoryNotifier(), + scanReady: make(chan struct{}), } - c.cond = sync.NewCond(&c.mu) - c.onAvailable.Store(func() {}) return c } -func (c *memoryQuotaController) setOnAvailable(fn func()) { - c.onAvailable.Store(fn) -} - -func (c *memoryQuotaController) notifyAvailable() { - c.onAvailable.Load().(func())() -} - func (c *memoryQuotaController) wakeAll() { - c.mu.Lock() - c.cond.Broadcast() - c.mu.Unlock() + c.eventNotifier.notify() + c.notifyScanAdmission() } func (c *memoryQuotaController) snapshot() (used, capacity uint64, level admissionLevel) { - c.mu.Lock() - defer c.mu.Unlock() - return c.used, c.capacity, c.level + c.scanMu.Lock() + defer c.scanMu.Unlock() + return c.used.Load(), c.capacity, c.level } func (c *memoryQuotaController) scanSnapshot() ( scanUsed uint64, - warmingScanUsed uint64, - warmingScanBudget uint64, scanEstimate uint64, hardLimit uint64, ) { - c.mu.Lock() - defer c.mu.Unlock() - return c.scanUsed, c.warmingScanUsed, c.warmingScanBudgetLocked(), - c.scanEstimate, c.hardLimitLocked() -} - -func (c *memoryQuotaController) addSubscription(span *subscribedSpan) { - c.mu.Lock() - c.subscriptions[span.subID] = newSubscriptionQuotaState() - c.mu.Unlock() -} - -func (c *memoryQuotaController) removeSubscription(span *subscribedSpan) { - c.mu.Lock() - state, ok := c.subscriptions[span.subID] - if !ok { - c.mu.Unlock() - return - } - delete(c.subscriptions, span.subID) - leases := make([]*memoryQuotaLease, 0, len(state.eventLeases)+len(state.scanLeases)) - for lease := range state.eventLeases { - leases = append(leases, lease) - } - for lease := range state.scanLeases { - leases = append(leases, lease) - } - // Wake event receivers so they can observe that the subscription was removed. - c.cond.Broadcast() - c.mu.Unlock() - - for _, lease := range leases { - lease.Release() - } - // Removing a subscription can make a pending request eligible even when the - // subscription itself did not own any lease. - c.notifyAvailable() -} - -func (c *memoryQuotaController) markSubscriptionInitialized() { - c.notifyAvailable() + c.scanMu.Lock() + defer c.scanMu.Unlock() + return c.scanUsed, c.scanEstimate, c.hardLimit } func (c *memoryQuotaController) acquireScan( region regionInfo, currentTs uint64, -) (*memoryQuotaLease, bool) { +) (bytes uint64, retry <-chan struct{}, admitted bool) { span := region.subscribedSpan - c.mu.Lock() - state, ok := c.subscriptions[span.subID] - if !ok { - // The subscription has already been removed. Let the request continue to - // the worker, where the normal stopped-subscription path will discard it. - c.mu.Unlock() - return newMemoryQuotaLease(func() {}), true + if span.stopped.Load() { + // Let stale tasks reach the worker's stopped-subscription cleanup path + // without consuming scan quota. + return 0, nil, true } + + c.scanMu.Lock() + defer c.scanMu.Unlock() c.refreshLevelLocked() if c.level == admissionFreezeAllNewScans { - c.mu.Unlock() - return nil, false + return 0, c.scanReady, false } - bytes := c.estimateScanSizeLocked(region, currentTs) warming := isWarmingScan(region, currentTs) - if c.isWarmingScanBlockedLocked(warming, bytes) { - c.mu.Unlock() - return nil, false - } - - lease := &memoryQuotaLease{} - lease.release = func() { - c.mu.Lock() - previousLevel := c.level - c.scanUsed = subtractFloor(c.scanUsed, bytes) - if warming { - c.warmingScanUsed = subtractFloor(c.warmingScanUsed, bytes) - } - delete(state.scanLeases, lease) - c.refreshLevelLocked() - shouldNotifyAdmission := c.level < previousLevel || - (warming && c.level == admissionNormal) - c.mu.Unlock() - - if shouldNotifyAdmission { - c.notifyAvailable() - } + // Admission is based on the pressure before accounting this scan. This lets + // one scan make progress even when its estimate alone exceeds the threshold. + if warming && c.level == admissionPauseWarming { + return 0, c.scanReady, false } + bytes = c.estimateScanSizeLocked(region, currentTs) c.scanUsed += bytes - if warming { - c.warmingScanUsed += bytes + c.refreshLevelLocked() + return bytes, nil, true +} + +func (c *memoryQuotaController) releaseScan(bytes uint64) { + if bytes == 0 { + return } - state.scanLeases[lease] = struct{}{} + c.scanMu.Lock() + previousLevel := c.level + c.scanUsed = subtractFloor(c.scanUsed, bytes) c.refreshLevelLocked() - c.mu.Unlock() - return lease, true + if c.level < previousLevel { + c.notifyScanAdmissionLocked() + } + c.scanMu.Unlock() } -func (c *memoryQuotaController) trackEvent( +// acquireEvent accounts one event batch. Below the hard limit its hot path is +// a context check and an atomic compare-and-swap; it does not allocate or take +// a mutex. +func (c *memoryQuotaController) acquireEvent( ctx context.Context, span *subscribedSpan, bytes uint64, -) *memoryQuotaLease { - c.mu.Lock() +) bool { if ctx.Err() != nil { - c.mu.Unlock() - return nil + return false } - state := c.subscriptions[span.subID] - if state == nil { - c.mu.Unlock() - return nil + if c.tryAcquireEvent(bytes) { + return true } - for c.used > 0 && wouldExceed(c.used, bytes, c.hardLimitLocked()) { - c.cond.Wait() - if ctx.Err() != nil { - c.mu.Unlock() - return nil + return c.eventNotifier.wait(ctx, span, func() bool { + return c.tryAcquireEvent(bytes) + }) +} + +func (c *memoryQuotaController) tryAcquireEvent(bytes uint64) bool { + for { + used := c.used.Load() + if used > 0 && wouldExceed(used, bytes, c.hardLimit) { + return false + } + if bytes > math.MaxUint64-used { + return false } - if c.subscriptions[span.subID] != state { - c.mu.Unlock() - return nil + if c.used.CompareAndSwap(used, used+bytes) { + return true } } +} + +func (c *memoryQuotaController) releaseEvent(bytes uint64) { + if bytes == 0 { + return + } + used := c.used.Add(^(bytes - 1)) + previousUsed := used + bytes + if crossesDown(previousUsed, used, c.resumeWarmingLimit) || + crossesDown(previousUsed, used, c.resumeAllLimit) { + c.refreshAdmissionAndNotify() + } + c.eventNotifier.notify() +} - c.used += bytes +func (c *memoryQuotaController) notifyScanAdmission() { + c.scanMu.Lock() + c.notifyScanAdmissionLocked() + c.scanMu.Unlock() +} + +func (c *memoryQuotaController) notifyScanAdmissionLocked() { + close(c.scanReady) + c.scanReady = make(chan struct{}) +} + +func (c *memoryQuotaController) refreshAdmissionAndNotify() { + c.scanMu.Lock() + previousLevel := c.level c.refreshLevelLocked() - lease := &memoryQuotaLease{} - lease.release = func() { - c.mu.Lock() - previousLevel := c.level - c.used = subtractFloor(c.used, bytes) - delete(state.eventLeases, lease) - c.refreshLevelLocked() - shouldNotifyAdmission := c.level < previousLevel - c.cond.Broadcast() - c.mu.Unlock() - - if shouldNotifyAdmission { - c.notifyAvailable() - } + if c.level < previousLevel { + c.notifyScanAdmissionLocked() } - state.eventLeases[lease] = struct{}{} - c.mu.Unlock() - return lease + c.scanMu.Unlock() } func (c *memoryQuotaController) estimateScanSizeLocked(region regionInfo, currentTs uint64) uint64 { @@ -360,34 +363,14 @@ func isWarmingScan(region regionInfo, currentTs uint64) bool { return regionScanLag(currentTs, region.resolvedTs()) >= lowLagRegionThreshold } -func (c *memoryQuotaController) isWarmingScanBlockedLocked(warming bool, bytes uint64) bool { - if !warming { - return false - } - if c.level == admissionPauseWarming { - return true - } - return wouldExceed(c.warmingScanUsed, bytes, c.warmingScanBudgetLocked()) -} - -func (c *memoryQuotaController) warmingScanBudgetLocked() uint64 { - budget := uint64(float64(c.capacity) * c.pauseWarmingRatio) - return max(budget, c.scanEstimate) -} - -func (c *memoryQuotaController) hardLimitLocked() uint64 { - return uint64(float64(c.capacity) * c.hardLimitRatio) -} - func (c *memoryQuotaController) refreshLevelLocked() { // scanUsed predicts the event memory an initial scan may produce, so adding // it to actual event bytes would count the same pressure twice. - pressure := max(c.used, c.scanUsed) - usage := float64(pressure) / float64(c.capacity) + pressure := max(c.used.Load(), c.scanUsed) switch c.level { case admissionFreezeAllNewScans: - if usage <= c.resumeAllRatio { - if usage >= c.pauseWarmingRatio { + if pressure <= c.resumeAllLimit { + if pressure >= c.pauseWarmingLimit { c.level = admissionPauseWarming } else { c.level = admissionNormal @@ -395,16 +378,16 @@ func (c *memoryQuotaController) refreshLevelLocked() { } case admissionPauseWarming: switch { - case usage >= c.freezeAllRatio: + case pressure >= c.freezeAllLimit: c.level = admissionFreezeAllNewScans - case usage <= c.resumeWarmingRatio: + case pressure <= c.resumeWarmingLimit: c.level = admissionNormal } default: switch { - case usage >= c.freezeAllRatio: + case pressure >= c.freezeAllLimit: c.level = admissionFreezeAllNewScans - case usage >= c.pauseWarmingRatio: + case pressure >= c.pauseWarmingLimit: c.level = admissionPauseWarming } } @@ -414,6 +397,10 @@ func wouldExceed(used, bytes, limit uint64) bool { return bytes > limit || used > limit-bytes } +func crossesDown(previous, current, threshold uint64) bool { + return previous > threshold && current <= threshold +} + func subtractFloor(value, delta uint64) uint64 { if value < delta { return 0 diff --git a/logservice/logpuller/memory_quota_test.go b/logservice/logpuller/memory_quota_test.go index 5f45602f43..1fac91945e 100644 --- a/logservice/logpuller/memory_quota_test.go +++ b/logservice/logpuller/memory_quota_test.go @@ -52,95 +52,80 @@ func TestMemoryQuotaAdmissionLevels(t *testing.T) { initializedSpan.initialized.Store(true) warmingTs := setTestQuotaSpanLag(warmingSpan, lowLagRegionThreshold+time.Minute) initializedTs := setTestQuotaSpanLag(initializedSpan, lowLagRegionThreshold+time.Minute) - quota.addSubscription(warmingSpan) - quota.addSubscription(initializedSpan) - - lowLease := quota.trackEvent(context.Background(), initializedSpan, 5) - pauseLease := quota.trackEvent(context.Background(), initializedSpan, 10) - require.NotNil(t, lowLease) - require.NotNil(t, pauseLease) - scanLease, admitted := quota.acquireScan(newTestQuotaRegion(warmingSpan), warmingTs) + + require.True(t, quota.acquireEvent(context.Background(), initializedSpan, 5)) + require.True(t, quota.acquireEvent(context.Background(), initializedSpan, 10)) + _, _, admitted := quota.acquireScan(newTestQuotaRegion(warmingSpan), warmingTs) require.False(t, admitted) - require.Nil(t, scanLease) - scanLease, admitted = quota.acquireScan( + scanBytes, _, admitted := quota.acquireScan( newTestQuotaRegion(initializedSpan), initializedTs) require.True(t, admitted) - scanLease.Release() + quota.releaseScan(scanBytes) - middleLease := quota.trackEvent(context.Background(), initializedSpan, 45) - freezeLease := quota.trackEvent(context.Background(), initializedSpan, 20) - require.NotNil(t, middleLease) - require.NotNil(t, freezeLease) - scanLease, admitted = quota.acquireScan( + require.True(t, quota.acquireEvent(context.Background(), initializedSpan, 45)) + require.True(t, quota.acquireEvent(context.Background(), initializedSpan, 20)) + _, _, admitted = quota.acquireScan( newTestQuotaRegion(initializedSpan), initializedTs) require.False(t, admitted) - require.Nil(t, scanLease) - freezeLease.Release() + quota.releaseEvent(20) _, _, level := quota.snapshot() require.Equal(t, admissionPauseWarming, level) - middleLease.Release() + quota.releaseEvent(45) _, _, level = quota.snapshot() require.Equal(t, admissionPauseWarming, level) - pauseLease.Release() + quota.releaseEvent(10) _, _, level = quota.snapshot() require.Equal(t, admissionNormal, level) - lowLease.Release() + quota.releaseEvent(5) } -func TestMemoryQuotaRemoveSubscriptionReleasesOwnedMemory(t *testing.T) { +func TestMemoryQuotaSpanStopKeepsOwnedMemoryUntilRelease(t *testing.T) { quota := newMemoryQuotaController(100, 10) span1 := newTestQuotaSpan(1) span2 := newTestQuotaSpan(2) - quota.addSubscription(span1) - quota.addSubscription(span2) - - lease1 := quota.trackEvent(context.Background(), span1, 30) - lease2 := quota.trackEvent(context.Background(), span2, 40) - require.NotNil(t, lease1) - require.NotNil(t, lease2) - scanLease, admitted := quota.acquireScan(newTestQuotaRegion(span1), span1.resolvedTs.Load()) + + require.True(t, quota.acquireEvent(context.Background(), span1, 30)) + require.True(t, quota.acquireEvent(context.Background(), span2, 40)) + scanBytes, _, admitted := quota.acquireScan( + newTestQuotaRegion(span1), span1.resolvedTs.Load()) require.True(t, admitted) - require.NotNil(t, scanLease) + require.NotZero(t, scanBytes) - quota.removeSubscription(span1) + span1.stopped.Store(true) + quota.wakeAll() used, _, _ := quota.snapshot() - require.Equal(t, uint64(40), used) - scanUsed, _, _, _, _ := quota.scanSnapshot() - require.Zero(t, scanUsed) - require.NotContains(t, quota.subscriptions, span1.subID) - quota.removeSubscription(span1) + require.Equal(t, uint64(70), used) + scanUsed, _, _ := quota.scanSnapshot() + require.Equal(t, scanBytes, scanUsed) - lease1.Release() - scanLease.Release() + quota.releaseEvent(30) + quota.releaseScan(scanBytes) used, _, _ = quota.snapshot() require.Equal(t, uint64(40), used) - // Late region tasks are allowed to reach the stopped-subscription cleanup - // path without recreating quota state. - scanLease, admitted = quota.acquireScan(newTestQuotaRegion(span1), span1.resolvedTs.Load()) + // Late tasks reach the stopped-subscription cleanup path without consuming + // scan quota. + scanBytes, _, admitted = quota.acquireScan( + newTestQuotaRegion(span1), span1.resolvedTs.Load()) require.True(t, admitted) - require.NotNil(t, scanLease) - scanLease.Release() - require.NotContains(t, quota.subscriptions, span1.subID) + require.Zero(t, scanBytes) - lease2.Release() + quota.releaseEvent(40) used, _, _ = quota.snapshot() require.Zero(t, used) } -func TestMemoryQuotaBlockedEventStopsWhenSubscriptionIsRemoved(t *testing.T) { +func TestMemoryQuotaBlockedEventStopsWhenSpanStops(t *testing.T) { quota := newMemoryQuotaController(100, 10) - quota.hardLimitRatio = 1 + quota.hardLimit = 100 span := newTestQuotaSpan(1) - quota.addSubscription(span) - lease := quota.trackEvent(context.Background(), span, 100) - require.NotNil(t, lease) - acquired := make(chan *memoryQuotaLease, 1) + require.True(t, quota.acquireEvent(context.Background(), span, 100)) + acquired := make(chan bool, 1) go func() { - acquired <- quota.trackEvent(context.Background(), span, 1) + acquired <- quota.acquireEvent(context.Background(), span, 1) }() select { @@ -149,27 +134,25 @@ func TestMemoryQuotaBlockedEventStopsWhenSubscriptionIsRemoved(t *testing.T) { case <-time.After(100 * time.Millisecond): } - quota.removeSubscription(span) + span.stopped.Store(true) + quota.wakeAll() select { - case blockedLease := <-acquired: - require.Nil(t, blockedLease) + case ok := <-acquired: + require.False(t, ok) case <-time.After(time.Second): - t.Fatal("removing the subscription did not wake the blocked event") + t.Fatal("stopping the subscription did not wake the blocked event") } - require.NotContains(t, quota.subscriptions, span.subID) - lease.Release() + quota.releaseEvent(100) } func TestMemoryQuotaBlockedEventResumesAfterRelease(t *testing.T) { quota := newMemoryQuotaController(100, 10) span := newTestQuotaSpan(1) - quota.addSubscription(span) - lease := quota.trackEvent(context.Background(), span, 200) - require.NotNil(t, lease) - acquired := make(chan *memoryQuotaLease, 1) + require.True(t, quota.acquireEvent(context.Background(), span, 200)) + acquired := make(chan bool, 1) go func() { - acquired <- quota.trackEvent(context.Background(), span, 1) + acquired <- quota.acquireEvent(context.Background(), span, 1) }() select { @@ -178,89 +161,127 @@ func TestMemoryQuotaBlockedEventResumesAfterRelease(t *testing.T) { case <-time.After(100 * time.Millisecond): } - lease.Release() + quota.releaseEvent(200) select { - case nextLease := <-acquired: - require.NotNil(t, nextLease) - nextLease.Release() + case ok := <-acquired: + require.True(t, ok) + quota.releaseEvent(1) case <-time.After(time.Second): t.Fatal("event memory did not resume after memory was released") } } -func TestMemoryQuotaWarmingScanBudget(t *testing.T) { - quota := newMemoryQuotaController(200, 10) +func TestMemoryQuotaBlockedEventStopsOnContextCancellation(t *testing.T) { + quota := newMemoryQuotaController(100, 10) + quota.hardLimit = 100 + span := newTestQuotaSpan(1) + ctx, cancel := context.WithCancel(context.Background()) + + require.True(t, quota.acquireEvent(context.Background(), span, 100)) + acquired := make(chan bool, 1) + go func() { + acquired <- quota.acquireEvent(ctx, span, 1) + }() + require.Eventually(t, func() bool { + return quota.eventNotifier.waiters.Load() == 1 + }, time.Second, time.Millisecond) + + // Cancellation must stop the waiter without a memory release or an explicit + // quota notification. + cancel() + select { + case ok := <-acquired: + require.False(t, ok) + case <-time.After(time.Second): + t.Fatal("context cancellation did not stop the blocked event") + } + quota.releaseEvent(100) +} + +func TestMemoryQuotaConcurrentWaitersDoNotLoseWakeups(t *testing.T) { + const waiterCount = 32 + quota := newMemoryQuotaController(100, 10) + quota.hardLimit = 1 + span := newTestQuotaSpan(1) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + // Hold the only available byte until every goroutine is waiting. Releasing + // it wakes all waiters; each successful waiter then releases it for the next. + require.True(t, quota.acquireEvent(ctx, span, 1)) + results := make(chan bool, waiterCount) + for range waiterCount { + go func() { + acquired := quota.acquireEvent(ctx, span, 1) + if acquired { + quota.releaseEvent(1) + } + results <- acquired + }() + } + require.Eventually(t, func() bool { + return quota.eventNotifier.waiters.Load() == waiterCount + }, time.Second, time.Millisecond) + + quota.releaseEvent(1) + for range waiterCount { + select { + case acquired := <-results: + require.True(t, acquired) + case <-ctx.Done(): + t.Fatal("event waiter did not make progress") + } + } + used, _, _ := quota.snapshot() + require.Zero(t, used) +} + +func TestMemoryQuotaWarmingScanUsesCurrentPressure(t *testing.T) { + quota := newMemoryQuotaController(100, 20) span := newTestQuotaSpan(1) currentTs := setTestQuotaSpanLag(span, lowLagRegionThreshold+time.Minute) - quota.addSubscription(span) region := newTestQuotaRegion(span) - lease1, admitted := quota.acquireScan(region, currentTs) - require.True(t, admitted) - lease2, admitted := quota.acquireScan(region, currentTs) + bytes1, _, admitted := quota.acquireScan(region, currentTs) require.True(t, admitted) + require.NotZero(t, bytes1) + scanUsed, _, _ := quota.scanSnapshot() + require.Greater(t, scanUsed, quota.pauseWarmingLimit) - lease3, admitted := quota.acquireScan(region, currentTs) + _, _, admitted = quota.acquireScan(region, currentTs) require.False(t, admitted) - require.Nil(t, lease3) - lease1.Release() - lease3, admitted = quota.acquireScan(region, currentTs) + quota.releaseScan(bytes1) + bytes2, _, admitted := quota.acquireScan(region, currentTs) require.True(t, admitted) - lease2.Release() - lease3.Release() + quota.releaseScan(bytes2) } func TestMemoryQuotaLowLagScanBypassesWarmingGate(t *testing.T) { quota := newMemoryQuotaController(100, 10) span := newTestQuotaSpan(1) currentTs := setTestQuotaSpanLag(span, lowLagRegionThreshold-time.Second) - quota.addSubscription(span) - pressureLease := quota.trackEvent(context.Background(), span, 20) - require.NotNil(t, pressureLease) - scanLease, admitted := quota.acquireScan(newTestQuotaRegion(span), currentTs) + require.True(t, quota.acquireEvent(context.Background(), span, 20)) + scanBytes, _, admitted := quota.acquireScan(newTestQuotaRegion(span), currentTs) require.True(t, admitted) - require.NotNil(t, scanLease) - _, warmingScanUsed, _, _, _ := quota.scanSnapshot() - require.Zero(t, warmingScanUsed) - - scanLease.Release() - pressureLease.Release() -} - -func TestMemoryQuotaRemovalNotifiesAdmissionWithoutLeases(t *testing.T) { - quota := newMemoryQuotaController(100, 10) - span := newTestQuotaSpan(1) - quota.addSubscription(span) + require.NotZero(t, scanBytes) + scanUsed, _, _ := quota.scanSnapshot() + require.NotZero(t, scanUsed) - notified := make(chan struct{}, 1) - quota.setOnAvailable(func() { - select { - case notified <- struct{}{}: - default: - } - }) - quota.removeSubscription(span) - select { - case <-notified: - case <-time.After(time.Second): - t.Fatal("subscription removal did not notify admission") - } + quota.releaseScan(scanBytes) + quota.releaseEvent(20) } -func TestAdmissionWaitsForMemoryAndReleasesScanLease(t *testing.T) { +func TestAdmissionWaitsForMemoryAndReleasesScanMemory(t *testing.T) { quota := newMemoryQuotaController(100, 10) span := newTestQuotaSpan(1) currentTs := setTestQuotaSpanLag(span, lowLagRegionThreshold+time.Minute) - quota.addSubscription(span) controller := newRegionAdmissionController(1, 1, quota, func() uint64 { return currentTs }) - quota.setOnAvailable(controller.notifyAvailable) - pressureLease := quota.trackEvent(context.Background(), span, 20) - require.NotNil(t, pressureLease) + require.True(t, quota.acquireEvent(context.Background(), span, 20)) region := newTestQuotaRegion(span) require.True(t, controller.submit(newRegionPriorityTask(region, currentTs, 1))) @@ -279,7 +300,7 @@ func TestAdmissionWaitsForMemoryAndReleasesScanLease(t *testing.T) { case <-time.After(100 * time.Millisecond): } - pressureLease.Release() + quota.releaseEvent(20) var resultValue popResult select { case resultValue = <-result: @@ -288,9 +309,63 @@ func TestAdmissionWaitsForMemoryAndReleasesScanLease(t *testing.T) { } require.NoError(t, resultValue.err) req := resultValue.req - scanUsed, _, _, _, _ := quota.scanSnapshot() + scanUsed, _, _ := quota.scanSnapshot() require.NotZero(t, scanUsed) require.True(t, req.abort()) - scanUsed, _, _, _, _ = quota.scanSnapshot() + scanUsed, _, _ = quota.scanSnapshot() require.Zero(t, scanUsed) } + +func TestAdmissionWakesWhenBlockedSpanStops(t *testing.T) { + quota := newMemoryQuotaController(100, 10) + span := newTestQuotaSpan(1) + currentTs := setTestQuotaSpanLag(span, lowLagRegionThreshold+time.Minute) + controller := newRegionAdmissionController(1, 1, quota, func() uint64 { + return currentTs + }) + + require.True(t, quota.acquireEvent(context.Background(), span, 20)) + require.True(t, controller.submit(newRegionPriorityTask( + newTestQuotaRegion(span), currentTs, 1))) + + type popResult struct { + req *regionReq + err error + } + result := make(chan popResult, 1) + go func() { + req, err := controller.pop(context.Background(), nil) + result <- popResult{req: req, err: err} + }() + select { + case <-result: + t.Fatal("warming scan should wait while memory is under pressure") + case <-time.After(100 * time.Millisecond): + } + + span.stopped.Store(true) + quota.wakeAll() + select { + case result := <-result: + require.NoError(t, result.err) + require.Zero(t, result.req.scanBytes) + require.True(t, result.req.abort()) + case <-time.After(time.Second): + t.Fatal("stopping the span did not wake scan admission") + } + quota.releaseEvent(20) +} + +func BenchmarkMemoryQuotaEventAccounting(b *testing.B) { + quota := newMemoryQuotaController(defaultLogPullerMemoryQuota, defaultScanBaseSize) + span := newTestQuotaSpan(1) + ctx := context.Background() + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + if !quota.acquireEvent(ctx, span, 1) { + b.Fatal("failed to acquire event memory") + } + quota.releaseEvent(1) + } +} diff --git a/logservice/logpuller/region_admission_controller.go b/logservice/logpuller/region_admission_controller.go index 3ab3804394..9b0dde1c74 100644 --- a/logservice/logpuller/region_admission_controller.go +++ b/logservice/logpuller/region_admission_controller.go @@ -36,7 +36,7 @@ type regionReq struct { regionInfo regionInfo createTime time.Time controller *regionAdmissionController - scanQuota *memoryQuotaLease + scanBytes uint64 released atomic.Bool } @@ -73,7 +73,7 @@ func (r *regionReq) release() bool { if !r.released.CompareAndSwap(false, true) { return false } - r.scanQuota.Release() + r.controller.memoryQuota.releaseScan(r.scanBytes) r.controller.release() return true } @@ -155,11 +155,11 @@ func (c *regionAdmissionController) submit(task *regionPriorityTask) bool { return true } -// pop waits for an eligible request. If interrupt is signaled first, it returns -// nil without an error so the worker can handle its control queue. +// pop waits for an eligible request. If controlReady is signaled first, it +// returns nil without an error so the worker can handle its control queue. func (c *regionAdmissionController) pop( ctx context.Context, - interrupt <-chan struct{}, + controlReady <-chan struct{}, ) (*regionReq, error) { for { c.state.Lock() @@ -167,7 +167,7 @@ func (c *regionAdmissionController) pop( c.state.Unlock() return nil, context.Canceled } - request, scanQuota := c.popEligibleLocked() + request, scanBytes, memoryReady := c.popEligibleLocked() if request != nil { c.state.inflight++ c.state.Unlock() @@ -175,14 +175,15 @@ func (c *regionAdmissionController) pop( regionInfo: request.regionInfo, createTime: time.Now(), controller: c, - scanQuota: scanQuota, + scanBytes: scanBytes, }, nil } c.state.Unlock() select { case <-c.notify: - case <-interrupt: + case <-memoryReady: + case <-controlReady: return nil, nil case <-ctx.Done(): return nil, ctx.Err() @@ -192,23 +193,24 @@ func (c *regionAdmissionController) pop( func (c *regionAdmissionController) popEligibleLocked() ( *regionPriorityTask, - *memoryQuotaLease, + uint64, + <-chan struct{}, ) { request, ok := c.state.pending.PeekTop() if !ok { - return nil, nil + return nil, 0, nil } if c.state.inflight >= c.windowFor(request) { - return nil, nil + return nil, 0, nil } - scanQuota, admitted := c.memoryQuota.acquireScan( + scanBytes, memoryReady, admitted := c.memoryQuota.acquireScan( request.regionInfo, c.currentTs()) if !admitted { - return nil, nil + return nil, 0, memoryReady } request, _ = c.state.pending.PopTop() - return request, scanQuota + return request, scanBytes, nil } func (c *regionAdmissionController) windowFor(request *regionPriorityTask) int { @@ -227,12 +229,6 @@ func (c *regionAdmissionController) release() { c.state.Unlock() } -func (c *regionAdmissionController) notifyAvailable() { - c.state.Lock() - c.notifyOneLocked() - c.state.Unlock() -} - func (c *regionAdmissionController) close() { c.state.Lock() if !c.state.closed { diff --git a/logservice/logpuller/region_admission_controller_test.go b/logservice/logpuller/region_admission_controller_test.go index a6277f80de..0fdeb3d949 100644 --- a/logservice/logpuller/region_admission_controller_test.go +++ b/logservice/logpuller/region_admission_controller_test.go @@ -82,9 +82,9 @@ func TestRegionAdmissionControllerNormalWindow(t *testing.T) { req1, err := controller.pop(t.Context(), nil) require.NoError(t, err) require.Equal(t, 1, controller.stats().inflight) - interrupt := make(chan struct{}) - close(interrupt) - req2, err := controller.pop(t.Context(), interrupt) + controlReady := make(chan struct{}) + close(controlReady) + req2, err := controller.pop(t.Context(), controlReady) require.Nil(t, req2) require.NoError(t, err) @@ -117,9 +117,9 @@ func TestRegionAdmissionControllerLowLagUsesMaxWindow(t *testing.T) { req2, err := controller.pop(t.Context(), nil) require.NoError(t, err) require.Equal(t, uint64(3), req2.regionInfo.verID.GetID()) - interrupt := make(chan struct{}) - close(interrupt) - req3, err := controller.pop(t.Context(), interrupt) + controlReady := make(chan struct{}) + close(controlReady) + req3, err := controller.pop(t.Context(), controlReady) require.Nil(t, req3) require.NoError(t, err) require.Equal(t, 2, controller.stats().inflight) diff --git a/logservice/logpuller/region_event_handler.go b/logservice/logpuller/region_event_handler.go index 92a649349a..a30b349418 100644 --- a/logservice/logpuller/region_event_handler.go +++ b/logservice/logpuller/region_event_handler.go @@ -54,23 +54,17 @@ type regionEvent struct { // Resolved-ts events: `resolvedTs` is set and `states` contains all related regions. states []*regionFeedState - entries *cdcpb.Event_Entries_ - resolvedTs uint64 - memoryQuota *memoryQuotaLease + entries *cdcpb.Event_Entries_ + resolvedTs uint64 + // memoryBytes is released when this event is dropped or after downstream + // finishes consuming the entries derived from it. + memoryBytes uint64 } -func (event *regionEvent) needMemoryQuota() bool { +func (event *regionEvent) needsMemoryAccounting() bool { return event.entries != nil } -func (event *regionEvent) releaseMemoryQuota() { - if event.memoryQuota == nil { - return - } - event.memoryQuota.Release() - event.memoryQuota = nil -} - func (event *regionEvent) getSize() int { size := int(unsafe.Sizeof(*event)) if event.entries != nil { @@ -133,11 +127,9 @@ func (h *regionEventHandler) Handle(span *subscribedSpan, events ...regionEvent) newResolvedTs := uint64(0) wasInitialized := span.initialized.Load() - quotaLeases := make([]*memoryQuotaLease, 0, len(events)) + memoryBytes := uint64(0) for _, event := range events { - if event.needMemoryQuota() { - quotaLeases = append(quotaLeases, event.memoryQuota) - } + memoryBytes += event.memoryBytes if len(event.states) == 1 && event.states[0].isStale() { hasError = true h.handleRegionError(event.states[0]) @@ -159,7 +151,7 @@ func (h *regionEventHandler) Handle(span *subscribedSpan, events ...regionEvent) } } if !wasInitialized && span.initialized.Load() { - h.eventSink.memoryQuota.markSubscriptionInitialized() + h.eventSink.memoryQuota.notifyScanAdmission() } tryAdvanceResolvedTs := func() { if newResolvedTs != 0 { @@ -167,9 +159,7 @@ func (h *regionEventHandler) Handle(span *subscribedSpan, events ...regionEvent) } } releaseMemoryQuota := func() { - for _, lease := range quotaLeases { - lease.Release() - } + h.eventSink.memoryQuota.releaseEvent(memoryBytes) } if len(span.kvEventsCache) > 0 { metricsEventCount.Add(float64(len(span.kvEventsCache))) @@ -253,7 +243,7 @@ func (h *regionEventHandler) GetType(event regionEvent) dynstream.EventType { } func (h *regionEventHandler) OnDrop(event regionEvent) interface{} { - event.releaseMemoryQuota() + h.eventSink.memoryQuota.releaseEvent(event.memoryBytes) // TODO: Distinguish between drop events caused by "path not found" errors and memory control. state := event.mustFirstState() fields := []zap.Field{ diff --git a/logservice/logpuller/region_event_handler_test.go b/logservice/logpuller/region_event_handler_test.go index dd4fc1d77b..9227f2bb8e 100644 --- a/logservice/logpuller/region_event_handler_test.go +++ b/logservice/logpuller/region_event_handler_test.go @@ -107,9 +107,8 @@ func TestHandleEventEntryEventOutOfOrder(t *testing.T) { }, } regionEvent := regionEvent{ - states: []*regionFeedState{state}, - entries: events, - memoryQuota: newMemoryQuotaLease(func() {}), + states: []*regionFeedState{state}, + entries: events, } ds.Push(subID, regionEvent) } @@ -128,9 +127,8 @@ func TestHandleEventEntryEventOutOfOrder(t *testing.T) { }, } regionEvent := regionEvent{ - states: []*regionFeedState{state}, - entries: events, - memoryQuota: newMemoryQuotaLease(func() {}), + states: []*regionFeedState{state}, + entries: events, } ds.Push(subID, regionEvent) } @@ -159,9 +157,8 @@ func TestHandleEventEntryEventOutOfOrder(t *testing.T) { }, } regionEvent := regionEvent{ - states: []*regionFeedState{state}, - entries: events, - memoryQuota: newMemoryQuotaLease(func() {}), + states: []*regionFeedState{state}, + entries: events, } ds.Push(subID, regionEvent) } @@ -187,9 +184,8 @@ func TestHandleEventEntryEventOutOfOrder(t *testing.T) { }, } regionEvent := regionEvent{ - states: []*regionFeedState{state}, - entries: events, - memoryQuota: newMemoryQuotaLease(func() {}), + states: []*regionFeedState{state}, + entries: events, } ds.Push(subID, regionEvent) } @@ -384,7 +380,6 @@ func TestHandleResolvedTsThrottled(t *testing.T) { func TestHandleEntriesReleasesMemoryAfterDownstreamCallback(t *testing.T) { quota := newMemoryQuotaController(1024, 8) span := newTestQuotaSpan(1) - quota.addSubscription(span) callbackCh := make(chan func(), 1) span.consumeKVEvents = func(_ []common.RawKVEntry, callback func()) bool { callbackCh <- callback @@ -402,8 +397,7 @@ func TestHandleEntriesReleasesMemoryAfterDownstreamCallback(t *testing.T) { lockedRangeState: lockedState, }, } - lease := quota.trackEvent(context.Background(), span, 10) - require.NotNil(t, lease) + require.True(t, quota.acquireEvent(context.Background(), span, 10)) handler := ®ionEventHandler{eventSink: ®ionEventSink{ ds: newMockRegionEventSinkStream(), memoryQuota: quota, @@ -411,7 +405,7 @@ func TestHandleEntriesReleasesMemoryAfterDownstreamCallback(t *testing.T) { await := handler.Handle(span, regionEvent{ states: []*regionFeedState{state}, - memoryQuota: lease, + memoryBytes: 10, entries: &cdcpb.Event_Entries_{Entries: &cdcpb.Event_Entries{ Entries: []*cdcpb.Event_Row{{ Type: cdcpb.Event_COMMITTED, @@ -441,13 +435,9 @@ func TestTryMarkSpanInitializedByResolvedTs(t *testing.T) { func TestSpanInitializationNotifiesMemoryAdmission(t *testing.T) { quota := newMemoryQuotaController(1024, 8) - notified := make(chan struct{}, 1) - quota.setOnAvailable(func() { - select { - case notified <- struct{}{}: - default: - } - }) + quota.scanMu.Lock() + notified := quota.scanReady + quota.scanMu.Unlock() const startTs = 100 rangeLock := regionlock.NewRangeLock(1, []byte("a"), []byte("z"), startTs) @@ -463,7 +453,6 @@ func TestSpanInitializationNotifiesMemoryAdmission(t *testing.T) { advanceResolvedTs: func(uint64) {}, } span.resolvedTs.Store(startTs) - quota.addSubscription(span) state := newRegionFeedState(regionInfo{ verID: tikv.NewRegionVerID(1, 1, 1), subscribedSpan: span, diff --git a/logservice/logpuller/region_event_sink.go b/logservice/logpuller/region_event_sink.go index 25987ff997..4a5b5b0bf6 100644 --- a/logservice/logpuller/region_event_sink.go +++ b/logservice/logpuller/region_event_sink.go @@ -15,8 +15,6 @@ package logpuller import ( "context" - "sync" - "sync/atomic" "github.com/pingcap/log" "github.com/pingcap/ticdc/pkg/metrics" @@ -24,16 +22,12 @@ import ( "go.uber.org/zap" ) -// regionEventSink delivers region events to dynstream and owns push-side flow control. +// regionEventSink delivers region events to dynstream and accounts their memory. type regionEventSink struct { ctx context.Context ds dynstream.DynamicStream[int, SubscriptionID, regionEvent, *subscribedSpan, *regionEventHandler] memoryQuota *memoryQuotaController - // the following three fields are used to manage feedback from ds and notify other goroutines - mu sync.Mutex - cond *sync.Cond - paused atomic.Bool } func newRegionEventSink( @@ -49,7 +43,6 @@ func newRegionEventSink( // TODO: Set `UseBuffer` to true until we refactor the `regionEventHandler.Handle` method so that it doesn't call any method of the dynamic stream. Currently, if `UseBuffer` is set to false, there will be a deadlock: // ds.handleLoop fetch events from `ch` -> regionEventHandler.Handle -> ds.RemovePath -> send event to `ch` option.UseBuffer = true - option.EnableMemoryControl = false ds := dynstream.NewParallelDynamicStream( "log-puller", ®ionEventHandler{eventSink: sink, failureHandler: failureHandler}, @@ -57,7 +50,6 @@ func newRegionEventSink( ) ds.Start() sink.ds = ds - sink.cond = sync.NewCond(&sink.mu) return sink } @@ -78,94 +70,31 @@ func (s *regionEventSink) Wake(subID SubscriptionID) { } func (s *regionEventSink) Push(subID SubscriptionID, event regionEvent) { - if event.needMemoryQuota() { + if event.needsMemoryAccounting() { span := event.mustFirstState().region.subscribedSpan - event.memoryQuota = s.memoryQuota.trackEvent(s.ctx, span, uint64(event.getSize())) - if event.memoryQuota == nil { + event.memoryBytes = uint64(event.getSize()) + if !s.memoryQuota.acquireEvent(s.ctx, span, event.memoryBytes) { return } } - // fast path - if !s.paused.Load() { - s.ds.Push(subID, event) - return - } - - // slow path: wait until paused is false - s.mu.Lock() - for s.paused.Load() { - select { - case <-s.ctx.Done(): - s.mu.Unlock() - event.releaseMemoryQuota() - return - default: - s.cond.Wait() - } - } - s.mu.Unlock() s.ds.Push(subID, event) } -func (s *regionEventSink) Run(ctx context.Context) error { - for { - select { - case <-ctx.Done(): - return nil - case feedback := <-s.ds.Feedback(): - switch feedback.FeedbackType { - case dynstream.PauseArea: - s.mu.Lock() - s.paused.Store(true) - s.mu.Unlock() - log.Info("subscription client pause push region event") - case dynstream.ResumeArea: - s.mu.Lock() - s.paused.Store(false) - s.cond.Broadcast() - s.mu.Unlock() - log.Info("subscription client resume push region event") - case dynstream.ReleasePath, dynstream.ResumePath: - // Ignore it, because it is no need to pause and resume a path in puller. - } - } - } -} - func (s *regionEventSink) UpdateMetrics() { dsMetrics := s.ds.GetMetrics() metricSubscriptionClientDSChannelSize.Set(float64(dsMetrics.EventChanSize)) metricSubscriptionClientDSPendingQueueLen.Set(float64(dsMetrics.PendingQueueLen)) used, capacity, _ := s.memoryQuota.snapshot() - scanUsed, warmingScanUsed, warmingScanBudget, scanEstimate, hardLimit := - s.memoryQuota.scanSnapshot() + scanUsed, scanEstimate, hardLimit := s.memoryQuota.scanSnapshot() metrics.LogPullerMemoryQuota.WithLabelValues("max").Set(float64(capacity)) metrics.LogPullerMemoryQuota.WithLabelValues("used").Set(float64(used)) metrics.LogPullerMemoryQuota.WithLabelValues("scan_used").Set(float64(scanUsed)) - metrics.LogPullerMemoryQuota.WithLabelValues("warming_scan_used").Set(float64(warmingScanUsed)) - metrics.LogPullerMemoryQuota.WithLabelValues("warming_scan_budget").Set(float64(warmingScanBudget)) metrics.LogPullerMemoryQuota.WithLabelValues("scan_estimate").Set(float64(scanEstimate)) metrics.LogPullerMemoryQuota.WithLabelValues("hard_limit").Set(float64(hardLimit)) - metrics.DynamicStreamMemoryUsage.WithLabelValues( - "log-puller", - "max", - "default", - "default", - ).Set(float64(capacity)) - metrics.DynamicStreamMemoryUsage.WithLabelValues( - "log-puller", - "used", - "default", - "default", - ).Set(float64(used)) } func (s *regionEventSink) Close() { s.memoryQuota.wakeAll() - s.mu.Lock() - s.paused.Store(false) - s.cond.Broadcast() - s.mu.Unlock() s.ds.Close() } diff --git a/logservice/logpuller/region_event_sink_test.go b/logservice/logpuller/region_event_sink_test.go index 43fd6f088d..7786e4b524 100644 --- a/logservice/logpuller/region_event_sink_test.go +++ b/logservice/logpuller/region_event_sink_test.go @@ -15,10 +15,7 @@ package logpuller import ( "context" - "sync" - "sync/atomic" "testing" - "time" "github.com/pingcap/kvproto/pkg/cdcpb" "github.com/pingcap/ticdc/pkg/metrics" @@ -28,18 +25,13 @@ import ( ) type mockRegionEventSinkStream struct { - feedbackCh chan dynstream.Feedback[int, SubscriptionID, *subscribedSpan] - pushCount atomic.Int32 - pushCh chan struct{} - eventCh chan regionEvent - metrics dynstream.Metrics[int, SubscriptionID] + eventCh chan regionEvent + metrics dynstream.Metrics[int, SubscriptionID] } func newMockRegionEventSinkStream() *mockRegionEventSinkStream { return &mockRegionEventSinkStream{ - feedbackCh: make(chan dynstream.Feedback[int, SubscriptionID, *subscribedSpan], 2), - pushCh: make(chan struct{}, 1), - eventCh: make(chan regionEvent, 1), + eventCh: make(chan regionEvent, 1), } } @@ -48,15 +40,13 @@ func (s *mockRegionEventSinkStream) Start() {} func (s *mockRegionEventSinkStream) Close() {} func (s *mockRegionEventSinkStream) Push(_ SubscriptionID, event regionEvent) { - s.pushCount.Add(1) - s.pushCh <- struct{}{} s.eventCh <- event } func (s *mockRegionEventSinkStream) Wake(_ SubscriptionID) {} func (s *mockRegionEventSinkStream) Feedback() <-chan dynstream.Feedback[int, SubscriptionID, *subscribedSpan] { - return s.feedbackCh + return nil } func (s *mockRegionEventSinkStream) AddPath(_ SubscriptionID, _ *subscribedSpan, _ ...dynstream.AreaSettings) error { @@ -75,66 +65,6 @@ func (s *mockRegionEventSinkStream) GetMetrics() dynstream.Metrics[int, Subscrip return s.metrics } -func TestRegionEventSinkRunPausesAndResumesPush(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - ds := newMockRegionEventSinkStream() - sink := ®ionEventSink{ - ctx: ctx, - ds: ds, - memoryQuota: newMemoryQuotaController(0, 0), - } - sink.cond = sync.NewCond(&sink.mu) - - runErrCh := make(chan error, 1) - go func() { - runErrCh <- sink.Run(ctx) - }() - - ds.feedbackCh <- dynstream.Feedback[int, SubscriptionID, *subscribedSpan]{ - FeedbackType: dynstream.PauseArea, - } - require.Eventually(t, sink.paused.Load, time.Second, 10*time.Millisecond) - - pushDone := make(chan struct{}) - go func() { - sink.Push(SubscriptionID(1), regionEvent{resolvedTs: 100}) - close(pushDone) - }() - - select { - case <-pushDone: - t.Fatal("Push should block while the sink is paused") - case <-time.After(100 * time.Millisecond): - } - require.Equal(t, int32(0), ds.pushCount.Load()) - - ds.feedbackCh <- dynstream.Feedback[int, SubscriptionID, *subscribedSpan]{ - FeedbackType: dynstream.ResumeArea, - } - require.Eventually(t, func() bool { return !sink.paused.Load() }, time.Second, 10*time.Millisecond) - - select { - case <-ds.pushCh: - case <-time.After(time.Second): - t.Fatal("Push should resume after ResumeArea feedback") - } - select { - case <-pushDone: - case <-time.After(time.Second): - t.Fatal("Push should return after ResumeArea feedback") - } - - cancel() - select { - case err := <-runErrCh: - require.NoError(t, err) - case <-time.After(time.Second): - t.Fatal("Run should exit after context cancellation") - } -} - func TestRegionEventSinkUpdateMetrics(t *testing.T) { t.Run("quota updates memory gauges", func(t *testing.T) { ds := newMockRegionEventSinkStream() @@ -144,33 +74,18 @@ func TestRegionEventSinkUpdateMetrics(t *testing.T) { } quota := newMemoryQuotaController(66, 8) span := newTestQuotaSpan(1) - quota.addSubscription(span) - lease := quota.trackEvent(context.Background(), span, 55) - require.NotNil(t, lease) - t.Cleanup(lease.Release) + require.True(t, quota.acquireEvent(context.Background(), span, 55)) + t.Cleanup(func() { quota.releaseEvent(55) }) sink := ®ionEventSink{ ctx: context.Background(), ds: ds, memoryQuota: quota, } - sink.cond = sync.NewCond(&sink.mu) sink.UpdateMetrics() require.Equal(t, float64(33), testutil.ToFloat64(metricSubscriptionClientDSChannelSize)) require.Equal(t, float64(44), testutil.ToFloat64(metricSubscriptionClientDSPendingQueueLen)) - require.Equal(t, float64(66), testutil.ToFloat64(metrics.DynamicStreamMemoryUsage.WithLabelValues( - "log-puller", - "max", - "default", - "default", - ))) - require.Equal(t, float64(55), testutil.ToFloat64(metrics.DynamicStreamMemoryUsage.WithLabelValues( - "log-puller", - "used", - "default", - "default", - ))) require.Equal(t, float64(66), testutil.ToFloat64( metrics.LogPullerMemoryQuota.WithLabelValues("max"))) require.Equal(t, float64(55), testutil.ToFloat64( @@ -181,7 +96,6 @@ func TestRegionEventSinkUpdateMetrics(t *testing.T) { func TestRegionEventSinkTracksEntriesUntilDrop(t *testing.T) { quota := newMemoryQuotaController(1024, 8) span := newTestQuotaSpan(1) - quota.addSubscription(span) state := ®ionFeedState{ region: regionInfo{subscribedSpan: span}, worker: ®ionRequestWorker{}, @@ -192,7 +106,6 @@ func TestRegionEventSinkTracksEntriesUntilDrop(t *testing.T) { ds: ds, memoryQuota: quota, } - sink.cond = sync.NewCond(&sink.mu) sink.Push(span.subID, regionEvent{ states: []*regionFeedState{state}, @@ -201,11 +114,11 @@ func TestRegionEventSinkTracksEntriesUntilDrop(t *testing.T) { }}, }) pushed := <-ds.eventCh - require.NotNil(t, pushed.memoryQuota) + require.NotZero(t, pushed.memoryBytes) used, _, _ := quota.snapshot() require.NotZero(t, used) - (®ionEventHandler{}).OnDrop(pushed) + (®ionEventHandler{eventSink: sink}).OnDrop(pushed) used, _, _ = quota.snapshot() require.Zero(t, used) } diff --git a/logservice/logpuller/region_request_scheduler.go b/logservice/logpuller/region_request_scheduler.go index d2e93081e3..9c3ba2a695 100644 --- a/logservice/logpuller/region_request_scheduler.go +++ b/logservice/logpuller/region_request_scheduler.go @@ -218,13 +218,6 @@ func (s *regionRequestScheduler) UpdateMetrics() { Set(float64(s.requestedRegionCount())) } -func (s *regionRequestScheduler) notifyAvailable() { - s.stores.Range(func(_, value any) bool { - value.(*regionRequestStore).notifyAvailable() - return true - }) -} - func (s *regionRequestScheduler) Close() { s.taskQueue.Close() } diff --git a/logservice/logpuller/region_request_store.go b/logservice/logpuller/region_request_store.go index 53a0d7a284..fe3f1202a5 100644 --- a/logservice/logpuller/region_request_store.go +++ b/logservice/logpuller/region_request_store.go @@ -87,9 +87,3 @@ func (s *regionRequestStore) requestedRegionCount() int { } return count } - -func (s *regionRequestStore) notifyAvailable() { - for _, worker := range s.workers { - worker.admission.notifyAvailable() - } -} diff --git a/logservice/logpuller/subscription_client.go b/logservice/logpuller/subscription_client.go index 28b26944c2..bfbe2f910b 100644 --- a/logservice/logpuller/subscription_client.go +++ b/logservice/logpuller/subscription_client.go @@ -190,7 +190,6 @@ func NewSubscriptionClient( subClient.failureHandler, subClient.memoryQuota, ) - subClient.memoryQuota.setOnAvailable(subClient.regionScheduler.notifyAvailable) return subClient } @@ -252,7 +251,6 @@ func (s *subscriptionClient) Subscribe( time.Duration(config.GetGlobalServerConfig().Debug.Puller.OldStartTsScanLowPriorityThreshold), ) s.spanRegistry.Add(rt) - s.memoryQuota.addSubscription(rt) s.eventSink.AddPath(rt) select { @@ -295,7 +293,6 @@ func (s *subscriptionClient) Run(ctx context.Context) error { // actual startup order. g.Go(func() error { return s.handleRangeTasks(ctx) }) g.Go(func() error { return s.regionScheduler.Run(ctx, g) }) - g.Go(func() error { return s.eventSink.Run(ctx) }) g.Go(func() error { return s.failureHandler.Run(ctx) }) g.Go(func() error { return s.spanRegistry.Run(ctx) }) g.Go(func() error { return s.handleResolveLockTasks(ctx) }) @@ -321,6 +318,8 @@ func (s *subscriptionClient) setTableStopped(rt *subscribedSpan) { // Set stopped to true so we can stop handling region events from the table, // then notify every existing worker to deregister the subscription. if rt.stopped.CompareAndSwap(false, true) { + // Wake event receivers and scan admission so they can observe stopped. + s.memoryQuota.wakeAll() s.regionScheduler.BroadcastDeregister(rt.subID, rt.filterLoop) if rt.rangeLock.Stop() { s.onTableDrained(rt) @@ -338,7 +337,6 @@ func (s *subscriptionClient) onTableDrained(rt *subscribedSpan) { zap.Uint64("subscriptionID", uint64(rt.subID)), zap.Error(err)) } - s.memoryQuota.removeSubscription(rt) s.spanRegistry.Remove(rt.subID) } From 7cf39e7bebdae88bf51ee0f436cc8d9ba5edd776 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Mon, 20 Jul 2026 10:57:30 +0800 Subject: [PATCH 08/26] refactor --- logservice/logpuller/memory_quota.go | 63 +++--- logservice/logpuller/memory_quota_test.go | 198 +++++++++++------- .../logpuller/region_admission_controller.go | 4 +- logservice/logpuller/region_event_handler.go | 6 +- .../logpuller/region_event_handler_test.go | 10 +- logservice/logpuller/region_event_sink.go | 13 +- .../logpuller/region_event_sink_test.go | 45 ++-- logservice/logpuller/subscription_client.go | 3 +- metrics/grafana/ticdc_new_arch.json | 127 ++++++++++- .../ticdc_new_arch_next_gen.json | 127 ++++++++++- pkg/metrics/log_puller.go | 25 +++ 11 files changed, 453 insertions(+), 168 deletions(-) diff --git a/logservice/logpuller/memory_quota.go b/logservice/logpuller/memory_quota.go index aa5396171e..9914b88ca3 100644 --- a/logservice/logpuller/memory_quota.go +++ b/logservice/logpuller/memory_quota.go @@ -20,6 +20,8 @@ import ( "sync" "sync/atomic" "time" + + "github.com/pingcap/ticdc/pkg/metrics" ) const ( @@ -193,28 +195,14 @@ func newMemoryQuotaController(capacity, scanBaseSize uint64) *memoryQuotaControl return c } -func (c *memoryQuotaController) wakeAll() { +// WakeAll wakes quota waiters so they can observe cancellation or a stopped span. +func (c *memoryQuotaController) WakeAll() { c.eventNotifier.notify() - c.notifyScanAdmission() -} - -func (c *memoryQuotaController) snapshot() (used, capacity uint64, level admissionLevel) { - c.scanMu.Lock() - defer c.scanMu.Unlock() - return c.used.Load(), c.capacity, c.level -} - -func (c *memoryQuotaController) scanSnapshot() ( - scanUsed uint64, - scanEstimate uint64, - hardLimit uint64, -) { - c.scanMu.Lock() - defer c.scanMu.Unlock() - return c.scanUsed, c.scanEstimate, c.hardLimit + c.NotifyScanAdmission() } -func (c *memoryQuotaController) acquireScan( +// AcquireScan admits one region scan and returns its memory estimate. +func (c *memoryQuotaController) AcquireScan( region regionInfo, currentTs uint64, ) (bytes uint64, retry <-chan struct{}, admitted bool) { @@ -244,7 +232,8 @@ func (c *memoryQuotaController) acquireScan( return bytes, nil, true } -func (c *memoryQuotaController) releaseScan(bytes uint64) { +// ReleaseScan releases the estimate owned by an admitted region scan. +func (c *memoryQuotaController) ReleaseScan(bytes uint64) { if bytes == 0 { return } @@ -258,10 +247,10 @@ func (c *memoryQuotaController) releaseScan(bytes uint64) { c.scanMu.Unlock() } -// acquireEvent accounts one event batch. Below the hard limit its hot path is +// AcquireEvent accounts one event batch. Below the hard limit its hot path is // a context check and an atomic compare-and-swap; it does not allocate or take // a mutex. -func (c *memoryQuotaController) acquireEvent( +func (c *memoryQuotaController) AcquireEvent( ctx context.Context, span *subscribedSpan, bytes uint64, @@ -273,9 +262,12 @@ func (c *memoryQuotaController) acquireEvent( return true } - return c.eventNotifier.wait(ctx, span, func() bool { + start := time.Now() + acquired := c.eventNotifier.wait(ctx, span, func() bool { return c.tryAcquireEvent(bytes) }) + metrics.LogPullerMemoryQuotaEventWaitDuration.Observe(time.Since(start).Seconds()) + return acquired } func (c *memoryQuotaController) tryAcquireEvent(bytes uint64) bool { @@ -293,7 +285,8 @@ func (c *memoryQuotaController) tryAcquireEvent(bytes uint64) bool { } } -func (c *memoryQuotaController) releaseEvent(bytes uint64) { +// ReleaseEvent releases event memory after downstream has consumed the event. +func (c *memoryQuotaController) ReleaseEvent(bytes uint64) { if bytes == 0 { return } @@ -306,12 +299,32 @@ func (c *memoryQuotaController) releaseEvent(bytes uint64) { c.eventNotifier.notify() } -func (c *memoryQuotaController) notifyScanAdmission() { +// NotifyScanAdmission wakes workers so they can recheck span state and admission. +func (c *memoryQuotaController) NotifyScanAdmission() { c.scanMu.Lock() c.notifyScanAdmissionLocked() c.scanMu.Unlock() } +// UpdateMetrics reports the current event-memory and scan-admission state. +func (c *memoryQuotaController) UpdateMetrics() { + c.scanMu.Lock() + used := c.used.Load() + scanUsed := c.scanUsed + level := c.level + scanEstimate := c.scanEstimate + c.scanMu.Unlock() + + metrics.LogPullerMemoryQuota.WithLabelValues("max").Set(float64(c.capacity)) + metrics.LogPullerMemoryQuota.WithLabelValues("used").Set(float64(used)) + metrics.LogPullerMemoryQuota.WithLabelValues("scan_used").Set(float64(scanUsed)) + metrics.LogPullerMemoryQuota.WithLabelValues("scan_estimate").Set(float64(scanEstimate)) + metrics.LogPullerMemoryQuota.WithLabelValues("hard_limit").Set(float64(c.hardLimit)) + metrics.LogPullerMemoryQuotaAdmissionLevel.Set(float64(level)) + metrics.LogPullerMemoryQuotaEventWaiterCount.Set( + float64(c.eventNotifier.waiters.Load())) +} + func (c *memoryQuotaController) notifyScanAdmissionLocked() { close(c.scanReady) c.scanReady = make(chan struct{}) diff --git a/logservice/logpuller/memory_quota_test.go b/logservice/logpuller/memory_quota_test.go index 1fac91945e..8b8b3cd1c6 100644 --- a/logservice/logpuller/memory_quota_test.go +++ b/logservice/logpuller/memory_quota_test.go @@ -20,10 +20,28 @@ import ( "time" "github.com/pingcap/ticdc/logservice/logpuller/regionlock" + "github.com/pingcap/ticdc/pkg/metrics" + "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/require" "github.com/tikv/client-go/v2/oracle" ) +type memoryQuotaTestState struct { + used uint64 + scanUsed uint64 + level admissionLevel +} + +func getMemoryQuotaTestState(quota *memoryQuotaController) memoryQuotaTestState { + quota.scanMu.Lock() + defer quota.scanMu.Unlock() + return memoryQuotaTestState{ + used: quota.used.Load(), + scanUsed: quota.scanUsed, + level: quota.level, + } +} + func newTestQuotaSpan(subID SubscriptionID) *subscribedSpan { span := &subscribedSpan{subID: subID} span.resolvedTs.Store(oracle.GoTimeToTS(time.Now())) @@ -45,6 +63,37 @@ func setTestQuotaSpanLag(span *subscribedSpan, lag time.Duration) uint64 { return oracle.GoTimeToTS(now) } +func TestMemoryQuotaUpdateMetrics(t *testing.T) { + quota := newMemoryQuotaController(66, 8) + span := newTestQuotaSpan(1) + require.True(t, quota.AcquireEvent(context.Background(), span, 55)) + t.Cleanup(func() { quota.ReleaseEvent(55) }) + + quota.scanMu.Lock() + quota.scanUsed = 7 + quota.level = admissionFreezeAllNewScans + quota.scanMu.Unlock() + quota.eventNotifier.waiters.Store(2) + t.Cleanup(func() { quota.eventNotifier.waiters.Store(0) }) + + quota.UpdateMetrics() + + require.Equal(t, float64(66), testutil.ToFloat64( + metrics.LogPullerMemoryQuota.WithLabelValues("max"))) + require.Equal(t, float64(55), testutil.ToFloat64( + metrics.LogPullerMemoryQuota.WithLabelValues("used"))) + require.Equal(t, float64(7), testutil.ToFloat64( + metrics.LogPullerMemoryQuota.WithLabelValues("scan_used"))) + require.Equal(t, float64(8), testutil.ToFloat64( + metrics.LogPullerMemoryQuota.WithLabelValues("scan_estimate"))) + require.Equal(t, float64(132), testutil.ToFloat64( + metrics.LogPullerMemoryQuota.WithLabelValues("hard_limit"))) + require.Equal(t, float64(admissionFreezeAllNewScans), + testutil.ToFloat64(metrics.LogPullerMemoryQuotaAdmissionLevel)) + require.Equal(t, float64(2), + testutil.ToFloat64(metrics.LogPullerMemoryQuotaEventWaiterCount)) +} + func TestMemoryQuotaAdmissionLevels(t *testing.T) { quota := newMemoryQuotaController(100, 10) warmingSpan := newTestQuotaSpan(1) @@ -53,32 +102,32 @@ func TestMemoryQuotaAdmissionLevels(t *testing.T) { warmingTs := setTestQuotaSpanLag(warmingSpan, lowLagRegionThreshold+time.Minute) initializedTs := setTestQuotaSpanLag(initializedSpan, lowLagRegionThreshold+time.Minute) - require.True(t, quota.acquireEvent(context.Background(), initializedSpan, 5)) - require.True(t, quota.acquireEvent(context.Background(), initializedSpan, 10)) - _, _, admitted := quota.acquireScan(newTestQuotaRegion(warmingSpan), warmingTs) + require.True(t, quota.AcquireEvent(context.Background(), initializedSpan, 5)) + require.True(t, quota.AcquireEvent(context.Background(), initializedSpan, 10)) + _, _, admitted := quota.AcquireScan(newTestQuotaRegion(warmingSpan), warmingTs) require.False(t, admitted) - scanBytes, _, admitted := quota.acquireScan( + scanBytes, _, admitted := quota.AcquireScan( newTestQuotaRegion(initializedSpan), initializedTs) require.True(t, admitted) - quota.releaseScan(scanBytes) + quota.ReleaseScan(scanBytes) - require.True(t, quota.acquireEvent(context.Background(), initializedSpan, 45)) - require.True(t, quota.acquireEvent(context.Background(), initializedSpan, 20)) - _, _, admitted = quota.acquireScan( + require.True(t, quota.AcquireEvent(context.Background(), initializedSpan, 45)) + require.True(t, quota.AcquireEvent(context.Background(), initializedSpan, 20)) + _, _, admitted = quota.AcquireScan( newTestQuotaRegion(initializedSpan), initializedTs) require.False(t, admitted) - quota.releaseEvent(20) - _, _, level := quota.snapshot() - require.Equal(t, admissionPauseWarming, level) - quota.releaseEvent(45) - _, _, level = quota.snapshot() - require.Equal(t, admissionPauseWarming, level) - quota.releaseEvent(10) - _, _, level = quota.snapshot() - require.Equal(t, admissionNormal, level) - quota.releaseEvent(5) + quota.ReleaseEvent(20) + state := getMemoryQuotaTestState(quota) + require.Equal(t, admissionPauseWarming, state.level) + quota.ReleaseEvent(45) + state = getMemoryQuotaTestState(quota) + require.Equal(t, admissionPauseWarming, state.level) + quota.ReleaseEvent(10) + state = getMemoryQuotaTestState(quota) + require.Equal(t, admissionNormal, state.level) + quota.ReleaseEvent(5) } func TestMemoryQuotaSpanStopKeepsOwnedMemoryUntilRelease(t *testing.T) { @@ -86,35 +135,34 @@ func TestMemoryQuotaSpanStopKeepsOwnedMemoryUntilRelease(t *testing.T) { span1 := newTestQuotaSpan(1) span2 := newTestQuotaSpan(2) - require.True(t, quota.acquireEvent(context.Background(), span1, 30)) - require.True(t, quota.acquireEvent(context.Background(), span2, 40)) - scanBytes, _, admitted := quota.acquireScan( + require.True(t, quota.AcquireEvent(context.Background(), span1, 30)) + require.True(t, quota.AcquireEvent(context.Background(), span2, 40)) + scanBytes, _, admitted := quota.AcquireScan( newTestQuotaRegion(span1), span1.resolvedTs.Load()) require.True(t, admitted) require.NotZero(t, scanBytes) span1.stopped.Store(true) - quota.wakeAll() - used, _, _ := quota.snapshot() - require.Equal(t, uint64(70), used) - scanUsed, _, _ := quota.scanSnapshot() - require.Equal(t, scanBytes, scanUsed) + quota.WakeAll() + state := getMemoryQuotaTestState(quota) + require.Equal(t, uint64(70), state.used) + require.Equal(t, scanBytes, state.scanUsed) - quota.releaseEvent(30) - quota.releaseScan(scanBytes) - used, _, _ = quota.snapshot() - require.Equal(t, uint64(40), used) + quota.ReleaseEvent(30) + quota.ReleaseScan(scanBytes) + state = getMemoryQuotaTestState(quota) + require.Equal(t, uint64(40), state.used) // Late tasks reach the stopped-subscription cleanup path without consuming // scan quota. - scanBytes, _, admitted = quota.acquireScan( + scanBytes, _, admitted = quota.AcquireScan( newTestQuotaRegion(span1), span1.resolvedTs.Load()) require.True(t, admitted) require.Zero(t, scanBytes) - quota.releaseEvent(40) - used, _, _ = quota.snapshot() - require.Zero(t, used) + quota.ReleaseEvent(40) + state = getMemoryQuotaTestState(quota) + require.Zero(t, state.used) } func TestMemoryQuotaBlockedEventStopsWhenSpanStops(t *testing.T) { @@ -122,10 +170,10 @@ func TestMemoryQuotaBlockedEventStopsWhenSpanStops(t *testing.T) { quota.hardLimit = 100 span := newTestQuotaSpan(1) - require.True(t, quota.acquireEvent(context.Background(), span, 100)) + require.True(t, quota.AcquireEvent(context.Background(), span, 100)) acquired := make(chan bool, 1) go func() { - acquired <- quota.acquireEvent(context.Background(), span, 1) + acquired <- quota.AcquireEvent(context.Background(), span, 1) }() select { @@ -135,24 +183,24 @@ func TestMemoryQuotaBlockedEventStopsWhenSpanStops(t *testing.T) { } span.stopped.Store(true) - quota.wakeAll() + quota.WakeAll() select { case ok := <-acquired: require.False(t, ok) case <-time.After(time.Second): t.Fatal("stopping the subscription did not wake the blocked event") } - quota.releaseEvent(100) + quota.ReleaseEvent(100) } func TestMemoryQuotaBlockedEventResumesAfterRelease(t *testing.T) { quota := newMemoryQuotaController(100, 10) span := newTestQuotaSpan(1) - require.True(t, quota.acquireEvent(context.Background(), span, 200)) + require.True(t, quota.AcquireEvent(context.Background(), span, 200)) acquired := make(chan bool, 1) go func() { - acquired <- quota.acquireEvent(context.Background(), span, 1) + acquired <- quota.AcquireEvent(context.Background(), span, 1) }() select { @@ -161,11 +209,11 @@ func TestMemoryQuotaBlockedEventResumesAfterRelease(t *testing.T) { case <-time.After(100 * time.Millisecond): } - quota.releaseEvent(200) + quota.ReleaseEvent(200) select { case ok := <-acquired: require.True(t, ok) - quota.releaseEvent(1) + quota.ReleaseEvent(1) case <-time.After(time.Second): t.Fatal("event memory did not resume after memory was released") } @@ -177,10 +225,10 @@ func TestMemoryQuotaBlockedEventStopsOnContextCancellation(t *testing.T) { span := newTestQuotaSpan(1) ctx, cancel := context.WithCancel(context.Background()) - require.True(t, quota.acquireEvent(context.Background(), span, 100)) + require.True(t, quota.AcquireEvent(context.Background(), span, 100)) acquired := make(chan bool, 1) go func() { - acquired <- quota.acquireEvent(ctx, span, 1) + acquired <- quota.AcquireEvent(ctx, span, 1) }() require.Eventually(t, func() bool { return quota.eventNotifier.waiters.Load() == 1 @@ -195,7 +243,7 @@ func TestMemoryQuotaBlockedEventStopsOnContextCancellation(t *testing.T) { case <-time.After(time.Second): t.Fatal("context cancellation did not stop the blocked event") } - quota.releaseEvent(100) + quota.ReleaseEvent(100) } func TestMemoryQuotaConcurrentWaitersDoNotLoseWakeups(t *testing.T) { @@ -208,13 +256,13 @@ func TestMemoryQuotaConcurrentWaitersDoNotLoseWakeups(t *testing.T) { // Hold the only available byte until every goroutine is waiting. Releasing // it wakes all waiters; each successful waiter then releases it for the next. - require.True(t, quota.acquireEvent(ctx, span, 1)) + require.True(t, quota.AcquireEvent(ctx, span, 1)) results := make(chan bool, waiterCount) for range waiterCount { go func() { - acquired := quota.acquireEvent(ctx, span, 1) + acquired := quota.AcquireEvent(ctx, span, 1) if acquired { - quota.releaseEvent(1) + quota.ReleaseEvent(1) } results <- acquired }() @@ -223,7 +271,7 @@ func TestMemoryQuotaConcurrentWaitersDoNotLoseWakeups(t *testing.T) { return quota.eventNotifier.waiters.Load() == waiterCount }, time.Second, time.Millisecond) - quota.releaseEvent(1) + quota.ReleaseEvent(1) for range waiterCount { select { case acquired := <-results: @@ -232,8 +280,8 @@ func TestMemoryQuotaConcurrentWaitersDoNotLoseWakeups(t *testing.T) { t.Fatal("event waiter did not make progress") } } - used, _, _ := quota.snapshot() - require.Zero(t, used) + state := getMemoryQuotaTestState(quota) + require.Zero(t, state.used) } func TestMemoryQuotaWarmingScanUsesCurrentPressure(t *testing.T) { @@ -242,19 +290,19 @@ func TestMemoryQuotaWarmingScanUsesCurrentPressure(t *testing.T) { currentTs := setTestQuotaSpanLag(span, lowLagRegionThreshold+time.Minute) region := newTestQuotaRegion(span) - bytes1, _, admitted := quota.acquireScan(region, currentTs) + bytes1, _, admitted := quota.AcquireScan(region, currentTs) require.True(t, admitted) require.NotZero(t, bytes1) - scanUsed, _, _ := quota.scanSnapshot() - require.Greater(t, scanUsed, quota.pauseWarmingLimit) + state := getMemoryQuotaTestState(quota) + require.Greater(t, state.scanUsed, quota.pauseWarmingLimit) - _, _, admitted = quota.acquireScan(region, currentTs) + _, _, admitted = quota.AcquireScan(region, currentTs) require.False(t, admitted) - quota.releaseScan(bytes1) - bytes2, _, admitted := quota.acquireScan(region, currentTs) + quota.ReleaseScan(bytes1) + bytes2, _, admitted := quota.AcquireScan(region, currentTs) require.True(t, admitted) - quota.releaseScan(bytes2) + quota.ReleaseScan(bytes2) } func TestMemoryQuotaLowLagScanBypassesWarmingGate(t *testing.T) { @@ -262,15 +310,15 @@ func TestMemoryQuotaLowLagScanBypassesWarmingGate(t *testing.T) { span := newTestQuotaSpan(1) currentTs := setTestQuotaSpanLag(span, lowLagRegionThreshold-time.Second) - require.True(t, quota.acquireEvent(context.Background(), span, 20)) - scanBytes, _, admitted := quota.acquireScan(newTestQuotaRegion(span), currentTs) + require.True(t, quota.AcquireEvent(context.Background(), span, 20)) + scanBytes, _, admitted := quota.AcquireScan(newTestQuotaRegion(span), currentTs) require.True(t, admitted) require.NotZero(t, scanBytes) - scanUsed, _, _ := quota.scanSnapshot() - require.NotZero(t, scanUsed) + state := getMemoryQuotaTestState(quota) + require.NotZero(t, state.scanUsed) - quota.releaseScan(scanBytes) - quota.releaseEvent(20) + quota.ReleaseScan(scanBytes) + quota.ReleaseEvent(20) } func TestAdmissionWaitsForMemoryAndReleasesScanMemory(t *testing.T) { @@ -281,7 +329,7 @@ func TestAdmissionWaitsForMemoryAndReleasesScanMemory(t *testing.T) { return currentTs }) - require.True(t, quota.acquireEvent(context.Background(), span, 20)) + require.True(t, quota.AcquireEvent(context.Background(), span, 20)) region := newTestQuotaRegion(span) require.True(t, controller.submit(newRegionPriorityTask(region, currentTs, 1))) @@ -300,7 +348,7 @@ func TestAdmissionWaitsForMemoryAndReleasesScanMemory(t *testing.T) { case <-time.After(100 * time.Millisecond): } - quota.releaseEvent(20) + quota.ReleaseEvent(20) var resultValue popResult select { case resultValue = <-result: @@ -309,11 +357,11 @@ func TestAdmissionWaitsForMemoryAndReleasesScanMemory(t *testing.T) { } require.NoError(t, resultValue.err) req := resultValue.req - scanUsed, _, _ := quota.scanSnapshot() - require.NotZero(t, scanUsed) + state := getMemoryQuotaTestState(quota) + require.NotZero(t, state.scanUsed) require.True(t, req.abort()) - scanUsed, _, _ = quota.scanSnapshot() - require.Zero(t, scanUsed) + state = getMemoryQuotaTestState(quota) + require.Zero(t, state.scanUsed) } func TestAdmissionWakesWhenBlockedSpanStops(t *testing.T) { @@ -324,7 +372,7 @@ func TestAdmissionWakesWhenBlockedSpanStops(t *testing.T) { return currentTs }) - require.True(t, quota.acquireEvent(context.Background(), span, 20)) + require.True(t, quota.AcquireEvent(context.Background(), span, 20)) require.True(t, controller.submit(newRegionPriorityTask( newTestQuotaRegion(span), currentTs, 1))) @@ -344,7 +392,7 @@ func TestAdmissionWakesWhenBlockedSpanStops(t *testing.T) { } span.stopped.Store(true) - quota.wakeAll() + quota.WakeAll() select { case result := <-result: require.NoError(t, result.err) @@ -353,7 +401,7 @@ func TestAdmissionWakesWhenBlockedSpanStops(t *testing.T) { case <-time.After(time.Second): t.Fatal("stopping the span did not wake scan admission") } - quota.releaseEvent(20) + quota.ReleaseEvent(20) } func BenchmarkMemoryQuotaEventAccounting(b *testing.B) { @@ -363,9 +411,9 @@ func BenchmarkMemoryQuotaEventAccounting(b *testing.B) { b.ReportAllocs() b.ResetTimer() for b.Loop() { - if !quota.acquireEvent(ctx, span, 1) { + if !quota.AcquireEvent(ctx, span, 1) { b.Fatal("failed to acquire event memory") } - quota.releaseEvent(1) + quota.ReleaseEvent(1) } } diff --git a/logservice/logpuller/region_admission_controller.go b/logservice/logpuller/region_admission_controller.go index 9b0dde1c74..1e7664473e 100644 --- a/logservice/logpuller/region_admission_controller.go +++ b/logservice/logpuller/region_admission_controller.go @@ -73,7 +73,7 @@ func (r *regionReq) release() bool { if !r.released.CompareAndSwap(false, true) { return false } - r.controller.memoryQuota.releaseScan(r.scanBytes) + r.controller.memoryQuota.ReleaseScan(r.scanBytes) r.controller.release() return true } @@ -204,7 +204,7 @@ func (c *regionAdmissionController) popEligibleLocked() ( return nil, 0, nil } - scanBytes, memoryReady, admitted := c.memoryQuota.acquireScan( + scanBytes, memoryReady, admitted := c.memoryQuota.AcquireScan( request.regionInfo, c.currentTs()) if !admitted { return nil, 0, memoryReady diff --git a/logservice/logpuller/region_event_handler.go b/logservice/logpuller/region_event_handler.go index a30b349418..704e346505 100644 --- a/logservice/logpuller/region_event_handler.go +++ b/logservice/logpuller/region_event_handler.go @@ -151,7 +151,7 @@ func (h *regionEventHandler) Handle(span *subscribedSpan, events ...regionEvent) } } if !wasInitialized && span.initialized.Load() { - h.eventSink.memoryQuota.notifyScanAdmission() + h.eventSink.memoryQuota.NotifyScanAdmission() } tryAdvanceResolvedTs := func() { if newResolvedTs != 0 { @@ -159,7 +159,7 @@ func (h *regionEventHandler) Handle(span *subscribedSpan, events ...regionEvent) } } releaseMemoryQuota := func() { - h.eventSink.memoryQuota.releaseEvent(memoryBytes) + h.eventSink.memoryQuota.ReleaseEvent(memoryBytes) } if len(span.kvEventsCache) > 0 { metricsEventCount.Add(float64(len(span.kvEventsCache))) @@ -243,7 +243,7 @@ func (h *regionEventHandler) GetType(event regionEvent) dynstream.EventType { } func (h *regionEventHandler) OnDrop(event regionEvent) interface{} { - h.eventSink.memoryQuota.releaseEvent(event.memoryBytes) + h.eventSink.memoryQuota.ReleaseEvent(event.memoryBytes) // TODO: Distinguish between drop events caused by "path not found" errors and memory control. state := event.mustFirstState() fields := []zap.Field{ diff --git a/logservice/logpuller/region_event_handler_test.go b/logservice/logpuller/region_event_handler_test.go index 9227f2bb8e..91749cf028 100644 --- a/logservice/logpuller/region_event_handler_test.go +++ b/logservice/logpuller/region_event_handler_test.go @@ -397,7 +397,7 @@ func TestHandleEntriesReleasesMemoryAfterDownstreamCallback(t *testing.T) { lockedRangeState: lockedState, }, } - require.True(t, quota.acquireEvent(context.Background(), span, 10)) + require.True(t, quota.AcquireEvent(context.Background(), span, 10)) handler := ®ionEventHandler{eventSink: ®ionEventSink{ ds: newMockRegionEventSinkStream(), memoryQuota: quota, @@ -415,13 +415,13 @@ func TestHandleEntriesReleasesMemoryAfterDownstreamCallback(t *testing.T) { }}, }) require.True(t, await) - used, _, _ := quota.snapshot() - require.Equal(t, uint64(10), used) + quotaState := getMemoryQuotaTestState(quota) + require.Equal(t, uint64(10), quotaState.used) callback := <-callbackCh callback() - used, _, _ = quota.snapshot() - require.Zero(t, used) + quotaState = getMemoryQuotaTestState(quota) + require.Zero(t, quotaState.used) } func TestTryMarkSpanInitializedByResolvedTs(t *testing.T) { diff --git a/logservice/logpuller/region_event_sink.go b/logservice/logpuller/region_event_sink.go index 4a5b5b0bf6..0c015db743 100644 --- a/logservice/logpuller/region_event_sink.go +++ b/logservice/logpuller/region_event_sink.go @@ -17,7 +17,6 @@ import ( "context" "github.com/pingcap/log" - "github.com/pingcap/ticdc/pkg/metrics" "github.com/pingcap/ticdc/utils/dynstream" "go.uber.org/zap" ) @@ -73,7 +72,7 @@ func (s *regionEventSink) Push(subID SubscriptionID, event regionEvent) { if event.needsMemoryAccounting() { span := event.mustFirstState().region.subscribedSpan event.memoryBytes = uint64(event.getSize()) - if !s.memoryQuota.acquireEvent(s.ctx, span, event.memoryBytes) { + if !s.memoryQuota.AcquireEvent(s.ctx, span, event.memoryBytes) { return } } @@ -84,17 +83,9 @@ func (s *regionEventSink) UpdateMetrics() { dsMetrics := s.ds.GetMetrics() metricSubscriptionClientDSChannelSize.Set(float64(dsMetrics.EventChanSize)) metricSubscriptionClientDSPendingQueueLen.Set(float64(dsMetrics.PendingQueueLen)) - - used, capacity, _ := s.memoryQuota.snapshot() - scanUsed, scanEstimate, hardLimit := s.memoryQuota.scanSnapshot() - metrics.LogPullerMemoryQuota.WithLabelValues("max").Set(float64(capacity)) - metrics.LogPullerMemoryQuota.WithLabelValues("used").Set(float64(used)) - metrics.LogPullerMemoryQuota.WithLabelValues("scan_used").Set(float64(scanUsed)) - metrics.LogPullerMemoryQuota.WithLabelValues("scan_estimate").Set(float64(scanEstimate)) - metrics.LogPullerMemoryQuota.WithLabelValues("hard_limit").Set(float64(hardLimit)) } func (s *regionEventSink) Close() { - s.memoryQuota.wakeAll() + s.memoryQuota.WakeAll() s.ds.Close() } diff --git a/logservice/logpuller/region_event_sink_test.go b/logservice/logpuller/region_event_sink_test.go index 7786e4b524..291d4d867c 100644 --- a/logservice/logpuller/region_event_sink_test.go +++ b/logservice/logpuller/region_event_sink_test.go @@ -18,7 +18,6 @@ import ( "testing" "github.com/pingcap/kvproto/pkg/cdcpb" - "github.com/pingcap/ticdc/pkg/metrics" "github.com/pingcap/ticdc/utils/dynstream" "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/require" @@ -66,31 +65,17 @@ func (s *mockRegionEventSinkStream) GetMetrics() dynstream.Metrics[int, Subscrip } func TestRegionEventSinkUpdateMetrics(t *testing.T) { - t.Run("quota updates memory gauges", func(t *testing.T) { - ds := newMockRegionEventSinkStream() - ds.metrics = dynstream.Metrics[int, SubscriptionID]{ - EventChanSize: 33, - PendingQueueLen: 44, - } - quota := newMemoryQuotaController(66, 8) - span := newTestQuotaSpan(1) - require.True(t, quota.acquireEvent(context.Background(), span, 55)) - t.Cleanup(func() { quota.releaseEvent(55) }) - - sink := ®ionEventSink{ - ctx: context.Background(), - ds: ds, - memoryQuota: quota, - } - sink.UpdateMetrics() - - require.Equal(t, float64(33), testutil.ToFloat64(metricSubscriptionClientDSChannelSize)) - require.Equal(t, float64(44), testutil.ToFloat64(metricSubscriptionClientDSPendingQueueLen)) - require.Equal(t, float64(66), testutil.ToFloat64( - metrics.LogPullerMemoryQuota.WithLabelValues("max"))) - require.Equal(t, float64(55), testutil.ToFloat64( - metrics.LogPullerMemoryQuota.WithLabelValues("used"))) - }) + ds := newMockRegionEventSinkStream() + ds.metrics = dynstream.Metrics[int, SubscriptionID]{ + EventChanSize: 33, + PendingQueueLen: 44, + } + sink := ®ionEventSink{ds: ds} + + sink.UpdateMetrics() + + require.Equal(t, float64(33), testutil.ToFloat64(metricSubscriptionClientDSChannelSize)) + require.Equal(t, float64(44), testutil.ToFloat64(metricSubscriptionClientDSPendingQueueLen)) } func TestRegionEventSinkTracksEntriesUntilDrop(t *testing.T) { @@ -115,10 +100,10 @@ func TestRegionEventSinkTracksEntriesUntilDrop(t *testing.T) { }) pushed := <-ds.eventCh require.NotZero(t, pushed.memoryBytes) - used, _, _ := quota.snapshot() - require.NotZero(t, used) + quotaState := getMemoryQuotaTestState(quota) + require.NotZero(t, quotaState.used) (®ionEventHandler{eventSink: sink}).OnDrop(pushed) - used, _, _ = quota.snapshot() - require.Zero(t, used) + quotaState = getMemoryQuotaTestState(quota) + require.Zero(t, quotaState.used) } diff --git a/logservice/logpuller/subscription_client.go b/logservice/logpuller/subscription_client.go index bfbe2f910b..af039a9ebd 100644 --- a/logservice/logpuller/subscription_client.go +++ b/logservice/logpuller/subscription_client.go @@ -212,6 +212,7 @@ func (s *subscriptionClient) updateMetrics(ctx context.Context) error { case <-ticker.C: s.regionScheduler.UpdateMetrics() s.eventSink.UpdateMetrics() + s.memoryQuota.UpdateMetrics() s.spanRegistry.UpdateMetrics() } } @@ -319,7 +320,7 @@ func (s *subscriptionClient) setTableStopped(rt *subscribedSpan) { // then notify every existing worker to deregister the subscription. if rt.stopped.CompareAndSwap(false, true) { // Wake event receivers and scan admission so they can observe stopped. - s.memoryQuota.wakeAll() + s.memoryQuota.WakeAll() s.regionScheduler.BroadcastDeregister(rt.subID, rt.filterLoop) if rt.rangeLock.Stop() { s.onTableDrained(rt) diff --git a/metrics/grafana/ticdc_new_arch.json b/metrics/grafana/ticdc_new_arch.json index 29123f9715..52cda78859 100644 --- a/metrics/grafana/ticdc_new_arch.json +++ b/metrics/grafana/ticdc_new_arch.json @@ -8119,7 +8119,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Log puller memory quota", + "description": "Actual event memory, estimated in-flight scan memory, and the configured memory limits.", "fieldConfig": { "defaults": {}, "overrides": [] @@ -8162,7 +8162,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(ticdc_dynamic_stream_memory_usage{k8s_cluster=~\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", module=~\"log-puller\"}) by (instance, type)", + "expr": "sum(ticdc_log_puller_memory_quota{k8s_cluster=~\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance, type)", "interval": "", "legendFormat": "{{instance}}-{{type}}", "refId": "A" @@ -8172,7 +8172,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Memory Quota", + "title": "Memory Quota Usage", "tooltip": { "shared": true, "sort": 0, @@ -8822,7 +8822,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Log puller memory quota tracked by the local memory quota controller.", + "description": "Scan admission level (0: normal, 1: pause warming scans, 2: pause all scans) and event receivers currently blocked at the memory hard limit.", "fieldConfig": { "defaults": {}, "overrides": [] @@ -8858,6 +8858,110 @@ "pointradius": 2, "points": false, "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/event-waiters$/", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "ticdc_log_puller_memory_quota_admission_level{k8s_cluster=~\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}", + "interval": "", + "legendFormat": "{{instance}}-admission-level", + "refId": "A" + }, + { + "exemplar": true, + "expr": "ticdc_log_puller_memory_quota_event_waiter_count{k8s_cluster=~\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}", + "interval": "", + "legendFormat": "{{instance}}-event-waiters", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Quota Throttling", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": "2", + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Time spent waiting to receive more events after log puller memory reaches the hard limit.", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 57 + }, + "hiddenSeries": false, + "id": 26002, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, @@ -8865,17 +8969,24 @@ "targets": [ { "exemplar": true, - "expr": "sum(ticdc_log_puller_memory_quota{k8s_cluster=~\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance, type)", + "expr": "histogram_quantile(0.99, sum(rate(ticdc_log_puller_memory_quota_event_wait_duration_bucket{k8s_cluster=~\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", "interval": "", - "legendFormat": "{{instance}}-{{type}}", + "legendFormat": "{{instance}}-p99", "refId": "A" + }, + { + "exemplar": true, + "expr": "sum(rate(ticdc_log_puller_memory_quota_event_wait_duration_sum{k8s_cluster=~\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance) / sum(rate(ticdc_log_puller_memory_quota_event_wait_duration_count{k8s_cluster=~\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", + "interval": "", + "legendFormat": "{{instance}}-avg", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Memory Quota Controller", + "title": "Memory Quota Event Wait Duration", "tooltip": { "shared": true, "sort": 0, @@ -8891,7 +9002,7 @@ }, "yaxes": [ { - "format": "bytes", + "format": "s", "logBase": 1, "min": "0", "show": true diff --git a/metrics/nextgengrafana/ticdc_new_arch_next_gen.json b/metrics/nextgengrafana/ticdc_new_arch_next_gen.json index 2bfe4b8015..b8452acca1 100644 --- a/metrics/nextgengrafana/ticdc_new_arch_next_gen.json +++ b/metrics/nextgengrafana/ticdc_new_arch_next_gen.json @@ -8119,7 +8119,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Log puller memory quota", + "description": "Actual event memory, estimated in-flight scan memory, and the configured memory limits.", "fieldConfig": { "defaults": {}, "overrides": [] @@ -8162,7 +8162,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(ticdc_dynamic_stream_memory_usage{k8s_cluster=~\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", module=~\"log-puller\"}) by (instance, type)", + "expr": "sum(ticdc_log_puller_memory_quota{k8s_cluster=~\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance, type)", "interval": "", "legendFormat": "{{instance}}-{{type}}", "refId": "A" @@ -8172,7 +8172,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Memory Quota", + "title": "Memory Quota Usage", "tooltip": { "shared": true, "sort": 0, @@ -8822,7 +8822,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Log puller memory quota tracked by the local memory quota controller.", + "description": "Scan admission level (0: normal, 1: pause warming scans, 2: pause all scans) and event receivers currently blocked at the memory hard limit.", "fieldConfig": { "defaults": {}, "overrides": [] @@ -8858,6 +8858,110 @@ "pointradius": 2, "points": false, "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/event-waiters$/", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "ticdc_log_puller_memory_quota_admission_level{k8s_cluster=~\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}", + "interval": "", + "legendFormat": "{{instance}}-admission-level", + "refId": "A" + }, + { + "exemplar": true, + "expr": "ticdc_log_puller_memory_quota_event_waiter_count{k8s_cluster=~\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}", + "interval": "", + "legendFormat": "{{instance}}-event-waiters", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Quota Throttling", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": "2", + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Time spent waiting to receive more events after log puller memory reaches the hard limit.", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 57 + }, + "hiddenSeries": false, + "id": 26002, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, @@ -8865,17 +8969,24 @@ "targets": [ { "exemplar": true, - "expr": "sum(ticdc_log_puller_memory_quota{k8s_cluster=~\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance, type)", + "expr": "histogram_quantile(0.99, sum(rate(ticdc_log_puller_memory_quota_event_wait_duration_bucket{k8s_cluster=~\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", "interval": "", - "legendFormat": "{{instance}}-{{type}}", + "legendFormat": "{{instance}}-p99", "refId": "A" + }, + { + "exemplar": true, + "expr": "sum(rate(ticdc_log_puller_memory_quota_event_wait_duration_sum{k8s_cluster=~\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance) / sum(rate(ticdc_log_puller_memory_quota_event_wait_duration_count{k8s_cluster=~\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", + "interval": "", + "legendFormat": "{{instance}}-avg", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Memory Quota Controller", + "title": "Memory Quota Event Wait Duration", "tooltip": { "shared": true, "sort": 0, @@ -8891,7 +9002,7 @@ }, "yaxes": [ { - "format": "bytes", + "format": "s", "logBase": 1, "min": "0", "show": true diff --git a/pkg/metrics/log_puller.go b/pkg/metrics/log_puller.go index be932f4a8c..aa45134c9f 100644 --- a/pkg/metrics/log_puller.go +++ b/pkg/metrics/log_puller.go @@ -71,6 +71,28 @@ var ( Name: "memory_quota", Help: "The log puller local memory quota usage.", }, []string{"type"}) + LogPullerMemoryQuotaAdmissionLevel = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: "ticdc", + Subsystem: "log_puller", + Name: "memory_quota_admission_level", + Help: "The log puller scan admission level: 0 normal, 1 pause warming scans, 2 pause all scans.", + }) + LogPullerMemoryQuotaEventWaiterCount = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: "ticdc", + Subsystem: "log_puller", + Name: "memory_quota_event_waiter_count", + Help: "The number of event receivers waiting at the log puller memory hard limit.", + }) + LogPullerMemoryQuotaEventWaitDuration = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Namespace: "ticdc", + Subsystem: "log_puller", + Name: "memory_quota_event_wait_duration", + Help: "The duration in seconds that an event receiver waits at the log puller memory hard limit.", + Buckets: prometheus.ExponentialBuckets(0.001, 2, 24), + }) SubscriptionClientResolvedTsLagGauge = prometheus.NewGauge( prometheus.GaugeOpts{ @@ -164,6 +186,9 @@ func initLogPullerMetrics(registry *prometheus.Registry) { registry.MustRegister(LogPullerMatcherCount) registry.MustRegister(LogPullerResolvedTsLag) registry.MustRegister(LogPullerMemoryQuota) + registry.MustRegister(LogPullerMemoryQuotaAdmissionLevel) + registry.MustRegister(LogPullerMemoryQuotaEventWaiterCount) + registry.MustRegister(LogPullerMemoryQuotaEventWaitDuration) registry.MustRegister(SubscriptionClientRequestedRegionCount) registry.MustRegister(RegionRequestFinishScanDuration) registry.MustRegister(SubscriptionClientSubscribedRegionCount) From 6c259463234570ffb7f5f7907af774508a45d007 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Tue, 4 Aug 2026 17:59:12 +0800 Subject: [PATCH 09/26] more fix --- logservice/logpuller/memory_quota.go | 43 ++-------- logservice/logpuller/memory_quota_test.go | 9 +- logservice/logpuller/priority_task.go | 18 +++- logservice/logpuller/region_request_worker.go | 5 +- logservice/logpuller/requested_store.go | 85 ------------------- 5 files changed, 29 insertions(+), 131 deletions(-) delete mode 100644 logservice/logpuller/requested_store.go diff --git a/logservice/logpuller/memory_quota.go b/logservice/logpuller/memory_quota.go index 9914b88ca3..3698bf2085 100644 --- a/logservice/logpuller/memory_quota.go +++ b/logservice/logpuller/memory_quota.go @@ -40,14 +40,6 @@ const ( // falls to 5% of the soft capacity. defaultResumeWarmingRatio = 0.05 - // defaultFreezeAllRatio pauses every new scan when memory pressure reaches - // 80% of the soft capacity. - defaultFreezeAllRatio = 0.8 - - // defaultResumeAllRatio allows new scans again after memory pressure falls - // to 60% of the soft capacity. - defaultResumeAllRatio = 0.6 - // defaultHardLimitRatio blocks receiving more events when accounted event // memory reaches twice the soft capacity. defaultHardLimitRatio = 2.0 @@ -70,7 +62,6 @@ type admissionLevel uint8 const ( admissionNormal admissionLevel = iota admissionPauseWarming - admissionFreezeAllNewScans ) // eventMemoryNotifier wakes event receivers that are waiting for memory. Each @@ -140,8 +131,9 @@ func (n *eventMemoryNotifier) notify() { // memoryQuotaController tracks event memory retained by downstream callbacks // and estimated memory for admitted initial scans. Event memory is allowed to // exceed the soft capacity, but the receive path waits at the hard limit. Scan -// admission first pauses uninitialized high-lag spans and freezes all new scans -// only under heavier pressure; both transitions use hysteresis when resuming. +// admission pauses uninitialized high-lag spans under memory pressure and uses +// hysteresis when resuming them. Other scans continue to make progress and are +// bounded by the region request window and the event-memory hard limit. type memoryQuotaController struct { capacity uint64 // used tracks event bytes retained until downstream finishes consuming them. @@ -166,8 +158,6 @@ type memoryQuotaController struct { pauseWarmingLimit uint64 resumeWarmingLimit uint64 - freezeAllLimit uint64 - resumeAllLimit uint64 hardLimit uint64 scanEstimate uint64 @@ -185,8 +175,6 @@ func newMemoryQuotaController(capacity, scanBaseSize uint64) *memoryQuotaControl level: admissionNormal, pauseWarmingLimit: uint64(math.Ceil(float64(capacity) * defaultPauseWarmingRatio)), resumeWarmingLimit: uint64(float64(capacity) * defaultResumeWarmingRatio), - freezeAllLimit: uint64(math.Ceil(float64(capacity) * defaultFreezeAllRatio)), - resumeAllLimit: uint64(float64(capacity) * defaultResumeAllRatio), hardLimit: uint64(float64(capacity) * defaultHardLimitRatio), scanEstimate: scanBaseSize, eventNotifier: newEventMemoryNotifier(), @@ -216,10 +204,6 @@ func (c *memoryQuotaController) AcquireScan( c.scanMu.Lock() defer c.scanMu.Unlock() c.refreshLevelLocked() - if c.level == admissionFreezeAllNewScans { - return 0, c.scanReady, false - } - warming := isWarmingScan(region, currentTs) // Admission is based on the pressure before accounting this scan. This lets // one scan make progress even when its estimate alone exceeds the threshold. @@ -292,8 +276,7 @@ func (c *memoryQuotaController) ReleaseEvent(bytes uint64) { } used := c.used.Add(^(bytes - 1)) previousUsed := used + bytes - if crossesDown(previousUsed, used, c.resumeWarmingLimit) || - crossesDown(previousUsed, used, c.resumeAllLimit) { + if crossesDown(previousUsed, used, c.resumeWarmingLimit) { c.refreshAdmissionAndNotify() } c.eventNotifier.notify() @@ -381,26 +364,12 @@ func (c *memoryQuotaController) refreshLevelLocked() { // it to actual event bytes would count the same pressure twice. pressure := max(c.used.Load(), c.scanUsed) switch c.level { - case admissionFreezeAllNewScans: - if pressure <= c.resumeAllLimit { - if pressure >= c.pauseWarmingLimit { - c.level = admissionPauseWarming - } else { - c.level = admissionNormal - } - } case admissionPauseWarming: - switch { - case pressure >= c.freezeAllLimit: - c.level = admissionFreezeAllNewScans - case pressure <= c.resumeWarmingLimit: + if pressure <= c.resumeWarmingLimit { c.level = admissionNormal } default: - switch { - case pressure >= c.freezeAllLimit: - c.level = admissionFreezeAllNewScans - case pressure >= c.pauseWarmingLimit: + if pressure >= c.pauseWarmingLimit { c.level = admissionPauseWarming } } diff --git a/logservice/logpuller/memory_quota_test.go b/logservice/logpuller/memory_quota_test.go index 8b8b3cd1c6..02be6cf86f 100644 --- a/logservice/logpuller/memory_quota_test.go +++ b/logservice/logpuller/memory_quota_test.go @@ -71,7 +71,7 @@ func TestMemoryQuotaUpdateMetrics(t *testing.T) { quota.scanMu.Lock() quota.scanUsed = 7 - quota.level = admissionFreezeAllNewScans + quota.level = admissionPauseWarming quota.scanMu.Unlock() quota.eventNotifier.waiters.Store(2) t.Cleanup(func() { quota.eventNotifier.waiters.Store(0) }) @@ -88,7 +88,7 @@ func TestMemoryQuotaUpdateMetrics(t *testing.T) { metrics.LogPullerMemoryQuota.WithLabelValues("scan_estimate"))) require.Equal(t, float64(132), testutil.ToFloat64( metrics.LogPullerMemoryQuota.WithLabelValues("hard_limit"))) - require.Equal(t, float64(admissionFreezeAllNewScans), + require.Equal(t, float64(admissionPauseWarming), testutil.ToFloat64(metrics.LogPullerMemoryQuotaAdmissionLevel)) require.Equal(t, float64(2), testutil.ToFloat64(metrics.LogPullerMemoryQuotaEventWaiterCount)) @@ -114,9 +114,10 @@ func TestMemoryQuotaAdmissionLevels(t *testing.T) { require.True(t, quota.AcquireEvent(context.Background(), initializedSpan, 45)) require.True(t, quota.AcquireEvent(context.Background(), initializedSpan, 20)) - _, _, admitted = quota.AcquireScan( + scanBytes, _, admitted = quota.AcquireScan( newTestQuotaRegion(initializedSpan), initializedTs) - require.False(t, admitted) + require.True(t, admitted) + quota.ReleaseScan(scanBytes) quota.ReleaseEvent(20) state := getMemoryQuotaTestState(quota) diff --git a/logservice/logpuller/priority_task.go b/logservice/logpuller/priority_task.go index 38cd813c8b..aad3aab16b 100644 --- a/logservice/logpuller/priority_task.go +++ b/logservice/logpuller/priority_task.go @@ -13,7 +13,14 @@ package logpuller -import "github.com/pingcap/kvproto/pkg/cdcpb" +import ( + "time" + + "github.com/pingcap/kvproto/pkg/cdcpb" + "github.com/tikv/client-go/v2/oracle" +) + +const lowLagRegionThreshold = 30 * time.Minute func normalizeScanPriority(priority cdcpb.ScanPriority) cdcpb.ScanPriority { if priority == cdcpb.ScanPriority_SCAN_PRIORITY_HIGH { @@ -68,3 +75,12 @@ func (pt *regionPriorityTask) LessThan(other *regionPriorityTask) bool { } return pt.sequence < other.sequence } + +func regionScanLag(currentTs, checkpointTs uint64) time.Duration { + currentTime := oracle.GetTimeFromTS(currentTs) + checkpointTime := oracle.GetTimeFromTS(checkpointTs) + if !currentTime.After(checkpointTime) { + return 0 + } + return currentTime.Sub(checkpointTime) +} diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index 8e333cf187..176c49cfea 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -39,10 +39,7 @@ const storeReconnectBackoff = time.Second // To generate a workerID in `newRegionRequestWorker`. var workerIDGen atomic.Uint64 -var ( - metricsResolvedTsCount = metrics.PullerEventCounter.WithLabelValues("resolved_ts") - metricBatchResolvedSize = metrics.BatchResolvedEventSize.WithLabelValues("event-store") -) +var metricBatchResolvedSize = metrics.BatchResolvedEventSize.WithLabelValues("event-store") type deregisterRequest struct { subID SubscriptionID diff --git a/logservice/logpuller/requested_store.go b/logservice/logpuller/requested_store.go deleted file mode 100644 index 97294a8e33..0000000000 --- a/logservice/logpuller/requested_store.go +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2026 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package logpuller - -import ( - "context" - "sync/atomic" - - "golang.org/x/sync/errgroup" -) - -// requestedStore owns the request workers connected to one TiKV store. The -// worker slice is complete before the store is published and is immutable -// afterwards, so task submission only needs an atomic round-robin counter. -type requestedStore struct { - storeAddr string - workers []*regionRequestWorker - nextWorker atomic.Uint64 -} - -func newRequestedStore( - upstream *upstreamHandle, - eventSink *regionEventSink, - failureHandler *regionFailureHandler, - storeAddr string, - workerCount int, - workerWindow int, - maxWindowMultiplier int, -) *requestedStore { - store := &requestedStore{ - storeAddr: storeAddr, - workers: make([]*regionRequestWorker, 0, workerCount), - } - for i := 0; i < workerCount; i++ { - store.workers = append(store.workers, newRegionRequestWorker( - upstream, eventSink, failureHandler, store, workerWindow, maxWindowMultiplier)) - } - return store -} - -func (s *requestedStore) run(ctx context.Context, group *errgroup.Group) { - for _, worker := range s.workers { - group.Go(func() error { return worker.Run(ctx) }) - } -} - -func (s *requestedStore) submit(task *regionPriorityTask) bool { - if len(s.workers) == 0 { - return false - } - index := (s.nextWorker.Add(1) - 1) % uint64(len(s.workers)) - return s.workers[index].admission.submit(task) -} - -func (s *requestedStore) broadcastDeregister(subID SubscriptionID, filterLoop bool) { - for _, worker := range s.workers { - worker.controlQueue.push(deregisterRequest{subID: subID, filterLoop: filterLoop}) - } -} - -func (s *requestedStore) close() { - for _, worker := range s.workers { - worker.admission.close() - } -} - -func (s *requestedStore) inflightCount() int { - count := 0 - for _, worker := range s.workers { - count += worker.admission.stats().inflight - } - return count -} From 3a860d0f8915efc8b8a227d4fe0da61bee742da8 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Tue, 4 Aug 2026 20:55:07 +0800 Subject: [PATCH 10/26] small fix --- logservice/eventstore/pebble.go | 2 +- logservice/logpuller/memory_quota.go | 32 ++++++++----------- logservice/logpuller/memory_quota_test.go | 2 +- .../region_admission_controller_test.go | 2 +- .../logpuller/region_event_handler_test.go | 4 +-- pkg/config/debug.go | 4 +-- 6 files changed, 20 insertions(+), 26 deletions(-) diff --git a/logservice/eventstore/pebble.go b/logservice/eventstore/pebble.go index 0d950c99d0..f5f48bf71b 100644 --- a/logservice/eventstore/pebble.go +++ b/logservice/eventstore/pebble.go @@ -42,7 +42,7 @@ func newPebbleOptions(dbNum int) *pebble.Options { MaxOpenFiles: maxOpenFilesPerDB, - MaxConcurrentCompactions: func() int { return 2 }, + MaxConcurrentCompactions: func() int { return 6 }, // Decrease compaction frequency L0CompactionThreshold: 20, diff --git a/logservice/logpuller/memory_quota.go b/logservice/logpuller/memory_quota.go index 3698bf2085..a499c1ab4e 100644 --- a/logservice/logpuller/memory_quota.go +++ b/logservice/logpuller/memory_quota.go @@ -25,10 +25,6 @@ import ( ) const ( - // defaultLogPullerMemoryQuota is the soft memory capacity shared by event - // accounting and initial-scan admission. - defaultLogPullerMemoryQuota uint64 = 1024 * 1024 * 1024 - // Admission ratios compare max(accounted event memory, estimated scan // memory) with the soft capacity. @@ -44,9 +40,6 @@ const ( // memory reaches twice the soft capacity. defaultHardLimitRatio = 2.0 - // defaultScanBaseSize is the minimum memory estimate for one admitted scan. - defaultScanBaseSize uint64 = 8 * 1024 * 1024 - // defaultScanLagUnit is the lag unit used by the logarithmic scan estimate. defaultScanLagUnit = 10 * time.Minute @@ -128,12 +121,19 @@ func (n *eventMemoryNotifier) notify() { n.mu.Unlock() } -// memoryQuotaController tracks event memory retained by downstream callbacks -// and estimated memory for admitted initial scans. Event memory is allowed to -// exceed the soft capacity, but the receive path waits at the hard limit. Scan -// admission pauses uninitialized high-lag spans under memory pressure and uses -// hysteresis when resuming them. Other scans continue to make progress and are -// bounded by the region request window and the event-memory hard limit. +// memoryQuotaController coordinates memory pressure from two sources: +// retained event memory and admitted initial scans. +// +// Event memory tracks bytes kept alive until downstream finishes consuming +// them. It may exceed the soft capacity temporarily, but the receive path +// blocks once it reaches the hard limit. +// +// Initial scans are charged by estimate instead of measured bytes. Each +// admitted scan starts from scanBaseSize, grows logarithmically with scan lag, +// and is capped at maxScanLagFactor times the base size. Scan admission +// compares max(event used, scan used) with the soft capacity: high-lag +// "warming" scans pause at pauseWarmingLimit and resume at +// resumeWarmingLimit, while lower-lag scans continue to make progress. type memoryQuotaController struct { capacity uint64 // used tracks event bytes retained until downstream finishes consuming them. @@ -164,12 +164,6 @@ type memoryQuotaController struct { } func newMemoryQuotaController(capacity, scanBaseSize uint64) *memoryQuotaController { - if capacity == 0 { - capacity = defaultLogPullerMemoryQuota - } - if scanBaseSize == 0 { - scanBaseSize = defaultScanBaseSize - } c := &memoryQuotaController{ capacity: capacity, level: admissionNormal, diff --git a/logservice/logpuller/memory_quota_test.go b/logservice/logpuller/memory_quota_test.go index 02be6cf86f..de409721a3 100644 --- a/logservice/logpuller/memory_quota_test.go +++ b/logservice/logpuller/memory_quota_test.go @@ -406,7 +406,7 @@ func TestAdmissionWakesWhenBlockedSpanStops(t *testing.T) { } func BenchmarkMemoryQuotaEventAccounting(b *testing.B) { - quota := newMemoryQuotaController(defaultLogPullerMemoryQuota, defaultScanBaseSize) + quota := newMemoryQuotaController(1024*1024*1024, 8*1024*1024) span := newTestQuotaSpan(1) ctx := context.Background() b.ReportAllocs() diff --git a/logservice/logpuller/region_admission_controller_test.go b/logservice/logpuller/region_admission_controller_test.go index 0fdeb3d949..770d849100 100644 --- a/logservice/logpuller/region_admission_controller_test.go +++ b/logservice/logpuller/region_admission_controller_test.go @@ -65,7 +65,7 @@ func newTestRegionAdmissionController( return newRegionAdmissionController( currentWindow, maxWindowMultiplier, - newMemoryQuotaController(0, 0), + newMemoryQuotaController(1024*1024*1024, 8*1024*1024), func() uint64 { return 0 }, ) } diff --git a/logservice/logpuller/region_event_handler_test.go b/logservice/logpuller/region_event_handler_test.go index 91749cf028..7c3db0c2cd 100644 --- a/logservice/logpuller/region_event_handler_test.go +++ b/logservice/logpuller/region_event_handler_test.go @@ -49,7 +49,7 @@ func TestHandleEventEntryEventOutOfOrder(t *testing.T) { // initialize option := dynstream.NewOption() handler := ®ionEventHandler{eventSink: ®ionEventSink{ - memoryQuota: newMemoryQuotaController(0, 0), + memoryQuota: newMemoryQuotaController(1024*1024*1024, 8*1024*1024), }} ds := dynstream.NewParallelDynamicStream("test", handler, option) ds.Start() @@ -208,7 +208,7 @@ func TestHandleResolvedTs(t *testing.T) { // initialize option := dynstream.NewOption() handler := ®ionEventHandler{eventSink: ®ionEventSink{ - memoryQuota: newMemoryQuotaController(0, 0), + memoryQuota: newMemoryQuotaController(1024*1024*1024, 8*1024*1024), }} ds := dynstream.NewParallelDynamicStream("test", handler, option) ds.Start() diff --git a/pkg/config/debug.go b/pkg/config/debug.go index df2fb31d27..12e0ba433c 100644 --- a/pkg/config/debug.go +++ b/pkg/config/debug.go @@ -24,9 +24,9 @@ import ( const ( // DefaultOldStartTsScanLowPriorityThreshold is the default lag threshold for // classifying scan tasks as low priority. - DefaultOldStartTsScanLowPriorityThreshold = 10 * time.Minute + DefaultOldStartTsScanLowPriorityThreshold = 10 * time.Minute defaultLogPullerMemoryQuota uint64 = 1024 * 1024 * 1024 - defaultLogPullerScanBaseSize uint64 = 8 * 1024 * 1024 + defaultLogPullerScanBaseSize uint64 = 8 * 1024 * 1024 ) // DebugConfig represents config for ticdc unexposed feature configurations From e8bf21851a143f01e7ad1561625db3d96295e3f7 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Tue, 4 Aug 2026 21:31:48 +0800 Subject: [PATCH 11/26] more fix --- logservice/logpuller/memory_quota.go | 24 +++--- logservice/logpuller/memory_quota_test.go | 76 +++++++++++-------- logservice/logpuller/priority_task.go | 14 ---- .../region_admission_controller_test.go | 17 +++-- logservice/logpuller/region_event_handler.go | 17 +++-- .../logpuller/region_event_handler_test.go | 47 ++++++++++-- .../region_request_scheduler_test.go | 10 +-- .../logpuller/region_request_store_test.go | 12 +-- .../logpuller/region_request_worker_test.go | 18 ++--- logservice/logpuller/span_registry.go | 12 --- .../logpuller/subscription_client_test.go | 34 +++++++-- pkg/metrics/log_puller.go | 8 -- 12 files changed, 165 insertions(+), 124 deletions(-) diff --git a/logservice/logpuller/memory_quota.go b/logservice/logpuller/memory_quota.go index a499c1ab4e..29b28874fa 100644 --- a/logservice/logpuller/memory_quota.go +++ b/logservice/logpuller/memory_quota.go @@ -22,12 +22,10 @@ import ( "time" "github.com/pingcap/ticdc/pkg/metrics" + "github.com/tikv/client-go/v2/oracle" ) const ( - // Admission ratios compare max(accounted event memory, estimated scan - // memory) with the soft capacity. - // defaultPauseWarmingRatio pauses new high-lag scans when memory pressure // reaches 15% of the soft capacity. defaultPauseWarmingRatio = 0.15 @@ -288,16 +286,11 @@ func (c *memoryQuotaController) UpdateMetrics() { c.scanMu.Lock() used := c.used.Load() scanUsed := c.scanUsed - level := c.level - scanEstimate := c.scanEstimate c.scanMu.Unlock() metrics.LogPullerMemoryQuota.WithLabelValues("max").Set(float64(c.capacity)) metrics.LogPullerMemoryQuota.WithLabelValues("used").Set(float64(used)) metrics.LogPullerMemoryQuota.WithLabelValues("scan_used").Set(float64(scanUsed)) - metrics.LogPullerMemoryQuota.WithLabelValues("scan_estimate").Set(float64(scanEstimate)) - metrics.LogPullerMemoryQuota.WithLabelValues("hard_limit").Set(float64(c.hardLimit)) - metrics.LogPullerMemoryQuotaAdmissionLevel.Set(float64(level)) metrics.LogPullerMemoryQuotaEventWaiterCount.Set( float64(c.eventNotifier.waiters.Load())) } @@ -345,12 +338,17 @@ func scanLagFactor(startTs, currentTs uint64) float64 { 1+defaultScanLagWeight*math.Log2(1+float64(lag)/float64(defaultScanLagUnit))) } -func isWarmingScan(region regionInfo, currentTs uint64) bool { - span := region.subscribedSpan - if span.initialized.Load() { - return false +func regionScanLag(currentTs, checkpointTs uint64) time.Duration { + currentTime := oracle.GetTimeFromTS(currentTs) + checkpointTime := oracle.GetTimeFromTS(checkpointTs) + if !currentTime.After(checkpointTime) { + return 0 } - return regionScanLag(currentTs, region.resolvedTs()) >= lowLagRegionThreshold + return currentTime.Sub(checkpointTime) +} + +func isWarmingScan(region regionInfo, _ uint64) bool { + return !isHighScanPriority(region.scanPriority) } func (c *memoryQuotaController) refreshLevelLocked() { diff --git a/logservice/logpuller/memory_quota_test.go b/logservice/logpuller/memory_quota_test.go index de409721a3..a808ec5be1 100644 --- a/logservice/logpuller/memory_quota_test.go +++ b/logservice/logpuller/memory_quota_test.go @@ -19,6 +19,7 @@ import ( "testing" "time" + "github.com/pingcap/kvproto/pkg/cdcpb" "github.com/pingcap/ticdc/logservice/logpuller/regionlock" "github.com/pingcap/ticdc/pkg/metrics" "github.com/prometheus/client_golang/prometheus/testutil" @@ -57,6 +58,15 @@ func newTestQuotaRegion(span *subscribedSpan) regionInfo { } } +func newTestQuotaRegionWithPriority( + span *subscribedSpan, + priority cdcpb.ScanPriority, +) regionInfo { + region := newTestQuotaRegion(span) + region.scanPriority = priority + return region +} + func setTestQuotaSpanLag(span *subscribedSpan, lag time.Duration) uint64 { now := time.Now() span.resolvedTs.Store(oracle.GoTimeToTS(now.Add(-lag))) @@ -71,7 +81,6 @@ func TestMemoryQuotaUpdateMetrics(t *testing.T) { quota.scanMu.Lock() quota.scanUsed = 7 - quota.level = admissionPauseWarming quota.scanMu.Unlock() quota.eventNotifier.waiters.Store(2) t.Cleanup(func() { quota.eventNotifier.waiters.Store(0) }) @@ -84,12 +93,6 @@ func TestMemoryQuotaUpdateMetrics(t *testing.T) { metrics.LogPullerMemoryQuota.WithLabelValues("used"))) require.Equal(t, float64(7), testutil.ToFloat64( metrics.LogPullerMemoryQuota.WithLabelValues("scan_used"))) - require.Equal(t, float64(8), testutil.ToFloat64( - metrics.LogPullerMemoryQuota.WithLabelValues("scan_estimate"))) - require.Equal(t, float64(132), testutil.ToFloat64( - metrics.LogPullerMemoryQuota.WithLabelValues("hard_limit"))) - require.Equal(t, float64(admissionPauseWarming), - testutil.ToFloat64(metrics.LogPullerMemoryQuotaAdmissionLevel)) require.Equal(t, float64(2), testutil.ToFloat64(metrics.LogPullerMemoryQuotaEventWaiterCount)) } @@ -97,25 +100,31 @@ func TestMemoryQuotaUpdateMetrics(t *testing.T) { func TestMemoryQuotaAdmissionLevels(t *testing.T) { quota := newMemoryQuotaController(100, 10) warmingSpan := newTestQuotaSpan(1) - initializedSpan := newTestQuotaSpan(2) - initializedSpan.initialized.Store(true) - warmingTs := setTestQuotaSpanLag(warmingSpan, lowLagRegionThreshold+time.Minute) - initializedTs := setTestQuotaSpanLag(initializedSpan, lowLagRegionThreshold+time.Minute) - - require.True(t, quota.AcquireEvent(context.Background(), initializedSpan, 5)) - require.True(t, quota.AcquireEvent(context.Background(), initializedSpan, 10)) - _, _, admitted := quota.AcquireScan(newTestQuotaRegion(warmingSpan), warmingTs) + highPrioritySpan := newTestQuotaSpan(2) + warmingTs := setTestQuotaSpanLag(warmingSpan, time.Hour) + highPriorityTs := setTestQuotaSpanLag(highPrioritySpan, time.Hour) + + require.True(t, quota.AcquireEvent(context.Background(), highPrioritySpan, 5)) + require.True(t, quota.AcquireEvent(context.Background(), highPrioritySpan, 10)) + _, _, admitted := quota.AcquireScan( + newTestQuotaRegionWithPriority(warmingSpan, cdcpb.ScanPriority_SCAN_PRIORITY_LOW), + warmingTs, + ) require.False(t, admitted) scanBytes, _, admitted := quota.AcquireScan( - newTestQuotaRegion(initializedSpan), initializedTs) + newTestQuotaRegionWithPriority(highPrioritySpan, cdcpb.ScanPriority_SCAN_PRIORITY_HIGH), + highPriorityTs, + ) require.True(t, admitted) quota.ReleaseScan(scanBytes) - require.True(t, quota.AcquireEvent(context.Background(), initializedSpan, 45)) - require.True(t, quota.AcquireEvent(context.Background(), initializedSpan, 20)) + require.True(t, quota.AcquireEvent(context.Background(), highPrioritySpan, 45)) + require.True(t, quota.AcquireEvent(context.Background(), highPrioritySpan, 20)) scanBytes, _, admitted = quota.AcquireScan( - newTestQuotaRegion(initializedSpan), initializedTs) + newTestQuotaRegionWithPriority(highPrioritySpan, cdcpb.ScanPriority_SCAN_PRIORITY_HIGH), + highPriorityTs, + ) require.True(t, admitted) quota.ReleaseScan(scanBytes) @@ -139,7 +148,9 @@ func TestMemoryQuotaSpanStopKeepsOwnedMemoryUntilRelease(t *testing.T) { require.True(t, quota.AcquireEvent(context.Background(), span1, 30)) require.True(t, quota.AcquireEvent(context.Background(), span2, 40)) scanBytes, _, admitted := quota.AcquireScan( - newTestQuotaRegion(span1), span1.resolvedTs.Load()) + newTestQuotaRegionWithPriority(span1, cdcpb.ScanPriority_SCAN_PRIORITY_HIGH), + span1.resolvedTs.Load(), + ) require.True(t, admitted) require.NotZero(t, scanBytes) @@ -157,7 +168,9 @@ func TestMemoryQuotaSpanStopKeepsOwnedMemoryUntilRelease(t *testing.T) { // Late tasks reach the stopped-subscription cleanup path without consuming // scan quota. scanBytes, _, admitted = quota.AcquireScan( - newTestQuotaRegion(span1), span1.resolvedTs.Load()) + newTestQuotaRegionWithPriority(span1, cdcpb.ScanPriority_SCAN_PRIORITY_HIGH), + span1.resolvedTs.Load(), + ) require.True(t, admitted) require.Zero(t, scanBytes) @@ -288,8 +301,8 @@ func TestMemoryQuotaConcurrentWaitersDoNotLoseWakeups(t *testing.T) { func TestMemoryQuotaWarmingScanUsesCurrentPressure(t *testing.T) { quota := newMemoryQuotaController(100, 20) span := newTestQuotaSpan(1) - currentTs := setTestQuotaSpanLag(span, lowLagRegionThreshold+time.Minute) - region := newTestQuotaRegion(span) + currentTs := setTestQuotaSpanLag(span, time.Hour) + region := newTestQuotaRegionWithPriority(span, cdcpb.ScanPriority_SCAN_PRIORITY_LOW) bytes1, _, admitted := quota.AcquireScan(region, currentTs) require.True(t, admitted) @@ -309,10 +322,13 @@ func TestMemoryQuotaWarmingScanUsesCurrentPressure(t *testing.T) { func TestMemoryQuotaLowLagScanBypassesWarmingGate(t *testing.T) { quota := newMemoryQuotaController(100, 10) span := newTestQuotaSpan(1) - currentTs := setTestQuotaSpanLag(span, lowLagRegionThreshold-time.Second) + currentTs := setTestQuotaSpanLag(span, time.Minute) require.True(t, quota.AcquireEvent(context.Background(), span, 20)) - scanBytes, _, admitted := quota.AcquireScan(newTestQuotaRegion(span), currentTs) + scanBytes, _, admitted := quota.AcquireScan( + newTestQuotaRegionWithPriority(span, cdcpb.ScanPriority_SCAN_PRIORITY_HIGH), + currentTs, + ) require.True(t, admitted) require.NotZero(t, scanBytes) state := getMemoryQuotaTestState(quota) @@ -325,14 +341,14 @@ func TestMemoryQuotaLowLagScanBypassesWarmingGate(t *testing.T) { func TestAdmissionWaitsForMemoryAndReleasesScanMemory(t *testing.T) { quota := newMemoryQuotaController(100, 10) span := newTestQuotaSpan(1) - currentTs := setTestQuotaSpanLag(span, lowLagRegionThreshold+time.Minute) + currentTs := setTestQuotaSpanLag(span, time.Hour) controller := newRegionAdmissionController(1, 1, quota, func() uint64 { return currentTs }) require.True(t, quota.AcquireEvent(context.Background(), span, 20)) - region := newTestQuotaRegion(span) - require.True(t, controller.submit(newRegionPriorityTask(region, currentTs, 1))) + region := newTestQuotaRegionWithPriority(span, cdcpb.ScanPriority_SCAN_PRIORITY_LOW) + require.True(t, controller.submit(newRegionPriorityTask(region, 1))) type popResult struct { req *regionReq @@ -368,14 +384,14 @@ func TestAdmissionWaitsForMemoryAndReleasesScanMemory(t *testing.T) { func TestAdmissionWakesWhenBlockedSpanStops(t *testing.T) { quota := newMemoryQuotaController(100, 10) span := newTestQuotaSpan(1) - currentTs := setTestQuotaSpanLag(span, lowLagRegionThreshold+time.Minute) + currentTs := setTestQuotaSpanLag(span, time.Hour) controller := newRegionAdmissionController(1, 1, quota, func() uint64 { return currentTs }) require.True(t, quota.AcquireEvent(context.Background(), span, 20)) require.True(t, controller.submit(newRegionPriorityTask( - newTestQuotaRegion(span), currentTs, 1))) + newTestQuotaRegionWithPriority(span, cdcpb.ScanPriority_SCAN_PRIORITY_LOW), 1))) type popResult struct { req *regionReq diff --git a/logservice/logpuller/priority_task.go b/logservice/logpuller/priority_task.go index aad3aab16b..d5a5b034e4 100644 --- a/logservice/logpuller/priority_task.go +++ b/logservice/logpuller/priority_task.go @@ -14,14 +14,9 @@ package logpuller import ( - "time" - "github.com/pingcap/kvproto/pkg/cdcpb" - "github.com/tikv/client-go/v2/oracle" ) -const lowLagRegionThreshold = 30 * time.Minute - func normalizeScanPriority(priority cdcpb.ScanPriority) cdcpb.ScanPriority { if priority == cdcpb.ScanPriority_SCAN_PRIORITY_HIGH { return cdcpb.ScanPriority_SCAN_PRIORITY_HIGH @@ -75,12 +70,3 @@ func (pt *regionPriorityTask) LessThan(other *regionPriorityTask) bool { } return pt.sequence < other.sequence } - -func regionScanLag(currentTs, checkpointTs uint64) time.Duration { - currentTime := oracle.GetTimeFromTS(currentTs) - checkpointTime := oracle.GetTimeFromTS(checkpointTs) - if !currentTime.After(checkpointTime) { - return 0 - } - return currentTime.Sub(checkpointTime) -} diff --git a/logservice/logpuller/region_admission_controller_test.go b/logservice/logpuller/region_admission_controller_test.go index 770d849100..226a1148cb 100644 --- a/logservice/logpuller/region_admission_controller_test.go +++ b/logservice/logpuller/region_admission_controller_test.go @@ -20,6 +20,7 @@ import ( "testing" "time" + "github.com/pingcap/kvproto/pkg/cdcpb" "github.com/pingcap/ticdc/heartbeatpb" "github.com/pingcap/ticdc/logservice/logpuller/regionlock" "github.com/stretchr/testify/require" @@ -38,6 +39,7 @@ func createTestRegionInfo(subID SubscriptionID, regionID uint64) regionInfo { span, nil, &subscribedSpan{subID: subID, startTs: 100, span: span}, + false, ) } @@ -54,7 +56,7 @@ func submitRegionForAdmission( currentTs uint64, ) { t.Helper() - task := newRegionPriorityTask(region, currentTs, region.verID.GetID()) + task := newRegionPriorityTask(region, region.verID.GetID()) require.True(t, controller.submit(task)) } @@ -132,11 +134,10 @@ func TestRegionAdmissionControllerLowLagUsesMaxWindow(t *testing.T) { require.True(t, req3.abort()) } -func TestRegionAdmissionControllerPrioritizesInitializedRegion(t *testing.T) { +func TestRegionAdmissionControllerPrioritizesHighPriorityRegion(t *testing.T) { controller := newTestRegionAdmissionController(1, 2) currentTs := oracle.GoTimeToTS(time.Now()) slowCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Hour)) - lowLagCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Minute)) submitRegionForAdmission(t, controller, prepareRegionForAdmission(createTestRegionInfo(1, 1), slowCheckpointTs), @@ -145,12 +146,12 @@ func TestRegionAdmissionControllerPrioritizesInitializedRegion(t *testing.T) { require.NoError(t, err) submitRegionForAdmission(t, controller, - prepareRegionForAdmission(createTestRegionInfo(1, 2), lowLagCheckpointTs), + prepareRegionForAdmission(createTestRegionInfo(1, 2), slowCheckpointTs), currentTs) - initializedRegion := prepareRegionForAdmission(createTestRegionInfo(1, 3), slowCheckpointTs) - initializedRegion.wasInitialized = true + highPriorityRegion := prepareRegionForAdmission(createTestRegionInfo(1, 3), slowCheckpointTs) + highPriorityRegion.scanPriority = cdcpb.ScanPriority_SCAN_PRIORITY_HIGH submitRegionForAdmission(t, controller, - initializedRegion, currentTs) + highPriorityRegion, currentTs) req2, err := controller.pop(t.Context(), nil) require.NoError(t, err) @@ -204,7 +205,7 @@ func TestRegionAdmissionControllerClose(t *testing.T) { controller := newTestRegionAdmissionController(1, 1) controller.close() region := prepareRegionForAdmission(createTestRegionInfo(1, 1), 1) - require.False(t, controller.submit(newRegionPriorityTask(region, 1, 1))) + require.False(t, controller.submit(newRegionPriorityTask(region, 1))) _, err := controller.pop(context.Background(), nil) require.ErrorIs(t, err, context.Canceled) diff --git a/logservice/logpuller/region_event_handler.go b/logservice/logpuller/region_event_handler.go index 704e346505..cd536847de 100644 --- a/logservice/logpuller/region_event_handler.go +++ b/logservice/logpuller/region_event_handler.go @@ -303,7 +303,7 @@ func handleEventEntries(span *subscribedSpan, state *regionFeedState, entries *c for _, entry := range entries.Entries.GetEntries() { switch entry.Type { case cdcpb.Event_INITIALIZED: - state.setInitialized() + span.markRegionInitialized(state) log.Debug("region is initialized", zap.Int64("tableID", span.span.TableID), zap.Uint64("regionID", regionID), @@ -410,7 +410,6 @@ func handleResolvedTs(span *subscribedSpan, state *regionFeedState, resolvedTs u } if shouldAdvance { - span.tryMarkInitialized(regionID, ts) lastResolvedTs := span.resolvedTs.Load() nextResolvedPhyTs := oracle.ExtractPhysical(ts) // Generally, we don't want to send duplicate resolved ts, @@ -418,7 +417,16 @@ func handleResolvedTs(span *subscribedSpan, state *regionFeedState, resolvedTs u // but when `ts` == `lastResolvedTs` == `span.startTs`, // the span may just be initialized and have not receive any resolved ts before, // so we also send ts in this case for quick notification to downstream. - if ts > lastResolvedTs || (ts == lastResolvedTs && lastResolvedTs == span.startTs) { + if ts > lastResolvedTs || + (span.initialized.Load() && ts == lastResolvedTs && lastResolvedTs == span.startTs) { + if lastResolvedTs == span.startTs && ts > span.startTs && !span.initialized.Load() { + log.Warn("should not happen: resolved ts advances before span is initialized", + zap.Uint64("subscriptionID", uint64(span.subID)), + zap.Int64("tableID", span.span.TableID), + zap.Uint64("regionID", regionID), + zap.Uint64("startTs", span.startTs), + zap.Uint64("resolvedTs", ts)) + } resolvedPhyTs := oracle.ExtractPhysical(lastResolvedTs) decreaseLag := float64(nextResolvedPhyTs-resolvedPhyTs) / 1e3 const largeResolvedTsAdvanceStepInSecs = 30 @@ -431,8 +439,7 @@ func handleResolvedTs(span *subscribedSpan, state *regionFeedState, resolvedTs u zap.Uint64("lastResolvedTs", lastResolvedTs), zap.Float64("decreaseLag(s)", decreaseLag)) } - span.resolvedTs.Store(ts) - span.resolvedTsUpdated.Store(time.Now().Unix()) + span.recordResolvedTs(ts) return ts } } diff --git a/logservice/logpuller/region_event_handler_test.go b/logservice/logpuller/region_event_handler_test.go index 7c3db0c2cd..1334acd2b6 100644 --- a/logservice/logpuller/region_event_handler_test.go +++ b/logservice/logpuller/region_event_handler_test.go @@ -88,6 +88,7 @@ func TestHandleEventEntryEventOutOfOrder(t *testing.T) { span, &tikv.RPCContext{}, subSpan, + false, ) region.lockedRangeState = ®ionlock.LockedRangeState{} state := newRegionFeedState(region, 1, worker, nil) @@ -424,13 +425,40 @@ func TestHandleEntriesReleasesMemoryAfterDownstreamCallback(t *testing.T) { require.Zero(t, quotaState.used) } -func TestTryMarkSpanInitializedByResolvedTs(t *testing.T) { - span := &subscribedSpan{subID: 1, startTs: 100} - require.False(t, span.tryMarkInitialized(1, 100)) +func TestSpanInitializedAfterFullRangeCoverage(t *testing.T) { + const startTs = 100 + span := &subscribedSpan{ + subID: 1, + startTs: startTs, + span: heartbeatpb.TableSpan{ + StartKey: []byte("a"), + EndKey: []byte("z"), + }, + } + firstState := newRegionFeedState(regionInfo{ + verID: tikv.NewRegionVerID(1, 1, 1), + span: heartbeatpb.TableSpan{ + StartKey: []byte("a"), + EndKey: []byte("m"), + }, + subscribedSpan: span, + lockedRangeState: ®ionlock.LockedRangeState{}, + }, uint64(span.subID), ®ionRequestWorker{}, nil) + secondState := newRegionFeedState(regionInfo{ + verID: tikv.NewRegionVerID(2, 1, 1), + span: heartbeatpb.TableSpan{ + StartKey: []byte("m"), + EndKey: []byte("z"), + }, + subscribedSpan: span, + lockedRangeState: ®ionlock.LockedRangeState{}, + }, uint64(span.subID), ®ionRequestWorker{}, nil) + + span.markRegionInitialized(firstState) require.False(t, span.initialized.Load()) - require.True(t, span.tryMarkInitialized(1, 101)) + + span.markRegionInitialized(secondState) require.True(t, span.initialized.Load()) - require.False(t, span.tryMarkInitialized(1, 102)) } func TestSpanInitializationNotifiesMemoryAdmission(t *testing.T) { @@ -443,11 +471,11 @@ func TestSpanInitializationNotifiesMemoryAdmission(t *testing.T) { rangeLock := regionlock.NewRangeLock(1, []byte("a"), []byte("z"), startTs) lockResult := rangeLock.LockRange(t.Context(), []byte("a"), []byte("z"), 1, 1) require.Equal(t, regionlock.LockRangeStatusSuccess, lockResult.Status) - lockResult.LockedRangeState.Initialized.Store(true) span := &subscribedSpan{ subID: 1, startTs: startTs, + span: heartbeatpb.TableSpan{StartKey: []byte("a"), EndKey: []byte("z")}, rangeLock: rangeLock, consumeKVEvents: func([]common.RawKVEntry, func()) bool { return false }, advanceResolvedTs: func(uint64) {}, @@ -455,14 +483,17 @@ func TestSpanInitializationNotifiesMemoryAdmission(t *testing.T) { span.resolvedTs.Store(startTs) state := newRegionFeedState(regionInfo{ verID: tikv.NewRegionVerID(1, 1, 1), + span: span.span, subscribedSpan: span, lockedRangeState: lockResult.LockedRangeState, }, uint64(span.subID), ®ionRequestWorker{}, nil) handler := ®ionEventHandler{eventSink: ®ionEventSink{memoryQuota: quota}} require.False(t, handler.Handle(span, regionEvent{ - states: []*regionFeedState{state}, - resolvedTs: startTs + 1, + states: []*regionFeedState{state}, + entries: &cdcpb.Event_Entries_{Entries: &cdcpb.Event_Entries{ + Entries: []*cdcpb.Event_Row{{Type: cdcpb.Event_INITIALIZED}}, + }}, })) require.True(t, span.initialized.Load()) select { diff --git a/logservice/logpuller/region_request_scheduler_test.go b/logservice/logpuller/region_request_scheduler_test.go index 86b6dd37be..7ae501c8f1 100644 --- a/logservice/logpuller/region_request_scheduler_test.go +++ b/logservice/logpuller/region_request_scheduler_test.go @@ -35,12 +35,12 @@ func TestRegionRequestSchedulerBroadcastDeregisterUsesWorkerControlQueue(t *test worker1 := ®ionRequestWorker{ storeAddr: "store-1", - admission: newRegionAdmissionController(1, 1), + admission: newTestRegionAdmissionController(1, 1), controlQueue: newControlQueue(), } worker2 := ®ionRequestWorker{ storeAddr: "store-2", - admission: newRegionAdmissionController(1, 1), + admission: newTestRegionAdmissionController(1, 1), controlQueue: newControlQueue(), } store1 := ®ionRequestStore{workers: []*regionRequestWorker{worker1}} @@ -72,8 +72,8 @@ func TestRegionRequestSchedulerBroadcastDeregisterUsesWorkerControlQueue(t *test func TestRegionRequestSchedulerRequestedRegionCountAggregatesStores(t *testing.T) { scheduler := ®ionRequestScheduler{} - worker1 := ®ionRequestWorker{admission: newRegionAdmissionController(1, 1)} - worker2 := ®ionRequestWorker{admission: newRegionAdmissionController(1, 1)} + worker1 := ®ionRequestWorker{admission: newTestRegionAdmissionController(2, 1)} + worker2 := ®ionRequestWorker{admission: newTestRegionAdmissionController(2, 1)} scheduler.stores.Store("store-1", ®ionRequestStore{workers: []*regionRequestWorker{worker1}}) scheduler.stores.Store("store-2", ®ionRequestStore{workers: []*regionRequestWorker{worker2}}) @@ -125,7 +125,7 @@ func TestRegionRequestSchedulerReschedulesRegionWhenStoreSubmitFails(t *testing. context.Background(), rawSpan.StartKey, rawSpan.EndKey, location.Region.GetID(), location.Region.GetVer()) require.Equal(t, regionlock.LockRangeStatusSuccess, lockRes.Status) - admission := newRegionAdmissionController(1, 1) + admission := newTestRegionAdmissionController(1, 1) admission.close() store := ®ionRequestStore{workers: []*regionRequestWorker{{admission: admission}}} diff --git a/logservice/logpuller/region_request_store_test.go b/logservice/logpuller/region_request_store_test.go index abc84add3e..e9d470e633 100644 --- a/logservice/logpuller/region_request_store_test.go +++ b/logservice/logpuller/region_request_store_test.go @@ -22,8 +22,8 @@ import ( ) func TestRegionRequestStoreDistributesRegionsAcrossWorkers(t *testing.T) { - worker1 := ®ionRequestWorker{admission: newRegionAdmissionController(1, 1)} - worker2 := ®ionRequestWorker{admission: newRegionAdmissionController(1, 1)} + worker1 := ®ionRequestWorker{admission: newTestRegionAdmissionController(1, 1)} + worker2 := ®ionRequestWorker{admission: newTestRegionAdmissionController(1, 1)} store := ®ionRequestStore{ workers: []*regionRequestWorker{worker1, worker2}, } @@ -39,8 +39,8 @@ func TestRegionRequestStoreDistributesRegionsAcrossWorkers(t *testing.T) { } func TestRegionRequestStoreRequestedRegionCountIncludesPendingAndInflight(t *testing.T) { - worker1 := ®ionRequestWorker{admission: newRegionAdmissionController(1, 1)} - worker2 := ®ionRequestWorker{admission: newRegionAdmissionController(1, 1)} + worker1 := ®ionRequestWorker{admission: newTestRegionAdmissionController(2, 1)} + worker2 := ®ionRequestWorker{admission: newTestRegionAdmissionController(2, 1)} store := ®ionRequestStore{ workers: []*regionRequestWorker{worker1, worker2}, } @@ -59,8 +59,8 @@ func TestRegionRequestStoreRequestedRegionCountIncludesPendingAndInflight(t *tes } func TestRegionRequestStoreCloseClosesWorkerAdmissions(t *testing.T) { - worker1 := ®ionRequestWorker{admission: newRegionAdmissionController(1, 1)} - worker2 := ®ionRequestWorker{admission: newRegionAdmissionController(1, 1)} + worker1 := ®ionRequestWorker{admission: newTestRegionAdmissionController(1, 1)} + worker2 := ®ionRequestWorker{admission: newTestRegionAdmissionController(1, 1)} store := ®ionRequestStore{ workers: []*regionRequestWorker{worker1, worker2}, } diff --git a/logservice/logpuller/region_request_worker_test.go b/logservice/logpuller/region_request_worker_test.go index 32228d8e7c..d647606e2e 100644 --- a/logservice/logpuller/region_request_worker_test.go +++ b/logservice/logpuller/region_request_worker_test.go @@ -272,7 +272,7 @@ func errCacheLen(handler *regionFailureHandler) int { } func TestRegionRequestWorkerIgnoresDuplicateActiveRegion(t *testing.T) { - admission := newRegionAdmissionController(10, 1) + admission := newTestRegionAdmissionController(10, 1) worker := ®ionRequestWorker{ admission: admission, storeAddr: "store-1", @@ -431,7 +431,7 @@ func benchmarkDispatchResolvedTsEvent(b *testing.B, regionCount int, useLegacy b } func TestWaitForRegionRequestDrainsIdleControlQueue(t *testing.T) { - admission := newRegionAdmissionController(1, 1) + admission := newTestRegionAdmissionController(1, 1) worker := ®ionRequestWorker{ admission: admission, controlQueue: newControlQueue(), @@ -513,7 +513,7 @@ func BenchmarkDispatchResolvedTsEventSmallBatchCurrent(b *testing.B) { } func TestStoppedStateRemovesSentRequest(t *testing.T) { - admission := newRegionAdmissionController(10, 1) + admission := newTestRegionAdmissionController(10, 1) worker := ®ionRequestWorker{ admission: admission, tracker: newRegionTracker(), @@ -539,7 +539,7 @@ func TestRunStreamFailurePushesTrackedRegionToEventSink(t *testing.T) { upstream: &upstreamHandle{pd: pdClient, credential: &security.Credential{}}, eventSink: ®ionEventSink{ds: ds}, failureHandler: handler, - admission: newRegionAdmissionController(10, 1), + admission: newTestRegionAdmissionController(10, 1), controlQueue: newControlQueue(), tracker: newRegionTracker(), storeAddr: "127.0.0.1:1", @@ -589,7 +589,7 @@ func TestRunStreamFailureReportsPendingRegionsToFailureHandler(t *testing.T) { upstream: &upstreamHandle{pd: pdClient, credential: &security.Credential{}}, eventSink: ®ionEventSink{ds: &mockDynamicStream{}}, failureHandler: handler, - admission: newRegionAdmissionController(10, 1), + admission: newTestRegionAdmissionController(10, 1), controlQueue: newControlQueue(), tracker: newRegionTracker(), storeAddr: "127.0.0.1:1", @@ -618,7 +618,7 @@ func TestRunStreamFailureReportsPendingRegionsToFailureHandler(t *testing.T) { } func TestProcessRegionSendTaskSendFailureCleansSentRequest(t *testing.T) { - admission := newRegionAdmissionController(10, 1) + admission := newTestRegionAdmissionController(10, 1) worker := ®ionRequestWorker{ admission: admission, controlQueue: newControlQueue(), @@ -649,7 +649,7 @@ func TestProcessRegionSendTaskSendFailureCleansSentRequest(t *testing.T) { } func TestProcessRegionSendTaskDoesNotSendRemovedRequest(t *testing.T) { - admission := newRegionAdmissionController(1, 1) + admission := newTestRegionAdmissionController(1, 1) worker := ®ionRequestWorker{ admission: admission, controlQueue: newControlQueue(), @@ -697,7 +697,7 @@ func TestProcessRegionSendTaskSendEOFIsRetriable(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - admission := newRegionAdmissionController(10, 1) + admission := newTestRegionAdmissionController(10, 1) worker := ®ionRequestWorker{ admission: admission, controlQueue: newControlQueue(), @@ -732,7 +732,7 @@ func TestProcessRegionSendTaskSendEOFIsRetriable(t *testing.T) { func TestProcessRegionSendTaskHandlesDeregisterFromControlQueue(t *testing.T) { ds := &mockRegionEventDynamicStream{} worker := ®ionRequestWorker{ - admission: newRegionAdmissionController(1, 1), + admission: newTestRegionAdmissionController(1, 1), controlQueue: newControlQueue(), storeAddr: "store-1", upstream: &upstreamHandle{clusterID: 42}, diff --git a/logservice/logpuller/span_registry.go b/logservice/logpuller/span_registry.go index 5c8f70fba8..797e42d833 100644 --- a/logservice/logpuller/span_registry.go +++ b/logservice/logpuller/span_registry.go @@ -190,18 +190,6 @@ func (span *subscribedSpan) resolveStaleLocks(targetTs uint64) { zap.Any("ranges", res)) } -func (span *subscribedSpan) tryMarkInitialized(regionID, resolvedTs uint64) bool { - if resolvedTs <= span.startTs || !span.initialized.CompareAndSwap(false, true) { - return false - } - log.Info("subscription client is initialized", - zap.Uint64("subscriptionID", uint64(span.subID)), - zap.Uint64("regionID", regionID), - zap.Uint64("resolvedTs", resolvedTs), - zap.Uint64("startTs", span.startTs)) - return true -} - func newSpanRegistry(pd pd.Client, pdClock pdutil.Clock) *spanRegistry { return &spanRegistry{ spans: make(map[SubscriptionID]*subscribedSpan), diff --git a/logservice/logpuller/subscription_client_test.go b/logservice/logpuller/subscription_client_test.go index 818a745e54..4c7f686af6 100644 --- a/logservice/logpuller/subscription_client_test.go +++ b/logservice/logpuller/subscription_client_test.go @@ -424,27 +424,49 @@ func (s *mockDynamicStream) GetMetrics() dynstream.Metrics[int, SubscriptionID] } func TestRegionEventSinkPushUnblocksOnClientClose(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + quota := newMemoryQuotaController(10, 8) + span := &subscribedSpan{subID: 1} + require.True(t, quota.AcquireEvent(ctx, span, 20)) + t.Cleanup(func() { quota.ReleaseEvent(20) }) + sink := ®ionEventSink{ - ds: &mockDynamicStream{}, + ctx: ctx, + ds: &mockDynamicStream{}, + memoryQuota: quota, } - sink.cond = sync.NewCond(&sink.mu) client := &subscriptionClient{eventSink: sink} client.regionScheduler = ®ionRequestScheduler{ taskQueue: priorityqueue.New[*regionPriorityTask](), } - client.ctx, client.cancel = context.WithCancel(context.Background()) + client.ctx = ctx + client.cancel = cancel - sink.paused.Store(true) + event := regionEvent{ + states: []*regionFeedState{{ + region: regionInfo{subscribedSpan: span}, + }}, + entries: &cdcpb.Event_Entries_{ + Entries: &cdcpb.Event_Entries{ + Entries: []*cdcpb.Event_Row{{ + Key: []byte("key"), + Value: []byte("value"), + }}, + }, + }, + } done := make(chan struct{}) go func() { - sink.Push(SubscriptionID(1), regionEvent{}) + sink.Push(SubscriptionID(1), event) close(done) }() select { case <-done: - t.Fatal("regionEventSink.Push should block when paused") + t.Fatal("regionEventSink.Push should block when event memory is exhausted") case <-time.After(100 * time.Millisecond): } diff --git a/pkg/metrics/log_puller.go b/pkg/metrics/log_puller.go index aa45134c9f..88504eb9a3 100644 --- a/pkg/metrics/log_puller.go +++ b/pkg/metrics/log_puller.go @@ -71,13 +71,6 @@ var ( Name: "memory_quota", Help: "The log puller local memory quota usage.", }, []string{"type"}) - LogPullerMemoryQuotaAdmissionLevel = prometheus.NewGauge( - prometheus.GaugeOpts{ - Namespace: "ticdc", - Subsystem: "log_puller", - Name: "memory_quota_admission_level", - Help: "The log puller scan admission level: 0 normal, 1 pause warming scans, 2 pause all scans.", - }) LogPullerMemoryQuotaEventWaiterCount = prometheus.NewGauge( prometheus.GaugeOpts{ Namespace: "ticdc", @@ -186,7 +179,6 @@ func initLogPullerMetrics(registry *prometheus.Registry) { registry.MustRegister(LogPullerMatcherCount) registry.MustRegister(LogPullerResolvedTsLag) registry.MustRegister(LogPullerMemoryQuota) - registry.MustRegister(LogPullerMemoryQuotaAdmissionLevel) registry.MustRegister(LogPullerMemoryQuotaEventWaiterCount) registry.MustRegister(LogPullerMemoryQuotaEventWaitDuration) registry.MustRegister(SubscriptionClientRequestedRegionCount) From beca99c24725325535f7843a45f0277fe10ff508 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Tue, 4 Aug 2026 21:42:53 +0800 Subject: [PATCH 12/26] small fix --- logservice/logpuller/region_event_handler.go | 1 - logservice/logpuller/region_request_worker.go | 5 ++++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/logservice/logpuller/region_event_handler.go b/logservice/logpuller/region_event_handler.go index cd536847de..0ae78d097c 100644 --- a/logservice/logpuller/region_event_handler.go +++ b/logservice/logpuller/region_event_handler.go @@ -29,7 +29,6 @@ import ( ) var ( - metricsResolvedTsCount = metrics.PullerEventCounter.WithLabelValues("resolved_ts") metricsEventCount = metrics.PullerEventCounter.WithLabelValues("event") metricRegionEventHandleDurationEntries = metrics.SubscriptionClientRegionEventHandleDuration.WithLabelValues("entries") diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index 176c49cfea..2ad23f0c45 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -39,7 +39,10 @@ const storeReconnectBackoff = time.Second // To generate a workerID in `newRegionRequestWorker`. var workerIDGen atomic.Uint64 -var metricBatchResolvedSize = metrics.BatchResolvedEventSize.WithLabelValues("event-store") +var ( + metricsResolvedTsCount = metrics.PullerEventCounter.WithLabelValues("resolved_ts") + metricBatchResolvedSize = metrics.BatchResolvedEventSize.WithLabelValues("event-store") +) type deregisterRequest struct { subID SubscriptionID From 492fad5bdd7921f8bff9bcfca030269232d50014 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Tue, 4 Aug 2026 22:05:59 +0800 Subject: [PATCH 13/26] more fix --- logservice/logpuller/memory_quota_test.go | 13 ++--- .../logpuller/region_admission_controller.go | 11 +++-- .../region_admission_controller_test.go | 4 +- logservice/logpuller/region_event_handler.go | 6 +-- .../logpuller/region_event_handler_test.go | 47 ++----------------- logservice/logpuller/region_request_worker.go | 4 +- 6 files changed, 24 insertions(+), 61 deletions(-) diff --git a/logservice/logpuller/memory_quota_test.go b/logservice/logpuller/memory_quota_test.go index a808ec5be1..1fe8226e51 100644 --- a/logservice/logpuller/memory_quota_test.go +++ b/logservice/logpuller/memory_quota_test.go @@ -22,6 +22,7 @@ import ( "github.com/pingcap/kvproto/pkg/cdcpb" "github.com/pingcap/ticdc/logservice/logpuller/regionlock" "github.com/pingcap/ticdc/pkg/metrics" + "github.com/pingcap/ticdc/pkg/pdutil" "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/require" "github.com/tikv/client-go/v2/oracle" @@ -342,9 +343,9 @@ func TestAdmissionWaitsForMemoryAndReleasesScanMemory(t *testing.T) { quota := newMemoryQuotaController(100, 10) span := newTestQuotaSpan(1) currentTs := setTestQuotaSpanLag(span, time.Hour) - controller := newRegionAdmissionController(1, 1, quota, func() uint64 { - return currentTs - }) + clock := pdutil.NewClock4Test().(*pdutil.Clock4Test) + clock.SetTS(currentTs) + controller := newRegionAdmissionController(1, 1, quota, clock) require.True(t, quota.AcquireEvent(context.Background(), span, 20)) region := newTestQuotaRegionWithPriority(span, cdcpb.ScanPriority_SCAN_PRIORITY_LOW) @@ -385,9 +386,9 @@ func TestAdmissionWakesWhenBlockedSpanStops(t *testing.T) { quota := newMemoryQuotaController(100, 10) span := newTestQuotaSpan(1) currentTs := setTestQuotaSpanLag(span, time.Hour) - controller := newRegionAdmissionController(1, 1, quota, func() uint64 { - return currentTs - }) + clock := pdutil.NewClock4Test().(*pdutil.Clock4Test) + clock.SetTS(currentTs) + controller := newRegionAdmissionController(1, 1, quota, clock) require.True(t, quota.AcquireEvent(context.Background(), span, 20)) require.True(t, controller.submit(newRegionPriorityTask( diff --git a/logservice/logpuller/region_admission_controller.go b/logservice/logpuller/region_admission_controller.go index 1e7664473e..63f3a150fc 100644 --- a/logservice/logpuller/region_admission_controller.go +++ b/logservice/logpuller/region_admission_controller.go @@ -22,6 +22,7 @@ import ( "github.com/pingcap/log" "github.com/pingcap/ticdc/pkg/metrics" + "github.com/pingcap/ticdc/pkg/pdutil" "github.com/pingcap/ticdc/utils/heap" "go.uber.org/zap" ) @@ -100,9 +101,9 @@ type regionAdmissionController struct { } // memoryQuota gates initial scans using the log puller's global memory - // pressure. currentTs is sampled when a request is admitted. + // pressure. pdClock is sampled when a request is admitted. memoryQuota *memoryQuotaController - currentTs func() uint64 + pdClock pdutil.Clock // notify wakes workers when a request is submitted or an admission slot is // released. The one-element buffer prevents a wakeup from being lost between // checking the admission condition and waiting on this channel. Notifications @@ -120,7 +121,7 @@ func newRegionAdmissionController( currentWindow int, maxWindowMultiplier int, memoryQuota *memoryQuotaController, - currentTs func() uint64, + pdClock pdutil.Clock, ) *regionAdmissionController { if currentWindow <= 0 { currentWindow = 1 @@ -136,7 +137,7 @@ func newRegionAdmissionController( currentWindow: currentWindow, maxWindow: maxWindow, memoryQuota: memoryQuota, - currentTs: currentTs, + pdClock: pdClock, notify: make(chan struct{}, 1), } controller.state.pending = heap.NewHeap[*regionPriorityTask]() @@ -205,7 +206,7 @@ func (c *regionAdmissionController) popEligibleLocked() ( } scanBytes, memoryReady, admitted := c.memoryQuota.AcquireScan( - request.regionInfo, c.currentTs()) + request.regionInfo, c.pdClock.CurrentTS()) if !admitted { return nil, 0, memoryReady } diff --git a/logservice/logpuller/region_admission_controller_test.go b/logservice/logpuller/region_admission_controller_test.go index 226a1148cb..485b527c63 100644 --- a/logservice/logpuller/region_admission_controller_test.go +++ b/logservice/logpuller/region_admission_controller_test.go @@ -23,6 +23,7 @@ import ( "github.com/pingcap/kvproto/pkg/cdcpb" "github.com/pingcap/ticdc/heartbeatpb" "github.com/pingcap/ticdc/logservice/logpuller/regionlock" + "github.com/pingcap/ticdc/pkg/pdutil" "github.com/stretchr/testify/require" "github.com/tikv/client-go/v2/oracle" "github.com/tikv/client-go/v2/tikv" @@ -64,11 +65,12 @@ func newTestRegionAdmissionController( currentWindow int, maxWindowMultiplier int, ) *regionAdmissionController { + clock := pdutil.NewClock4Test() return newRegionAdmissionController( currentWindow, maxWindowMultiplier, newMemoryQuotaController(1024*1024*1024, 8*1024*1024), - func() uint64 { return 0 }, + clock, ) } diff --git a/logservice/logpuller/region_event_handler.go b/logservice/logpuller/region_event_handler.go index 0ae78d097c..359e915675 100644 --- a/logservice/logpuller/region_event_handler.go +++ b/logservice/logpuller/region_event_handler.go @@ -29,7 +29,7 @@ import ( ) var ( - metricsEventCount = metrics.PullerEventCounter.WithLabelValues("event") + metricsEventCount = metrics.PullerEventCounter.WithLabelValues("event") metricRegionEventHandleDurationEntries = metrics.SubscriptionClientRegionEventHandleDuration.WithLabelValues("entries") metricRegionEventHandleDurationResolved = metrics.SubscriptionClientRegionEventHandleDuration.WithLabelValues("resolved") @@ -125,7 +125,6 @@ func (h *regionEventHandler) Handle(span *subscribedSpan, events ...regionEvent) } newResolvedTs := uint64(0) - wasInitialized := span.initialized.Load() memoryBytes := uint64(0) for _, event := range events { memoryBytes += event.memoryBytes @@ -149,9 +148,6 @@ func (h *regionEventHandler) Handle(span *subscribedSpan, events ...regionEvent) log.Panic("should not reach", zap.Any("event", event), zap.Any("events", events)) } } - if !wasInitialized && span.initialized.Load() { - h.eventSink.memoryQuota.NotifyScanAdmission() - } tryAdvanceResolvedTs := func() { if newResolvedTs != 0 { span.advanceResolvedTs(newResolvedTs) diff --git a/logservice/logpuller/region_event_handler_test.go b/logservice/logpuller/region_event_handler_test.go index 1334acd2b6..38a2b50f2e 100644 --- a/logservice/logpuller/region_event_handler_test.go +++ b/logservice/logpuller/region_event_handler_test.go @@ -23,6 +23,7 @@ import ( "github.com/pingcap/ticdc/heartbeatpb" "github.com/pingcap/ticdc/logservice/logpuller/regionlock" "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/pdutil" "github.com/pingcap/ticdc/utils/dynstream" "github.com/stretchr/testify/require" "github.com/tikv/client-go/v2/tikv" @@ -238,6 +239,7 @@ func TestHandleResolvedTs(t *testing.T) { consumeKVEvents: consumeKVEvents, advanceResolvedTs: advanceResolvedTs, advanceInterval: 0, + priorityPolicy: newScanPriorityPolicy(pdutil.NewClock4Test(), 30*time.Minute), } ds.AddPath(subID1, subSpan, dynstream.AreaSettings{}) state1.region.subscribedSpan = subSpan @@ -261,6 +263,7 @@ func TestHandleResolvedTs(t *testing.T) { consumeKVEvents: consumeKVEvents, advanceResolvedTs: advanceResolvedTs, advanceInterval: 0, + priorityPolicy: newScanPriorityPolicy(pdutil.NewClock4Test(), 30*time.Minute), } ds.AddPath(subID2, subSpan, dynstream.AreaSettings{}) state2.region.subscribedSpan = subSpan @@ -284,6 +287,7 @@ func TestHandleResolvedTs(t *testing.T) { consumeKVEvents: consumeKVEvents, advanceResolvedTs: advanceResolvedTs, advanceInterval: 0, + priorityPolicy: newScanPriorityPolicy(pdutil.NewClock4Test(), 30*time.Minute), } ds.AddPath(subID3, subSpan, dynstream.AreaSettings{}) state3.region.subscribedSpan = subSpan @@ -361,6 +365,7 @@ func TestHandleResolvedTsThrottled(t *testing.T) { subID: SubscriptionID(1), rangeLock: l, advanceInterval: 100, + priorityPolicy: newScanPriorityPolicy(pdutil.NewClock4Test(), 30*time.Minute), } span.lastAdvanceTime.Store(0) worker := ®ionRequestWorker{tracker: newRegionTracker()} @@ -460,45 +465,3 @@ func TestSpanInitializedAfterFullRangeCoverage(t *testing.T) { span.markRegionInitialized(secondState) require.True(t, span.initialized.Load()) } - -func TestSpanInitializationNotifiesMemoryAdmission(t *testing.T) { - quota := newMemoryQuotaController(1024, 8) - quota.scanMu.Lock() - notified := quota.scanReady - quota.scanMu.Unlock() - - const startTs = 100 - rangeLock := regionlock.NewRangeLock(1, []byte("a"), []byte("z"), startTs) - lockResult := rangeLock.LockRange(t.Context(), []byte("a"), []byte("z"), 1, 1) - require.Equal(t, regionlock.LockRangeStatusSuccess, lockResult.Status) - - span := &subscribedSpan{ - subID: 1, - startTs: startTs, - span: heartbeatpb.TableSpan{StartKey: []byte("a"), EndKey: []byte("z")}, - rangeLock: rangeLock, - consumeKVEvents: func([]common.RawKVEntry, func()) bool { return false }, - advanceResolvedTs: func(uint64) {}, - } - span.resolvedTs.Store(startTs) - state := newRegionFeedState(regionInfo{ - verID: tikv.NewRegionVerID(1, 1, 1), - span: span.span, - subscribedSpan: span, - lockedRangeState: lockResult.LockedRangeState, - }, uint64(span.subID), ®ionRequestWorker{}, nil) - handler := ®ionEventHandler{eventSink: ®ionEventSink{memoryQuota: quota}} - - require.False(t, handler.Handle(span, regionEvent{ - states: []*regionFeedState{state}, - entries: &cdcpb.Event_Entries_{Entries: &cdcpb.Event_Entries{ - Entries: []*cdcpb.Event_Row{{Type: cdcpb.Event_INITIALIZED}}, - }}, - })) - require.True(t, span.initialized.Load()) - select { - case <-notified: - case <-time.After(time.Second): - t.Fatal("span initialization did not notify memory admission") - } -} diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index 2ad23f0c45..3943c067fb 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -40,7 +40,7 @@ const storeReconnectBackoff = time.Second var workerIDGen atomic.Uint64 var ( - metricsResolvedTsCount = metrics.PullerEventCounter.WithLabelValues("resolved_ts") + metricsResolvedTsCount = metrics.PullerEventCounter.WithLabelValues("resolved_ts") metricBatchResolvedSize = metrics.BatchResolvedEventSize.WithLabelValues("event-store") ) @@ -125,7 +125,7 @@ func newRegionRequestWorker( currentWindow, maxWindowMultiplier, memoryQuota, - upstream.pdClock.CurrentTS, + upstream.pdClock, ), controlQueue: newControlQueue(), tracker: newRegionTracker(), From b7ebb2be3a7d9f6497fb102047506a4f7f0a3c88 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Tue, 4 Aug 2026 22:15:28 +0800 Subject: [PATCH 14/26] more refactor --- logservice/logpuller/memory_quota.go | 44 +++++++++++------------ logservice/logpuller/memory_quota_test.go | 20 +++++------ 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/logservice/logpuller/memory_quota.go b/logservice/logpuller/memory_quota.go index 29b28874fa..88dcf170f5 100644 --- a/logservice/logpuller/memory_quota.go +++ b/logservice/logpuller/memory_quota.go @@ -26,13 +26,13 @@ import ( ) const ( - // defaultPauseWarmingRatio pauses new high-lag scans when memory pressure - // reaches 15% of the soft capacity. - defaultPauseWarmingRatio = 0.15 + // defaultPauseLowPriorityRatio pauses new low-priority scans when memory + // pressure reaches 15% of the soft capacity. + defaultPauseLowPriorityRatio = 0.15 - // defaultResumeWarmingRatio resumes high-lag scans after memory pressure - // falls to 5% of the soft capacity. - defaultResumeWarmingRatio = 0.05 + // defaultResumeLowPriorityRatio resumes low-priority scans after memory + // pressure falls to 5% of the soft capacity. + defaultResumeLowPriorityRatio = 0.05 // defaultHardLimitRatio blocks receiving more events when accounted event // memory reaches twice the soft capacity. @@ -52,7 +52,7 @@ type admissionLevel uint8 const ( admissionNormal admissionLevel = iota - admissionPauseWarming + admissionPauseLowPriority ) // eventMemoryNotifier wakes event receivers that are waiting for memory. Each @@ -129,9 +129,9 @@ func (n *eventMemoryNotifier) notify() { // Initial scans are charged by estimate instead of measured bytes. Each // admitted scan starts from scanBaseSize, grows logarithmically with scan lag, // and is capped at maxScanLagFactor times the base size. Scan admission -// compares max(event used, scan used) with the soft capacity: high-lag -// "warming" scans pause at pauseWarmingLimit and resume at -// resumeWarmingLimit, while lower-lag scans continue to make progress. +// compares max(event used, scan used) with the soft capacity: low-priority +// scans pause at pauseLowPriorityLimit and resume at +// resumeLowPriorityLimit, while high-priority scans continue to make progress. type memoryQuotaController struct { capacity uint64 // used tracks event bytes retained until downstream finishes consuming them. @@ -154,8 +154,8 @@ type memoryQuotaController struct { // synchronous broadcast to every store and request worker. scanReady chan struct{} - pauseWarmingLimit uint64 - resumeWarmingLimit uint64 + pauseLowPriorityLimit uint64 + resumeLowPriorityLimit uint64 hardLimit uint64 scanEstimate uint64 @@ -165,8 +165,8 @@ func newMemoryQuotaController(capacity, scanBaseSize uint64) *memoryQuotaControl c := &memoryQuotaController{ capacity: capacity, level: admissionNormal, - pauseWarmingLimit: uint64(math.Ceil(float64(capacity) * defaultPauseWarmingRatio)), - resumeWarmingLimit: uint64(float64(capacity) * defaultResumeWarmingRatio), + pauseLowPriorityLimit: uint64(math.Ceil(float64(capacity) * defaultPauseLowPriorityRatio)), + resumeLowPriorityLimit: uint64(float64(capacity) * defaultResumeLowPriorityRatio), hardLimit: uint64(float64(capacity) * defaultHardLimitRatio), scanEstimate: scanBaseSize, eventNotifier: newEventMemoryNotifier(), @@ -196,10 +196,10 @@ func (c *memoryQuotaController) AcquireScan( c.scanMu.Lock() defer c.scanMu.Unlock() c.refreshLevelLocked() - warming := isWarmingScan(region, currentTs) + lowPriority := isLowPriorityScan(region, currentTs) // Admission is based on the pressure before accounting this scan. This lets // one scan make progress even when its estimate alone exceeds the threshold. - if warming && c.level == admissionPauseWarming { + if lowPriority && c.level == admissionPauseLowPriority { return 0, c.scanReady, false } bytes = c.estimateScanSizeLocked(region, currentTs) @@ -268,7 +268,7 @@ func (c *memoryQuotaController) ReleaseEvent(bytes uint64) { } used := c.used.Add(^(bytes - 1)) previousUsed := used + bytes - if crossesDown(previousUsed, used, c.resumeWarmingLimit) { + if crossesDown(previousUsed, used, c.resumeLowPriorityLimit) { c.refreshAdmissionAndNotify() } c.eventNotifier.notify() @@ -347,7 +347,7 @@ func regionScanLag(currentTs, checkpointTs uint64) time.Duration { return currentTime.Sub(checkpointTime) } -func isWarmingScan(region regionInfo, _ uint64) bool { +func isLowPriorityScan(region regionInfo, _ uint64) bool { return !isHighScanPriority(region.scanPriority) } @@ -356,13 +356,13 @@ func (c *memoryQuotaController) refreshLevelLocked() { // it to actual event bytes would count the same pressure twice. pressure := max(c.used.Load(), c.scanUsed) switch c.level { - case admissionPauseWarming: - if pressure <= c.resumeWarmingLimit { + case admissionPauseLowPriority: + if pressure <= c.resumeLowPriorityLimit { c.level = admissionNormal } default: - if pressure >= c.pauseWarmingLimit { - c.level = admissionPauseWarming + if pressure >= c.pauseLowPriorityLimit { + c.level = admissionPauseLowPriority } } } diff --git a/logservice/logpuller/memory_quota_test.go b/logservice/logpuller/memory_quota_test.go index 1fe8226e51..4f43ccf565 100644 --- a/logservice/logpuller/memory_quota_test.go +++ b/logservice/logpuller/memory_quota_test.go @@ -100,16 +100,16 @@ func TestMemoryQuotaUpdateMetrics(t *testing.T) { func TestMemoryQuotaAdmissionLevels(t *testing.T) { quota := newMemoryQuotaController(100, 10) - warmingSpan := newTestQuotaSpan(1) + lowPrioritySpan := newTestQuotaSpan(1) highPrioritySpan := newTestQuotaSpan(2) - warmingTs := setTestQuotaSpanLag(warmingSpan, time.Hour) + lowPriorityTs := setTestQuotaSpanLag(lowPrioritySpan, time.Hour) highPriorityTs := setTestQuotaSpanLag(highPrioritySpan, time.Hour) require.True(t, quota.AcquireEvent(context.Background(), highPrioritySpan, 5)) require.True(t, quota.AcquireEvent(context.Background(), highPrioritySpan, 10)) _, _, admitted := quota.AcquireScan( - newTestQuotaRegionWithPriority(warmingSpan, cdcpb.ScanPriority_SCAN_PRIORITY_LOW), - warmingTs, + newTestQuotaRegionWithPriority(lowPrioritySpan, cdcpb.ScanPriority_SCAN_PRIORITY_LOW), + lowPriorityTs, ) require.False(t, admitted) @@ -131,10 +131,10 @@ func TestMemoryQuotaAdmissionLevels(t *testing.T) { quota.ReleaseEvent(20) state := getMemoryQuotaTestState(quota) - require.Equal(t, admissionPauseWarming, state.level) + require.Equal(t, admissionPauseLowPriority, state.level) quota.ReleaseEvent(45) state = getMemoryQuotaTestState(quota) - require.Equal(t, admissionPauseWarming, state.level) + require.Equal(t, admissionPauseLowPriority, state.level) quota.ReleaseEvent(10) state = getMemoryQuotaTestState(quota) require.Equal(t, admissionNormal, state.level) @@ -299,7 +299,7 @@ func TestMemoryQuotaConcurrentWaitersDoNotLoseWakeups(t *testing.T) { require.Zero(t, state.used) } -func TestMemoryQuotaWarmingScanUsesCurrentPressure(t *testing.T) { +func TestMemoryQuotaLowPriorityScanUsesCurrentPressure(t *testing.T) { quota := newMemoryQuotaController(100, 20) span := newTestQuotaSpan(1) currentTs := setTestQuotaSpanLag(span, time.Hour) @@ -309,7 +309,7 @@ func TestMemoryQuotaWarmingScanUsesCurrentPressure(t *testing.T) { require.True(t, admitted) require.NotZero(t, bytes1) state := getMemoryQuotaTestState(quota) - require.Greater(t, state.scanUsed, quota.pauseWarmingLimit) + require.Greater(t, state.scanUsed, quota.pauseLowPriorityLimit) _, _, admitted = quota.AcquireScan(region, currentTs) require.False(t, admitted) @@ -362,7 +362,7 @@ func TestAdmissionWaitsForMemoryAndReleasesScanMemory(t *testing.T) { }() select { case <-result: - t.Fatal("warming scan should wait while memory is under pressure") + t.Fatal("low-priority scan should wait while memory is under pressure") case <-time.After(100 * time.Millisecond): } @@ -405,7 +405,7 @@ func TestAdmissionWakesWhenBlockedSpanStops(t *testing.T) { }() select { case <-result: - t.Fatal("warming scan should wait while memory is under pressure") + t.Fatal("low-priority scan should wait while memory is under pressure") case <-time.After(100 * time.Millisecond): } From f5261f3cb6bb7e105f6f34e89c1a76e5c3863979 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Wed, 5 Aug 2026 09:32:55 +0800 Subject: [PATCH 15/26] small fix --- logservice/logpuller/priority_task.go | 4 +--- .../logpuller/region_admission_controller.go | 16 ++++++++-------- .../region_admission_controller_test.go | 12 ++++++------ 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/logservice/logpuller/priority_task.go b/logservice/logpuller/priority_task.go index d5a5b034e4..38cd813c8b 100644 --- a/logservice/logpuller/priority_task.go +++ b/logservice/logpuller/priority_task.go @@ -13,9 +13,7 @@ package logpuller -import ( - "github.com/pingcap/kvproto/pkg/cdcpb" -) +import "github.com/pingcap/kvproto/pkg/cdcpb" func normalizeScanPriority(priority cdcpb.ScanPriority) cdcpb.ScanPriority { if priority == cdcpb.ScanPriority_SCAN_PRIORITY_HIGH { diff --git a/logservice/logpuller/region_admission_controller.go b/logservice/logpuller/region_admission_controller.go index 63f3a150fc..5e7e54b257 100644 --- a/logservice/logpuller/region_admission_controller.go +++ b/logservice/logpuller/region_admission_controller.go @@ -74,8 +74,7 @@ func (r *regionReq) release() bool { if !r.released.CompareAndSwap(false, true) { return false } - r.controller.memoryQuota.ReleaseScan(r.scanBytes) - r.controller.release() + r.controller.release(r.scanBytes) return true } @@ -101,7 +100,7 @@ type regionAdmissionController struct { } // memoryQuota gates initial scans using the log puller's global memory - // pressure. pdClock is sampled when a request is admitted. + // pressure. pdClock provides the current TS used for scan estimation. memoryQuota *memoryQuotaController pdClock pdutil.Clock // notify wakes workers when a request is submitted or an admission slot is @@ -156,11 +155,11 @@ func (c *regionAdmissionController) submit(task *regionPriorityTask) bool { return true } -// pop waits for an eligible request. If controlReady is signaled first, it -// returns nil without an error so the worker can handle its control queue. +// pop waits for an eligible request. If interrupt is signaled first, it +// returns nil without an error so the worker can handle the interrupt source. func (c *regionAdmissionController) pop( ctx context.Context, - controlReady <-chan struct{}, + interrupt <-chan struct{}, ) (*regionReq, error) { for { c.state.Lock() @@ -184,7 +183,7 @@ func (c *regionAdmissionController) pop( select { case <-c.notify: case <-memoryReady: - case <-controlReady: + case <-interrupt: return nil, nil case <-ctx.Done(): return nil, ctx.Err() @@ -221,7 +220,8 @@ func (c *regionAdmissionController) windowFor(request *regionPriorityTask) int { return c.currentWindow } -func (c *regionAdmissionController) release() { +func (c *regionAdmissionController) release(scanBytes uint64) { + c.memoryQuota.ReleaseScan(scanBytes) c.state.Lock() if c.state.inflight > 0 { c.state.inflight-- diff --git a/logservice/logpuller/region_admission_controller_test.go b/logservice/logpuller/region_admission_controller_test.go index 485b527c63..c1c393916b 100644 --- a/logservice/logpuller/region_admission_controller_test.go +++ b/logservice/logpuller/region_admission_controller_test.go @@ -86,9 +86,9 @@ func TestRegionAdmissionControllerNormalWindow(t *testing.T) { req1, err := controller.pop(t.Context(), nil) require.NoError(t, err) require.Equal(t, 1, controller.stats().inflight) - controlReady := make(chan struct{}) - close(controlReady) - req2, err := controller.pop(t.Context(), controlReady) + interrupt := make(chan struct{}) + close(interrupt) + req2, err := controller.pop(t.Context(), interrupt) require.Nil(t, req2) require.NoError(t, err) @@ -121,9 +121,9 @@ func TestRegionAdmissionControllerLowLagUsesMaxWindow(t *testing.T) { req2, err := controller.pop(t.Context(), nil) require.NoError(t, err) require.Equal(t, uint64(3), req2.regionInfo.verID.GetID()) - controlReady := make(chan struct{}) - close(controlReady) - req3, err := controller.pop(t.Context(), controlReady) + interrupt := make(chan struct{}) + close(interrupt) + req3, err := controller.pop(t.Context(), interrupt) require.Nil(t, req3) require.NoError(t, err) require.Equal(t, 2, controller.stats().inflight) From 9c6044cecacb22b5aa02c69896639eb32ddf88dc Mon Sep 17 00:00:00 2001 From: lidezhu Date: Wed, 5 Aug 2026 10:32:24 +0800 Subject: [PATCH 16/26] more fix and metrics --- logservice/logpuller/memory_quota.go | 17 ++++--- logservice/logpuller/memory_quota_test.go | 4 ++ .../logpuller/region_admission_controller.go | 20 +++++++++ logservice/logpuller/region_event_handler.go | 3 ++ metrics/grafana/ticdc_new_arch.json | 45 +++++++++++-------- .../ticdc_new_arch_next_gen.json | 45 +++++++++++-------- pkg/metrics/log_puller.go | 17 +++++++ 7 files changed, 106 insertions(+), 45 deletions(-) diff --git a/logservice/logpuller/memory_quota.go b/logservice/logpuller/memory_quota.go index 88dcf170f5..fbc0bc6c86 100644 --- a/logservice/logpuller/memory_quota.go +++ b/logservice/logpuller/memory_quota.go @@ -141,6 +141,7 @@ type memoryQuotaController struct { // eventNotifier owns the wait protocol used after the hard limit is reached. eventNotifier *eventMemoryNotifier + scanWaiters atomic.Int64 // scanMu guards scan admission state and scanReady. Scan admission happens // once per region rather than once per event batch, so it is intentionally @@ -156,21 +157,21 @@ type memoryQuotaController struct { pauseLowPriorityLimit uint64 resumeLowPriorityLimit uint64 - hardLimit uint64 + hardLimit uint64 scanEstimate uint64 } func newMemoryQuotaController(capacity, scanBaseSize uint64) *memoryQuotaController { c := &memoryQuotaController{ - capacity: capacity, - level: admissionNormal, + capacity: capacity, + level: admissionNormal, pauseLowPriorityLimit: uint64(math.Ceil(float64(capacity) * defaultPauseLowPriorityRatio)), resumeLowPriorityLimit: uint64(float64(capacity) * defaultResumeLowPriorityRatio), - hardLimit: uint64(float64(capacity) * defaultHardLimitRatio), - scanEstimate: scanBaseSize, - eventNotifier: newEventMemoryNotifier(), - scanReady: make(chan struct{}), + hardLimit: uint64(float64(capacity) * defaultHardLimitRatio), + scanEstimate: scanBaseSize, + eventNotifier: newEventMemoryNotifier(), + scanReady: make(chan struct{}), } return c } @@ -293,6 +294,8 @@ func (c *memoryQuotaController) UpdateMetrics() { metrics.LogPullerMemoryQuota.WithLabelValues("scan_used").Set(float64(scanUsed)) metrics.LogPullerMemoryQuotaEventWaiterCount.Set( float64(c.eventNotifier.waiters.Load())) + metrics.LogPullerMemoryQuotaScanWaiterCount.Set( + float64(c.scanWaiters.Load())) } func (c *memoryQuotaController) notifyScanAdmissionLocked() { diff --git a/logservice/logpuller/memory_quota_test.go b/logservice/logpuller/memory_quota_test.go index 4f43ccf565..41525f7b1e 100644 --- a/logservice/logpuller/memory_quota_test.go +++ b/logservice/logpuller/memory_quota_test.go @@ -85,6 +85,8 @@ func TestMemoryQuotaUpdateMetrics(t *testing.T) { quota.scanMu.Unlock() quota.eventNotifier.waiters.Store(2) t.Cleanup(func() { quota.eventNotifier.waiters.Store(0) }) + quota.scanWaiters.Store(3) + t.Cleanup(func() { quota.scanWaiters.Store(0) }) quota.UpdateMetrics() @@ -96,6 +98,8 @@ func TestMemoryQuotaUpdateMetrics(t *testing.T) { metrics.LogPullerMemoryQuota.WithLabelValues("scan_used"))) require.Equal(t, float64(2), testutil.ToFloat64(metrics.LogPullerMemoryQuotaEventWaiterCount)) + require.Equal(t, float64(3), + testutil.ToFloat64(metrics.LogPullerMemoryQuotaScanWaiterCount)) } func TestMemoryQuotaAdmissionLevels(t *testing.T) { diff --git a/logservice/logpuller/region_admission_controller.go b/logservice/logpuller/region_admission_controller.go index 5e7e54b257..bd4537b89f 100644 --- a/logservice/logpuller/region_admission_controller.go +++ b/logservice/logpuller/region_admission_controller.go @@ -180,14 +180,34 @@ func (c *regionAdmissionController) pop( } c.state.Unlock() + waitingForMemory := memoryReady != nil + if waitingForMemory { + c.memoryQuota.scanWaiters.Add(1) + } + waitStart := time.Time{} + if waitingForMemory { + waitStart = time.Now() + } select { case <-c.notify: case <-memoryReady: case <-interrupt: + if waitingForMemory { + c.memoryQuota.scanWaiters.Add(-1) + metrics.LogPullerMemoryQuotaScanWaitDuration.Observe(time.Since(waitStart).Seconds()) + } return nil, nil case <-ctx.Done(): + if waitingForMemory { + c.memoryQuota.scanWaiters.Add(-1) + metrics.LogPullerMemoryQuotaScanWaitDuration.Observe(time.Since(waitStart).Seconds()) + } return nil, ctx.Err() } + if waitingForMemory { + c.memoryQuota.scanWaiters.Add(-1) + metrics.LogPullerMemoryQuotaScanWaitDuration.Observe(time.Since(waitStart).Seconds()) + } } } diff --git a/logservice/logpuller/region_event_handler.go b/logservice/logpuller/region_event_handler.go index 359e915675..c8024b7020 100644 --- a/logservice/logpuller/region_event_handler.go +++ b/logservice/logpuller/region_event_handler.go @@ -65,6 +65,9 @@ func (event *regionEvent) needsMemoryAccounting() bool { } func (event *regionEvent) getSize() int { + if event == nil { + return 0 + } size := int(unsafe.Sizeof(*event)) if event.entries != nil { size += int(unsafe.Sizeof(*event.entries)) diff --git a/metrics/grafana/ticdc_new_arch.json b/metrics/grafana/ticdc_new_arch.json index 52cda78859..c92a6c9381 100644 --- a/metrics/grafana/ticdc_new_arch.json +++ b/metrics/grafana/ticdc_new_arch.json @@ -8822,7 +8822,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Scan admission level (0: normal, 1: pause warming scans, 2: pause all scans) and event receivers currently blocked at the memory hard limit.", + "description": "Event receivers blocked at the memory hard limit and region scans blocked at the scan admission gate.", "fieldConfig": { "defaults": {}, "overrides": [] @@ -8858,28 +8858,23 @@ "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/event-waiters$/", - "yaxis": 2 - } - ], + "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, - "expr": "ticdc_log_puller_memory_quota_admission_level{k8s_cluster=~\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}", + "expr": "ticdc_log_puller_memory_quota_event_waiter_count{k8s_cluster=~\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}", "interval": "", - "legendFormat": "{{instance}}-admission-level", + "legendFormat": "{{instance}}-event-waiters", "refId": "A" }, { "exemplar": true, - "expr": "ticdc_log_puller_memory_quota_event_waiter_count{k8s_cluster=~\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}", + "expr": "ticdc_log_puller_memory_quota_scan_waiter_count{k8s_cluster=~\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}", "interval": "", - "legendFormat": "{{instance}}-event-waiters", + "legendFormat": "{{instance}}-scan-waiters", "refId": "B" } ], @@ -8887,7 +8882,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Memory Quota Throttling", + "title": "Memory Quota Waiters", "tooltip": { "shared": true, "sort": 0, @@ -8905,15 +8900,13 @@ { "format": "short", "logBase": 1, - "max": "2", "min": "0", "show": true }, { "format": "short", "logBase": 1, - "min": "0", - "show": true + "show": false } ], "yaxis": { @@ -8926,7 +8919,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Time spent waiting to receive more events after log puller memory reaches the hard limit.", + "description": "Time spent waiting at the event hard limit or the scan admission gate.", "fieldConfig": { "defaults": {}, "overrides": [] @@ -8971,22 +8964,36 @@ "exemplar": true, "expr": "histogram_quantile(0.99, sum(rate(ticdc_log_puller_memory_quota_event_wait_duration_bucket{k8s_cluster=~\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", "interval": "", - "legendFormat": "{{instance}}-p99", + "legendFormat": "{{instance}}-event-p99", "refId": "A" }, { "exemplar": true, "expr": "sum(rate(ticdc_log_puller_memory_quota_event_wait_duration_sum{k8s_cluster=~\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance) / sum(rate(ticdc_log_puller_memory_quota_event_wait_duration_count{k8s_cluster=~\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", "interval": "", - "legendFormat": "{{instance}}-avg", + "legendFormat": "{{instance}}-event-avg", "refId": "B" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_log_puller_memory_quota_scan_wait_duration_bucket{k8s_cluster=~\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", + "interval": "", + "legendFormat": "{{instance}}-scan-p99", + "refId": "C" + }, + { + "exemplar": true, + "expr": "sum(rate(ticdc_log_puller_memory_quota_scan_wait_duration_sum{k8s_cluster=~\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance) / sum(rate(ticdc_log_puller_memory_quota_scan_wait_duration_count{k8s_cluster=~\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", + "interval": "", + "legendFormat": "{{instance}}-scan-avg", + "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Memory Quota Event Wait Duration", + "title": "Memory Quota Wait Duration", "tooltip": { "shared": true, "sort": 0, diff --git a/metrics/nextgengrafana/ticdc_new_arch_next_gen.json b/metrics/nextgengrafana/ticdc_new_arch_next_gen.json index b8452acca1..b684f1d915 100644 --- a/metrics/nextgengrafana/ticdc_new_arch_next_gen.json +++ b/metrics/nextgengrafana/ticdc_new_arch_next_gen.json @@ -8822,7 +8822,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Scan admission level (0: normal, 1: pause warming scans, 2: pause all scans) and event receivers currently blocked at the memory hard limit.", + "description": "Event receivers blocked at the memory hard limit and region scans blocked at the scan admission gate.", "fieldConfig": { "defaults": {}, "overrides": [] @@ -8858,28 +8858,23 @@ "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/event-waiters$/", - "yaxis": 2 - } - ], + "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, - "expr": "ticdc_log_puller_memory_quota_admission_level{k8s_cluster=~\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}", + "expr": "ticdc_log_puller_memory_quota_event_waiter_count{k8s_cluster=~\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}", "interval": "", - "legendFormat": "{{instance}}-admission-level", + "legendFormat": "{{instance}}-event-waiters", "refId": "A" }, { "exemplar": true, - "expr": "ticdc_log_puller_memory_quota_event_waiter_count{k8s_cluster=~\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}", + "expr": "ticdc_log_puller_memory_quota_scan_waiter_count{k8s_cluster=~\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}", "interval": "", - "legendFormat": "{{instance}}-event-waiters", + "legendFormat": "{{instance}}-scan-waiters", "refId": "B" } ], @@ -8887,7 +8882,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Memory Quota Throttling", + "title": "Memory Quota Waiters", "tooltip": { "shared": true, "sort": 0, @@ -8905,15 +8900,13 @@ { "format": "short", "logBase": 1, - "max": "2", "min": "0", "show": true }, { "format": "short", "logBase": 1, - "min": "0", - "show": true + "show": false } ], "yaxis": { @@ -8926,7 +8919,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Time spent waiting to receive more events after log puller memory reaches the hard limit.", + "description": "Time spent waiting at the event hard limit or the scan admission gate.", "fieldConfig": { "defaults": {}, "overrides": [] @@ -8971,22 +8964,36 @@ "exemplar": true, "expr": "histogram_quantile(0.99, sum(rate(ticdc_log_puller_memory_quota_event_wait_duration_bucket{k8s_cluster=~\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", "interval": "", - "legendFormat": "{{instance}}-p99", + "legendFormat": "{{instance}}-event-p99", "refId": "A" }, { "exemplar": true, "expr": "sum(rate(ticdc_log_puller_memory_quota_event_wait_duration_sum{k8s_cluster=~\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance) / sum(rate(ticdc_log_puller_memory_quota_event_wait_duration_count{k8s_cluster=~\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", "interval": "", - "legendFormat": "{{instance}}-avg", + "legendFormat": "{{instance}}-event-avg", "refId": "B" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_log_puller_memory_quota_scan_wait_duration_bucket{k8s_cluster=~\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", + "interval": "", + "legendFormat": "{{instance}}-scan-p99", + "refId": "C" + }, + { + "exemplar": true, + "expr": "sum(rate(ticdc_log_puller_memory_quota_scan_wait_duration_sum{k8s_cluster=~\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance) / sum(rate(ticdc_log_puller_memory_quota_scan_wait_duration_count{k8s_cluster=~\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", + "interval": "", + "legendFormat": "{{instance}}-scan-avg", + "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Memory Quota Event Wait Duration", + "title": "Memory Quota Wait Duration", "tooltip": { "shared": true, "sort": 0, diff --git a/pkg/metrics/log_puller.go b/pkg/metrics/log_puller.go index 88504eb9a3..34af5325ab 100644 --- a/pkg/metrics/log_puller.go +++ b/pkg/metrics/log_puller.go @@ -86,6 +86,21 @@ var ( Help: "The duration in seconds that an event receiver waits at the log puller memory hard limit.", Buckets: prometheus.ExponentialBuckets(0.001, 2, 24), }) + LogPullerMemoryQuotaScanWaiterCount = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: "ticdc", + Subsystem: "log_puller", + Name: "memory_quota_scan_waiter_count", + Help: "The number of region scans waiting at the log puller memory quota gate.", + }) + LogPullerMemoryQuotaScanWaitDuration = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Namespace: "ticdc", + Subsystem: "log_puller", + Name: "memory_quota_scan_wait_duration", + Help: "The duration in seconds that a region scan waits at the log puller memory quota gate.", + Buckets: prometheus.ExponentialBuckets(0.001, 2, 24), + }) SubscriptionClientResolvedTsLagGauge = prometheus.NewGauge( prometheus.GaugeOpts{ @@ -181,6 +196,8 @@ func initLogPullerMetrics(registry *prometheus.Registry) { registry.MustRegister(LogPullerMemoryQuota) registry.MustRegister(LogPullerMemoryQuotaEventWaiterCount) registry.MustRegister(LogPullerMemoryQuotaEventWaitDuration) + registry.MustRegister(LogPullerMemoryQuotaScanWaiterCount) + registry.MustRegister(LogPullerMemoryQuotaScanWaitDuration) registry.MustRegister(SubscriptionClientRequestedRegionCount) registry.MustRegister(RegionRequestFinishScanDuration) registry.MustRegister(SubscriptionClientSubscribedRegionCount) From 92912686a4546340babe4babd236a4ceb1726150 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Wed, 5 Aug 2026 11:54:38 +0800 Subject: [PATCH 17/26] more fix --- logservice/logpuller/region_event_handler.go | 4 ++-- logservice/logpuller/region_event_sink.go | 2 ++ logservice/logpuller/region_request_scheduler.go | 2 +- logservice/logpuller/region_request_worker_test.go | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/logservice/logpuller/region_event_handler.go b/logservice/logpuller/region_event_handler.go index c8024b7020..49dab3e46c 100644 --- a/logservice/logpuller/region_event_handler.go +++ b/logservice/logpuller/region_event_handler.go @@ -55,8 +55,8 @@ type regionEvent struct { entries *cdcpb.Event_Entries_ resolvedTs uint64 - // memoryBytes is released when this event is dropped or after downstream - // finishes consuming the entries derived from it. + // memoryBytes is released when this event is dropped or when the derived KV + // events no longer need to be retained by the log puller. memoryBytes uint64 } diff --git a/logservice/logpuller/region_event_sink.go b/logservice/logpuller/region_event_sink.go index 0c015db743..64c837f320 100644 --- a/logservice/logpuller/region_event_sink.go +++ b/logservice/logpuller/region_event_sink.go @@ -72,6 +72,8 @@ func (s *regionEventSink) Push(subID SubscriptionID, event regionEvent) { if event.needsMemoryAccounting() { span := event.mustFirstState().region.subscribedSpan event.memoryBytes = uint64(event.getSize()) + // AcquireEvent only returns false after shutdown or when the + // subscription has already been stopped. if !s.memoryQuota.AcquireEvent(s.ctx, span, event.memoryBytes) { return } diff --git a/logservice/logpuller/region_request_scheduler.go b/logservice/logpuller/region_request_scheduler.go index 9c3ba2a695..2118b246ba 100644 --- a/logservice/logpuller/region_request_scheduler.go +++ b/logservice/logpuller/region_request_scheduler.go @@ -207,7 +207,7 @@ func (s *regionRequestScheduler) BroadcastDeregister( func (s *regionRequestScheduler) requestedRegionCount() int { count := 0 s.stores.Range(func(_, value any) bool { - count += value.(*regionRequestStore).inflightCount() + count += value.(*regionRequestStore).requestedRegionCount() return true }) return count diff --git a/logservice/logpuller/region_request_worker_test.go b/logservice/logpuller/region_request_worker_test.go index d647606e2e..6362bf7ebe 100644 --- a/logservice/logpuller/region_request_worker_test.go +++ b/logservice/logpuller/region_request_worker_test.go @@ -143,7 +143,7 @@ func TestRunStreamCancelsBlockingReceiveWhenSenderExits(t *testing.T) { defer pdClient.Close() cluster.AddStore(1, storeAddr) - admission := newRegionAdmissionController(1, 1) + admission := newTestRegionAdmissionController(1, 1) worker := ®ionRequestWorker{ admission: admission, controlQueue: newControlQueue(), From e91461529481c1ac1c4881bc82e97f95ddc32299 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Wed, 12 Aug 2026 19:51:20 +0800 Subject: [PATCH 18/26] fix test --- .../logpuller/region_admission_controller_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/logservice/logpuller/region_admission_controller_test.go b/logservice/logpuller/region_admission_controller_test.go index c1c393916b..acab949286 100644 --- a/logservice/logpuller/region_admission_controller_test.go +++ b/logservice/logpuller/region_admission_controller_test.go @@ -99,11 +99,10 @@ func TestRegionAdmissionControllerNormalWindow(t *testing.T) { require.True(t, req2.abort()) } -func TestRegionAdmissionControllerLowLagUsesMaxWindow(t *testing.T) { +func TestRegionAdmissionControllerHighPriorityUsesMaxWindow(t *testing.T) { controller := newTestRegionAdmissionController(1, 2) currentTs := oracle.GoTimeToTS(time.Now()) slowCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Hour)) - lowLagCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Minute)) submitRegionForAdmission(t, controller, prepareRegionForAdmission(createTestRegionInfo(1, 1), slowCheckpointTs), @@ -114,9 +113,10 @@ func TestRegionAdmissionControllerLowLagUsesMaxWindow(t *testing.T) { submitRegionForAdmission(t, controller, prepareRegionForAdmission(createTestRegionInfo(1, 2), slowCheckpointTs), currentTs) - submitRegionForAdmission(t, controller, - prepareRegionForAdmission(createTestRegionInfo(1, 3), lowLagCheckpointTs), - currentTs) + highPriorityRegion := prepareRegionForAdmission( + createTestRegionInfo(1, 3), slowCheckpointTs) + highPriorityRegion.scanPriority = cdcpb.ScanPriority_SCAN_PRIORITY_HIGH + submitRegionForAdmission(t, controller, highPriorityRegion, currentTs) req2, err := controller.pop(t.Context(), nil) require.NoError(t, err) From 4a8d179a443c319cf58887071d2869139fa1d385 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Wed, 12 Aug 2026 20:07:21 +0800 Subject: [PATCH 19/26] f --- logservice/logpuller/memory_quota.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/logservice/logpuller/memory_quota.go b/logservice/logpuller/memory_quota.go index fbc0bc6c86..909842e8f7 100644 --- a/logservice/logpuller/memory_quota.go +++ b/logservice/logpuller/memory_quota.go @@ -315,10 +315,7 @@ func (c *memoryQuotaController) refreshAdmissionAndNotify() { func (c *memoryQuotaController) estimateScanSizeLocked(region regionInfo, currentTs uint64) uint64 { raw := float64(c.scanEstimate) * scanLagFactor(region.resolvedTs(), currentTs) - estimate := uint64(raw) - if estimate < c.scanEstimate { - estimate = c.scanEstimate - } + estimate := max(uint64(raw), c.scanEstimate) maxEstimate := uint64(math.MaxUint64) if c.scanEstimate <= math.MaxUint64/defaultMaxScanLagFactor { maxEstimate = c.scanEstimate * defaultMaxScanLagFactor From 175504774668deaa1593a528311ff000b00a65a9 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Thu, 13 Aug 2026 11:02:18 +0800 Subject: [PATCH 20/26] add design doc --- ...026-08-12-ticdc-log-puller-memory-quota.md | 427 ++++++++++++++++++ 1 file changed, 427 insertions(+) create mode 100644 docs/design/2026-08-12-ticdc-log-puller-memory-quota.md diff --git a/docs/design/2026-08-12-ticdc-log-puller-memory-quota.md b/docs/design/2026-08-12-ticdc-log-puller-memory-quota.md new file mode 100644 index 0000000000..b8ded617ec --- /dev/null +++ b/docs/design/2026-08-12-ticdc-log-puller-memory-quota.md @@ -0,0 +1,427 @@ +# TiCDC Log Puller Memory Quota Design + +This document describes the memory quota mechanism implemented by the +new-architecture Log Puller. The mechanism is owned by +[`memoryQuotaController`](../../logservice/logpuller/memory_quota.go) and is +shared by the Region event receive path and Region initial-scan admission. + +Related code includes: + +- [`region_event_sink.go`](../../logservice/logpuller/region_event_sink.go): + accounts entry events before they enter the dynamic stream. +- [`region_event_handler.go`](../../logservice/logpuller/region_event_handler.go): + releases event memory after downstream consumption or event drop. +- [`region_admission_controller.go`](../../logservice/logpuller/region_admission_controller.go): + acquires and releases estimated initial-scan memory. +- [`scan_priority.go`](../../logservice/logpuller/scan_priority.go): determines + whether a Region scan has high or low priority. +- [`pkg/config/debug.go`](../../pkg/config/debug.go): defines the Log Puller + memory quota configuration. +- [`pkg/metrics/log_puller.go`](../../pkg/metrics/log_puller.go): defines the + quota metrics. + +## 1. Background + +The Log Puller has two sources of memory pressure: + +1. TiKV entry events that have been received but are still retained by the Log + Puller or its downstream consumer. +2. Region initial scans that have been admitted but have not yet completed. + +The first source is measurable after an event arrives. The second must be +controlled before data arrives, so it is represented by a memory estimate. +Controlling only one source is insufficient: + +- Limiting only buffered events reacts too late when many initial scans start + concurrently. +- Limiting only initial scans does not protect the process when downstream + consumption stalls and received events remain retained for a long time. + +The controller therefore combines event accounting with scan admission while +keeping their hot paths and wake-up conditions separate. + +## 2. Goals and non-goals + +The design has the following goals: + +1. Bound the growth of retained Region entry events when downstream is slow. +2. Reduce the number of new low-priority initial scans before event memory + reaches the receive-path hard limit. +3. Preserve progress for high-priority recovery and caught-up workloads. +4. Keep event accounting inexpensive because it runs once per received entry + batch. +5. Wake blocked goroutines without lost notifications during release, + cancellation, or subscription shutdown. +6. Make ownership explicit so every successful acquisition has exactly one + release path. + +The mechanism is not intended to: + +- Measure the complete Go heap or process RSS. +- Enforce a strict upper bound equal to the configured soft quota. +- Provide a separate quota or fairness policy for each subscription. +- Reclaim events or cancel active scans when pressure increases. +- Replace the Region request window maintained by each request worker. + +## 3. Architecture + +One `subscriptionClient` creates one `memoryQuotaController` and shares it with +the event sink and all Region request workers. + +```mermaid +flowchart LR + T[TiKV event stream] --> S[regionEventSink] + S -->|AcquireEvent| Q[memoryQuotaController] + S --> D[dynamic stream] + D --> H[regionEventHandler] + H --> C[downstream consumer] + H -->|ReleaseEvent| Q + + R[Region request scheduler] --> A[regionAdmissionController] + A -->|AcquireScan| Q + A --> W[Region request worker] + W -->|initial scan finishes or aborts| A + A -->|ReleaseScan| Q +``` + +The controller tracks two values: + +| Value | Meaning | Accounting model | +| --- | --- | --- | +| `used` | Bytes retained by received entry events. | Estimated from the actual event batch after it arrives. | +| `scanUsed` | Predicted bytes for admitted, unfinished initial scans. | Reserved before a scan starts and released when it finishes or aborts. | + +The combined pressure is: + +```text +pressure = max(used, scanUsed) +``` + +The values are deliberately not added. `scanUsed` predicts event memory that +an initial scan may produce, while `used` measures that memory after events +arrive. Adding them would increasingly count the same pressure twice while a +scan is producing events. + +## 4. Configuration and thresholds + +The settings are under the server debug puller configuration: + +```toml +[debug.puller] +memory-quota = 1073741824 +scan-base-size = 8388608 +``` + +| Setting | Default | Meaning | +| --- | ---: | --- | +| `memory-quota` | 1 GiB | Local soft capacity, denoted by `Q`. | +| `scan-base-size` | 8 MiB | Base estimate for one admitted initial scan, denoted by `B`. | + +Zero values are replaced by these defaults during configuration validation. +The following thresholds are derived internally: + +| Threshold | Value | Purpose | +| --- | ---: | --- | +| Pause low-priority scans | `ceil(0.15 * Q)` | Enter scan throttling early. | +| Resume low-priority scans | `floor(0.05 * Q)` | Resume with hysteresis. | +| Event receive hard limit | `2 * Q` | Block additional entry events. | +| Maximum scan estimate | `16 * B` | Bound one scan's predicted charge. | + +With the defaults, low-priority scan admission pauses around 153.6 MiB, +resumes around 51.2 MiB, and event receiving blocks around 2 GiB. + +`memory-quota` is called a soft capacity because high-priority scans may pass +the scan gate and already-owned event memory is not discarded. The receive +hard limit is a separate safety threshold rather than the value exported as +the configured quota. + +## 5. Event memory accounting + +### 5.1 What is accounted + +Only Region events containing TiKV entries acquire event memory. Resolved-ts +events and Region error notifications do not. + +For an entry event, `regionEvent.getSize()` estimates: + +- The `regionEvent` value. +- Entry wrapper structures. +- Every row structure. +- Key, value, and old-value byte slices. +- The slice of Region state pointers. + +This is an accounting estimate, not a heap profiler. It intentionally does not +include every allocator, runtime, dynamic-stream, or downstream data-structure +overhead. + +### 5.2 Acquisition and the hard limit + +`regionEventSink.Push()` calls `AcquireEvent()` before pushing an entry event +into the dynamic stream. Below the hard limit, acquisition uses an atomic +compare-and-swap loop and takes no mutex. + +For current usage `U`, batch size `E`, and hard limit `H = 2Q`: + +```text +U == 0 -> admit E +U > 0 and U + E <= H -> admit E +U > 0 and U + E > H -> wait +``` + +The empty-controller exception allows one oversized event batch to make +progress even when that batch alone is larger than the hard limit. Without it, +such a batch could never be admitted. Consequently, the hard limit is a +backpressure point, not an absolute maximum. + +If admission cannot proceed, the receiver waits until one of these conditions +is true: + +- Event memory is released and the acquisition retry succeeds. +- The subscription is stopped. +- The subscription client context is canceled. + +While waiting, `AcquireEvent()` returns `false` when it observes either of the +latter two cases, and the event is not pushed into the dynamic stream. The +uncontended fast path checks context cancellation but does not inspect the span +state; normal Region deregistration prevents stopped spans from continuing to +produce events. + +### 5.3 Ownership and release + +A successful event acquisition remains owned until one of these terminal +paths releases it: + +| Path | Release point | +| --- | --- | +| Dynamic stream drops the event | `regionEventHandler.OnDrop()` | +| Downstream consumes asynchronously | The downstream wake callback, after the KV event cache is cleared and resolved-ts is advanced | +| Downstream does not retain the batch | Immediately after synchronous handling | +| Entry event produces no retained KV events | At the end of handler processing | + +When several dynamic-stream events are handled as a batch, their +`memoryBytes` values are summed and released together. Stopping a subscription +does not erase already-owned memory; the normal drop, callback, or handler path +still performs the release. + +### 5.4 Event waiter notification + +Event receivers use a close-and-replace channel protocol: + +1. A receiver registers itself as a waiter. +2. It loads the current `ready` channel under the notifier mutex. +3. It retries acquisition and checks whether the span has stopped. +4. Only then does it block on `ready` or context cancellation. + +`ReleaseEvent()` closes the current channel and installs a new one for future +waiters. The retry after registration prevents a release immediately before or +during registration from becoming a lost wake-up. An atomic waiter count is +only a fast-path hint used to avoid unnecessary broadcasts. + +## 6. Initial-scan admission + +### 6.1 Scan estimate + +An initial scan reserves estimated memory before the Region request is sent. +For base size `B` and resolved-ts lag `L`, the estimate is: + +```text +lagFactor = min(16, 1 + 0.22 * log2(1 + L / 10 minutes)) +estimate = clamp(B * lagFactor, B, 16 * B) +``` + +The logarithmic factor gives older scans a larger charge without allowing one +Region to monopolize the entire quota. A scan with no positive lag is charged +`B`; the maximum charge is `16B`. + +The estimate is predictive. It is not adjusted to match the exact bytes later +received from that Region. + +### 6.2 Priority and admission states + +The scan priority policy marks a Region HIGH when any of these conditions is +true: + +- The request inherited HIGH priority from an earlier attempt. +- The Region resolved-ts is within the configured old-start-ts lag threshold. +- The subscribed span has caught up once; this state is sticky across later + retries. + +Other scans are LOW priority. Priority is also sent to TiKV/CSE and controls +the local Region request queue and request-worker window. + +The memory quota controller has two admission states: + +| State | LOW priority | HIGH priority | +| --- | --- | --- | +| `normal` | Admitted | Admitted | +| `pauseLowPriority` | Waits on `scanReady` | Admitted | + +The transition rules use `pressure = max(used, scanUsed)`: + +```mermaid +stateDiagram-v2 + [*] --> normal + normal --> pauseLowPriority: pressure >= 15% of Q + pauseLowPriority --> normal: pressure <= 5% of Q +``` + +Hysteresis prevents scans from repeatedly stopping and starting around one +threshold. HIGH priority scans are an escape path: they remain eligible while +LOW priority backlog is paused, subject to the request worker's maximum +window. + +Admission is decided using pressure before adding the new scan estimate. This +allows one LOW priority scan to cross the pause threshold and make progress; +subsequent LOW priority scans wait. HIGH priority scans can continue increasing +`scanUsed` beyond the soft threshold. + +### 6.3 Interaction with the Region request window + +Memory admission is applied after the per-worker Region request window check. +Both conditions must allow a scan: + +1. The worker must have an available ordinary or maximum-window slot. +2. The global memory quota must admit the scan. + +LOW priority requests use the ordinary window. HIGH priority requests can use +the larger window configured by `region-request-max-window-multiplier` and also +bypass `pauseLowPriority`. These two controls serve different purposes: the +window bounds per-worker concurrency, while the quota coordinates memory +pressure across all workers. + +### 6.4 Scan lease lifecycle + +Successful admission returns a byte charge stored in a `regionReq` lease. The +lease is released when: + +- The Region emits its initialization completion event. +- The request is canceled because the subscription stopped. +- The store stream fails or exits. +- Another request cleanup path aborts the scan. + +`finish()` and `abort()` share an atomic compare-and-swap, so concurrent cleanup +paths release the scan estimate and request-window slot exactly once. + +If a queued task belongs to an already-stopped span, `AcquireScan()` admits it +with a zero-byte lease. This lets the task reach the normal stopped-request +cleanup path without consuming quota or remaining blocked forever. + +### 6.5 Scan waiter notification + +Rejected scans wait on the current `scanReady` channel. The controller closes +and replaces this channel when a transition can make scans eligible: + +- Event usage falls far enough to change `pauseLowPriority` to `normal`. +- Releasing a scan estimate changes the state to `normal`. +- Subscription stop or client shutdown explicitly calls `WakeAll()`. + +The waiting admission loop always rechecks the worker window, span state, and +memory state after waking. The channel is a broadcast signal, not a reservation +for a particular worker. + +## 7. Shutdown and cancellation + +Stopping a subscription first marks its span as stopped and then calls +`WakeAll()`: + +- Blocked event receivers wake and recheck acquisition, cancellation, and the + stopped span state. +- Blocked scan admissions wake and receive a zero-byte lease for cleanup. + +Closing the event sink also calls `WakeAll()` so receivers can observe context +cancellation. Wake-up does not release memory on behalf of an owner. Existing +event and scan charges remain until their corresponding drop, callback, +finish, or abort path runs. + +This separation is important: notification changes scheduling, while release +changes accounting. + +## 8. Concurrency model and invariants + +The implementation separates synchronization by access frequency: + +| State | Synchronization | +| --- | --- | +| Event `used` | `atomic.Uint64` and compare-and-swap | +| Event waiter count | Atomic counter | +| Event ready channel | `eventMemoryNotifier.mu` | +| Scan usage, admission level, and ready channel | `scanMu` | +| Scan waiter count | Atomic counter | +| Per-worker pending queue and inflight window | `regionAdmissionController.state` mutex | + +The main invariants are: + +1. Every successful nonzero acquisition has one terminal release. +2. `used` changes only through `AcquireEvent()` and `ReleaseEvent()`. +3. `scanUsed` changes only while holding `scanMu`. +4. A `regionReq` releases its scan charge and worker slot at most once. +5. Waiters recheck their predicate after registration and after every wake-up. +6. Notifications never transfer ownership and do not imply successful + admission. +7. Stopping a span wakes blocked work but does not invalidate ownership already + handed to downstream code. + +## 9. Observability + +The subscription client updates quota metrics every ten seconds. + +| Metric | Meaning | +| --- | --- | +| `ticdc_log_puller_memory_quota{type="max"}` | Configured soft capacity `Q`. | +| `ticdc_log_puller_memory_quota{type="used"}` | Accounted retained event bytes. | +| `ticdc_log_puller_memory_quota{type="scan_used"}` | Estimated bytes reserved by active initial scans. | +| `ticdc_log_puller_memory_quota_event_waiter_count` | Event receivers currently waiting at the hard limit. | +| `ticdc_log_puller_memory_quota_scan_waiter_count` | Region scans currently waiting at the scan gate. | +| `ticdc_log_puller_memory_quota_event_wait_duration` | Event receive wait duration histogram. | +| `ticdc_log_puller_memory_quota_scan_wait_duration` | Scan admission wait duration histogram. | + +The Grafana dashboards expose three panels: + +- **Memory Quota Usage** for `max`, `used`, and `scan_used`. +- **Memory Quota Waiters** for current event and scan waiters. +- **Memory Quota Wait Duration** for average and P99 wait latency. + +Operationally: + +- Rising `scan_used` followed by scan waiters means LOW priority initial scans + are being intentionally paced. +- Rising `used` with event waiters means downstream retention has reached the + receive hard limit. +- Persistent HIGH `scan_used` without scan waiters can be expected when active + requests are HIGH priority, because they bypass the soft scan gate. + +## 10. Limitations and trade-offs + +### 10.1 Approximate rather than exact accounting + +Event size is estimated from selected Go structures and payload bytes, and +scan memory is predicted from lag. The metrics should be interpreted as Log +Puller quota state, not exact heap usage. + +### 10.2 Progress over a strict cap + +One oversized event can enter an empty controller, and HIGH priority scans can +continue above the soft quota. These exceptions avoid deadlock and protect +recovery progress, at the cost of allowing temporary overshoot. + +### 10.3 Global rather than per-subscription fairness + +All subscriptions using the client share the controller. The design protects +the Log Puller as a whole but does not reserve memory for a particular +subscription or prevent one active workload from consuming most of the +accounted memory. + +### 10.4 Predictive and actual pressure overlap + +Using `max(used, scanUsed)` avoids double counting, but it is intentionally +conservative in only one dimension at a time. If scan estimates and retained +events represent unrelated workloads, their combined real memory can be +higher than the reported pressure. The separate event hard limit remains the +last receive-path backpressure point. + +### 10.5 No forced reclamation + +The controller blocks new work and waits for current owners to release memory. +It does not discard downstream-owned events or revoke active scan leases. This +keeps ownership and correctness simple, but recovery depends on the downstream +callback and request cleanup paths continuing to run. From 484eddc45f0e73fe5f457f910897113a72449055 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Thu, 13 Aug 2026 13:32:04 +0800 Subject: [PATCH 21/26] small fix --- .../2026-08-12-ticdc-log-puller-memory-quota.md | 9 +++++---- logservice/logpuller/memory_quota.go | 2 +- logservice/logpuller/memory_quota_test.go | 2 +- metrics/grafana/ticdc_new_arch.json | 13 ++++++++++--- metrics/nextgengrafana/ticdc_new_arch_next_gen.json | 13 ++++++++++--- pkg/config/debug.go | 6 +++++- 6 files changed, 32 insertions(+), 13 deletions(-) diff --git a/docs/design/2026-08-12-ticdc-log-puller-memory-quota.md b/docs/design/2026-08-12-ticdc-log-puller-memory-quota.md index b8ded617ec..0c96eb669d 100644 --- a/docs/design/2026-08-12-ticdc-log-puller-memory-quota.md +++ b/docs/design/2026-08-12-ticdc-log-puller-memory-quota.md @@ -369,7 +369,7 @@ The subscription client updates quota metrics every ten seconds. | --- | --- | | `ticdc_log_puller_memory_quota{type="max"}` | Configured soft capacity `Q`. | | `ticdc_log_puller_memory_quota{type="used"}` | Accounted retained event bytes. | -| `ticdc_log_puller_memory_quota{type="scan_used"}` | Estimated bytes reserved by active initial scans. | +| `ticdc_log_puller_memory_quota{type="scan_estimated"}` | Estimated bytes reserved by active initial scans. | | `ticdc_log_puller_memory_quota_event_waiter_count` | Event receivers currently waiting at the hard limit. | | `ticdc_log_puller_memory_quota_scan_waiter_count` | Region scans currently waiting at the scan gate. | | `ticdc_log_puller_memory_quota_event_wait_duration` | Event receive wait duration histogram. | @@ -377,17 +377,18 @@ The subscription client updates quota metrics every ten seconds. The Grafana dashboards expose three panels: -- **Memory Quota Usage** for `max`, `used`, and `scan_used`. +- **Memory Quota** for the existing Dynamic Stream quota values and + `scan_estimated`. - **Memory Quota Waiters** for current event and scan waiters. - **Memory Quota Wait Duration** for average and P99 wait latency. Operationally: -- Rising `scan_used` followed by scan waiters means LOW priority initial scans +- Rising `scan_estimated` followed by scan waiters means LOW priority initial scans are being intentionally paced. - Rising `used` with event waiters means downstream retention has reached the receive hard limit. -- Persistent HIGH `scan_used` without scan waiters can be expected when active +- Persistent HIGH `scan_estimated` without scan waiters can be expected when active requests are HIGH priority, because they bypass the soft scan gate. ## 10. Limitations and trade-offs diff --git a/logservice/logpuller/memory_quota.go b/logservice/logpuller/memory_quota.go index 909842e8f7..70f6d12d56 100644 --- a/logservice/logpuller/memory_quota.go +++ b/logservice/logpuller/memory_quota.go @@ -291,7 +291,7 @@ func (c *memoryQuotaController) UpdateMetrics() { metrics.LogPullerMemoryQuota.WithLabelValues("max").Set(float64(c.capacity)) metrics.LogPullerMemoryQuota.WithLabelValues("used").Set(float64(used)) - metrics.LogPullerMemoryQuota.WithLabelValues("scan_used").Set(float64(scanUsed)) + metrics.LogPullerMemoryQuota.WithLabelValues("scan_estimated").Set(float64(scanUsed)) metrics.LogPullerMemoryQuotaEventWaiterCount.Set( float64(c.eventNotifier.waiters.Load())) metrics.LogPullerMemoryQuotaScanWaiterCount.Set( diff --git a/logservice/logpuller/memory_quota_test.go b/logservice/logpuller/memory_quota_test.go index 41525f7b1e..530acc08da 100644 --- a/logservice/logpuller/memory_quota_test.go +++ b/logservice/logpuller/memory_quota_test.go @@ -95,7 +95,7 @@ func TestMemoryQuotaUpdateMetrics(t *testing.T) { require.Equal(t, float64(55), testutil.ToFloat64( metrics.LogPullerMemoryQuota.WithLabelValues("used"))) require.Equal(t, float64(7), testutil.ToFloat64( - metrics.LogPullerMemoryQuota.WithLabelValues("scan_used"))) + metrics.LogPullerMemoryQuota.WithLabelValues("scan_estimated"))) require.Equal(t, float64(2), testutil.ToFloat64(metrics.LogPullerMemoryQuotaEventWaiterCount)) require.Equal(t, float64(3), diff --git a/metrics/grafana/ticdc_new_arch.json b/metrics/grafana/ticdc_new_arch.json index c92a6c9381..74e37d99a6 100644 --- a/metrics/grafana/ticdc_new_arch.json +++ b/metrics/grafana/ticdc_new_arch.json @@ -8119,7 +8119,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Actual event memory, estimated in-flight scan memory, and the configured memory limits.", + "description": "Log puller memory quota", "fieldConfig": { "defaults": {}, "overrides": [] @@ -8162,17 +8162,24 @@ "targets": [ { "exemplar": true, - "expr": "sum(ticdc_log_puller_memory_quota{k8s_cluster=~\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance, type)", + "expr": "sum(ticdc_dynamic_stream_memory_usage{k8s_cluster=~\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", module=~\"log-puller\"}) by (instance, type)", "interval": "", "legendFormat": "{{instance}}-{{type}}", "refId": "A" + }, + { + "exemplar": true, + "expr": "sum(ticdc_log_puller_memory_quota{k8s_cluster=~\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", type=\"scan_estimated\"}) by (instance)", + "interval": "", + "legendFormat": "{{instance}}-scan_estimated", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Memory Quota Usage", + "title": "Memory Quota", "tooltip": { "shared": true, "sort": 0, diff --git a/metrics/nextgengrafana/ticdc_new_arch_next_gen.json b/metrics/nextgengrafana/ticdc_new_arch_next_gen.json index b684f1d915..d92a1536e6 100644 --- a/metrics/nextgengrafana/ticdc_new_arch_next_gen.json +++ b/metrics/nextgengrafana/ticdc_new_arch_next_gen.json @@ -8119,7 +8119,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Actual event memory, estimated in-flight scan memory, and the configured memory limits.", + "description": "Log puller memory quota", "fieldConfig": { "defaults": {}, "overrides": [] @@ -8162,17 +8162,24 @@ "targets": [ { "exemplar": true, - "expr": "sum(ticdc_log_puller_memory_quota{k8s_cluster=~\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance, type)", + "expr": "sum(ticdc_dynamic_stream_memory_usage{k8s_cluster=~\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", module=~\"log-puller\"}) by (instance, type)", "interval": "", "legendFormat": "{{instance}}-{{type}}", "refId": "A" + }, + { + "exemplar": true, + "expr": "sum(ticdc_log_puller_memory_quota{k8s_cluster=~\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", type=\"scan_estimated\"}) by (instance)", + "interval": "", + "legendFormat": "{{instance}}-scan_estimated", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Memory Quota Usage", + "title": "Memory Quota", "tooltip": { "shared": true, "sort": 0, diff --git a/pkg/config/debug.go b/pkg/config/debug.go index 12e0ba433c..7b96ae4d17 100644 --- a/pkg/config/debug.go +++ b/pkg/config/debug.go @@ -88,7 +88,11 @@ type PullerConfig struct { OldStartTsScanLowPriorityThreshold TomlDuration `toml:"old-start-ts-scan-low-priority-threshold" json:"old_start_ts_scan_low_priority_threshold"` // MemoryQuota is the log puller's local soft memory limit in bytes. MemoryQuota uint64 `toml:"memory-quota" json:"memory_quota"` - // ScanBaseSize is the base memory estimate for one admitted initial scan. + // ScanBaseSize is the base memory estimate reserved for one admitted initial + // scan. The actual estimate grows logarithmically with scan lag, up to a + // bounded multiple of this value. The estimate contributes to MemoryQuota + // pressure and throttles new low-priority scans; it is not an actual memory + // allocation or a per-scan hard limit. ScanBaseSize uint64 `toml:"scan-base-size" json:"scan_base_size"` } From 3de3c043fdf43fe5138dc3353995d17807d2bf57 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Thu, 13 Aug 2026 13:35:00 +0800 Subject: [PATCH 22/26] small fix --- pkg/config/debug.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/pkg/config/debug.go b/pkg/config/debug.go index 7b96ae4d17..f9ffe4aec1 100644 --- a/pkg/config/debug.go +++ b/pkg/config/debug.go @@ -24,9 +24,14 @@ import ( const ( // DefaultOldStartTsScanLowPriorityThreshold is the default lag threshold for // classifying scan tasks as low priority. - DefaultOldStartTsScanLowPriorityThreshold = 10 * time.Minute - defaultLogPullerMemoryQuota uint64 = 1024 * 1024 * 1024 - defaultLogPullerScanBaseSize uint64 = 8 * 1024 * 1024 + DefaultOldStartTsScanLowPriorityThreshold = 10 * time.Minute + + // DefaultLogPullerMemoryQuota is the default Log Puller soft memory limit. + DefaultLogPullerMemoryQuota uint64 = 1024 * 1024 * 1024 + + // DefaultLogPullerScanBaseSize is the default base memory estimate for one + // initial scan. + DefaultLogPullerScanBaseSize uint64 = 8 * 1024 * 1024 ) // DebugConfig represents config for ticdc unexposed feature configurations @@ -106,8 +111,8 @@ func NewDefaultPullerConfig() *PullerConfig { RegionRequestMaxWindowMultiplier: 4, // Allows high-priority scans to use up to 4 * PendingRegionRequestQueueSize. OldStartTsScanLowPriorityThreshold: TomlDuration( DefaultOldStartTsScanLowPriorityThreshold), - MemoryQuota: defaultLogPullerMemoryQuota, - ScanBaseSize: defaultLogPullerScanBaseSize, + MemoryQuota: DefaultLogPullerMemoryQuota, + ScanBaseSize: DefaultLogPullerScanBaseSize, } } From 9136d2be2ca5b54b7a7540f92133a72670f2dc4c Mon Sep 17 00:00:00 2001 From: lidezhu Date: Thu, 13 Aug 2026 14:16:59 +0800 Subject: [PATCH 23/26] fix test --- logservice/logpuller/subscription_client_test.go | 1 + metrics/grafana/ticdc_new_arch.json | 4 ++-- metrics/nextgengrafana/ticdc_new_arch_next_gen.json | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/logservice/logpuller/subscription_client_test.go b/logservice/logpuller/subscription_client_test.go index 4c7f686af6..33d9a827cd 100644 --- a/logservice/logpuller/subscription_client_test.go +++ b/logservice/logpuller/subscription_client_test.go @@ -306,6 +306,7 @@ func TestResolveLockTaskDroppedWhenChannelFull(t *testing.T) { func TestStopTaskUsesSubscribedSpanFilterLoop(t *testing.T) { client := &subscriptionClient{ resolveLockTaskCh: make(chan resolveLockTask, 1), + memoryQuota: newMemoryQuotaController(1024, 8), } client.ctx, client.cancel = context.WithCancel(context.Background()) defer client.cancel() diff --git a/metrics/grafana/ticdc_new_arch.json b/metrics/grafana/ticdc_new_arch.json index 74e37d99a6..cb01262efe 100644 --- a/metrics/grafana/ticdc_new_arch.json +++ b/metrics/grafana/ticdc_new_arch.json @@ -8169,9 +8169,9 @@ }, { "exemplar": true, - "expr": "sum(ticdc_log_puller_memory_quota{k8s_cluster=~\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", type=\"scan_estimated\"}) by (instance)", + "expr": "sum(ticdc_log_puller_memory_quota{k8s_cluster=~\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance, type)", "interval": "", - "legendFormat": "{{instance}}-scan_estimated", + "legendFormat": "{{instance}}-quota-{{type}}", "refId": "B" } ], diff --git a/metrics/nextgengrafana/ticdc_new_arch_next_gen.json b/metrics/nextgengrafana/ticdc_new_arch_next_gen.json index d92a1536e6..d11e96e9c1 100644 --- a/metrics/nextgengrafana/ticdc_new_arch_next_gen.json +++ b/metrics/nextgengrafana/ticdc_new_arch_next_gen.json @@ -8169,9 +8169,9 @@ }, { "exemplar": true, - "expr": "sum(ticdc_log_puller_memory_quota{k8s_cluster=~\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", type=\"scan_estimated\"}) by (instance)", + "expr": "sum(ticdc_log_puller_memory_quota{k8s_cluster=~\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance, type)", "interval": "", - "legendFormat": "{{instance}}-scan_estimated", + "legendFormat": "{{instance}}-quota-{{type}}", "refId": "B" } ], From a1c3a74ff44afa8c8bdda04f0e7d5af1c57dbe0b Mon Sep 17 00:00:00 2001 From: lidezhu Date: Thu, 13 Aug 2026 14:45:09 +0800 Subject: [PATCH 24/26] address comment --- ...026-08-12-ticdc-log-puller-memory-quota.md | 5 ++- logservice/logpuller/memory_quota.go | 21 +++++++++-- logservice/logpuller/memory_quota_test.go | 37 +++++++++++++++++++ pkg/config/debug_test.go | 11 +++++- 4 files changed, 66 insertions(+), 8 deletions(-) diff --git a/docs/design/2026-08-12-ticdc-log-puller-memory-quota.md b/docs/design/2026-08-12-ticdc-log-puller-memory-quota.md index 0c96eb669d..e4ef5e3bce 100644 --- a/docs/design/2026-08-12-ticdc-log-puller-memory-quota.md +++ b/docs/design/2026-08-12-ticdc-log-puller-memory-quota.md @@ -377,8 +377,9 @@ The subscription client updates quota metrics every ten seconds. The Grafana dashboards expose three panels: -- **Memory Quota** for the existing Dynamic Stream quota values and - `scan_estimated`. +- **Memory Quota** for logpuller quota values from + `ticdc_log_puller_memory_quota`. For compatibility, the panel also reads the + legacy log-puller series from `ticdc_dynamic_stream_memory_usage`. - **Memory Quota Waiters** for current event and scan waiters. - **Memory Quota Wait Duration** for average and P99 wait latency. diff --git a/logservice/logpuller/memory_quota.go b/logservice/logpuller/memory_quota.go index 70f6d12d56..f2228f2ad8 100644 --- a/logservice/logpuller/memory_quota.go +++ b/logservice/logpuller/memory_quota.go @@ -163,12 +163,16 @@ type memoryQuotaController struct { } func newMemoryQuotaController(capacity, scanBaseSize uint64) *memoryQuotaController { + hardLimit := uint64(math.MaxUint64) + if capacity <= math.MaxUint64/uint64(defaultHardLimitRatio) { + hardLimit = capacity * uint64(defaultHardLimitRatio) + } c := &memoryQuotaController{ capacity: capacity, level: admissionNormal, pauseLowPriorityLimit: uint64(math.Ceil(float64(capacity) * defaultPauseLowPriorityRatio)), resumeLowPriorityLimit: uint64(float64(capacity) * defaultResumeLowPriorityRatio), - hardLimit: uint64(float64(capacity) * defaultHardLimitRatio), + hardLimit: hardLimit, scanEstimate: scanBaseSize, eventNotifier: newEventMemoryNotifier(), scanReady: make(chan struct{}), @@ -267,8 +271,14 @@ func (c *memoryQuotaController) ReleaseEvent(bytes uint64) { if bytes == 0 { return } - used := c.used.Add(^(bytes - 1)) - previousUsed := used + bytes + var previousUsed, used uint64 + for { + previousUsed = c.used.Load() + used = subtractFloor(previousUsed, bytes) + if c.used.CompareAndSwap(previousUsed, used) { + break + } + } if crossesDown(previousUsed, used, c.resumeLowPriorityLimit) { c.refreshAdmissionAndNotify() } @@ -315,7 +325,10 @@ func (c *memoryQuotaController) refreshAdmissionAndNotify() { func (c *memoryQuotaController) estimateScanSizeLocked(region regionInfo, currentTs uint64) uint64 { raw := float64(c.scanEstimate) * scanLagFactor(region.resolvedTs(), currentTs) - estimate := max(uint64(raw), c.scanEstimate) + estimate := uint64(math.MaxUint64) + if raw < float64(math.MaxUint64) { + estimate = max(uint64(raw), c.scanEstimate) + } maxEstimate := uint64(math.MaxUint64) if c.scanEstimate <= math.MaxUint64/defaultMaxScanLagFactor { maxEstimate = c.scanEstimate * defaultMaxScanLagFactor diff --git a/logservice/logpuller/memory_quota_test.go b/logservice/logpuller/memory_quota_test.go index 530acc08da..8ec1eab982 100644 --- a/logservice/logpuller/memory_quota_test.go +++ b/logservice/logpuller/memory_quota_test.go @@ -16,6 +16,7 @@ package logpuller import ( "context" + "math" "testing" "time" @@ -145,6 +146,42 @@ func TestMemoryQuotaAdmissionLevels(t *testing.T) { quota.ReleaseEvent(5) } +func TestMemoryQuotaReleaseEventClampsToZero(t *testing.T) { + quota := newMemoryQuotaController(100, 10) + span := newTestQuotaSpan(1) + currentTs := setTestQuotaSpanLag(span, time.Hour) + + require.True(t, quota.AcquireEvent(context.Background(), span, 20)) + _, _, admitted := quota.AcquireScan( + newTestQuotaRegionWithPriority(span, cdcpb.ScanPriority_SCAN_PRIORITY_LOW), + currentTs, + ) + require.False(t, admitted) + + quota.ReleaseEvent(30) + state := getMemoryQuotaTestState(quota) + require.Zero(t, state.used) + require.Equal(t, admissionNormal, state.level) + + require.True(t, quota.AcquireEvent(context.Background(), span, 1)) + quota.ReleaseEvent(1) +} + +func TestMemoryQuotaDerivedLimitsSaturate(t *testing.T) { + quota := newMemoryQuotaController(math.MaxUint64, math.MaxUint64/2+1) + require.Equal(t, uint64(math.MaxUint64), quota.hardLimit) + + span := newTestQuotaSpan(1) + currentTs := setTestQuotaSpanLag(span, 24*time.Hour) + scanBytes, _, admitted := quota.AcquireScan( + newTestQuotaRegionWithPriority(span, cdcpb.ScanPriority_SCAN_PRIORITY_HIGH), + currentTs, + ) + require.True(t, admitted) + require.Equal(t, uint64(math.MaxUint64), scanBytes) + quota.ReleaseScan(scanBytes) +} + func TestMemoryQuotaSpanStopKeepsOwnedMemoryUntilRelease(t *testing.T) { quota := newMemoryQuotaController(100, 10) span1 := newTestQuotaSpan(1) diff --git a/pkg/config/debug_test.go b/pkg/config/debug_test.go index 82904dcabf..3f2b3428fa 100644 --- a/pkg/config/debug_test.go +++ b/pkg/config/debug_test.go @@ -24,16 +24,23 @@ func TestPullerConfigValidateAndAdjustRegionRequestWindow(t *testing.T) { defaultCfg := NewDefaultPullerConfig() require.Equal(t, 32, defaultCfg.PendingRegionRequestQueueSize) require.Equal(t, 4, defaultCfg.RegionRequestMaxWindowMultiplier) + require.Equal( + t, + TomlDuration(DefaultOldStartTsScanLowPriorityThreshold), + defaultCfg.OldStartTsScanLowPriorityThreshold, + ) require.Equal(t, uint64(1024*1024*1024), defaultCfg.MemoryQuota) require.Equal(t, uint64(8*1024*1024), defaultCfg.ScanBaseSize) cfg := &PullerConfig{ - PendingRegionRequestQueueSize: -1, - RegionRequestMaxWindowMultiplier: 0, + PendingRegionRequestQueueSize: -1, + RegionRequestMaxWindowMultiplier: 0, + OldStartTsScanLowPriorityThreshold: 0, } cfg.ValidateAndAdjust() require.Equal(t, defaultCfg.PendingRegionRequestQueueSize, cfg.PendingRegionRequestQueueSize) require.Equal(t, defaultCfg.RegionRequestMaxWindowMultiplier, cfg.RegionRequestMaxWindowMultiplier) + require.Equal(t, defaultCfg.OldStartTsScanLowPriorityThreshold, cfg.OldStartTsScanLowPriorityThreshold) require.Equal(t, defaultCfg.MemoryQuota, cfg.MemoryQuota) require.Equal(t, defaultCfg.ScanBaseSize, cfg.ScanBaseSize) } From 4409069e10cde6507570131906c439907f90a47c Mon Sep 17 00:00:00 2001 From: lidezhu Date: Thu, 13 Aug 2026 21:08:36 +0800 Subject: [PATCH 25/26] add some defensive check --- logservice/logpuller/region_event_handler.go | 11 ++++++++- .../logpuller/region_event_handler_test.go | 24 +++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/logservice/logpuller/region_event_handler.go b/logservice/logpuller/region_event_handler.go index 49dab3e46c..24ad6f4140 100644 --- a/logservice/logpuller/region_event_handler.go +++ b/logservice/logpuller/region_event_handler.go @@ -243,7 +243,16 @@ func (h *regionEventHandler) GetType(event regionEvent) dynstream.EventType { func (h *regionEventHandler) OnDrop(event regionEvent) interface{} { h.eventSink.memoryQuota.ReleaseEvent(event.memoryBytes) // TODO: Distinguish between drop events caused by "path not found" errors and memory control. - state := event.mustFirstState() + if len(event.states) == 0 || event.states[0] == nil { + log.Error("drop invalid region event", + zap.Bool("hasEntries", event.entries != nil), + zap.Uint64("resolvedTs", event.resolvedTs), + zap.Int("states", len(event.states)), + zap.Uint64("memoryBytes", event.memoryBytes)) + return nil + } + + state := event.states[0] fields := []zap.Field{ zap.Bool("hasEntries", event.entries != nil), zap.Uint64("resolvedTs", event.resolvedTs), diff --git a/logservice/logpuller/region_event_handler_test.go b/logservice/logpuller/region_event_handler_test.go index 38a2b50f2e..b4cf4237dc 100644 --- a/logservice/logpuller/region_event_handler_test.go +++ b/logservice/logpuller/region_event_handler_test.go @@ -430,6 +430,30 @@ func TestHandleEntriesReleasesMemoryAfterDownstreamCallback(t *testing.T) { require.Zero(t, quotaState.used) } +func TestOnDropInvalidEventReleasesMemory(t *testing.T) { + testCases := []struct { + name string + states []*regionFeedState + }{ + {name: "empty states"}, + {name: "nil state", states: []*regionFeedState{nil}}, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + quota := newMemoryQuotaController(1024, 8) + span := newTestQuotaSpan(1) + require.True(t, quota.AcquireEvent(context.Background(), span, 10)) + handler := ®ionEventHandler{eventSink: ®ionEventSink{memoryQuota: quota}} + + require.NotPanics(t, func() { + handler.OnDrop(regionEvent{states: testCase.states, memoryBytes: 10}) + }) + require.Zero(t, getMemoryQuotaTestState(quota).used) + }) + } +} + func TestSpanInitializedAfterFullRangeCoverage(t *testing.T) { const startTs = 100 span := &subscribedSpan{ From 4b75ff8e727e1e6d071df27383d94123a13824f1 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Fri, 14 Aug 2026 10:12:06 +0800 Subject: [PATCH 26/26] add test --- .../logpuller/region_event_sink_test.go | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/logservice/logpuller/region_event_sink_test.go b/logservice/logpuller/region_event_sink_test.go index 291d4d867c..ceab22fabb 100644 --- a/logservice/logpuller/region_event_sink_test.go +++ b/logservice/logpuller/region_event_sink_test.go @@ -16,11 +16,14 @@ package logpuller import ( "context" "testing" + "time" "github.com/pingcap/kvproto/pkg/cdcpb" + "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/utils/dynstream" "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/require" + "github.com/tikv/client-go/v2/tikv" ) type mockRegionEventSinkStream struct { @@ -107,3 +110,57 @@ func TestRegionEventSinkTracksEntriesUntilDrop(t *testing.T) { quotaState = getMemoryQuotaTestState(quota) require.Zero(t, quotaState.used) } + +func TestRegionEventSinkRemovePathReleasesQueuedEventMemory(t *testing.T) { + quota := newMemoryQuotaController(1024*1024, 8) + sink := newRegionEventSink(context.Background(), nil, quota) + defer sink.Close() + + span := newTestQuotaSpan(1) + span.resolvedTs.Store(100) + callbackCh := make(chan func(), 1) + span.consumeKVEvents = func(_ []common.RawKVEntry, callback func()) bool { + callbackCh <- callback + return true + } + span.advanceResolvedTs = func(uint64) {} + sink.AddPath(span) + + worker := ®ionRequestWorker{} + region := newTestQuotaRegion(span) + region.rpcCtx = &tikv.RPCContext{} + state := newRegionFeedState( + region, + uint64(span.subID), + worker, + nil, + ) + newEvent := func(commitTs uint64) regionEvent { + return regionEvent{ + states: []*regionFeedState{state}, + entries: &cdcpb.Event_Entries_{Entries: &cdcpb.Event_Entries{ + Entries: []*cdcpb.Event_Row{{ + Type: cdcpb.Event_COMMITTED, + OpType: cdcpb.Event_Row_PUT, + CommitTs: commitTs, + }}, + }}, + } + } + + sink.Push(span.subID, newEvent(101)) + callback := <-callbackCh + firstEventUsed := getMemoryQuotaTestState(quota).used + require.NotZero(t, firstEventUsed) + + // The first event blocks the path until callback is invoked, so this event + // remains queued when the path is removed. + sink.Push(span.subID, newEvent(102)) + require.Greater(t, getMemoryQuotaTestState(quota).used, firstEventUsed) + require.NoError(t, sink.RemovePath(span.subID)) + + callback() + require.Eventually(t, func() bool { + return getMemoryQuotaTestState(quota).used == 0 + }, time.Second, 10*time.Millisecond) +}