From b009a234abe8ffab6180e09074da77a76d988b48 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Fri, 10 Jul 2026 19:21:57 +0800 Subject: [PATCH 01/71] logpuller: make region request worker lifecycle explicit --- logservice/logpuller/region_event_handler.go | 2 +- .../logpuller/region_event_handler_test.go | 17 +- logservice/logpuller/region_req_cache.go | 341 ------------ logservice/logpuller/region_req_cache_test.go | 336 ------------ logservice/logpuller/region_request_cache.go | 269 ++++++++++ .../logpuller/region_request_cache_test.go | 148 ++++++ logservice/logpuller/region_request_worker.go | 484 ++++++++---------- .../logpuller/region_request_worker_test.go | 180 ++++--- logservice/logpuller/region_state.go | 53 +- logservice/logpuller/region_tracker.go | 118 +++++ logservice/logpuller/subscription_client.go | 56 +- .../logpuller/subscription_client_test.go | 47 +- utils/notifyqueue/notify_queue.go | 77 +++ utils/notifyqueue/notify_queue_test.go | 83 +++ 14 files changed, 1081 insertions(+), 1130 deletions(-) delete mode 100644 logservice/logpuller/region_req_cache.go delete mode 100644 logservice/logpuller/region_req_cache_test.go create mode 100644 logservice/logpuller/region_request_cache.go create mode 100644 logservice/logpuller/region_request_cache_test.go create mode 100644 logservice/logpuller/region_tracker.go create mode 100644 utils/notifyqueue/notify_queue.go create mode 100644 utils/notifyqueue/notify_queue_test.go diff --git a/logservice/logpuller/region_event_handler.go b/logservice/logpuller/region_event_handler.go index 777af088bd..91c4466bd0 100644 --- a/logservice/logpuller/region_event_handler.go +++ b/logservice/logpuller/region_event_handler.go @@ -255,7 +255,7 @@ func (h *regionEventHandler) handleRegionError(state *regionFeedState) { zap.Error(err)) } if stepsToRemoved { - worker.takeRegionState(SubscriptionID(state.requestID), state.getRegionID()) + worker.tracker.RemoveRegion(SubscriptionID(state.requestID), state.getRegionID()) h.failureHandler.Report(newRegionErrorInfo(state.getRegionInfo(), err)) } } diff --git a/logservice/logpuller/region_event_handler_test.go b/logservice/logpuller/region_event_handler_test.go index e94d176ed0..2a15ef75f1 100644 --- a/logservice/logpuller/region_event_handler_test.go +++ b/logservice/logpuller/region_event_handler_test.go @@ -79,6 +79,7 @@ func TestHandleEventEntryEventOutOfOrder(t *testing.T) { worker := ®ionRequestWorker{ requestCache: &requestCache{}, + tracker: newRegionTracker(0), } region := newRegionInfo( tikv.RegionVerID{}, @@ -88,8 +89,7 @@ func TestHandleEventEntryEventOutOfOrder(t *testing.T) { false, ) region.lockedRangeState = ®ionlock.LockedRangeState{} - state := newRegionFeedState(region, 1, worker) - state.start() + state := newRegionFeedState(region, 1, worker, nil) // Receive prewrite2 with empty value. { @@ -218,9 +218,9 @@ func TestHandleResolvedTs(t *testing.T) { subID1 := SubscriptionID(1) worker := ®ionRequestWorker{ requestCache: &requestCache{}, + tracker: newRegionTracker(0), } - state1 := newRegionFeedState(regionInfo{verID: tikv.NewRegionVerID(1, 1, 1)}, uint64(subID1), worker) - state1.start() + state1 := newRegionFeedState(regionInfo{verID: tikv.NewRegionVerID(1, 1, 1)}, uint64(subID1), worker, nil) { span := heartbeatpb.TableSpan{ TableID: 100, @@ -243,8 +243,7 @@ func TestHandleResolvedTs(t *testing.T) { } subID2 := SubscriptionID(2) - state2 := newRegionFeedState(regionInfo{verID: tikv.NewRegionVerID(2, 2, 2)}, uint64(subID2), worker) - state2.start() + state2 := newRegionFeedState(regionInfo{verID: tikv.NewRegionVerID(2, 2, 2)}, uint64(subID2), worker, nil) { span := heartbeatpb.TableSpan{ TableID: 100, @@ -267,8 +266,7 @@ func TestHandleResolvedTs(t *testing.T) { } subID3 := SubscriptionID(3) - state3 := newRegionFeedState(regionInfo{verID: tikv.NewRegionVerID(3, 3, 3)}, uint64(subID3), worker) - state3.start() + state3 := newRegionFeedState(regionInfo{verID: tikv.NewRegionVerID(3, 3, 3)}, uint64(subID3), worker, nil) { span := heartbeatpb.TableSpan{ TableID: 100, @@ -361,6 +359,7 @@ func TestHandleResolvedTsThrottled(t *testing.T) { advanceInterval: 100, } span.lastAdvanceTime.Store(0) + worker := ®ionRequestWorker{tracker: newRegionTracker(0)} state := newRegionFeedState( regionInfo{ verID: tikv.NewRegionVerID(1, 1, 1), @@ -368,9 +367,9 @@ func TestHandleResolvedTsThrottled(t *testing.T) { lockedRangeState: res1.LockedRangeState, }, 1, + worker, nil, ) - state.start() require.Equal(t, uint64(200), handleResolvedTs(span, state, 300)) } diff --git a/logservice/logpuller/region_req_cache.go b/logservice/logpuller/region_req_cache.go deleted file mode 100644 index b4478cab07..0000000000 --- a/logservice/logpuller/region_req_cache.go +++ /dev/null @@ -1,341 +0,0 @@ -// Copyright 2025 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package logpuller - -import ( - "context" - "sync" - "time" - - "github.com/pingcap/log" - "github.com/pingcap/ticdc/pkg/metrics" - "go.uber.org/atomic" - "go.uber.org/zap" -) - -const ( - checkStaleRequestInterval = time.Second * 10 - requestGCLifeTime = time.Minute * 180 - addReqRetryInterval = time.Millisecond * 1 - addReqRetryLimit = 3 - abnormalRequestDurationInSec = 60 * 60 * 2 // 2 hours -) - -// regionReq represents a wrapped region request with state -type regionReq struct { - regionInfo regionInfo - createTime time.Time -} - -func newRegionReq(region regionInfo) regionReq { - return regionReq{ - regionInfo: region, - createTime: time.Now(), - } -} - -func (r *regionReq) isStale() bool { - return time.Since(r.createTime) > requestGCLifeTime -} - -// requestCache manages region requests with flow control -type requestCache struct { - // pending requests waiting to be sent - pendingQueue chan regionReq - - // sent requests waiting for initialization (subscriptionID -> regions -> regionReq) - sentRequests struct { - sync.RWMutex - regionReqs map[SubscriptionID]map[uint64]regionReq - } - - // pendingCount is a flow control slot counter. - // A slot is acquired when a request is successfully enqueued into pendingQueue (see add), - // and is released when the request is finished/removed (resolve/markStopped/markDone/clear). - // pop and markSent don't change it. If markSent overwrites an existing request for the same region, - // it will release a slot for the replaced request to avoid leaking pendingCount. - pendingCount atomic.Int64 - // maximum number of pending requests allowed - maxPendingCount int64 - - // channel to signal when space becomes available - spaceAvailable chan struct{} - - lastCheckStaleRequestTime atomic.Time -} - -func newRequestCache(maxPendingCount int) *requestCache { - res := &requestCache{ - pendingQueue: make(chan regionReq, maxPendingCount), // Large buffer to reduce blocking - sentRequests: struct { - sync.RWMutex - regionReqs map[SubscriptionID]map[uint64]regionReq - }{regionReqs: make(map[SubscriptionID]map[uint64]regionReq)}, - pendingCount: atomic.Int64{}, - maxPendingCount: int64(maxPendingCount), - spaceAvailable: make(chan struct{}, 16), // Buffered to avoid blocking - } - - res.lastCheckStaleRequestTime.Store(time.Now()) - return res -} - -// add adds a new region request to the cache -// It blocks if pendingCount >= maxPendingCount until there's space or ctx is cancelled -func (c *requestCache) add(ctx context.Context, region regionInfo, force bool) (bool, error) { - start := time.Now() - ticker := time.NewTicker(addReqRetryInterval) - defer ticker.Stop() - addReqRetryLimit := addReqRetryLimit - - for { - current := c.pendingCount.Load() - if current < c.maxPendingCount || force { - // Try to add the request - req := newRegionReq(region) - select { - case <-ctx.Done(): - return false, ctx.Err() - case c.pendingQueue <- req: - c.pendingCount.Inc() - cost := time.Since(start) - metrics.SubscriptionClientAddRegionRequestDuration.Observe(cost.Seconds()) - return true, nil - case <-ticker.C: - addReqRetryLimit-- - if addReqRetryLimit <= 0 { - return false, nil - } - continue - } - } - - // Wait for space to become available - select { - case <-ticker.C: - addReqRetryLimit-- - if addReqRetryLimit <= 0 { - return false, nil - } - continue - case <-c.spaceAvailable: - continue - case <-ctx.Done(): - return false, ctx.Err() - } - } -} - -// pop gets the next pending request. -// Note: it doesn't change pendingCount. The slot acquired in add() should be released later -// (e.g. resolve/markStopped/markDone). -func (c *requestCache) pop(ctx context.Context) (regionReq, error) { - select { - case req := <-c.pendingQueue: - return req, nil - case <-ctx.Done(): - return regionReq{}, ctx.Err() - } -} - -// markSent marks a request as sent and adds it to sent requests. -// It doesn't change pendingCount: the slot is released when the request is finished/removed. -func (c *requestCache) markSent(req regionReq) { - c.sentRequests.Lock() - defer c.sentRequests.Unlock() - - m, ok := c.sentRequests.regionReqs[req.regionInfo.subscribedSpan.subID] - - if !ok { - m = make(map[uint64]regionReq) - c.sentRequests.regionReqs[req.regionInfo.subscribedSpan.subID] = m - } - - if oldReq, exists := m[req.regionInfo.verID.GetID()]; exists { - log.Warn("region request overwritten", - zap.Uint64("subID", uint64(req.regionInfo.subscribedSpan.subID)), - zap.Uint64("regionID", req.regionInfo.verID.GetID()), - zap.Float64("oldAgeSec", time.Since(oldReq.createTime).Seconds()), - zap.Float64("newAgeSec", time.Since(req.createTime).Seconds()), - zap.Int("pendingCount", int(c.pendingCount.Load())), - zap.Int("pendingQueueLen", len(c.pendingQueue))) - c.markDone() - } - m[req.regionInfo.verID.GetID()] = req -} - -// markStopped removes a sent request and releases a slot. -func (c *requestCache) markStopped(subID SubscriptionID, regionID uint64) { - c.sentRequests.Lock() - defer c.sentRequests.Unlock() - - regionReqs, ok := c.sentRequests.regionReqs[subID] - if !ok { - return - } - - _, exists := regionReqs[regionID] - if !exists { - return - } - - delete(regionReqs, regionID) - if len(regionReqs) == 0 { - delete(c.sentRequests.regionReqs, subID) - } - c.markDone() -} - -// resolve marks a region as initialized and removes it from sent requests -func (c *requestCache) resolve(subscriptionID SubscriptionID, regionID uint64) bool { - c.sentRequests.Lock() - defer c.sentRequests.Unlock() - regionReqs, ok := c.sentRequests.regionReqs[subscriptionID] - if !ok { - return false - } - - req, exists := regionReqs[regionID] - if !exists { - return false - } - - // Check if the subscription ID matches - if req.regionInfo.subscribedSpan.subID == subscriptionID { - delete(regionReqs, regionID) - c.markDone() - cost := time.Since(req.createTime).Seconds() - if cost > 0 && cost < abnormalRequestDurationInSec { - log.Debug("cdc resolve region request", zap.Uint64("subID", uint64(subscriptionID)), zap.Uint64("regionID", regionID), zap.Float64("cost", cost), zap.Int("pendingCount", int(c.pendingCount.Load())), zap.Int("pendingQueueLen", len(c.pendingQueue))) - metrics.RegionRequestFinishScanDuration.Observe(cost) - } else { - log.Info("region request duration abnormal, skip metric", zap.Float64("cost", cost), zap.Uint64("regionID", regionID)) - } - return true - } - - return false -} - -// clearStaleRequest clears stale requests from the cache -// Note: Sometimes, the CDC sends the same region request to TiKV multiple times. In such cases, this method is needed to reduce the pendingSize. -func (c *requestCache) clearStaleRequest() { - if time.Since(c.lastCheckStaleRequestTime.Load()) < checkStaleRequestInterval { - return - } - c.sentRequests.Lock() - defer c.sentRequests.Unlock() - reqCount := 0 - for subID, regionReqs := range c.sentRequests.regionReqs { - for regionID, regionReq := range regionReqs { - if regionReq.regionInfo.isStopped() || - regionReq.regionInfo.subscribedSpan.stopped.Load() || - regionReq.regionInfo.lockedRangeState.Initialized.Load() || - regionReq.isStale() { - c.markDone() - log.Warn("region worker delete stale region request", - zap.Uint64("subID", uint64(subID)), - zap.Uint64("regionID", regionID), - zap.Int("pendingCount", int(c.pendingCount.Load())), - zap.Int("pendingQueueLen", len(c.pendingQueue)), - zap.Bool("isRegionStopped", regionReq.regionInfo.isStopped()), - zap.Bool("isSubscribedSpanStopped", regionReq.regionInfo.subscribedSpan.stopped.Load()), - zap.Bool("isStale", regionReq.isStale()), - zap.Time("createTime", regionReq.createTime)) - delete(regionReqs, regionID) - } else { - reqCount++ - } - } - if len(regionReqs) == 0 { - delete(c.sentRequests.regionReqs, subID) - } - } - - // If there are no in-cache region requests but pendingCount isn't 0, it means pendingCount is stale. - // Reset it to avoid blocking add() forever. - if reqCount == 0 && len(c.pendingQueue) == 0 && c.pendingCount.Load() != 0 { - log.Info("region worker pending request count is not equal to actual region request count, correct it", - zap.Int("pendingCount", int(c.pendingCount.Load())), - zap.Int("actualReqCount", reqCount), - zap.Int("pendingQueueLen", len(c.pendingQueue))) - c.pendingCount.Store(0) - // Notify waiting add operations that there's space available. - select { - case c.spaceAvailable <- struct{}{}: - default: - } - } - - c.lastCheckStaleRequestTime.Store(time.Now()) -} - -// clear removes all requests and returns them -func (c *requestCache) clear() []regionInfo { - var regions []regionInfo - - // Drain pending requests from channel -LOOP: - for { - select { - case req := <-c.pendingQueue: - regions = append(regions, req.regionInfo) - c.markDone() - default: - break LOOP - } - } - - c.sentRequests.Lock() - defer c.sentRequests.Unlock() - - for subID, regionReqs := range c.sentRequests.regionReqs { - for regionID := range regionReqs { - regions = append(regions, regionReqs[regionID].regionInfo) - delete(regionReqs, regionID) - c.markDone() - } - delete(c.sentRequests.regionReqs, subID) - } - return regions -} - -// getPendingCount returns the current pending count -func (c *requestCache) getPendingCount() int { - return int(c.pendingCount.Load()) -} - -func (c *requestCache) markDone() { - // Decrement pendingCount by 1, but never let it go below 0. - // Do it with CAS to avoid clobbering concurrent Inc() calls. - for { - old := c.pendingCount.Load() - if old == 0 { - break - } else if old < 0 { - if c.pendingCount.CompareAndSwap(old, 0) { - break - } - } else { - if c.pendingCount.CompareAndSwap(old, old-1) { - break - } - } - } - // Notify waiting add operations that there's space available. - select { - case c.spaceAvailable <- struct{}{}: - default: // If channel is full, skip notification - } -} diff --git a/logservice/logpuller/region_req_cache_test.go b/logservice/logpuller/region_req_cache_test.go deleted file mode 100644 index 62706a8542..0000000000 --- a/logservice/logpuller/region_req_cache_test.go +++ /dev/null @@ -1,336 +0,0 @@ -// Copyright 2025 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package logpuller - -import ( - "context" - "testing" - "time" - - "github.com/pingcap/ticdc/heartbeatpb" - "github.com/stretchr/testify/require" - "github.com/tikv/client-go/v2/tikv" -) - -func createTestRegionInfo(subID SubscriptionID, regionID uint64) regionInfo { - verID := tikv.NewRegionVerID(regionID, 1, 1) - - span := heartbeatpb.TableSpan{ - TableID: 1, - StartKey: []byte("start"), - EndKey: []byte("end"), - } - - subscribedSpan := &subscribedSpan{ - subID: subID, - startTs: 100, - span: span, - } - - return newRegionInfo(verID, span, nil, subscribedSpan, false) -} - -func TestRequestCacheAdd_NormalCase(t *testing.T) { - cache := newRequestCache(10) - ctx := context.Background() - - region := createTestRegionInfo(1, 1) - - ok, err := cache.add(ctx, region, false) - require.NoError(t, err) - require.True(t, ok) - require.Equal(t, 1, cache.getPendingCount()) - - // Verify the request was added to the queue - req, err := cache.pop(ctx) - require.NoError(t, err) - require.NotNil(t, req) - require.Equal(t, region.verID.GetID(), req.regionInfo.verID.GetID()) - require.Equal(t, region.subscribedSpan.subID, req.regionInfo.subscribedSpan.subID) -} - -func TestRequestCacheAdd_ForceFlag(t *testing.T) { - cache := newRequestCache(1) - ctx := context.Background() - - // Fill up the cache - region1 := createTestRegionInfo(1, 1) - ok, err := cache.add(ctx, region1, false) - require.True(t, ok) - require.NoError(t, err) - require.Equal(t, 1, cache.getPendingCount()) - - // Try to add another request without force - should fail due to retry limit - region2 := createTestRegionInfo(1, 2) - ok, err = cache.add(ctx, region2, false) - require.False(t, ok) - require.NoError(t, err) - - // With force=true, it should still fail because the channel is full - // The force flag only bypasses the pendingCount check, not the channel capacity - region3 := createTestRegionInfo(1, 3) - ok, err = cache.add(ctx, region3, true) - require.False(t, ok) - require.NoError(t, err) - - // consume the pending queue ann add with force - req, err := cache.pop(ctx) - require.NoError(t, err) - require.NotNil(t, req) - require.Equal(t, region1.verID.GetID(), req.regionInfo.verID.GetID()) - require.Equal(t, region1.subscribedSpan.subID, req.regionInfo.subscribedSpan.subID) - cache.markSent(req) - require.Equal(t, 1, cache.getPendingCount()) - - ok, err = cache.add(ctx, region3, true) - require.True(t, ok) - require.NoError(t, err) - // It is 2 since region1 is unresolved - require.Equal(t, 2, cache.getPendingCount()) - - // resolve region1 - cache.resolve(region1.subscribedSpan.subID, region1.verID.GetID()) - require.Equal(t, 1, cache.getPendingCount()) -} - -func TestRequestCacheAdd_ContextCancellation(t *testing.T) { - cache := newRequestCache(1) - - // Fill up the cache - region1 := createTestRegionInfo(1, 1) - ctx1 := context.Background() - ok, err := cache.add(ctx1, region1, false) - require.True(t, ok) - require.NoError(t, err) - - // Try to add another request with a cancelled context - ctx2, cancel := context.WithCancel(context.Background()) - cancel() // Cancel immediately - - region2 := createTestRegionInfo(1, 2) - ok, err = cache.add(ctx2, region2, false) - require.False(t, ok) - require.Error(t, err) - require.Equal(t, context.Canceled, err) -} - -func TestRequestCacheAdd_RetryLimitExceeded(t *testing.T) { - cache := newRequestCache(1) - ctx := context.Background() - - // Fill up the cache - region1 := createTestRegionInfo(1, 1) - ok, err := cache.add(ctx, region1, false) - require.True(t, ok) - require.NoError(t, err) - - // Try to add another request - should eventually hit retry limit - region2 := createTestRegionInfo(1, 2) - ok, err = cache.add(ctx, region2, false) - require.False(t, ok) - require.NoError(t, err) -} - -func TestRequestCacheAdd_SpaceAvailableNotification(t *testing.T) { - cache := newRequestCache(2) - ctx := context.Background() - - // Fill up the cache - region1 := createTestRegionInfo(1, 1) - ok, err := cache.add(ctx, region1, false) - require.True(t, ok) - require.NoError(t, err) - require.Equal(t, 1, cache.getPendingCount()) - - region2 := createTestRegionInfo(1, 2) - ok, err = cache.add(ctx, region2, false) - require.True(t, ok) - require.NoError(t, err) - require.Equal(t, 2, cache.getPendingCount()) - - // Pop a request and mark it as sent, then resolve it to free up space - req, err := cache.pop(ctx) - require.NoError(t, err) - require.NotNil(t, req) - require.Equal(t, 2, cache.getPendingCount()) // pop doesn't change pendingCount - // Mark as sent - cache.markSent(req) - require.Equal(t, 2, cache.getPendingCount()) - - // Resolve the request to free up space - success := cache.resolve(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID()) - require.True(t, success) - require.Equal(t, 1, cache.getPendingCount()) - - // Now we should be able to add another request - region3 := createTestRegionInfo(1, 3) - ok, err = cache.add(ctx, region3, false) - require.True(t, ok) - require.NoError(t, err) - require.Equal(t, 2, cache.getPendingCount()) -} - -func TestRequestCacheAdd_ConcurrentAdds(t *testing.T) { - cache := newRequestCache(10) - ctx := context.Background() - - const numGoroutines = 5 - done := make(chan error, numGoroutines) - - // Start multiple goroutines adding requests concurrently - for i := 0; i < numGoroutines; i++ { - go func(id int) { - region := createTestRegionInfo(SubscriptionID(id%3), uint64(id)) - ok, err := cache.add(ctx, region, false) - require.True(t, ok) - require.NoError(t, err) - done <- err - }(i) - } - - // Wait for all goroutines to complete - for i := 0; i < numGoroutines; i++ { - select { - case err := <-done: - require.NoError(t, err) - case <-time.After(1 * time.Second): - t.Fatal("Timeout waiting for concurrent adds to complete") - } - } - - require.Equal(t, numGoroutines, cache.getPendingCount()) -} - -func TestRequestCacheAdd_StaleRequestCleanup(t *testing.T) { - cache := newRequestCache(10) - ctx := context.Background() - - // Add a request and mark it as sent - region := createTestRegionInfo(1, 1) - ok, err := cache.add(ctx, region, false) - require.True(t, ok) - require.NoError(t, err) - - req, err := cache.pop(ctx) - require.NoError(t, err) - require.NotNil(t, req) - - // Mark as sent - cache.markSent(req) - require.Equal(t, 1, cache.getPendingCount()) - - // Manually set the request as stale by modifying createTime - cache.sentRequests.Lock() - regionReqs := cache.sentRequests.regionReqs[req.regionInfo.subscribedSpan.subID] - regionReqs[req.regionInfo.verID.GetID()] = regionReq{ - regionInfo: req.regionInfo, - createTime: time.Now().Add(-requestGCLifeTime - time.Second), // Make it stale - } - cache.sentRequests.Unlock() - - // Manually set lastCheckStaleRequestTime to bypass the time interval check - cache.lastCheckStaleRequestTime.Store(time.Now().Add(-checkStaleRequestInterval - time.Second)) - - // Manually trigger stale cleanup by calling clearStaleRequest - cache.clearStaleRequest() - - // The stale request should be cleaned up - require.Equal(t, 0, cache.getPendingCount()) -} - -func TestRequestCacheAdd_WithStoppedRegion(t *testing.T) { - cache := newRequestCache(10) - ctx := context.Background() - - // Create a region info with stopped state (lockedRangeState = nil) - region := createTestRegionInfo(1, 1) - region.lockedRangeState = nil // This makes it stopped - - ok, err := cache.add(ctx, region, false) - require.True(t, ok) - require.NoError(t, err) - require.Equal(t, 1, cache.getPendingCount()) - - req, err := cache.pop(ctx) - require.NoError(t, err) - require.NotNil(t, req) - - // Mark as sent - cache.markSent(req) - require.Equal(t, 1, cache.getPendingCount()) - - // Manually set lastCheckStaleRequestTime to bypass the time interval check - cache.lastCheckStaleRequestTime.Store(time.Now().Add(-checkStaleRequestInterval - time.Second)) - - // Manually trigger cleanup of stopped region - cache.clearStaleRequest() - - // The stopped region should be cleaned up - require.Equal(t, 0, cache.getPendingCount()) -} - -func TestRequestCacheMarkSent_DuplicateReleaseSlot(t *testing.T) { - cache := newRequestCache(10) - ctx := context.Background() - - region := createTestRegionInfo(1, 1) - - ok, err := cache.add(ctx, region, false) - require.True(t, ok) - require.NoError(t, err) - - // Add a duplicate request for the same region. It should not leak pendingCount even if - // markSent overwrites the existing entry. - ok, err = cache.add(ctx, region, false) - require.True(t, ok) - require.NoError(t, err) - require.Equal(t, 2, cache.getPendingCount()) - - req1, err := cache.pop(ctx) - require.NoError(t, err) - cache.markSent(req1) - require.Equal(t, 2, cache.getPendingCount()) - - req2, err := cache.pop(ctx) - require.NoError(t, err) - cache.markSent(req2) - require.Equal(t, 1, cache.getPendingCount()) - - // Finish the remaining tracked request. - require.True(t, cache.resolve(region.subscribedSpan.subID, region.verID.GetID())) - require.Equal(t, 0, cache.getPendingCount()) -} - -func TestRequestCacheMarkStopped_ReleasesSlot(t *testing.T) { - cache := newRequestCache(10) - ctx := context.Background() - - region := createTestRegionInfo(1, 1) - - ok, err := cache.add(ctx, region, false) - require.True(t, ok) - require.NoError(t, err) - require.Equal(t, 1, cache.getPendingCount()) - - req, err := cache.pop(ctx) - require.NoError(t, err) - - cache.markSent(req) - require.Equal(t, 1, cache.getPendingCount()) - require.Contains(t, cache.sentRequests.regionReqs, req.regionInfo.subscribedSpan.subID) - - cache.markStopped(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID()) - require.Equal(t, 0, cache.getPendingCount()) - require.NotContains(t, cache.sentRequests.regionReqs, req.regionInfo.subscribedSpan.subID) -} diff --git a/logservice/logpuller/region_request_cache.go b/logservice/logpuller/region_request_cache.go new file mode 100644 index 0000000000..bf2c5cad7b --- /dev/null +++ b/logservice/logpuller/region_request_cache.go @@ -0,0 +1,269 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package logpuller + +import ( + "context" + "sync" + "time" + + "github.com/pingcap/log" + "github.com/pingcap/ticdc/pkg/metrics" + "github.com/pingcap/ticdc/utils/notifyqueue" + "go.uber.org/zap" +) + +const ( + addReqRetryInterval = time.Millisecond + addReqRetryLimit = 3 + abnormalRequestDurationInSec = 60 * 60 * 2 // 2 hours +) + +type regionReqStage uint8 + +const ( + regionReqStageQueued regionReqStage = iota + regionReqStageProcessing + regionReqStageSent +) + +// regionReq tracks one region request from admission to cleanup. +type regionReq struct { + regionInfo regionInfo + createTime time.Time + + // stage is guarded by requestCache.mu. + stage regionReqStage +} + +func newRegionReq(region regionInfo) *regionReq { + return ®ionReq{ + regionInfo: region, + createTime: time.Now(), + stage: regionReqStageQueued, + } +} + +// requestCache owns the lifecycle of worker-local region requests. +// Requests move through queued, processing and sent before explicit cleanup. +type requestCache struct { + mu sync.Mutex + + maxPendingCount int + // requests is the source of truth for every live request in this worker. + requests map[*regionReq]struct{} + queue *notifyqueue.Queue[*regionReq] + + spaceAvailable chan struct{} +} + +func newRequestCache(maxPendingCount int) *requestCache { + return &requestCache{ + maxPendingCount: maxPendingCount, + requests: make(map[*regionReq]struct{}), + queue: notifyqueue.New[*regionReq](), + spaceAvailable: make(chan struct{}, 1), + } +} + +func (c *requestCache) add(ctx context.Context, region regionInfo, force bool) (bool, error) { + start := time.Now() + ticker := time.NewTicker(addReqRetryInterval) + defer ticker.Stop() + retries := addReqRetryLimit + + for { + if c.tryAdd(region, force) { + metrics.SubscriptionClientAddRegionRequestDuration.Observe(time.Since(start).Seconds()) + return true, nil + } + + select { + case <-ticker.C: + retries-- + if retries <= 0 { + return false, nil + } + case <-c.spaceAvailable: + case <-ctx.Done(): + return false, ctx.Err() + } + } +} + +func (c *requestCache) tryAdd(region regionInfo, force bool) bool { + c.mu.Lock() + defer c.mu.Unlock() + + if len(c.requests) >= c.maxPendingCount && !force { + return false + } + + req := newRegionReq(region) + c.requests[req] = struct{}{} + c.queue.Push(req) + return true +} + +func (c *requestCache) pop(ctx context.Context) (*regionReq, error) { + for { + if req := c.tryPop(); req != nil { + return req, nil + } + + select { + case <-c.queue.Ready(): + case <-ctx.Done(): + return nil, ctx.Err() + } + } +} + +func (c *requestCache) tryPop() *regionReq { + c.mu.Lock() + defer c.mu.Unlock() + + for { + req, ok := c.queue.TryPop() + if !ok { + return nil + } + if _, ok := c.requests[req]; !ok { + log.Warn("request cache pops a removed request", + zap.Uint64("subID", uint64(req.regionInfo.subscribedSpan.subID)), + zap.Uint64("regionID", req.regionInfo.verID.GetID()), + zap.Uint8("stage", uint8(req.stage))) + continue + } + if req.stage != regionReqStageQueued { + log.Warn("request cache pops a non-queued request", + zap.Uint64("subID", uint64(req.regionInfo.subscribedSpan.subID)), + zap.Uint64("regionID", req.regionInfo.verID.GetID()), + zap.Uint8("stage", uint8(req.stage))) + continue + } + req.stage = regionReqStageProcessing + return req + } +} + +func (c *requestCache) markSent(req *regionReq) { + c.mu.Lock() + defer c.mu.Unlock() + if _, ok := c.requests[req]; ok && req.stage == regionReqStageProcessing { + req.stage = regionReqStageSent + } +} + +func (c *requestCache) finishScan(req *regionReq) bool { + if !c.remove(req) { + return false + } + + cost := time.Since(req.createTime).Seconds() + if cost > 0 && cost < abnormalRequestDurationInSec { + log.Debug("cdc resolve region request", + zap.Uint64("subID", uint64(req.regionInfo.subscribedSpan.subID)), + zap.Uint64("regionID", req.regionInfo.verID.GetID()), + zap.Float64("cost", cost), + zap.Int("pendingCount", c.pendingCount())) + metrics.RegionRequestFinishScanDuration.Observe(cost) + return true + } + log.Info("region request duration abnormal, skip metric", + zap.Float64("cost", cost), + zap.Uint64("regionID", req.regionInfo.verID.GetID())) + return true +} + +func (c *requestCache) abortScan(req *regionReq) bool { + return c.remove(req) +} + +func (c *requestCache) remove(req *regionReq) bool { + if req == nil { + return false + } + + c.mu.Lock() + removed := c.removeLocked(req) + c.mu.Unlock() + + if removed { + c.notifySpace() + } + return removed +} + +// drainUnsentRegions removes queued and processing requests. Sent requests are +// owned by regionTracker and are cleaned by their regionFeedState. +func (c *requestCache) drainUnsentRegions() []regionInfo { + c.mu.Lock() + regions := make([]regionInfo, 0, len(c.requests)) + for req := range c.requests { + if req.stage == regionReqStageSent { + continue + } + if c.removeLocked(req) { + regions = append(regions, req.regionInfo) + } + } + c.queue.Drain() + c.mu.Unlock() + + if len(regions) > 0 { + c.notifySpace() + } + return regions +} + +func (c *requestCache) close() { + c.mu.Lock() + removed := len(c.requests) > 0 + clear(c.requests) + c.queue.Drain() + c.mu.Unlock() + + if removed { + c.notifySpace() + } +} + +func (c *requestCache) pendingCount() int { + c.mu.Lock() + defer c.mu.Unlock() + return len(c.requests) +} + +func (c *requestCache) removeLocked(req *regionReq) bool { + if req == nil { + return false + } + if _, ok := c.requests[req]; !ok { + return false + } + delete(c.requests, req) + return true +} + +func (c *requestCache) ready() <-chan struct{} { + return c.queue.Ready() +} + +func (c *requestCache) notifySpace() { + select { + case c.spaceAvailable <- struct{}{}: + default: + } +} diff --git a/logservice/logpuller/region_request_cache_test.go b/logservice/logpuller/region_request_cache_test.go new file mode 100644 index 0000000000..d4fe55616c --- /dev/null +++ b/logservice/logpuller/region_request_cache_test.go @@ -0,0 +1,148 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package logpuller + +import ( + "context" + "testing" + + "github.com/pingcap/ticdc/heartbeatpb" + "github.com/stretchr/testify/require" + "github.com/tikv/client-go/v2/tikv" +) + +func createTestRegionInfo(subID SubscriptionID, regionID uint64) regionInfo { + span := heartbeatpb.TableSpan{ + TableID: 1, + StartKey: []byte("start"), + EndKey: []byte("end"), + } + return newRegionInfo( + tikv.NewRegionVerID(regionID, 1, 1), + span, + nil, + &subscribedSpan{subID: subID, startTs: 100, span: span}, + false, + ) +} + +func TestRequestCacheLifecycle(t *testing.T) { + cache := newRequestCache(1) + region := createTestRegionInfo(1, 1) + + ok, err := cache.add(t.Context(), region, false) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, 1, cache.pendingCount()) + + req, err := cache.pop(t.Context()) + require.NoError(t, err) + require.Equal(t, regionReqStageProcessing, req.stage) + require.Equal(t, 1, cache.pendingCount()) + + cache.markSent(req) + require.Equal(t, regionReqStageSent, req.stage) + require.True(t, cache.finishScan(req)) + require.False(t, cache.finishScan(req)) + require.Equal(t, 0, cache.pendingCount()) +} + +func TestRequestCacheCapacityAndForceAdd(t *testing.T) { + cache := newRequestCache(1) + + ok, err := cache.add(t.Context(), createTestRegionInfo(1, 1), false) + require.NoError(t, err) + require.True(t, ok) + + ok, err = cache.add(t.Context(), createTestRegionInfo(1, 2), false) + require.NoError(t, err) + require.False(t, ok) + + ok, err = cache.add(t.Context(), createTestRegionInfo(1, 3), true) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, 2, cache.pendingCount()) +} + +func TestRequestCacheAddHonorsCancellationWhenFull(t *testing.T) { + cache := newRequestCache(1) + ok, err := cache.add(t.Context(), createTestRegionInfo(1, 1), false) + require.NoError(t, err) + require.True(t, ok) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + ok, err = cache.add(ctx, createTestRegionInfo(1, 2), false) + require.False(t, ok) + require.ErrorIs(t, err, context.Canceled) +} + +func TestRequestCacheTracksDuplicateRegionsIndependently(t *testing.T) { + cache := newRequestCache(2) + region := createTestRegionInfo(1, 1) + + ok, err := cache.add(t.Context(), region, false) + require.NoError(t, err) + require.True(t, ok) + ok, err = cache.add(t.Context(), region, false) + require.NoError(t, err) + require.True(t, ok) + + req1, err := cache.pop(t.Context()) + require.NoError(t, err) + req2, err := cache.pop(t.Context()) + require.NoError(t, err) + require.NotSame(t, req1, req2) + require.Equal(t, 2, cache.pendingCount()) +} + +func TestRequestCacheDrainOnlyUnsentRegions(t *testing.T) { + cache := newRequestCache(3) + for regionID := uint64(1); regionID <= 3; regionID++ { + ok, err := cache.add(t.Context(), createTestRegionInfo(1, regionID), false) + require.NoError(t, err) + require.True(t, ok) + } + + sentReq, err := cache.pop(t.Context()) + require.NoError(t, err) + cache.markSent(sentReq) + processingReq, err := cache.pop(t.Context()) + require.NoError(t, err) + require.Equal(t, regionReqStageProcessing, processingReq.stage) + + regions := cache.drainUnsentRegions() + require.Len(t, regions, 2) + require.Equal(t, 1, cache.pendingCount()) + require.True(t, cache.abortScan(sentReq)) + require.Equal(t, 0, cache.pendingCount()) +} + +func TestRequestCacheCloseRemovesAllRequests(t *testing.T) { + cache := newRequestCache(2) + for regionID := uint64(1); regionID <= 2; regionID++ { + ok, err := cache.add(t.Context(), createTestRegionInfo(1, regionID), false) + require.NoError(t, err) + require.True(t, ok) + } + + req, err := cache.pop(t.Context()) + require.NoError(t, err) + cache.markSent(req) + cache.close() + + require.Equal(t, 0, cache.pendingCount()) + require.Nil(t, cache.tryPop()) + require.False(t, cache.abortScan(req)) +} diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index 142f4c6493..9fa559dc19 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -25,172 +25,168 @@ import ( "github.com/pingcap/kvproto/pkg/kvrpcpb" "github.com/pingcap/log" cerror "github.com/pingcap/ticdc/pkg/errors" - "github.com/pingcap/ticdc/pkg/security" "github.com/pingcap/ticdc/pkg/util" "github.com/pingcap/ticdc/pkg/version" + "github.com/pingcap/ticdc/utils/notifyqueue" "go.uber.org/zap" "golang.org/x/sync/errgroup" grpcstatus "google.golang.org/grpc/status" ) +const storeReconnectBackoff = time.Second + // To generate a workerID in `newRegionRequestWorker`. var workerIDGen atomic.Uint64 -type regionFeedStates map[uint64]*regionFeedState +type deregisterRequest struct { + subID SubscriptionID + filterLoop bool +} -// regionRequestWorker is responsible for sending region requests to a specific TiKV store. -type regionRequestWorker struct { - workerID uint64 +type controlQueue struct { + mu sync.Mutex + queue *notifyqueue.Queue[deregisterRequest] +} - client *subscriptionClient +func newControlQueue() *controlQueue { + return &controlQueue{queue: notifyqueue.New[deregisterRequest]()} +} + +func (q *controlQueue) push(req deregisterRequest) { + q.mu.Lock() + defer q.mu.Unlock() + q.queue.Push(req) +} - store *requestedStore +func (q *controlQueue) tryPop() (deregisterRequest, bool) { + q.mu.Lock() + defer q.mu.Unlock() + return q.queue.TryPop() +} - // we must always get a region to request before create a grpc stream. - // only in this way we can avoid to try to connect to an offline store infinitely. - preFetchForConnecting *regionInfo +func (q *controlQueue) len() int { + q.mu.Lock() + defer q.mu.Unlock() + return q.queue.Len() +} - // request cache with flow control - requestCache *requestCache +func (q *controlQueue) ready() <-chan struct{} { + return q.queue.Ready() +} - // all regions maintained by this worker. - requestedRegions struct { - sync.RWMutex +// regionRequestWorker owns one TiKV event-feed stream and the requests sent +// through it, including reconnect cleanup and subscription deregistration. +type regionRequestWorker struct { + workerID uint64 - subscriptions map[SubscriptionID]regionFeedStates - } + client *subscriptionClient + store *requestedStore + + requestCache *requestCache + controlQueue *controlQueue + tracker *regionTracker } func newRegionRequestWorker( - ctx context.Context, client *subscriptionClient, - credential *security.Credential, - g *errgroup.Group, store *requestedStore, - requestCacheSize int, + requestCache *requestCache, ) *regionRequestWorker { - worker := ®ionRequestWorker{ - workerID: workerIDGen.Add(1), + workerID := workerIDGen.Add(1) + return ®ionRequestWorker{ + workerID: workerID, client: client, store: store, - requestCache: newRequestCache(requestCacheSize), + requestCache: requestCache, + controlQueue: newControlQueue(), + tracker: newRegionTracker(workerID), } - worker.requestedRegions.subscriptions = make(map[SubscriptionID]regionFeedStates) +} - waitForPreFetching := func() error { - if worker.preFetchForConnecting != nil { - log.Panic("preFetchForConnecting should be nil", - zap.Uint64("workerID", worker.workerID), - zap.String("addr", store.storeAddr)) +func (s *regionRequestWorker) Run(ctx context.Context) error { + for { + // Do not connect an idle worker to an unavailable store indefinitely. + firstReq, err := s.requestCache.pop(ctx) + if err != nil { + return err } - for { - req, err := worker.requestCache.pop(ctx) - if err != nil { - return err - } - if req.regionInfo.isStopped() { - worker.requestCache.markDone() - continue - } - worker.preFetchForConnecting = new(regionInfo) - *worker.preFetchForConnecting = req.regionInfo - return nil + + regionErr := error(&storeStreamErr{}) + if err := s.checkStoreVersion(ctx); err != nil { + regionErr = err + } else if err := s.runStream(ctx, firstReq); err != nil { + regionErr = err + } + if ctx.Err() != nil { + return ctx.Err() } - } - g.Go(func() error { - for { - if err := waitForPreFetching(); err != nil { - return err - } - var regionErr error - if err := version.CheckStoreVersion(ctx, worker.client.pd); err != nil { - if errors.Cause(err) == context.Canceled { - return nil - } - log.Error("event feed check store version fails", - zap.Uint64("workerID", worker.workerID), - zap.String("addr", worker.store.storeAddr), - zap.Error(err)) - if cerror.Is(err, cerror.ErrGetAllStoresFailed) { - regionErr = &getStoreErr{} - } else { - regionErr = &storeStreamErr{} - } - } else { - if canceled := worker.run(ctx, credential); canceled { - return nil - } - regionErr = &storeStreamErr{} - } - for subID, m := range worker.clearRegionStates() { - for _, state := range m { - state.markStopped(regionErr) - regionEvent := regionEvent{ - states: []*regionFeedState{state}, - } - worker.client.pushRegionEventToDS(subID, regionEvent) - } - } - // The store may fail forever, so we need try to re-schedule all pending regions. - for _, region := range worker.clearPendingRegions() { - if region.isStopped() { - // It means it's a special task for stopping the table. - continue - } - client.onRegionFail(newRegionErrorInfo(region, regionErr)) - } - if err := util.Hang(ctx, time.Second); err != nil { - return err + // Sent requests are owned by their states. Stopping those states removes + // their requests before unsent requests are drained below. + for subID, states := range s.tracker.Drain() { + for _, state := range states { + state.markStopped(regionErr) + s.client.eventSink.Push(subID, regionEvent{states: []*regionFeedState{state}}) } } - }) + for _, region := range s.requestCache.drainUnsentRegions() { + s.client.onRegionFail(newRegionErrorInfo(region, regionErr)) + } - return worker + if err := util.Hang(ctx, storeReconnectBackoff); err != nil { + return err + } + } } -func (s *regionRequestWorker) run(ctx context.Context, credential *security.Credential) (canceled bool) { - isCanceled := func() bool { - select { - case <-ctx.Done(): - return true - default: - return false - } +func (s *regionRequestWorker) checkStoreVersion(ctx context.Context) error { + err := version.CheckStoreVersion(ctx, s.client.pd) + if err == nil { + return nil + } + if ctx.Err() != nil { + return ctx.Err() } + log.Error("event feed check store version fails", + zap.Uint64("workerID", s.workerID), + zap.String("addr", s.store.storeAddr), + zap.Error(err)) + if cerror.Is(err, cerror.ErrGetAllStoresFailed) { + return &getStoreErr{} + } + return &storeStreamErr{} +} +func (s *regionRequestWorker) runStream(ctx context.Context, firstReq *regionReq) (err error) { log.Info("region request worker going to create grpc stream", zap.Uint64("workerID", s.workerID), zap.String("addr", s.store.storeAddr)) - defer func() { log.Info("region request worker exits", zap.Uint64("workerID", s.workerID), zap.String("addr", s.store.storeAddr), - zap.Bool("canceled", canceled)) + zap.Error(err)) }() g, gctx := errgroup.WithContext(ctx) - conn, err := Connect(gctx, credential, s.store.storeAddr) + conn, err := Connect(gctx, s.client.credential, s.store.storeAddr) if err != nil { log.Warn("region request worker create grpc stream failed", zap.Uint64("workerID", s.workerID), zap.String("addr", s.store.storeAddr), zap.Error(err)) - // Close the connection if it was partially created to prevent goroutine leaks if conn != nil && conn.Conn != nil { _ = conn.Conn.Close() } - return isCanceled() + if ctx.Err() != nil { + return ctx.Err() + } + return &storeStreamErr{} } - defer func() { - _ = conn.Conn.Close() - }() + defer func() { _ = conn.Conn.Close() }() - g.Go(func() error { - return s.receiveAndDispatchChangeEvents(conn) - }) - g.Go(func() error { return s.processRegionSendTask(gctx, conn) }) + g.Go(func() error { return s.receiveAndDispatchChangeEvents(conn) }) + g.Go(func() error { return s.processRegionSendTask(gctx, conn, firstReq) }) failpoint.Inject("InjectForceReconnect", func() { timer := time.After(10 * time.Second) @@ -202,8 +198,14 @@ func (s *regionRequestWorker) run(ctx context.Context, credential *security.Cred }) }) - _ = g.Wait() - return isCanceled() + err = g.Wait() + if err != nil { + if ctx.Err() != nil { + return ctx.Err() + } + return &storeStreamErr{} + } + return nil } func normalizeStreamError(err error) error { @@ -213,7 +215,6 @@ func normalizeStreamError(err error) error { return errors.Trace(err) } -// receiveAndDispatchChangeEventsToProcessor receives events from the grpc stream and dispatches them to ds. func (s *regionRequestWorker) receiveAndDispatchChangeEvents(conn *ConnAndClient) error { for { changeEvent, err := conn.Client.Recv() @@ -238,11 +239,9 @@ func (s *regionRequestWorker) dispatchRegionChangeEvents(events []*cdcpb.Event) for _, event := range events { regionID := event.RegionId subscriptionID := SubscriptionID(event.RequestId) - state := s.getRegionState(subscriptionID, regionID) + state := s.tracker.Get(subscriptionID, regionID) if state != nil { - regionEvent := regionEvent{ - states: []*regionFeedState{state}, - } + regionEvent := regionEvent{states: []*regionFeedState{state}} switch eventData := event.Event.(type) { case *cdcpb.Event_Entries_: if eventData == nil { @@ -254,7 +253,6 @@ func (s *regionRequestWorker) dispatchRegionChangeEvents(events []*cdcpb.Event) } regionEvent.entries = eventData case *cdcpb.Event_Admin_: - // ignore continue case *cdcpb.Event_Error: log.Debug("region request worker receives a region error", @@ -266,26 +264,25 @@ func (s *regionRequestWorker) dispatchRegionChangeEvents(events []*cdcpb.Event) case *cdcpb.Event_ResolvedTs: regionEvent.resolvedTs = eventData.ResolvedTs case *cdcpb.Event_LongTxn_: - // ignore continue default: log.Panic("unknown event type", zap.Any("event", event)) } - s.client.pushRegionEventToDS(subscriptionID, regionEvent) - } else { - switch event.Event.(type) { - case *cdcpb.Event_Error: - // it is normal to receive region error after deregister a subscription - log.Debug("region request worker receives an error for a stale region, ignore it", - zap.Uint64("workerID", s.workerID), - zap.Uint64("subscriptionID", uint64(subscriptionID)), - zap.Uint64("regionID", event.RegionId)) - default: - log.Warn("region request worker receives a region event for an untracked region", - zap.Uint64("workerID", s.workerID), - zap.Uint64("subscriptionID", uint64(subscriptionID)), - zap.Uint64("regionID", event.RegionId)) - } + s.client.eventSink.Push(subscriptionID, regionEvent) + continue + } + + switch event.Event.(type) { + case *cdcpb.Event_Error: + log.Debug("region request worker receives an error for a stale region, ignore it", + zap.Uint64("workerID", s.workerID), + zap.Uint64("subscriptionID", uint64(subscriptionID)), + zap.Uint64("regionID", event.RegionId)) + default: + log.Warn("region request worker receives a region event for an untracked region", + zap.Uint64("workerID", s.workerID), + zap.Uint64("subscriptionID", uint64(subscriptionID)), + zap.Uint64("regionID", event.RegionId)) } } } @@ -294,7 +291,6 @@ func (s *regionRequestWorker) dispatchResolvedTsEvent(resolvedTsEvent *cdcpb.Res subscriptionID := SubscriptionID(resolvedTsEvent.RequestId) metricsResolvedTsCount.Add(float64(len(resolvedTsEvent.Regions))) s.client.metrics.batchResolvedSize.Observe(float64(len(resolvedTsEvent.Regions))) - // TODO: resolvedTsEvent.Ts be 0 is impossible, we need find the root cause. if resolvedTsEvent.Ts == 0 { log.Warn("region request worker receives a resolved ts event with zero value, ignore it", zap.Uint64("workerID", s.workerID), @@ -302,34 +298,27 @@ func (s *regionRequestWorker) dispatchResolvedTsEvent(resolvedTsEvent *cdcpb.Res zap.Any("regionIDs", resolvedTsEvent.Regions)) return } - // Avoid allocating a huge states slice when resolvedTsEvent.Regions is large. - // Push resolved-ts events in batches to reduce peak memory usage and improve GC behavior. + const resolvedTsStateBatchSize = 1024 - capHint := len(resolvedTsEvent.Regions) - if capHint > resolvedTsStateBatchSize { - capHint = resolvedTsStateBatchSize - } + capHint := min(len(resolvedTsEvent.Regions), resolvedTsStateBatchSize) resolvedStates := make([]*regionFeedState, 0, capHint) flush := func() { if len(resolvedStates) == 0 { return } - s.client.pushRegionEventToDS(subscriptionID, regionEvent{ + s.client.eventSink.Push(subscriptionID, regionEvent{ resolvedTs: resolvedTsEvent.Ts, states: resolvedStates, }) resolvedStates = nil } for i, regionID := range resolvedTsEvent.Regions { - if state := s.getRegionState(subscriptionID, regionID); state != nil { + if state := s.tracker.Get(subscriptionID, regionID); state != nil { resolvedStates = append(resolvedStates, state) if len(resolvedStates) >= resolvedTsStateBatchSize { flush() if i+1 < len(resolvedTsEvent.Regions) { - capHint = len(resolvedTsEvent.Regions) - (i + 1) - if capHint > resolvedTsStateBatchSize { - capHint = resolvedTsStateBatchSize - } + capHint = min(len(resolvedTsEvent.Regions)-(i+1), resolvedTsStateBatchSize) resolvedStates = make([]*regionFeedState, 0, capHint) } } @@ -344,10 +333,10 @@ func (s *regionRequestWorker) dispatchResolvedTsEvent(resolvedTsEvent *cdcpb.Res flush() } -// processRegionSendTask receives region requests from the channel and sends them to the remote store. func (s *regionRequestWorker) processRegionSendTask( ctx context.Context, conn *ConnAndClient, + firstReq *regionReq, ) error { doSend := func(req *cdcpb.ChangeDataRequest) error { if err := conn.Client.Send(req); err != nil { @@ -359,81 +348,80 @@ func (s *regionRequestWorker) processRegionSendTask( zap.Error(err)) return normalizeStreamError(err) } - // TODO: add a metric? return nil } - - // Handle pre-fetched region first - region := *s.preFetchForConnecting - s.preFetchForConnecting = nil - regionReq := newRegionReq(region) - var err error - for { - region := regionReq.regionInfo - subID := region.subscribedSpan.subID - log.Debug("region request worker gets a singleRegionInfo", - zap.Uint64("workerID", s.workerID), - zap.Uint64("subscriptionID", uint64(subID)), - zap.Uint64("regionID", region.verID.GetID()), - zap.String("addr", s.store.storeAddr), - zap.Bool("bdrMode", region.filterLoop)) - - // It means it's a special task for stopping the table. - if region.isStopped() { - req := &cdcpb.ChangeDataRequest{ - Header: &cdcpb.Header{ClusterId: s.client.clusterID, TicdcVersion: version.ReleaseSemver()}, - RequestId: uint64(subID), - Request: &cdcpb.ChangeDataRequest_Deregister_{ - Deregister: &cdcpb.ChangeDataRequest_Deregister{}, - }, - FilterLoop: region.filterLoop, + sendDeregister := func(req deregisterRequest) error { + changeDataReq := &cdcpb.ChangeDataRequest{ + Header: &cdcpb.Header{ClusterId: s.client.clusterID, TicdcVersion: version.ReleaseSemver()}, + RequestId: uint64(req.subID), + Request: &cdcpb.ChangeDataRequest_Deregister_{ + Deregister: &cdcpb.ChangeDataRequest_Deregister{}, + }, + FilterLoop: req.filterLoop, + } + if err := doSend(changeDataReq); err != nil { + return err + } + for _, state := range s.tracker.RemoveSubscription(req.subID) { + state.markStopped(&requestCancelledErr{}) + s.client.eventSink.Push(req.subID, regionEvent{states: []*regionFeedState{state}}) + } + return nil + } + drainControl := func() error { + for { + req, ok := s.controlQueue.tryPop() + if !ok { + return nil } - s.requestCache.markDone() - if err := doSend(req); err != nil { + if err := sendDeregister(req); err != nil { return err } - for _, state := range s.takeRegionStates(subID) { - state.markStopped(&requestCancelledErr{}) - regionEvent := regionEvent{ - states: []*regionFeedState{state}, + } + } + + regionReq := firstReq + for { + if regionReq != nil { + region := regionReq.regionInfo + subID := region.subscribedSpan.subID + log.Debug("region request worker gets a singleRegionInfo", + zap.Uint64("workerID", s.workerID), + zap.Uint64("subscriptionID", uint64(subID)), + zap.Uint64("regionID", region.verID.GetID()), + zap.String("addr", s.store.storeAddr), + zap.Bool("bdrMode", region.filterLoop)) + + if region.subscribedSpan.stopped.Load() { + s.client.onRegionFail(newRegionErrorInfo(region, &storeStreamErr{})) + s.requestCache.abortScan(regionReq) + } else { + state := newRegionFeedState(region, uint64(subID), s, regionReq) + s.tracker.Track(subID, region.verID.GetID(), state) + // Make the request and its state visible in the same order. A fast + // region error can then clean the request without racing markSent. + s.requestCache.markSent(regionReq) + if err := doSend(s.createRegionRequest(region)); err != nil { + state.markStopped(err) + return err } - s.client.pushRegionEventToDS(subID, regionEvent) - } - } else if region.subscribedSpan.stopped.Load() { - // It can be skipped directly because there must be no pending states from - // the stopped subscribedTable, or the special singleRegionInfo for stopping - // the table will be handled later. - s.client.onRegionFail(newRegionErrorInfo(region, &storeStreamErr{})) - s.requestCache.markDone() - } else { - state := newRegionFeedState(region, uint64(subID), s) - state.start() - s.addRegionState(subID, region.verID.GetID(), state) - // Mark the request as sent before sending it. - // Otherwise there is a race with the receiver goroutine: - // 1. addRegionState makes the region visible to error handling. - // 2. doSend sends the request. - // 3. the receiver goroutine may receive a region error immediately. - // 4. markStopped runs before markSent, so requestCache.markStopped cannot - // find the request in sentRequests. - // 5. the sender goroutine then calls markSent and leaves a stale sent - // request behind, even though the region has already been - // unlocked/rescheduled. - // - // Tracking the request before Send keeps requestedRegions and - // sentRequests visible in the same order and avoids leaving stale - // requests in cleanup. - s.requestCache.markSent(regionReq) - if err := doSend(s.createRegionRequest(region)); err != nil { - state.markStopped(err) - return err } + regionReq = nil + continue } - // Try to get from cache - regionReq, err = s.requestCache.pop(ctx) - if err != nil { + + if err := drainControl(); err != nil { return err } + if regionReq = s.requestCache.tryPop(); regionReq != nil { + continue + } + select { + case <-s.controlQueue.ready(): + case <-s.requestCache.ready(): + case <-ctx.Done(): + return ctx.Err() + } } } @@ -451,78 +439,6 @@ func (s *regionRequestWorker) createRegionRequest(region regionInfo) *cdcpb.Chan } } -func (s *regionRequestWorker) addRegionState(subscriptionID SubscriptionID, regionID uint64, state *regionFeedState) { - s.requestedRegions.Lock() - defer s.requestedRegions.Unlock() - states := s.requestedRegions.subscriptions[subscriptionID] - if states == nil { - states = make(regionFeedStates) - s.requestedRegions.subscriptions[subscriptionID] = states - } - - states[regionID] = state -} - -func (s *regionRequestWorker) getRegionState(subscriptionID SubscriptionID, regionID uint64) *regionFeedState { - s.requestedRegions.RLock() - defer s.requestedRegions.RUnlock() - if states, ok := s.requestedRegions.subscriptions[subscriptionID]; ok { - return states[regionID] - } - return nil -} - -func (s *regionRequestWorker) takeRegionState(subscriptionID SubscriptionID, regionID uint64) *regionFeedState { - s.requestedRegions.Lock() - defer s.requestedRegions.Unlock() - if statesMap, ok := s.requestedRegions.subscriptions[subscriptionID]; ok { - state := statesMap[regionID] - delete(statesMap, regionID) - if len(statesMap) == 0 { - delete(s.requestedRegions.subscriptions, subscriptionID) - } - return state - } - return nil -} - -func (s *regionRequestWorker) takeRegionStates(subscriptionID SubscriptionID) regionFeedStates { - s.requestedRegions.Lock() - defer s.requestedRegions.Unlock() - states := s.requestedRegions.subscriptions[subscriptionID] - delete(s.requestedRegions.subscriptions, subscriptionID) - return states -} - -func (s *regionRequestWorker) clearRegionStates() map[SubscriptionID]regionFeedStates { - s.requestedRegions.Lock() - defer s.requestedRegions.Unlock() - subscriptions := s.requestedRegions.subscriptions - s.requestedRegions.subscriptions = make(map[SubscriptionID]regionFeedStates) - return subscriptions -} - -// add adds a region request to the worker's cache -// It blocks if the cache is full until there's space or ctx is cancelled func (s *regionRequestWorker) add(ctx context.Context, region regionInfo, force bool) (bool, error) { return s.requestCache.add(ctx, region, force) } - -func (s *regionRequestWorker) clearPendingRegions() []regionInfo { - var regions []regionInfo - - // Clear pre-fetched region - if s.preFetchForConnecting != nil { - region := *s.preFetchForConnecting - s.preFetchForConnecting = nil - regions = append(regions, region) - // The pre-fetched region was popped from pendingQueue but hasn't been marked as sent or done yet. - // Release its pendingCount slot to avoid leaking flow control credits on worker failures. - s.requestCache.markDone() - } - - // Clear all regions from cache - cacheRegions := s.requestCache.clear() - regions = append(regions, cacheRegions...) - return regions -} diff --git a/logservice/logpuller/region_request_worker_test.go b/logservice/logpuller/region_request_worker_test.go index 268045c492..8c895138b8 100644 --- a/logservice/logpuller/region_request_worker_test.go +++ b/logservice/logpuller/region_request_worker_test.go @@ -17,6 +17,7 @@ import ( "context" "io" "testing" + "time" "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/cdcpb" @@ -35,9 +36,15 @@ import ( type mockEventFeedV2Client struct { sendErr error recvErr error + sendCh chan *cdcpb.ChangeDataRequest } -func (m *mockEventFeedV2Client) Send(*cdcpb.ChangeDataRequest) error { return m.sendErr } +func (m *mockEventFeedV2Client) Send(req *cdcpb.ChangeDataRequest) error { + if m.sendCh != nil { + m.sendCh <- req + } + return m.sendErr +} func (m *mockEventFeedV2Client) Recv() (*cdcpb.ChangeDataEvent, error) { return nil, m.recvErr } func (m *mockEventFeedV2Client) Header() (metadata.MD, error) { return metadata.MD{}, nil } func (m *mockEventFeedV2Client) Trailer() metadata.MD { return metadata.MD{} } @@ -58,48 +65,48 @@ func prepareRegionForSendTest(region regionInfo) regionInfo { } func TestRegionStatesOperation(t *testing.T) { - worker := ®ionRequestWorker{} - worker.requestedRegions.subscriptions = make(map[SubscriptionID]regionFeedStates) - - require.Nil(t, worker.getRegionState(1, 2)) - require.Nil(t, worker.takeRegionState(1, 2)) - - worker.addRegionState(1, 2, ®ionFeedState{}) - require.NotNil(t, worker.getRegionState(1, 2)) - require.NotNil(t, worker.takeRegionState(1, 2)) - require.Nil(t, worker.getRegionState(1, 2)) - require.Equal(t, 0, len(worker.requestedRegions.subscriptions)) - - worker.addRegionState(1, 2, ®ionFeedState{}) - require.NotNil(t, worker.getRegionState(1, 2)) - require.NotNil(t, worker.takeRegionState(1, 2)) - require.Nil(t, worker.getRegionState(1, 2)) - require.Equal(t, 0, len(worker.requestedRegions.subscriptions)) + tracker := newRegionTracker(0) + + require.Nil(t, tracker.Get(1, 2)) + require.Nil(t, tracker.RemoveRegion(1, 2)) + + tracker.Track(1, 2, ®ionFeedState{}) + require.NotNil(t, tracker.Get(1, 2)) + require.NotNil(t, tracker.RemoveRegion(1, 2)) + require.Nil(t, tracker.Get(1, 2)) + + tracker.Track(1, 2, ®ionFeedState{}) + require.NotNil(t, tracker.Get(1, 2)) + require.NotNil(t, tracker.RemoveRegion(1, 2)) + require.Nil(t, tracker.Get(1, 2)) } -func TestClearPendingRegionsReleaseSlotForPreFetchedRegion(t *testing.T) { +func TestRegionTrackerOverwriteAbortsOldRequest(t *testing.T) { worker := ®ionRequestWorker{ requestCache: newRequestCache(10), + tracker: newRegionTracker(0), } - - ctx := context.Background() region := createTestRegionInfo(1, 1) - ok, err := worker.requestCache.add(ctx, region, false) + ok, err := worker.requestCache.add(t.Context(), region, false) require.NoError(t, err) require.True(t, ok) - - req, err := worker.requestCache.pop(ctx) + req1, err := worker.requestCache.pop(t.Context()) require.NoError(t, err) - require.Equal(t, 1, worker.requestCache.getPendingCount()) + worker.requestCache.markSent(req1) + state1 := newRegionFeedState(region, uint64(region.subscribedSpan.subID), worker, req1) + worker.tracker.Track(region.subscribedSpan.subID, region.verID.GetID(), state1) - worker.preFetchForConnecting = new(regionInfo) - *worker.preFetchForConnecting = req.regionInfo + ok, err = worker.requestCache.add(t.Context(), region, false) + require.NoError(t, err) + require.True(t, ok) + req2, err := worker.requestCache.pop(t.Context()) + require.NoError(t, err) + state2 := newRegionFeedState(region, uint64(region.subscribedSpan.subID), worker, req2) + worker.tracker.Track(region.subscribedSpan.subID, region.verID.GetID(), state2) - regions := worker.clearPendingRegions() - require.Len(t, regions, 1) - require.Nil(t, worker.preFetchForConnecting) - require.Equal(t, 0, worker.requestCache.getPendingCount()) + require.Equal(t, 1, worker.requestCache.pendingCount()) + require.Same(t, state2, worker.tracker.Get(region.subscribedSpan.subID, region.verID.GetID())) } type pushedResolvedEvent struct { @@ -159,17 +166,15 @@ func newDispatchResolvedTsTestWorker(regionCount int) (*regionRequestWorker, *mo }, eventSink: ®ionEventSink{ds: ds}, }, - } - worker.requestedRegions.subscriptions = map[SubscriptionID]regionFeedStates{ - 1: make(regionFeedStates, regionCount), + tracker: newRegionTracker(0), } regions := make([]uint64, regionCount) for i := 0; i < regionCount; i++ { regionID := uint64(i + 1) regions[i] = regionID - worker.requestedRegions.subscriptions[1][regionID] = ®ionFeedState{ + worker.tracker.Track(1, regionID, ®ionFeedState{ requestID: 1, - } + }) } return worker, ds, &cdcpb.ResolvedTs{ @@ -195,7 +200,7 @@ func dispatchResolvedTsEventLegacyForBenchmark(s *regionRequestWorker, resolvedT resolvedStates = make([]*regionFeedState, 0, resolvedTsStateBatchSize) } for _, regionID := range resolvedTsEvent.Regions { - if state := s.getRegionState(subscriptionID, regionID); state != nil { + if state := s.tracker.Get(subscriptionID, regionID); state != nil { resolvedStates = append(resolvedStates, state) if len(resolvedStates) >= resolvedTsStateBatchSize { flush() @@ -269,59 +274,47 @@ func BenchmarkDispatchResolvedTsEventSmallBatchCurrent(b *testing.B) { benchmarkDispatchResolvedTsEvent(b, 16, false) } -func TestClearPendingRegionsDoesNotReturnStoppedSentRegion(t *testing.T) { +func TestStoppedStateRemovesSentRequest(t *testing.T) { worker := ®ionRequestWorker{ requestCache: newRequestCache(10), + tracker: newRegionTracker(0), } - worker.requestedRegions.subscriptions = make(map[SubscriptionID]regionFeedStates) - - ctx := context.Background() region := createTestRegionInfo(1, 1) - ok, err := worker.requestCache.add(ctx, region, false) + ok, err := worker.requestCache.add(t.Context(), region, false) require.NoError(t, err) require.True(t, ok) - - req, err := worker.requestCache.pop(ctx) + req, err := worker.requestCache.pop(t.Context()) require.NoError(t, err) - state := newRegionFeedState(req.regionInfo, uint64(req.regionInfo.subscribedSpan.subID), worker) - state.start() - worker.addRegionState(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID(), state) - - // Simulate the race we are fixing in processRegionSendTask: - // once a request is visible in sentRequests, a fast region error may mark the - // region stopped before worker cleanup runs. In that case, markStopped should - // remove the sent request immediately, so clearPendingRegions must not return - // the stale region again during worker shutdown. + state := newRegionFeedState(req.regionInfo, uint64(req.regionInfo.subscribedSpan.subID), worker, req) + worker.tracker.Track(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID(), state) worker.requestCache.markSent(req) state.markStopped(errors.New("send request to store error")) - worker.takeRegionState(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID()) + worker.tracker.RemoveRegion(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID()) - require.Equal(t, 0, worker.requestCache.getPendingCount()) - require.Empty(t, worker.clearPendingRegions()) + require.Equal(t, 0, worker.requestCache.pendingCount()) + require.Empty(t, worker.requestCache.drainUnsentRegions()) } func TestProcessRegionSendTaskSendFailureCleansSentRequest(t *testing.T) { worker := ®ionRequestWorker{ requestCache: newRequestCache(10), + controlQueue: newControlQueue(), store: &requestedStore{storeAddr: "store-1"}, client: &subscriptionClient{}, + tracker: newRegionTracker(0), } - worker.requestedRegions.subscriptions = make(map[SubscriptionID]regionFeedStates) - ctx := context.Background() region := prepareRegionForSendTest(createTestRegionInfo(1, 1)) - ok, err := worker.requestCache.add(ctx, region, false) + ok, err := worker.requestCache.add(t.Context(), region, false) require.NoError(t, err) require.True(t, ok) - require.Equal(t, 1, worker.requestCache.getPendingCount()) + require.Equal(t, 1, worker.requestCache.pendingCount()) - req, err := worker.requestCache.pop(ctx) + req, err := worker.requestCache.pop(t.Context()) require.NoError(t, err) - worker.preFetchForConnecting = new(regionInfo) - *worker.preFetchForConnecting = req.regionInfo sendErr := errors.New("send failed") conn := &ConnAndClient{ @@ -329,11 +322,10 @@ func TestProcessRegionSendTaskSendFailureCleansSentRequest(t *testing.T) { Conn: &grpc.ClientConn{}, } - err = worker.processRegionSendTask(ctx, conn) + err = worker.processRegionSendTask(t.Context(), conn, req) require.ErrorIs(t, err, sendErr) - require.Equal(t, 0, worker.requestCache.getPendingCount()) - require.Empty(t, worker.requestCache.sentRequests.regionReqs) - state := worker.getRegionState(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID()) + require.Equal(t, 0, worker.requestCache.pendingCount()) + state := worker.tracker.Get(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID()) require.True(t, state == nil || state.isStale(), "region state should be removed or marked stale after send failure") } @@ -356,35 +348,31 @@ func TestProcessRegionSendTaskSendEOFIsRetriable(t *testing.T) { t.Run(tc.name, func(t *testing.T) { worker := ®ionRequestWorker{ requestCache: newRequestCache(10), + controlQueue: newControlQueue(), store: &requestedStore{storeAddr: "store-1"}, client: &subscriptionClient{}, + tracker: newRegionTracker(0), } - worker.requestedRegions.subscriptions = make(map[SubscriptionID]regionFeedStates) - - ctx := context.Background() region := prepareRegionForSendTest(createTestRegionInfo(1, 1)) - ok, err := worker.requestCache.add(ctx, region, false) + ok, err := worker.requestCache.add(t.Context(), region, false) require.NoError(t, err) require.True(t, ok) - req, err := worker.requestCache.pop(ctx) + req, err := worker.requestCache.pop(t.Context()) require.NoError(t, err) - worker.preFetchForConnecting = new(regionInfo) - *worker.preFetchForConnecting = req.regionInfo conn := &ConnAndClient{ Client: &mockEventFeedV2Client{sendErr: tc.sendErr}, Conn: &grpc.ClientConn{}, } - err = worker.processRegionSendTask(ctx, conn) + err = worker.processRegionSendTask(t.Context(), conn, req) var streamErr *storeStreamErr require.ErrorAs(t, err, &streamErr) - require.Equal(t, 0, worker.requestCache.getPendingCount()) - require.Empty(t, worker.requestCache.sentRequests.regionReqs) + require.Equal(t, 0, worker.requestCache.pendingCount()) - state := worker.getRegionState(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID()) + state := worker.tracker.Get(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID()) require.NotNil(t, state) require.True(t, state.isStale()) @@ -394,6 +382,42 @@ func TestProcessRegionSendTaskSendEOFIsRetriable(t *testing.T) { } } +func TestProcessRegionSendTaskHandlesDeregisterFromControlQueue(t *testing.T) { + ds := &mockRegionEventDynamicStream{} + worker := ®ionRequestWorker{ + requestCache: newRequestCache(1), + controlQueue: newControlQueue(), + store: &requestedStore{storeAddr: "store-1"}, + client: &subscriptionClient{ + eventSink: ®ionEventSink{ds: ds}, + }, + tracker: newRegionTracker(0), + } + state := ®ionFeedState{worker: worker} + worker.tracker.Track(1, 1, state) + worker.controlQueue.push(deregisterRequest{subID: 1, filterLoop: true}) + + ctx, cancel := context.WithCancel(context.Background()) + sendCh := make(chan *cdcpb.ChangeDataRequest, 1) + done := make(chan error, 1) + go func() { + done <- worker.processRegionSendTask(ctx, &ConnAndClient{ + Client: &mockEventFeedV2Client{sendCh: sendCh}, + Conn: &grpc.ClientConn{}, + }, nil) + }() + + req := <-sendCh + require.Equal(t, uint64(1), req.RequestId) + require.True(t, req.FilterLoop) + require.NotNil(t, req.GetDeregister()) + require.Eventually(t, func() bool { + return worker.tracker.Get(1, 1) == nil + }, time.Second, 10*time.Millisecond) + cancel() + require.ErrorIs(t, <-done, context.Canceled) +} + func TestReceiveAndDispatchChangeEventsEOFIsRetriable(t *testing.T) { testCases := []struct { name string diff --git a/logservice/logpuller/region_state.go b/logservice/logpuller/region_state.go index e9c21a7aad..01831afbbe 100644 --- a/logservice/logpuller/region_state.go +++ b/logservice/logpuller/region_state.go @@ -48,11 +48,6 @@ type regionInfo struct { filterLoop bool } -func (s *regionInfo) isStopped() bool { - // lockedRange only nil when the region's subscribedTable is stopped. - return s.lockedRangeState == nil -} - func newRegionInfo( verID tikv.RegionVerID, span heartbeatpb.TableSpan, @@ -102,20 +97,28 @@ type regionFeedState struct { // `err` is used to retrieve errors generated outside. err error } + regionReq struct { + sync.Mutex + request *regionReq + } worker *regionRequestWorker } -func newRegionFeedState(region regionInfo, requestID uint64, worker *regionRequestWorker) *regionFeedState { - return ®ionFeedState{ +func newRegionFeedState( + region regionInfo, + requestID uint64, + worker *regionRequestWorker, + request *regionReq, +) *regionFeedState { + state := ®ionFeedState{ region: region, requestID: requestID, + matcher: newMatcher(), worker: worker, } -} - -func (s *regionFeedState) start() { - s.matcher = newMatcher() + state.regionReq.request = request + return state } // mark regionFeedState as stopped with the given error if possible. @@ -126,7 +129,7 @@ func (s *regionFeedState) markStopped(err error) { s.state.v = stateStopped s.state.err = err } - s.worker.requestCache.markStopped(s.region.subscribedSpan.subID, s.region.verID.GetID()) + s.abortScanIfNeeded() } // mark regionFeedState as removed if possible. @@ -138,7 +141,7 @@ func (s *regionFeedState) markRemoved() (changed bool) { changed = true s.matcher.clear() } - s.worker.requestCache.markStopped(s.region.subscribedSpan.subID, s.region.verID.GetID()) + s.abortScanIfNeeded() return } @@ -162,7 +165,29 @@ func (s *regionFeedState) isInitialized() bool { func (s *regionFeedState) setInitialized() { s.region.lockedRangeState.Initialized.Store(true) - s.worker.requestCache.resolve(s.region.subscribedSpan.subID, s.region.verID.GetID()) + s.finishScan() +} + +func (s *regionFeedState) finishScan() { + s.regionReq.Lock() + request := s.regionReq.request + s.regionReq.request = nil + s.regionReq.Unlock() + + if request != nil { + s.worker.requestCache.finishScan(request) + } +} + +func (s *regionFeedState) abortScanIfNeeded() { + s.regionReq.Lock() + request := s.regionReq.request + s.regionReq.request = nil + s.regionReq.Unlock() + + if request != nil { + s.worker.requestCache.abortScan(request) + } } func (s *regionFeedState) getRegionID() uint64 { diff --git a/logservice/logpuller/region_tracker.go b/logservice/logpuller/region_tracker.go new file mode 100644 index 0000000000..7b5ed93c89 --- /dev/null +++ b/logservice/logpuller/region_tracker.go @@ -0,0 +1,118 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package logpuller + +import ( + "sync" + + "github.com/pingcap/log" + "go.uber.org/zap" +) + +type trackedRegionStates map[uint64]*regionFeedState + +// regionTracker owns the region states tracked by one region request worker. +type regionTracker struct { + sync.RWMutex + workerID uint64 + + regionsBySubscription map[SubscriptionID]trackedRegionStates +} + +func newRegionTracker(workerID uint64) *regionTracker { + return ®ionTracker{ + workerID: workerID, + regionsBySubscription: make(map[SubscriptionID]trackedRegionStates), + } +} + +// Track records a region after the worker picks its request and before the +// request is sent to TiKV. An overwritten state no longer has another owner +// that can clean up its request, so Track aborts it explicitly. +func (t *regionTracker) Track(subscriptionID SubscriptionID, regionID uint64, state *regionFeedState) { + t.Lock() + regions := t.regionsBySubscription[subscriptionID] + if regions == nil { + regions = make(trackedRegionStates) + t.regionsBySubscription[subscriptionID] = regions + } + oldState := regions[regionID] + regions[regionID] = state + t.Unlock() + + if oldState == nil { + return + } + log.Warn("region request state overwritten", + zap.Uint64("workerID", t.workerID), + zap.Uint64("subscriptionID", uint64(subscriptionID)), + zap.Uint64("regionID", regionID)) + oldState.abortScanIfNeeded() +} + +func (t *regionTracker) Get(subscriptionID SubscriptionID, regionID uint64) *regionFeedState { + t.RLock() + defer t.RUnlock() + + if regions, ok := t.regionsBySubscription[subscriptionID]; ok { + return regions[regionID] + } + return nil +} + +func (t *regionTracker) RemoveRegion(subscriptionID SubscriptionID, regionID uint64) *regionFeedState { + t.Lock() + var state *regionFeedState + if regions, ok := t.regionsBySubscription[subscriptionID]; ok { + state = regions[regionID] + delete(regions, regionID) + if len(regions) == 0 { + delete(t.regionsBySubscription, subscriptionID) + } + } + t.Unlock() + return state +} + +func (t *regionTracker) RemoveSubscription(subscriptionID SubscriptionID) []*regionFeedState { + t.Lock() + regions := t.regionsBySubscription[subscriptionID] + delete(t.regionsBySubscription, subscriptionID) + t.Unlock() + + return collectTrackedRegionStates(regions) +} + +func (t *regionTracker) Drain() map[SubscriptionID][]*regionFeedState { + t.Lock() + regionsBySubscription := t.regionsBySubscription + t.regionsBySubscription = make(map[SubscriptionID]trackedRegionStates) + t.Unlock() + + statesBySubscription := make(map[SubscriptionID][]*regionFeedState, len(regionsBySubscription)) + for subID, regions := range regionsBySubscription { + statesBySubscription[subID] = collectTrackedRegionStates(regions) + } + return statesBySubscription +} + +func collectTrackedRegionStates(regions trackedRegionStates) []*regionFeedState { + states := make([]*regionFeedState, 0, len(regions)) + for _, state := range regions { + if state != nil { + states = append(states, state) + } + } + return states +} diff --git a/logservice/logpuller/subscription_client.go b/logservice/logpuller/subscription_client.go index 15d5002108..cfaf2bcfa0 100644 --- a/logservice/logpuller/subscription_client.go +++ b/logservice/logpuller/subscription_client.go @@ -214,8 +214,7 @@ func (s *subscriptionClient) updateMetrics(ctx context.Context) error { store := value.(*requestedStore) store.requestWorkers.RLock() for _, worker := range store.requestWorkers.s { - worker.requestCache.clearStaleRequest() - pendingRegionReqCount += worker.requestCache.getPendingCount() + pendingRegionReqCount += worker.requestCache.pendingCount() } store.requestWorkers.RUnlock() return true @@ -327,11 +326,10 @@ func (s *subscriptionClient) setTableStopped(rt *subscribedSpan) { log.Info("subscription client starts to stop table", zap.Uint64("subscriptionID", uint64(rt.subID))) - // Set stopped to true so we can stop handling region events from the table. - // Then send a special singleRegionInfo to regionRouter to deregister the table - // from all TiKV instances. + // Set stopped to true so we can stop handling region events from the table, + // then notify every existing worker to deregister the subscription. if rt.stopped.CompareAndSwap(false, true) { - s.regionTaskQueue.Push(NewRegionPriorityTask(TaskHighPrior, regionInfo{subscribedSpan: rt, filterLoop: rt.filterLoop}, s.pdClock.CurrentTS())) + s.broadcastDeregister(rt.subID, rt.filterLoop) if rt.rangeLock.Stop() { s.onTableDrained(rt) } @@ -390,7 +388,6 @@ func (s *subscriptionClient) handleRegions(ctx context.Context, eg *errgroup.Gro rs = &requestedStore{storeAddr: storeAddr} rs.requestWorkers.s = make([]*regionRequestWorker, 0, s.config.RegionRequestWorkerPerStore) - s.stores.Store(storeAddr, rs) perWorkerQueueSize := pendingRegionRequestQueueSize / int(s.config.RegionRequestWorkerPerStore) if perWorkerQueueSize <= 0 { @@ -402,10 +399,16 @@ func (s *subscriptionClient) handleRegions(ctx context.Context, eg *errgroup.Gro rs.requestWorkers.Lock() for i := uint(0); i < s.config.RegionRequestWorkerPerStore; i++ { - requestWorker := newRegionRequestWorker(ctx, s, s.credential, eg, rs, perWorkerQueueSize) + requestWorker := newRegionRequestWorker(s, rs, newRequestCache(perWorkerQueueSize)) rs.requestWorkers.s = append(rs.requestWorkers.s, requestWorker) } rs.requestWorkers.Unlock() + + // Publish the store only after its immutable worker list is complete. + s.stores.Store(storeAddr, rs) + for _, requestWorker := range rs.requestWorkers.s { + eg.Go(func() error { return requestWorker.Run(ctx) }) + } return rs } @@ -415,7 +418,7 @@ func (s *subscriptionClient) handleRegions(ctx context.Context, eg *errgroup.Gro rs.requestWorkers.RLock() for _, w := range rs.requestWorkers.s { - w.requestCache.clear() + w.requestCache.close() } rs.requestWorkers.RUnlock() @@ -439,19 +442,6 @@ func (s *subscriptionClient) handleRegions(ctx context.Context, eg *errgroup.Gro } region := regionTask.GetRegionInfo() - if region.isStopped() { - enqueued, err := s.enqueueRegionToAllStores(ctx, region) - if err != nil { - return err - } - if !enqueued { - log.Debug("enqueue stop request failed, retry later", - zap.Uint64("subscriptionID", uint64(region.subscribedSpan.subID))) - s.regionTaskQueue.Push(regionTask) - } - continue - } - region, ok := s.attachRPCContextForRegion(ctx, region) // If attachRPCContextForRegion fails, the region will be re-scheduled. if !ok { @@ -484,30 +474,16 @@ func (s *subscriptionClient) handleRegions(ctx context.Context, eg *errgroup.Gro } } -func (s *subscriptionClient) enqueueRegionToAllStores(ctx context.Context, region regionInfo) (bool, error) { - enqueued := true - var firstErr error +func (s *subscriptionClient) broadcastDeregister(subID SubscriptionID, filterLoop bool) { s.stores.Range(func(_ any, value any) bool { rs := value.(*requestedStore) rs.requestWorkers.RLock() - workers := rs.requestWorkers.s - rs.requestWorkers.RUnlock() - for _, worker := range workers { - ok, err := worker.add(ctx, region, true) - if err != nil { - firstErr = err - enqueued = false - return false - } - if !ok { - enqueued = false - // It is likely the store is busy, no need to try other workers in this store now. - break - } + for _, worker := range rs.requestWorkers.s { + worker.controlQueue.push(deregisterRequest{subID: subID, filterLoop: filterLoop}) } + rs.requestWorkers.RUnlock() return true }) - return enqueued, firstErr } func (s *subscriptionClient) attachRPCContextForRegion(ctx context.Context, region regionInfo) (regionInfo, bool) { diff --git a/logservice/logpuller/subscription_client_test.go b/logservice/logpuller/subscription_client_test.go index 95d0921f4d..87977adb7f 100644 --- a/logservice/logpuller/subscription_client_test.go +++ b/logservice/logpuller/subscription_client_test.go @@ -105,11 +105,12 @@ func TestGenerateResolveLockTask(t *testing.T) { worker := ®ionRequestWorker{ requestCache: &requestCache{}, + tracker: newRegionTracker(0), } // Lock another range, no task will be triggered before initialized. res = span.rangeLock.LockRange(context.Background(), []byte{'c'}, []byte{'d'}, 2, 100) require.Equal(t, regionlock.LockRangeStatusSuccess, res.Status) - state := newRegionFeedState(regionInfo{lockedRangeState: res.LockedRangeState, subscribedSpan: span}, 1, worker) + state := newRegionFeedState(regionInfo{lockedRangeState: res.LockedRangeState, subscribedSpan: span}, 1, worker, nil) span.resolveStaleLocks(200) select { case <-client.resolveLockTaskCh: @@ -301,11 +302,9 @@ func TestResolveLockTaskDroppedWhenChannelFull(t *testing.T) { func TestStopTaskUsesSubscribedSpanFilterLoop(t *testing.T) { client := &subscriptionClient{ resolveLockTaskCh: make(chan resolveLockTask, 1), - regionTaskQueue: priorityqueue.New[PriorityTask](), } client.ctx, client.cancel = context.WithCancel(context.Background()) defer client.cancel() - client.pdClock = pdutil.NewClock4Test() rawSpan := heartbeatpb.TableSpan{ TableID: 1, @@ -329,16 +328,17 @@ func TestStopTaskUsesSubscribedSpanFilterLoop(t *testing.T) { res := span.rangeLock.LockRange(context.Background(), rawSpan.StartKey, rawSpan.EndKey, 1, 1) require.Equal(t, regionlock.LockRangeStatusSuccess, res.Status) + worker := ®ionRequestWorker{controlQueue: newControlQueue()} + store := &requestedStore{storeAddr: "store-1"} + store.requestWorkers.s = []*regionRequestWorker{worker} + client.stores.Store(store.storeAddr, store) client.setTableStopped(span) - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - task, err := client.regionTaskQueue.Pop(ctx) - require.NoError(t, err) - region := task.GetRegionInfo() - require.True(t, region.isStopped()) - require.True(t, region.filterLoop) + req, ok := worker.controlQueue.tryPop() + require.True(t, ok) + require.Equal(t, SubscriptionID(1), req.subID) + require.True(t, req.filterLoop) } func TestOnRegionFailQueuesCanceledErrorCache(t *testing.T) { @@ -451,12 +451,12 @@ func TestPushRegionEventToDSUnblocksOnClose(t *testing.T) { } } -func TestEnqueueRegionToAllStoresRetryWhenCacheFull(t *testing.T) { - ctx := context.Background() +func TestBroadcastDeregisterUsesWorkerControlQueue(t *testing.T) { client := &subscriptionClient{} worker := ®ionRequestWorker{ requestCache: newRequestCache(1), + controlQueue: newControlQueue(), } store := &requestedStore{storeAddr: "store-1"} store.requestWorkers.s = []*regionRequestWorker{worker} @@ -466,24 +466,17 @@ func TestEnqueueRegionToAllStoresRetryWhenCacheFull(t *testing.T) { subscribedSpan: &subscribedSpan{subID: SubscriptionID(2)}, lockedRangeState: ®ionlock.LockedRangeState{}, } - ok, err := worker.add(ctx, dummyRegion, true) + ok, err := worker.add(t.Context(), dummyRegion, true) require.NoError(t, err) require.True(t, ok) - stopRegion := regionInfo{ - subscribedSpan: &subscribedSpan{subID: SubscriptionID(1)}, - } - enqueued, err := client.enqueueRegionToAllStores(ctx, stopRegion) - require.NoError(t, err) - require.False(t, enqueued) - - <-worker.requestCache.pendingQueue - worker.requestCache.markDone() - - enqueued, err = client.enqueueRegionToAllStores(ctx, stopRegion) - require.NoError(t, err) - require.True(t, enqueued) - require.Equal(t, 1, len(worker.requestCache.pendingQueue)) + client.broadcastDeregister(SubscriptionID(1), true) + require.Equal(t, 1, worker.controlQueue.len()) + req, ok := worker.controlQueue.tryPop() + require.True(t, ok) + require.Equal(t, SubscriptionID(1), req.subID) + require.True(t, req.filterLoop) + require.Equal(t, 1, worker.requestCache.pendingCount()) } func TestSubscriptionWithFailedTiKV(t *testing.T) { diff --git a/utils/notifyqueue/notify_queue.go b/utils/notifyqueue/notify_queue.go new file mode 100644 index 0000000000..331aaf4b02 --- /dev/null +++ b/utils/notifyqueue/notify_queue.go @@ -0,0 +1,77 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package notifyqueue provides a FIFO queue with a selectable ready signal. +package notifyqueue + +import "github.com/pingcap/ticdc/utils/deque" + +// Queue is a FIFO queue with a wake-up channel. +// +// Queue is not safe for concurrent use. Callers must serialize Push, TryPop, +// Drain, and Len by themselves. +// +// Ready returns a wake-up hint. Receiving from it does not guarantee TryPop +// will return an item, and callers must re-check the queue state after wake-up. +type Queue[T any] struct { + queue *deque.Deque[T] + ready chan struct{} +} + +// New creates an empty Queue. +func New[T any]() *Queue[T] { + return &Queue[T]{ + queue: deque.NewDequeDefault[T](), + ready: make(chan struct{}, 1), + } +} + +// Push appends an item and signals Ready. +func (q *Queue[T]) Push(item T) { + q.queue.PushBack(item) + q.signal() +} + +// TryPop pops one item from the front of the queue. +func (q *Queue[T]) TryPop() (T, bool) { + return q.queue.PopFront() +} + +// Drain removes and returns all queued items in FIFO order. +func (q *Queue[T]) Drain() []T { + items := make([]T, 0, q.queue.Length()) + for { + item, ok := q.queue.PopFront() + if !ok { + return items + } + items = append(items, item) + } +} + +// Len returns the number of queued items. +func (q *Queue[T]) Len() int { + return q.queue.Length() +} + +// Ready returns a wake-up channel for consumers. +func (q *Queue[T]) Ready() <-chan struct{} { + return q.ready +} + +func (q *Queue[T]) signal() { + select { + case q.ready <- struct{}{}: + default: + } +} diff --git a/utils/notifyqueue/notify_queue_test.go b/utils/notifyqueue/notify_queue_test.go new file mode 100644 index 0000000000..1532f23d3d --- /dev/null +++ b/utils/notifyqueue/notify_queue_test.go @@ -0,0 +1,83 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package notifyqueue + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestQueuePushPopAndReady(t *testing.T) { + q := New[int]() + + select { + case <-q.Ready(): + t.Fatal("empty queue should not be ready") + default: + } + + q.Push(1) + q.Push(2) + require.Equal(t, 2, q.Len()) + + select { + case <-q.Ready(): + default: + t.Fatal("push should signal ready") + } + + v, ok := q.TryPop() + require.True(t, ok) + require.Equal(t, 1, v) + v, ok = q.TryPop() + require.True(t, ok) + require.Equal(t, 2, v) + require.Equal(t, 0, q.Len()) + + _, ok = q.TryPop() + require.False(t, ok) +} + +func TestQueueReadyIsCoalesced(t *testing.T) { + q := New[int]() + + q.Push(1) + q.Push(2) + + select { + case <-q.Ready(): + default: + t.Fatal("push should signal ready") + } + + select { + case <-q.Ready(): + t.Fatal("ready signal should be coalesced") + default: + } +} + +func TestQueueDrain(t *testing.T) { + q := New[int]() + q.Push(1) + q.Push(2) + q.Push(3) + + require.Equal(t, []int{1, 2, 3}, q.Drain()) + require.Equal(t, 0, q.Len()) + + _, ok := q.TryPop() + require.False(t, ok) +} From e78fc0b3270a778ded03e45a1f63dd25046aedfa Mon Sep 17 00:00:00 2001 From: lidezhu Date: Fri, 10 Jul 2026 20:31:52 +0800 Subject: [PATCH 02/71] avoid rename --- .../logpuller/{region_request_cache.go => region_req_cache.go} | 0 .../{region_request_cache_test.go => region_req_cache_test.go} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename logservice/logpuller/{region_request_cache.go => region_req_cache.go} (100%) rename logservice/logpuller/{region_request_cache_test.go => region_req_cache_test.go} (100%) diff --git a/logservice/logpuller/region_request_cache.go b/logservice/logpuller/region_req_cache.go similarity index 100% rename from logservice/logpuller/region_request_cache.go rename to logservice/logpuller/region_req_cache.go diff --git a/logservice/logpuller/region_request_cache_test.go b/logservice/logpuller/region_req_cache_test.go similarity index 100% rename from logservice/logpuller/region_request_cache_test.go rename to logservice/logpuller/region_req_cache_test.go From 6292a1b7c5ffda653657eb24e9f8c8e157e6ac45 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Fri, 10 Jul 2026 21:04:39 +0800 Subject: [PATCH 03/71] fix --- logservice/logpuller/region_event_handler.go | 2 +- .../logpuller/region_event_handler_test.go | 6 +- logservice/logpuller/region_request_worker.go | 23 ++++- .../logpuller/region_request_worker_test.go | 43 +++------ logservice/logpuller/region_tracker.go | 94 ++++++++++--------- logservice/logpuller/region_tracker_test.go | 60 ++++++++++++ .../logpuller/subscription_client_test.go | 2 +- 7 files changed, 148 insertions(+), 82 deletions(-) create mode 100644 logservice/logpuller/region_tracker_test.go diff --git a/logservice/logpuller/region_event_handler.go b/logservice/logpuller/region_event_handler.go index 91c4466bd0..787b27be01 100644 --- a/logservice/logpuller/region_event_handler.go +++ b/logservice/logpuller/region_event_handler.go @@ -255,7 +255,7 @@ func (h *regionEventHandler) handleRegionError(state *regionFeedState) { zap.Error(err)) } if stepsToRemoved { - worker.tracker.RemoveRegion(SubscriptionID(state.requestID), state.getRegionID()) + worker.tracker.RemoveIf(SubscriptionID(state.requestID), state.getRegionID(), state) h.failureHandler.Report(newRegionErrorInfo(state.getRegionInfo(), err)) } } diff --git a/logservice/logpuller/region_event_handler_test.go b/logservice/logpuller/region_event_handler_test.go index 2a15ef75f1..219767a900 100644 --- a/logservice/logpuller/region_event_handler_test.go +++ b/logservice/logpuller/region_event_handler_test.go @@ -79,7 +79,7 @@ func TestHandleEventEntryEventOutOfOrder(t *testing.T) { worker := ®ionRequestWorker{ requestCache: &requestCache{}, - tracker: newRegionTracker(0), + tracker: newRegionTracker(), } region := newRegionInfo( tikv.RegionVerID{}, @@ -218,7 +218,7 @@ func TestHandleResolvedTs(t *testing.T) { subID1 := SubscriptionID(1) worker := ®ionRequestWorker{ requestCache: &requestCache{}, - tracker: newRegionTracker(0), + tracker: newRegionTracker(), } state1 := newRegionFeedState(regionInfo{verID: tikv.NewRegionVerID(1, 1, 1)}, uint64(subID1), worker, nil) { @@ -359,7 +359,7 @@ func TestHandleResolvedTsThrottled(t *testing.T) { advanceInterval: 100, } span.lastAdvanceTime.Store(0) - worker := ®ionRequestWorker{tracker: newRegionTracker(0)} + worker := ®ionRequestWorker{tracker: newRegionTracker()} state := newRegionFeedState( regionInfo{ verID: tikv.NewRegionVerID(1, 1, 1), diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index 9fa559dc19..19827fa226 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -99,7 +99,7 @@ func newRegionRequestWorker( store: store, requestCache: requestCache, controlQueue: newControlQueue(), - tracker: newRegionTracker(workerID), + tracker: newRegionTracker(), } } @@ -362,7 +362,7 @@ func (s *regionRequestWorker) processRegionSendTask( if err := doSend(changeDataReq); err != nil { return err } - for _, state := range s.tracker.RemoveSubscription(req.subID) { + for _, state := range s.tracker.TakeSubscription(req.subID) { state.markStopped(&requestCancelledErr{}) s.client.eventSink.Push(req.subID, regionEvent{states: []*regionFeedState{state}}) } @@ -397,7 +397,7 @@ func (s *regionRequestWorker) processRegionSendTask( s.requestCache.abortScan(regionReq) } else { state := newRegionFeedState(region, uint64(subID), s, regionReq) - s.tracker.Track(subID, region.verID.GetID(), state) + s.replaceRegionState(subID, region.verID.GetID(), state) // Make the request and its state visible in the same order. A fast // region error can then clean the request without racing markSent. s.requestCache.markSent(regionReq) @@ -439,6 +439,23 @@ func (s *regionRequestWorker) createRegionRequest(region regionInfo) *cdcpb.Chan } } +func (s *regionRequestWorker) replaceRegionState( + subscriptionID SubscriptionID, + regionID uint64, + state *regionFeedState, +) { + oldState := s.tracker.Replace(subscriptionID, regionID, state) + if oldState == nil { + return + } + + log.Warn("region request state overwritten", + zap.Uint64("workerID", s.workerID), + zap.Uint64("subscriptionID", uint64(subscriptionID)), + zap.Uint64("regionID", regionID)) + oldState.abortScanIfNeeded() +} + func (s *regionRequestWorker) add(ctx context.Context, region regionInfo, force bool) (bool, error) { return s.requestCache.add(ctx, region, force) } diff --git a/logservice/logpuller/region_request_worker_test.go b/logservice/logpuller/region_request_worker_test.go index 8c895138b8..5bbb79156c 100644 --- a/logservice/logpuller/region_request_worker_test.go +++ b/logservice/logpuller/region_request_worker_test.go @@ -64,27 +64,10 @@ func prepareRegionForSendTest(region regionInfo) regionInfo { return region } -func TestRegionStatesOperation(t *testing.T) { - tracker := newRegionTracker(0) - - require.Nil(t, tracker.Get(1, 2)) - require.Nil(t, tracker.RemoveRegion(1, 2)) - - tracker.Track(1, 2, ®ionFeedState{}) - require.NotNil(t, tracker.Get(1, 2)) - require.NotNil(t, tracker.RemoveRegion(1, 2)) - require.Nil(t, tracker.Get(1, 2)) - - tracker.Track(1, 2, ®ionFeedState{}) - require.NotNil(t, tracker.Get(1, 2)) - require.NotNil(t, tracker.RemoveRegion(1, 2)) - require.Nil(t, tracker.Get(1, 2)) -} - -func TestRegionTrackerOverwriteAbortsOldRequest(t *testing.T) { +func TestRegionRequestWorkerReplaceRegionStateAbortsOldRequest(t *testing.T) { worker := ®ionRequestWorker{ requestCache: newRequestCache(10), - tracker: newRegionTracker(0), + tracker: newRegionTracker(), } region := createTestRegionInfo(1, 1) @@ -95,7 +78,7 @@ func TestRegionTrackerOverwriteAbortsOldRequest(t *testing.T) { require.NoError(t, err) worker.requestCache.markSent(req1) state1 := newRegionFeedState(region, uint64(region.subscribedSpan.subID), worker, req1) - worker.tracker.Track(region.subscribedSpan.subID, region.verID.GetID(), state1) + worker.replaceRegionState(region.subscribedSpan.subID, region.verID.GetID(), state1) ok, err = worker.requestCache.add(t.Context(), region, false) require.NoError(t, err) @@ -103,7 +86,7 @@ func TestRegionTrackerOverwriteAbortsOldRequest(t *testing.T) { req2, err := worker.requestCache.pop(t.Context()) require.NoError(t, err) state2 := newRegionFeedState(region, uint64(region.subscribedSpan.subID), worker, req2) - worker.tracker.Track(region.subscribedSpan.subID, region.verID.GetID(), state2) + worker.replaceRegionState(region.subscribedSpan.subID, region.verID.GetID(), state2) require.Equal(t, 1, worker.requestCache.pendingCount()) require.Same(t, state2, worker.tracker.Get(region.subscribedSpan.subID, region.verID.GetID())) @@ -166,13 +149,13 @@ func newDispatchResolvedTsTestWorker(regionCount int) (*regionRequestWorker, *mo }, eventSink: ®ionEventSink{ds: ds}, }, - tracker: newRegionTracker(0), + tracker: newRegionTracker(), } regions := make([]uint64, regionCount) for i := 0; i < regionCount; i++ { regionID := uint64(i + 1) regions[i] = regionID - worker.tracker.Track(1, regionID, ®ionFeedState{ + worker.tracker.Replace(1, regionID, ®ionFeedState{ requestID: 1, }) } @@ -277,7 +260,7 @@ func BenchmarkDispatchResolvedTsEventSmallBatchCurrent(b *testing.B) { func TestStoppedStateRemovesSentRequest(t *testing.T) { worker := ®ionRequestWorker{ requestCache: newRequestCache(10), - tracker: newRegionTracker(0), + tracker: newRegionTracker(), } region := createTestRegionInfo(1, 1) @@ -288,10 +271,10 @@ func TestStoppedStateRemovesSentRequest(t *testing.T) { require.NoError(t, err) state := newRegionFeedState(req.regionInfo, uint64(req.regionInfo.subscribedSpan.subID), worker, req) - worker.tracker.Track(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID(), state) + worker.tracker.Replace(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID(), state) worker.requestCache.markSent(req) state.markStopped(errors.New("send request to store error")) - worker.tracker.RemoveRegion(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID()) + worker.tracker.RemoveIf(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID(), state) require.Equal(t, 0, worker.requestCache.pendingCount()) require.Empty(t, worker.requestCache.drainUnsentRegions()) @@ -303,7 +286,7 @@ func TestProcessRegionSendTaskSendFailureCleansSentRequest(t *testing.T) { controlQueue: newControlQueue(), store: &requestedStore{storeAddr: "store-1"}, client: &subscriptionClient{}, - tracker: newRegionTracker(0), + tracker: newRegionTracker(), } region := prepareRegionForSendTest(createTestRegionInfo(1, 1)) @@ -351,7 +334,7 @@ func TestProcessRegionSendTaskSendEOFIsRetriable(t *testing.T) { controlQueue: newControlQueue(), store: &requestedStore{storeAddr: "store-1"}, client: &subscriptionClient{}, - tracker: newRegionTracker(0), + tracker: newRegionTracker(), } region := prepareRegionForSendTest(createTestRegionInfo(1, 1)) @@ -391,10 +374,10 @@ func TestProcessRegionSendTaskHandlesDeregisterFromControlQueue(t *testing.T) { client: &subscriptionClient{ eventSink: ®ionEventSink{ds: ds}, }, - tracker: newRegionTracker(0), + tracker: newRegionTracker(), } state := ®ionFeedState{worker: worker} - worker.tracker.Track(1, 1, state) + worker.tracker.Replace(1, 1, state) worker.controlQueue.push(deregisterRequest{subID: 1, filterLoop: true}) ctx, cancel := context.WithCancel(context.Background()) diff --git a/logservice/logpuller/region_tracker.go b/logservice/logpuller/region_tracker.go index 7b5ed93c89..d545444513 100644 --- a/logservice/logpuller/region_tracker.go +++ b/logservice/logpuller/region_tracker.go @@ -13,35 +13,44 @@ package logpuller -import ( - "sync" - - "github.com/pingcap/log" - "go.uber.org/zap" -) +import "sync" type trackedRegionStates map[uint64]*regionFeedState // regionTracker owns the region states tracked by one region request worker. type regionTracker struct { - sync.RWMutex - workerID uint64 + mu sync.RWMutex regionsBySubscription map[SubscriptionID]trackedRegionStates } -func newRegionTracker(workerID uint64) *regionTracker { +func newRegionTracker() *regionTracker { return ®ionTracker{ - workerID: workerID, regionsBySubscription: make(map[SubscriptionID]trackedRegionStates), } } -// Track records a region after the worker picks its request and before the -// request is sent to TiKV. An overwritten state no longer has another owner -// that can clean up its request, so Track aborts it explicitly. -func (t *regionTracker) Track(subscriptionID SubscriptionID, regionID uint64, state *regionFeedState) { - t.Lock() +// Get returns the state tracked by a subscription and region. +func (t *regionTracker) Get(subscriptionID SubscriptionID, regionID uint64) *regionFeedState { + t.mu.RLock() + defer t.mu.RUnlock() + + if regions, ok := t.regionsBySubscription[subscriptionID]; ok { + return regions[regionID] + } + return nil +} + +// Replace records a region state and returns the state previously tracked by +// the same subscription and region, if any. +func (t *regionTracker) Replace( + subscriptionID SubscriptionID, + regionID uint64, + state *regionFeedState, +) *regionFeedState { + t.mu.Lock() + defer t.mu.Unlock() + regions := t.regionsBySubscription[subscriptionID] if regions == nil { regions = make(trackedRegionStates) @@ -49,56 +58,53 @@ func (t *regionTracker) Track(subscriptionID SubscriptionID, regionID uint64, st } oldState := regions[regionID] regions[regionID] = state - t.Unlock() - - if oldState == nil { - return - } - log.Warn("region request state overwritten", - zap.Uint64("workerID", t.workerID), - zap.Uint64("subscriptionID", uint64(subscriptionID)), - zap.Uint64("regionID", regionID)) - oldState.abortScanIfNeeded() + return oldState } -func (t *regionTracker) Get(subscriptionID SubscriptionID, regionID uint64) *regionFeedState { - t.RLock() - defer t.RUnlock() - - if regions, ok := t.regionsBySubscription[subscriptionID]; ok { - return regions[regionID] +// RemoveIf removes a region only when it is still tracked by the expected +// state. It prevents delayed events for an old state from removing its +// replacement. +func (t *regionTracker) RemoveIf( + subscriptionID SubscriptionID, + regionID uint64, + expected *regionFeedState, +) bool { + if expected == nil { + return false } - return nil -} -func (t *regionTracker) RemoveRegion(subscriptionID SubscriptionID, regionID uint64) *regionFeedState { - t.Lock() - var state *regionFeedState + t.mu.Lock() + defer t.mu.Unlock() + if regions, ok := t.regionsBySubscription[subscriptionID]; ok { - state = regions[regionID] + if regions[regionID] != expected { + return false + } delete(regions, regionID) if len(regions) == 0 { delete(t.regionsBySubscription, subscriptionID) } + return true } - t.Unlock() - return state + return false } -func (t *regionTracker) RemoveSubscription(subscriptionID SubscriptionID) []*regionFeedState { - t.Lock() +// TakeSubscription removes and returns all states tracked by a subscription. +func (t *regionTracker) TakeSubscription(subscriptionID SubscriptionID) []*regionFeedState { + t.mu.Lock() regions := t.regionsBySubscription[subscriptionID] delete(t.regionsBySubscription, subscriptionID) - t.Unlock() + t.mu.Unlock() return collectTrackedRegionStates(regions) } +// Drain removes and returns all tracked states grouped by subscription. func (t *regionTracker) Drain() map[SubscriptionID][]*regionFeedState { - t.Lock() + t.mu.Lock() regionsBySubscription := t.regionsBySubscription t.regionsBySubscription = make(map[SubscriptionID]trackedRegionStates) - t.Unlock() + t.mu.Unlock() statesBySubscription := make(map[SubscriptionID][]*regionFeedState, len(regionsBySubscription)) for subID, regions := range regionsBySubscription { diff --git a/logservice/logpuller/region_tracker_test.go b/logservice/logpuller/region_tracker_test.go new file mode 100644 index 0000000000..5586c0789b --- /dev/null +++ b/logservice/logpuller/region_tracker_test.go @@ -0,0 +1,60 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package logpuller + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestRegionTrackerOperations(t *testing.T) { + tracker := newRegionTracker() + state1 := ®ionFeedState{} + state2 := ®ionFeedState{} + state3 := ®ionFeedState{} + + require.Nil(t, tracker.Get(1, 1)) + require.Nil(t, tracker.Replace(1, 1, state1)) + require.Same(t, state1, tracker.Get(1, 1)) + require.Nil(t, tracker.Replace(1, 2, state2)) + require.Nil(t, tracker.Replace(2, 3, state3)) + + require.ElementsMatch(t, []*regionFeedState{state1, state2}, tracker.TakeSubscription(1)) + require.Nil(t, tracker.Get(1, 1)) + require.Nil(t, tracker.Get(1, 2)) + require.Empty(t, tracker.TakeSubscription(1)) + + drained := tracker.Drain() + require.Equal(t, map[SubscriptionID][]*regionFeedState{ + 2: {state3}, + }, drained) + require.Nil(t, tracker.Get(2, 3)) + require.Empty(t, tracker.Drain()) +} + +func TestRegionTrackerRemoveIfPreservesReplacement(t *testing.T) { + tracker := newRegionTracker() + oldState := ®ionFeedState{} + newState := ®ionFeedState{} + + require.Nil(t, tracker.Replace(1, 1, oldState)) + require.Same(t, oldState, tracker.Replace(1, 1, newState)) + + require.False(t, tracker.RemoveIf(1, 1, oldState)) + require.Same(t, newState, tracker.Get(1, 1)) + require.True(t, tracker.RemoveIf(1, 1, newState)) + require.Nil(t, tracker.Get(1, 1)) + require.False(t, tracker.RemoveIf(1, 1, newState)) +} diff --git a/logservice/logpuller/subscription_client_test.go b/logservice/logpuller/subscription_client_test.go index 87977adb7f..41dc761bd1 100644 --- a/logservice/logpuller/subscription_client_test.go +++ b/logservice/logpuller/subscription_client_test.go @@ -105,7 +105,7 @@ func TestGenerateResolveLockTask(t *testing.T) { worker := ®ionRequestWorker{ requestCache: &requestCache{}, - tracker: newRegionTracker(0), + tracker: newRegionTracker(), } // Lock another range, no task will be triggered before initialized. res = span.rangeLock.LockRange(context.Background(), []byte{'c'}, []byte{'d'}, 2, 100) From 5b18360670a245129eac8c77e44210257eb53691 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Fri, 10 Jul 2026 21:07:58 +0800 Subject: [PATCH 04/71] small refactor --- logservice/logpuller/region_tracker.go | 56 +++++++++++++------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/logservice/logpuller/region_tracker.go b/logservice/logpuller/region_tracker.go index d545444513..7c5b107bb0 100644 --- a/logservice/logpuller/region_tracker.go +++ b/logservice/logpuller/region_tracker.go @@ -15,18 +15,18 @@ package logpuller import "sync" -type trackedRegionStates map[uint64]*regionFeedState +type regionStatesByID map[uint64]*regionFeedState // regionTracker owns the region states tracked by one region request worker. type regionTracker struct { mu sync.RWMutex - regionsBySubscription map[SubscriptionID]trackedRegionStates + statesBySubscription map[SubscriptionID]regionStatesByID } func newRegionTracker() *regionTracker { return ®ionTracker{ - regionsBySubscription: make(map[SubscriptionID]trackedRegionStates), + statesBySubscription: make(map[SubscriptionID]regionStatesByID), } } @@ -35,8 +35,8 @@ func (t *regionTracker) Get(subscriptionID SubscriptionID, regionID uint64) *reg t.mu.RLock() defer t.mu.RUnlock() - if regions, ok := t.regionsBySubscription[subscriptionID]; ok { - return regions[regionID] + if states, ok := t.statesBySubscription[subscriptionID]; ok { + return states[regionID] } return nil } @@ -51,13 +51,13 @@ func (t *regionTracker) Replace( t.mu.Lock() defer t.mu.Unlock() - regions := t.regionsBySubscription[subscriptionID] - if regions == nil { - regions = make(trackedRegionStates) - t.regionsBySubscription[subscriptionID] = regions + states := t.statesBySubscription[subscriptionID] + if states == nil { + states = make(regionStatesByID) + t.statesBySubscription[subscriptionID] = states } - oldState := regions[regionID] - regions[regionID] = state + oldState := states[regionID] + states[regionID] = state return oldState } @@ -76,13 +76,13 @@ func (t *regionTracker) RemoveIf( t.mu.Lock() defer t.mu.Unlock() - if regions, ok := t.regionsBySubscription[subscriptionID]; ok { - if regions[regionID] != expected { + if states, ok := t.statesBySubscription[subscriptionID]; ok { + if states[regionID] != expected { return false } - delete(regions, regionID) - if len(regions) == 0 { - delete(t.regionsBySubscription, subscriptionID) + delete(states, regionID) + if len(states) == 0 { + delete(t.statesBySubscription, subscriptionID) } return true } @@ -92,30 +92,30 @@ func (t *regionTracker) RemoveIf( // TakeSubscription removes and returns all states tracked by a subscription. func (t *regionTracker) TakeSubscription(subscriptionID SubscriptionID) []*regionFeedState { t.mu.Lock() - regions := t.regionsBySubscription[subscriptionID] - delete(t.regionsBySubscription, subscriptionID) + states := t.statesBySubscription[subscriptionID] + delete(t.statesBySubscription, subscriptionID) t.mu.Unlock() - return collectTrackedRegionStates(regions) + return collectRegionStates(states) } // Drain removes and returns all tracked states grouped by subscription. func (t *regionTracker) Drain() map[SubscriptionID][]*regionFeedState { t.mu.Lock() - regionsBySubscription := t.regionsBySubscription - t.regionsBySubscription = make(map[SubscriptionID]trackedRegionStates) + statesBySubscription := t.statesBySubscription + t.statesBySubscription = make(map[SubscriptionID]regionStatesByID) t.mu.Unlock() - statesBySubscription := make(map[SubscriptionID][]*regionFeedState, len(regionsBySubscription)) - for subID, regions := range regionsBySubscription { - statesBySubscription[subID] = collectTrackedRegionStates(regions) + drainedStates := make(map[SubscriptionID][]*regionFeedState, len(statesBySubscription)) + for subID, states := range statesBySubscription { + drainedStates[subID] = collectRegionStates(states) } - return statesBySubscription + return drainedStates } -func collectTrackedRegionStates(regions trackedRegionStates) []*regionFeedState { - states := make([]*regionFeedState, 0, len(regions)) - for _, state := range regions { +func collectRegionStates(statesByID regionStatesByID) []*regionFeedState { + states := make([]*regionFeedState, 0, len(statesByID)) + for _, state := range statesByID { if state != nil { states = append(states, state) } From ecfa2f4be8f6c653617a349f6abeb9a53e081b1f Mon Sep 17 00:00:00 2001 From: lidezhu Date: Fri, 10 Jul 2026 21:16:42 +0800 Subject: [PATCH 05/71] refactor --- logservice/logpuller/region_tracker.go | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/logservice/logpuller/region_tracker.go b/logservice/logpuller/region_tracker.go index 7c5b107bb0..61a0176987 100644 --- a/logservice/logpuller/region_tracker.go +++ b/logservice/logpuller/region_tracker.go @@ -13,7 +13,11 @@ package logpuller -import "sync" +import ( + "maps" + "slices" + "sync" +) type regionStatesByID map[uint64]*regionFeedState @@ -96,7 +100,7 @@ func (t *regionTracker) TakeSubscription(subscriptionID SubscriptionID) []*regio delete(t.statesBySubscription, subscriptionID) t.mu.Unlock() - return collectRegionStates(states) + return slices.Collect(maps.Values(states)) } // Drain removes and returns all tracked states grouped by subscription. @@ -108,17 +112,7 @@ func (t *regionTracker) Drain() map[SubscriptionID][]*regionFeedState { drainedStates := make(map[SubscriptionID][]*regionFeedState, len(statesBySubscription)) for subID, states := range statesBySubscription { - drainedStates[subID] = collectRegionStates(states) + drainedStates[subID] = slices.Collect(maps.Values(states)) } return drainedStates } - -func collectRegionStates(statesByID regionStatesByID) []*regionFeedState { - states := make([]*regionFeedState, 0, len(statesByID)) - for _, state := range statesByID { - if state != nil { - states = append(states, state) - } - } - return states -} From 4daae4fd3aa0ef31af1cab1de47b92c934bad0e9 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Fri, 10 Jul 2026 21:29:00 +0800 Subject: [PATCH 06/71] more refactor --- logservice/logpuller/region_req_cache.go | 8 +- logservice/logpuller/region_req_cache_test.go | 101 +++++++++++++++++- logservice/logpuller/region_request_worker.go | 43 +++++++- .../logpuller/region_request_worker_test.go | 83 +++++++++++++- logservice/logpuller/region_state.go | 20 +--- 5 files changed, 229 insertions(+), 26 deletions(-) diff --git a/logservice/logpuller/region_req_cache.go b/logservice/logpuller/region_req_cache.go index bf2c5cad7b..099e6f53d7 100644 --- a/logservice/logpuller/region_req_cache.go +++ b/logservice/logpuller/region_req_cache.go @@ -158,12 +158,14 @@ func (c *requestCache) tryPop() *regionReq { } } -func (c *requestCache) markSent(req *regionReq) { +func (c *requestCache) markSent(req *regionReq) bool { c.mu.Lock() defer c.mu.Unlock() - if _, ok := c.requests[req]; ok && req.stage == regionReqStageProcessing { - req.stage = regionReqStageSent + if _, ok := c.requests[req]; !ok || req.stage != regionReqStageProcessing { + return false } + req.stage = regionReqStageSent + return true } func (c *requestCache) finishScan(req *regionReq) bool { diff --git a/logservice/logpuller/region_req_cache_test.go b/logservice/logpuller/region_req_cache_test.go index d4fe55616c..c8d9b00635 100644 --- a/logservice/logpuller/region_req_cache_test.go +++ b/logservice/logpuller/region_req_cache_test.go @@ -15,7 +15,9 @@ package logpuller import ( "context" + "sync" "testing" + "time" "github.com/pingcap/ticdc/heartbeatpb" "github.com/stretchr/testify/require" @@ -51,7 +53,7 @@ func TestRequestCacheLifecycle(t *testing.T) { require.Equal(t, regionReqStageProcessing, req.stage) require.Equal(t, 1, cache.pendingCount()) - cache.markSent(req) + require.True(t, cache.markSent(req)) require.Equal(t, regionReqStageSent, req.stage) require.True(t, cache.finishScan(req)) require.False(t, cache.finishScan(req)) @@ -117,7 +119,7 @@ func TestRequestCacheDrainOnlyUnsentRegions(t *testing.T) { sentReq, err := cache.pop(t.Context()) require.NoError(t, err) - cache.markSent(sentReq) + require.True(t, cache.markSent(sentReq)) processingReq, err := cache.pop(t.Context()) require.NoError(t, err) require.Equal(t, regionReqStageProcessing, processingReq.stage) @@ -139,10 +141,103 @@ func TestRequestCacheCloseRemovesAllRequests(t *testing.T) { req, err := cache.pop(t.Context()) require.NoError(t, err) - cache.markSent(req) + require.True(t, cache.markSent(req)) cache.close() require.Equal(t, 0, cache.pendingCount()) require.Nil(t, cache.tryPop()) require.False(t, cache.abortScan(req)) } + +func TestRequestCacheConcurrentAdds(t *testing.T) { + const requestCount = 16 + cache := newRequestCache(requestCount) + start := make(chan struct{}) + results := make(chan bool, requestCount) + + for regionID := uint64(1); regionID <= requestCount; regionID++ { + go func() { + <-start + ok, err := cache.add(t.Context(), createTestRegionInfo(1, regionID), false) + results <- ok && err == nil + }() + } + close(start) + + for range requestCount { + require.True(t, <-results) + } + require.Equal(t, requestCount, cache.pendingCount()) +} + +func TestRequestCacheSpaceAvailableWakesAdd(t *testing.T) { + cache := newRequestCache(1) + require.True(t, cache.tryAdd(createTestRegionInfo(1, 1), false)) + firstReq := cache.tryPop() + require.NotNil(t, firstReq) + + type addResult struct { + ok bool + err error + } + started := make(chan struct{}) + resultCh := make(chan addResult, 1) + go func() { + close(started) + ok, err := cache.add(t.Context(), createTestRegionInfo(1, 2), false) + resultCh <- addResult{ok: ok, err: err} + }() + <-started + require.True(t, cache.abortScan(firstReq)) + + select { + case result := <-resultCh: + require.NoError(t, result.err) + require.True(t, result.ok) + case <-time.After(time.Second): + t.Fatal("add was not woken after request cache space became available") + } + require.Equal(t, uint64(2), cache.tryPop().regionInfo.verID.GetID()) +} + +func TestRequestCacheMarkSentRejectsInvalidTransition(t *testing.T) { + cache := newRequestCache(1) + require.True(t, cache.tryAdd(createTestRegionInfo(1, 1), false)) + req := cache.tryPop() + require.NotNil(t, req) + + require.True(t, cache.markSent(req)) + require.False(t, cache.markSent(req)) + require.True(t, cache.abortScan(req)) + require.False(t, cache.markSent(req)) +} + +func TestRegionFeedStateCleansRequestOnce(t *testing.T) { + cache := newRequestCache(1) + require.True(t, cache.tryAdd(createTestRegionInfo(1, 1), false)) + req := cache.tryPop() + require.NotNil(t, req) + require.True(t, cache.markSent(req)) + + worker := ®ionRequestWorker{requestCache: cache} + state := newRegionFeedState(req.regionInfo, uint64(req.regionInfo.subscribedSpan.subID), worker, req) + start := make(chan struct{}) + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + <-start + state.finishScan() + }() + go func() { + defer wg.Done() + <-start + state.abortScanIfNeeded() + }() + close(start) + wg.Wait() + + require.Nil(t, state.regionReq.Load()) + require.Zero(t, cache.pendingCount()) + require.False(t, cache.abortScan(req)) +} diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index 19827fa226..c30301c091 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -70,6 +70,16 @@ func (q *controlQueue) len() int { return q.queue.Len() } +func (q *controlQueue) drain() { + q.mu.Lock() + defer q.mu.Unlock() + for { + if _, ok := q.queue.TryPop(); !ok { + return + } + } +} + func (q *controlQueue) ready() <-chan struct{} { return q.queue.Ready() } @@ -106,7 +116,7 @@ func newRegionRequestWorker( func (s *regionRequestWorker) Run(ctx context.Context) error { for { // Do not connect an idle worker to an unavailable store indefinitely. - firstReq, err := s.requestCache.pop(ctx) + firstReq, err := s.waitForRegionRequest(ctx) if err != nil { return err } @@ -132,6 +142,9 @@ func (s *regionRequestWorker) Run(ctx context.Context) error { for _, region := range s.requestCache.drainUnsentRegions() { s.client.onRegionFail(newRegionErrorInfo(region, regionErr)) } + // The failed stream no longer owns remote registrations, so queued + // deregistration requests are obsolete. + s.controlQueue.drain() if err := util.Hang(ctx, storeReconnectBackoff); err != nil { return err @@ -139,6 +152,26 @@ func (s *regionRequestWorker) Run(ctx context.Context) error { } } +func (s *regionRequestWorker) waitForRegionRequest(ctx context.Context) (*regionReq, error) { + for { + // Without a stream there are no remote registrations to deregister. + s.controlQueue.drain() + if req := s.requestCache.tryPop(); req != nil { + // Drop controls that raced with selecting the first request. Any later + // controls will be handled by the stream send loop. + s.controlQueue.drain() + return req, nil + } + + select { + case <-s.controlQueue.ready(): + case <-s.requestCache.ready(): + case <-ctx.Done(): + return nil, ctx.Err() + } + } +} + func (s *regionRequestWorker) checkStoreVersion(ctx context.Context) error { err := version.CheckStoreVersion(ctx, s.client.pd) if err == nil { @@ -400,7 +433,13 @@ func (s *regionRequestWorker) processRegionSendTask( s.replaceRegionState(subID, region.verID.GetID(), state) // Make the request and its state visible in the same order. A fast // region error can then clean the request without racing markSent. - s.requestCache.markSent(regionReq) + if !s.requestCache.markSent(regionReq) { + log.Warn("region request transition to sent failed", + zap.Uint64("workerID", s.workerID), + zap.Uint64("subscriptionID", uint64(subID)), + zap.Uint64("regionID", region.verID.GetID())) + return &storeStreamErr{} + } if err := doSend(s.createRegionRequest(region)); err != nil { state.markStopped(err) return err diff --git a/logservice/logpuller/region_request_worker_test.go b/logservice/logpuller/region_request_worker_test.go index 5bbb79156c..9ea8ff1642 100644 --- a/logservice/logpuller/region_request_worker_test.go +++ b/logservice/logpuller/region_request_worker_test.go @@ -76,7 +76,7 @@ func TestRegionRequestWorkerReplaceRegionStateAbortsOldRequest(t *testing.T) { require.True(t, ok) req1, err := worker.requestCache.pop(t.Context()) require.NoError(t, err) - worker.requestCache.markSent(req1) + require.True(t, worker.requestCache.markSent(req1)) state1 := newRegionFeedState(region, uint64(region.subscribedSpan.subID), worker, req1) worker.replaceRegionState(region.subscribedSpan.subID, region.verID.GetID(), state1) @@ -104,6 +104,15 @@ type mockRegionEventDynamicStream struct { pushed []pushedResolvedEvent } +type countingRegionEventDynamicStream struct { + mockRegionEventDynamicStream + pushCount int +} + +func (m *countingRegionEventDynamicStream) Push(_ SubscriptionID, _ regionEvent) { + m.pushCount++ +} + func (m *mockRegionEventDynamicStream) Start() {} func (m *mockRegionEventDynamicStream) Close() {} @@ -194,7 +203,9 @@ func dispatchResolvedTsEventLegacyForBenchmark(s *regionRequestWorker, resolvedT } func benchmarkDispatchResolvedTsEvent(b *testing.B, regionCount int, useLegacy bool) { - worker, ds, event := newDispatchResolvedTsTestWorker(regionCount) + worker, _, event := newDispatchResolvedTsTestWorker(regionCount) + ds := &countingRegionEventDynamicStream{} + worker.client.eventSink.ds = ds b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { @@ -210,6 +221,44 @@ func benchmarkDispatchResolvedTsEvent(b *testing.B, regionCount int, useLegacy b } } +func TestWaitForRegionRequestDrainsIdleControlQueue(t *testing.T) { + worker := ®ionRequestWorker{ + requestCache: newRequestCache(1), + controlQueue: newControlQueue(), + } + + type waitResult struct { + req *regionReq + err error + } + resultCh := make(chan waitResult, 1) + go func() { + req, err := worker.waitForRegionRequest(t.Context()) + resultCh <- waitResult{req: req, err: err} + }() + + for subID := SubscriptionID(1); subID <= 100; subID++ { + worker.controlQueue.push(deregisterRequest{subID: subID}) + } + require.Eventually(t, func() bool { + return worker.controlQueue.len() == 0 + }, time.Second, 10*time.Millisecond) + + ok, err := worker.requestCache.add(t.Context(), createTestRegionInfo(1, 1), false) + require.NoError(t, err) + require.True(t, ok) + + select { + case result := <-resultCh: + require.NoError(t, result.err) + require.NotNil(t, result.req) + require.Equal(t, uint64(1), result.req.regionInfo.verID.GetID()) + case <-time.After(time.Second): + t.Fatal("worker did not receive the first region request") + } + require.Zero(t, worker.controlQueue.len()) +} + func TestDispatchResolvedTsEventSingleRegion(t *testing.T) { worker, ds, event := newDispatchResolvedTsTestWorker(1) worker.dispatchResolvedTsEvent(event) @@ -272,7 +321,7 @@ func TestStoppedStateRemovesSentRequest(t *testing.T) { state := newRegionFeedState(req.regionInfo, uint64(req.regionInfo.subscribedSpan.subID), worker, req) worker.tracker.Replace(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID(), state) - worker.requestCache.markSent(req) + require.True(t, worker.requestCache.markSent(req)) state.markStopped(errors.New("send request to store error")) worker.tracker.RemoveIf(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID(), state) @@ -312,6 +361,34 @@ func TestProcessRegionSendTaskSendFailureCleansSentRequest(t *testing.T) { require.True(t, state == nil || state.isStale(), "region state should be removed or marked stale after send failure") } +func TestProcessRegionSendTaskDoesNotSendRemovedRequest(t *testing.T) { + worker := ®ionRequestWorker{ + requestCache: newRequestCache(1), + controlQueue: newControlQueue(), + store: &requestedStore{storeAddr: "store-1"}, + client: &subscriptionClient{}, + tracker: newRegionTracker(), + } + region := prepareRegionForSendTest(createTestRegionInfo(1, 1)) + require.True(t, worker.requestCache.tryAdd(region, false)) + req := worker.requestCache.tryPop() + require.NotNil(t, req) + require.True(t, worker.requestCache.abortScan(req)) + + sendCh := make(chan *cdcpb.ChangeDataRequest, 1) + err := worker.processRegionSendTask(t.Context(), &ConnAndClient{ + Client: &mockEventFeedV2Client{sendCh: sendCh}, + Conn: &grpc.ClientConn{}, + }, req) + var streamErr *storeStreamErr + require.ErrorAs(t, err, &streamErr) + select { + case sentReq := <-sendCh: + t.Fatalf("removed request was sent: %+v", sentReq) + default: + } +} + func TestProcessRegionSendTaskSendEOFIsRetriable(t *testing.T) { testCases := []struct { name string diff --git a/logservice/logpuller/region_state.go b/logservice/logpuller/region_state.go index 01831afbbe..8cd74b1eaf 100644 --- a/logservice/logpuller/region_state.go +++ b/logservice/logpuller/region_state.go @@ -15,6 +15,7 @@ package logpuller import ( "sync" + "sync/atomic" "github.com/pingcap/ticdc/heartbeatpb" "github.com/pingcap/ticdc/logservice/logpuller/regionlock" @@ -97,10 +98,7 @@ type regionFeedState struct { // `err` is used to retrieve errors generated outside. err error } - regionReq struct { - sync.Mutex - request *regionReq - } + regionReq atomic.Pointer[regionReq] worker *regionRequestWorker } @@ -117,7 +115,7 @@ func newRegionFeedState( matcher: newMatcher(), worker: worker, } - state.regionReq.request = request + state.regionReq.Store(request) return state } @@ -169,22 +167,14 @@ func (s *regionFeedState) setInitialized() { } func (s *regionFeedState) finishScan() { - s.regionReq.Lock() - request := s.regionReq.request - s.regionReq.request = nil - s.regionReq.Unlock() - + request := s.regionReq.Swap(nil) if request != nil { s.worker.requestCache.finishScan(request) } } func (s *regionFeedState) abortScanIfNeeded() { - s.regionReq.Lock() - request := s.regionReq.request - s.regionReq.request = nil - s.regionReq.Unlock() - + request := s.regionReq.Swap(nil) if request != nil { s.worker.requestCache.abortScan(request) } From 68d414a199625867a6282e51e60ab7d38817a546 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Sat, 11 Jul 2026 13:11:58 +0800 Subject: [PATCH 07/71] more refactor --- logservice/logpuller/region_req_cache.go | 71 ++--- logservice/logpuller/region_req_cache_test.go | 37 ++- logservice/logpuller/region_request_worker.go | 244 ++++++++++-------- .../logpuller/region_request_worker_test.go | 47 ++-- logservice/logpuller/region_state.go | 4 +- logservice/logpuller/region_tracker.go | 18 +- logservice/logpuller/region_tracker_test.go | 20 +- 7 files changed, 219 insertions(+), 222 deletions(-) diff --git a/logservice/logpuller/region_req_cache.go b/logservice/logpuller/region_req_cache.go index 099e6f53d7..0658847120 100644 --- a/logservice/logpuller/region_req_cache.go +++ b/logservice/logpuller/region_req_cache.go @@ -30,38 +30,26 @@ const ( abnormalRequestDurationInSec = 60 * 60 * 2 // 2 hours ) -type regionReqStage uint8 - -const ( - regionReqStageQueued regionReqStage = iota - regionReqStageProcessing - regionReqStageSent -) - // regionReq tracks one region request from admission to cleanup. type regionReq struct { regionInfo regionInfo createTime time.Time - - // stage is guarded by requestCache.mu. - stage regionReqStage } func newRegionReq(region regionInfo) *regionReq { return ®ionReq{ regionInfo: region, createTime: time.Now(), - stage: regionReqStageQueued, } } -// requestCache owns the lifecycle of worker-local region requests. -// Requests move through queued, processing and sent before explicit cleanup. +// requestCache owns flow-control slots from enqueue until the initial region +// scan finishes or the request is aborted. type requestCache struct { mu sync.Mutex maxPendingCount int - // requests is the source of truth for every live request in this worker. + // requests is the source of truth for every live request. requests map[*regionReq]struct{} queue *notifyqueue.Queue[*regionReq] @@ -139,36 +127,21 @@ func (c *requestCache) tryPop() *regionReq { if !ok { return nil } - if _, ok := c.requests[req]; !ok { - log.Warn("request cache pops a removed request", - zap.Uint64("subID", uint64(req.regionInfo.subscribedSpan.subID)), - zap.Uint64("regionID", req.regionInfo.verID.GetID()), - zap.Uint8("stage", uint8(req.stage))) - continue + if _, ok := c.requests[req]; ok { + return req } - if req.stage != regionReqStageQueued { - log.Warn("request cache pops a non-queued request", - zap.Uint64("subID", uint64(req.regionInfo.subscribedSpan.subID)), - zap.Uint64("regionID", req.regionInfo.verID.GetID()), - zap.Uint8("stage", uint8(req.stage))) - continue - } - req.stage = regionReqStageProcessing - return req + c.logRemovedRequest(req) } } -func (c *requestCache) markSent(req *regionReq) bool { +func (c *requestCache) isPending(req *regionReq) bool { c.mu.Lock() defer c.mu.Unlock() - if _, ok := c.requests[req]; !ok || req.stage != regionReqStageProcessing { - return false - } - req.stage = regionReqStageSent - return true + _, ok := c.requests[req] + return ok } -func (c *requestCache) finishScan(req *regionReq) bool { +func (c *requestCache) resolve(req *regionReq) bool { if !c.remove(req) { return false } @@ -189,10 +162,6 @@ func (c *requestCache) finishScan(req *regionReq) bool { return true } -func (c *requestCache) abortScan(req *regionReq) bool { - return c.remove(req) -} - func (c *requestCache) remove(req *regionReq) bool { if req == nil { return false @@ -208,19 +177,15 @@ func (c *requestCache) remove(req *regionReq) bool { return removed } -// drainUnsentRegions removes queued and processing requests. Sent requests are -// owned by regionTracker and are cleaned by their regionFeedState. -func (c *requestCache) drainUnsentRegions() []regionInfo { +// drain removes all remaining requests. A worker must stop its tracked region +// states first, so only requests that were never sent remain here. +func (c *requestCache) drain() []regionInfo { c.mu.Lock() regions := make([]regionInfo, 0, len(c.requests)) for req := range c.requests { - if req.stage == regionReqStageSent { - continue - } - if c.removeLocked(req) { - regions = append(regions, req.regionInfo) - } + regions = append(regions, req.regionInfo) } + clear(c.requests) c.queue.Drain() c.mu.Unlock() @@ -259,6 +224,12 @@ func (c *requestCache) removeLocked(req *regionReq) bool { return true } +func (c *requestCache) logRemovedRequest(req *regionReq) { + log.Warn("request cache pops a removed request", + zap.Uint64("subID", uint64(req.regionInfo.subscribedSpan.subID)), + zap.Uint64("regionID", req.regionInfo.verID.GetID())) +} + func (c *requestCache) ready() <-chan struct{} { return c.queue.Ready() } diff --git a/logservice/logpuller/region_req_cache_test.go b/logservice/logpuller/region_req_cache_test.go index c8d9b00635..c34d906957 100644 --- a/logservice/logpuller/region_req_cache_test.go +++ b/logservice/logpuller/region_req_cache_test.go @@ -50,13 +50,11 @@ func TestRequestCacheLifecycle(t *testing.T) { req, err := cache.pop(t.Context()) require.NoError(t, err) - require.Equal(t, regionReqStageProcessing, req.stage) require.Equal(t, 1, cache.pendingCount()) - require.True(t, cache.markSent(req)) - require.Equal(t, regionReqStageSent, req.stage) - require.True(t, cache.finishScan(req)) - require.False(t, cache.finishScan(req)) + require.True(t, cache.isPending(req)) + require.True(t, cache.resolve(req)) + require.False(t, cache.resolve(req)) require.Equal(t, 0, cache.pendingCount()) } @@ -109,7 +107,7 @@ func TestRequestCacheTracksDuplicateRegionsIndependently(t *testing.T) { require.Equal(t, 2, cache.pendingCount()) } -func TestRequestCacheDrainOnlyUnsentRegions(t *testing.T) { +func TestRequestCacheDrainRemainingRequests(t *testing.T) { cache := newRequestCache(3) for regionID := uint64(1); regionID <= 3; regionID++ { ok, err := cache.add(t.Context(), createTestRegionInfo(1, regionID), false) @@ -117,17 +115,15 @@ func TestRequestCacheDrainOnlyUnsentRegions(t *testing.T) { require.True(t, ok) } - sentReq, err := cache.pop(t.Context()) + finishedReq, err := cache.pop(t.Context()) require.NoError(t, err) - require.True(t, cache.markSent(sentReq)) + require.True(t, cache.remove(finishedReq)) processingReq, err := cache.pop(t.Context()) require.NoError(t, err) - require.Equal(t, regionReqStageProcessing, processingReq.stage) + require.NotNil(t, processingReq) - regions := cache.drainUnsentRegions() + regions := cache.drain() require.Len(t, regions, 2) - require.Equal(t, 1, cache.pendingCount()) - require.True(t, cache.abortScan(sentReq)) require.Equal(t, 0, cache.pendingCount()) } @@ -141,12 +137,11 @@ func TestRequestCacheCloseRemovesAllRequests(t *testing.T) { req, err := cache.pop(t.Context()) require.NoError(t, err) - require.True(t, cache.markSent(req)) cache.close() require.Equal(t, 0, cache.pendingCount()) require.Nil(t, cache.tryPop()) - require.False(t, cache.abortScan(req)) + require.False(t, cache.remove(req)) } func TestRequestCacheConcurrentAdds(t *testing.T) { @@ -188,7 +183,7 @@ func TestRequestCacheSpaceAvailableWakesAdd(t *testing.T) { resultCh <- addResult{ok: ok, err: err} }() <-started - require.True(t, cache.abortScan(firstReq)) + require.True(t, cache.remove(firstReq)) select { case result := <-resultCh: @@ -200,16 +195,15 @@ func TestRequestCacheSpaceAvailableWakesAdd(t *testing.T) { require.Equal(t, uint64(2), cache.tryPop().regionInfo.verID.GetID()) } -func TestRequestCacheMarkSentRejectsInvalidTransition(t *testing.T) { +func TestRequestCacheTracksOnlyPendingRequests(t *testing.T) { cache := newRequestCache(1) require.True(t, cache.tryAdd(createTestRegionInfo(1, 1), false)) req := cache.tryPop() require.NotNil(t, req) - require.True(t, cache.markSent(req)) - require.False(t, cache.markSent(req)) - require.True(t, cache.abortScan(req)) - require.False(t, cache.markSent(req)) + require.True(t, cache.isPending(req)) + require.True(t, cache.remove(req)) + require.False(t, cache.isPending(req)) } func TestRegionFeedStateCleansRequestOnce(t *testing.T) { @@ -217,7 +211,6 @@ func TestRegionFeedStateCleansRequestOnce(t *testing.T) { require.True(t, cache.tryAdd(createTestRegionInfo(1, 1), false)) req := cache.tryPop() require.NotNil(t, req) - require.True(t, cache.markSent(req)) worker := ®ionRequestWorker{requestCache: cache} state := newRegionFeedState(req.regionInfo, uint64(req.regionInfo.subscribedSpan.subID), worker, req) @@ -239,5 +232,5 @@ func TestRegionFeedStateCleansRequestOnce(t *testing.T) { require.Nil(t, state.regionReq.Load()) require.Zero(t, cache.pendingCount()) - require.False(t, cache.abortScan(req)) + require.False(t, cache.remove(req)) } diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index c30301c091..edd068c269 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -131,20 +131,7 @@ func (s *regionRequestWorker) Run(ctx context.Context) error { return ctx.Err() } - // Sent requests are owned by their states. Stopping those states removes - // their requests before unsent requests are drained below. - for subID, states := range s.tracker.Drain() { - for _, state := range states { - state.markStopped(regionErr) - s.client.eventSink.Push(subID, regionEvent{states: []*regionFeedState{state}}) - } - } - for _, region := range s.requestCache.drainUnsentRegions() { - s.client.onRegionFail(newRegionErrorInfo(region, regionErr)) - } - // The failed stream no longer owns remote registrations, so queued - // deregistration requests are obsolete. - s.controlQueue.drain() + s.failStreamRegions(regionErr) if err := util.Hang(ctx, storeReconnectBackoff); err != nil { return err @@ -152,24 +139,41 @@ func (s *regionRequestWorker) Run(ctx context.Context) error { } } -func (s *regionRequestWorker) waitForRegionRequest(ctx context.Context) (*regionReq, error) { - for { - // Without a stream there are no remote registrations to deregister. - s.controlQueue.drain() - if req := s.requestCache.tryPop(); req != nil { - // Drop controls that raced with selecting the first request. Any later - // controls will be handled by the stream send loop. - s.controlQueue.drain() - return req, nil +// failStreamRegions transfers every request owned by a failed stream to the +// recovery pipeline. Sent requests go through their state; unsent requests can +// be reported directly because no state exists for them yet. +func (s *regionRequestWorker) failStreamRegions(err error) { + for _, states := range s.tracker.Drain() { + for _, state := range states { + s.notifyRegionError(state, err) } + } + for _, region := range s.requestCache.drain() { + s.client.onRegionFail(newRegionErrorInfo(region, err)) + } + // The failed stream no longer owns remote registrations. + s.controlQueue.drain() +} - select { - case <-s.controlQueue.ready(): - case <-s.requestCache.ready(): - case <-ctx.Done(): - return nil, ctx.Err() - } +func (s *regionRequestWorker) notifyRegionError(state *regionFeedState, err error) { + state.markStopped(err) + s.client.eventSink.Push( + SubscriptionID(state.requestID), + regionEvent{states: []*regionFeedState{state}}, + ) +} + +func (s *regionRequestWorker) waitForRegionRequest(ctx context.Context) (*regionReq, error) { + // Without a stream there are no remote registrations to deregister. + s.controlQueue.drain() + req, err := s.requestCache.pop(ctx) + if err != nil { + return nil, err } + // Drop controls that raced with selecting the first request. Any later + // controls will be handled by the stream send loop. + s.controlQueue.drain() + return req, nil } func (s *regionRequestWorker) checkStoreVersion(ctx context.Context) error { @@ -293,7 +297,8 @@ func (s *regionRequestWorker) dispatchRegionChangeEvents(events []*cdcpb.Event) zap.Uint64("subscriptionID", uint64(subscriptionID)), zap.Uint64("regionID", event.RegionId), zap.Any("error", eventData.Error)) - state.markStopped(&eventError{err: eventData.Error}) + s.notifyRegionError(state, &eventError{err: eventData.Error}) + continue case *cdcpb.Event_ResolvedTs: regionEvent.resolvedTs = eventData.ResolvedTs case *cdcpb.Event_LongTxn_: @@ -366,90 +371,120 @@ func (s *regionRequestWorker) dispatchResolvedTsEvent(resolvedTsEvent *cdcpb.Res flush() } -func (s *regionRequestWorker) processRegionSendTask( - ctx context.Context, +func (s *regionRequestWorker) sendChangeDataRequest( conn *ConnAndClient, - firstReq *regionReq, + req *cdcpb.ChangeDataRequest, ) error { - doSend := func(req *cdcpb.ChangeDataRequest) error { - if err := conn.Client.Send(req); err != nil { - log.Warn("region request worker send request to grpc stream failed", - zap.Uint64("workerID", s.workerID), - zap.Uint64("subscriptionID", req.RequestId), - zap.Uint64("regionID", req.RegionId), - zap.String("addr", s.store.storeAddr), - zap.Error(err)) - return normalizeStreamError(err) - } - return nil + if err := conn.Client.Send(req); err != nil { + log.Warn("region request worker send request to grpc stream failed", + zap.Uint64("workerID", s.workerID), + zap.Uint64("subscriptionID", req.RequestId), + zap.Uint64("regionID", req.RegionId), + zap.String("addr", s.store.storeAddr), + zap.Error(err)) + return normalizeStreamError(err) } - sendDeregister := func(req deregisterRequest) error { - changeDataReq := &cdcpb.ChangeDataRequest{ - Header: &cdcpb.Header{ClusterId: s.client.clusterID, TicdcVersion: version.ReleaseSemver()}, - RequestId: uint64(req.subID), - Request: &cdcpb.ChangeDataRequest_Deregister_{ - Deregister: &cdcpb.ChangeDataRequest_Deregister{}, - }, - FilterLoop: req.filterLoop, + return nil +} + +func (s *regionRequestWorker) sendDeregisterRequest( + conn *ConnAndClient, + req deregisterRequest, +) error { + changeDataReq := &cdcpb.ChangeDataRequest{ + Header: &cdcpb.Header{ClusterId: s.client.clusterID, TicdcVersion: version.ReleaseSemver()}, + RequestId: uint64(req.subID), + Request: &cdcpb.ChangeDataRequest_Deregister_{ + Deregister: &cdcpb.ChangeDataRequest_Deregister{}, + }, + FilterLoop: req.filterLoop, + } + if err := s.sendChangeDataRequest(conn, changeDataReq); err != nil { + return err + } + for _, state := range s.tracker.TakeSubscription(req.subID) { + s.notifyRegionError(state, &requestCancelledErr{}) + } + return nil +} + +func (s *regionRequestWorker) drainControlQueue(conn *ConnAndClient) error { + for { + req, ok := s.controlQueue.tryPop() + if !ok { + return nil } - if err := doSend(changeDataReq); err != nil { + if err := s.sendDeregisterRequest(conn, req); err != nil { return err } - for _, state := range s.tracker.TakeSubscription(req.subID) { - state.markStopped(&requestCancelledErr{}) - s.client.eventSink.Push(req.subID, regionEvent{states: []*regionFeedState{state}}) - } + } +} + +func (s *regionRequestWorker) sendRegionRequest(conn *ConnAndClient, req *regionReq) error { + region := req.regionInfo + subID := region.subscribedSpan.subID + log.Debug("region request worker gets a singleRegionInfo", + zap.Uint64("workerID", s.workerID), + zap.Uint64("subscriptionID", uint64(subID)), + zap.Uint64("regionID", region.verID.GetID()), + zap.String("addr", s.store.storeAddr), + zap.Bool("bdrMode", region.filterLoop)) + + if region.subscribedSpan.stopped.Load() { + s.requestCache.remove(req) + s.client.onRegionFail(newRegionErrorInfo(region, &storeStreamErr{})) return nil } - drainControl := func() error { - for { - req, ok := s.controlQueue.tryPop() - if !ok { - return nil - } - if err := sendDeregister(req); err != nil { - return err - } - } + + // The request must still own a cache slot before it becomes visible to the + // receiver. Both are published before Send, so a fast response observes a + // complete state. + if !s.requestCache.isPending(req) { + log.Warn("region request transition to sent failed", + zap.Uint64("workerID", s.workerID), + zap.Uint64("subscriptionID", uint64(subID)), + zap.Uint64("regionID", region.verID.GetID())) + return &storeStreamErr{} + } + state := newRegionFeedState(region, uint64(subID), s, req) + if !s.tracker.Add(subID, region.verID.GetID(), state) { + // RangeLock normally prevents duplicate active regions. Keep the existing + // owner, including its range-lock ownership, if that invariant is ever + // violated. Only the duplicate request's flow-control slot is released. + state.abortScanIfNeeded() + state.matcher.clear() + log.Warn("duplicate active region request ignored", + zap.Uint64("workerID", s.workerID), + zap.Uint64("subscriptionID", uint64(subID)), + zap.Uint64("regionID", region.verID.GetID())) + return nil + } + if err := s.sendChangeDataRequest(conn, s.createRegionRequest(region)); err != nil { + // Transport failures are always recoverable at the region level. Preserve + // the stream error as the function result, but classify the region for + // rescheduling instead of exposing an arbitrary gRPC error downstream. + state.markStopped(&storeStreamErr{}) + return err } + return nil +} +func (s *regionRequestWorker) processRegionSendTask( + ctx context.Context, + conn *ConnAndClient, + firstReq *regionReq, +) error { regionReq := firstReq for { if regionReq != nil { - region := regionReq.regionInfo - subID := region.subscribedSpan.subID - log.Debug("region request worker gets a singleRegionInfo", - zap.Uint64("workerID", s.workerID), - zap.Uint64("subscriptionID", uint64(subID)), - zap.Uint64("regionID", region.verID.GetID()), - zap.String("addr", s.store.storeAddr), - zap.Bool("bdrMode", region.filterLoop)) - - if region.subscribedSpan.stopped.Load() { - s.client.onRegionFail(newRegionErrorInfo(region, &storeStreamErr{})) - s.requestCache.abortScan(regionReq) - } else { - state := newRegionFeedState(region, uint64(subID), s, regionReq) - s.replaceRegionState(subID, region.verID.GetID(), state) - // Make the request and its state visible in the same order. A fast - // region error can then clean the request without racing markSent. - if !s.requestCache.markSent(regionReq) { - log.Warn("region request transition to sent failed", - zap.Uint64("workerID", s.workerID), - zap.Uint64("subscriptionID", uint64(subID)), - zap.Uint64("regionID", region.verID.GetID())) - return &storeStreamErr{} - } - if err := doSend(s.createRegionRequest(region)); err != nil { - state.markStopped(err) - return err - } + if err := s.sendRegionRequest(conn, regionReq); err != nil { + return err } regionReq = nil continue } - if err := drainControl(); err != nil { + if err := s.drainControlQueue(conn); err != nil { return err } if regionReq = s.requestCache.tryPop(); regionReq != nil { @@ -478,23 +513,6 @@ func (s *regionRequestWorker) createRegionRequest(region regionInfo) *cdcpb.Chan } } -func (s *regionRequestWorker) replaceRegionState( - subscriptionID SubscriptionID, - regionID uint64, - state *regionFeedState, -) { - oldState := s.tracker.Replace(subscriptionID, regionID, state) - if oldState == nil { - return - } - - log.Warn("region request state overwritten", - zap.Uint64("workerID", s.workerID), - zap.Uint64("subscriptionID", uint64(subscriptionID)), - zap.Uint64("regionID", regionID)) - oldState.abortScanIfNeeded() -} - func (s *regionRequestWorker) add(ctx context.Context, region regionInfo, force bool) (bool, error) { return s.requestCache.add(ctx, region, force) } diff --git a/logservice/logpuller/region_request_worker_test.go b/logservice/logpuller/region_request_worker_test.go index 9ea8ff1642..1faa60870d 100644 --- a/logservice/logpuller/region_request_worker_test.go +++ b/logservice/logpuller/region_request_worker_test.go @@ -64,32 +64,46 @@ func prepareRegionForSendTest(region regionInfo) regionInfo { return region } -func TestRegionRequestWorkerReplaceRegionStateAbortsOldRequest(t *testing.T) { +func TestRegionRequestWorkerIgnoresDuplicateActiveRegion(t *testing.T) { worker := ®ionRequestWorker{ requestCache: newRequestCache(10), + store: &requestedStore{storeAddr: "store-1"}, + client: &subscriptionClient{}, tracker: newRegionTracker(), } - region := createTestRegionInfo(1, 1) + region := prepareRegionForSendTest(createTestRegionInfo(1, 1)) ok, err := worker.requestCache.add(t.Context(), region, false) require.NoError(t, err) require.True(t, ok) req1, err := worker.requestCache.pop(t.Context()) require.NoError(t, err) - require.True(t, worker.requestCache.markSent(req1)) state1 := newRegionFeedState(region, uint64(region.subscribedSpan.subID), worker, req1) - worker.replaceRegionState(region.subscribedSpan.subID, region.verID.GetID(), state1) + require.True(t, worker.tracker.Add(region.subscribedSpan.subID, region.verID.GetID(), state1)) ok, err = worker.requestCache.add(t.Context(), region, false) require.NoError(t, err) require.True(t, ok) req2, err := worker.requestCache.pop(t.Context()) require.NoError(t, err) - state2 := newRegionFeedState(region, uint64(region.subscribedSpan.subID), worker, req2) - worker.replaceRegionState(region.subscribedSpan.subID, region.verID.GetID(), state2) + sendCh := make(chan *cdcpb.ChangeDataRequest, 1) + err = worker.sendRegionRequest(&ConnAndClient{ + Client: &mockEventFeedV2Client{sendCh: sendCh}, + Conn: &grpc.ClientConn{}, + }, req2) + require.NoError(t, err) require.Equal(t, 1, worker.requestCache.pendingCount()) - require.Same(t, state2, worker.tracker.Get(region.subscribedSpan.subID, region.verID.GetID())) + require.Same(t, state1, worker.tracker.Get(region.subscribedSpan.subID, region.verID.GetID())) + require.False(t, state1.isStale()) + select { + case <-sendCh: + t.Fatal("duplicate region request must not be sent") + default: + } + + state1.abortScanIfNeeded() + state1.matcher.clear() } type pushedResolvedEvent struct { @@ -164,7 +178,7 @@ func newDispatchResolvedTsTestWorker(regionCount int) (*regionRequestWorker, *mo for i := 0; i < regionCount; i++ { regionID := uint64(i + 1) regions[i] = regionID - worker.tracker.Replace(1, regionID, ®ionFeedState{ + worker.tracker.Add(1, regionID, ®ionFeedState{ requestID: 1, }) } @@ -240,9 +254,6 @@ func TestWaitForRegionRequestDrainsIdleControlQueue(t *testing.T) { for subID := SubscriptionID(1); subID <= 100; subID++ { worker.controlQueue.push(deregisterRequest{subID: subID}) } - require.Eventually(t, func() bool { - return worker.controlQueue.len() == 0 - }, time.Second, 10*time.Millisecond) ok, err := worker.requestCache.add(t.Context(), createTestRegionInfo(1, 1), false) require.NoError(t, err) @@ -320,13 +331,12 @@ func TestStoppedStateRemovesSentRequest(t *testing.T) { require.NoError(t, err) state := newRegionFeedState(req.regionInfo, uint64(req.regionInfo.subscribedSpan.subID), worker, req) - worker.tracker.Replace(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID(), state) - require.True(t, worker.requestCache.markSent(req)) + require.True(t, worker.tracker.Add(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID(), state)) state.markStopped(errors.New("send request to store error")) worker.tracker.RemoveIf(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID(), state) require.Equal(t, 0, worker.requestCache.pendingCount()) - require.Empty(t, worker.requestCache.drainUnsentRegions()) + require.Empty(t, worker.requestCache.drain()) } func TestProcessRegionSendTaskSendFailureCleansSentRequest(t *testing.T) { @@ -358,7 +368,10 @@ func TestProcessRegionSendTaskSendFailureCleansSentRequest(t *testing.T) { require.ErrorIs(t, err, sendErr) require.Equal(t, 0, worker.requestCache.pendingCount()) state := worker.tracker.Get(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID()) - require.True(t, state == nil || state.isStale(), "region state should be removed or marked stale after send failure") + require.NotNil(t, state) + require.True(t, state.isStale()) + var streamErr *storeStreamErr + require.ErrorAs(t, state.takeError(), &streamErr) } func TestProcessRegionSendTaskDoesNotSendRemovedRequest(t *testing.T) { @@ -373,7 +386,7 @@ func TestProcessRegionSendTaskDoesNotSendRemovedRequest(t *testing.T) { require.True(t, worker.requestCache.tryAdd(region, false)) req := worker.requestCache.tryPop() require.NotNil(t, req) - require.True(t, worker.requestCache.abortScan(req)) + require.True(t, worker.requestCache.remove(req)) sendCh := make(chan *cdcpb.ChangeDataRequest, 1) err := worker.processRegionSendTask(t.Context(), &ConnAndClient{ @@ -454,7 +467,7 @@ func TestProcessRegionSendTaskHandlesDeregisterFromControlQueue(t *testing.T) { tracker: newRegionTracker(), } state := ®ionFeedState{worker: worker} - worker.tracker.Replace(1, 1, state) + require.True(t, worker.tracker.Add(1, 1, state)) worker.controlQueue.push(deregisterRequest{subID: 1, filterLoop: true}) ctx, cancel := context.WithCancel(context.Background()) diff --git a/logservice/logpuller/region_state.go b/logservice/logpuller/region_state.go index 8cd74b1eaf..042da211aa 100644 --- a/logservice/logpuller/region_state.go +++ b/logservice/logpuller/region_state.go @@ -169,14 +169,14 @@ func (s *regionFeedState) setInitialized() { func (s *regionFeedState) finishScan() { request := s.regionReq.Swap(nil) if request != nil { - s.worker.requestCache.finishScan(request) + s.worker.requestCache.resolve(request) } } func (s *regionFeedState) abortScanIfNeeded() { request := s.regionReq.Swap(nil) if request != nil { - s.worker.requestCache.abortScan(request) + s.worker.requestCache.remove(request) } } diff --git a/logservice/logpuller/region_tracker.go b/logservice/logpuller/region_tracker.go index 61a0176987..866ee49f73 100644 --- a/logservice/logpuller/region_tracker.go +++ b/logservice/logpuller/region_tracker.go @@ -45,13 +45,13 @@ func (t *regionTracker) Get(subscriptionID SubscriptionID, regionID uint64) *reg return nil } -// Replace records a region state and returns the state previously tracked by -// the same subscription and region, if any. -func (t *regionTracker) Replace( +// Add records a region state unless the same subscription and region is +// already tracked. +func (t *regionTracker) Add( subscriptionID SubscriptionID, regionID uint64, state *regionFeedState, -) *regionFeedState { +) bool { t.mu.Lock() defer t.mu.Unlock() @@ -60,14 +60,16 @@ func (t *regionTracker) Replace( states = make(regionStatesByID) t.statesBySubscription[subscriptionID] = states } - oldState := states[regionID] + if _, ok := states[regionID]; ok { + return false + } states[regionID] = state - return oldState + return true } // RemoveIf removes a region only when it is still tracked by the expected -// state. It prevents delayed events for an old state from removing its -// replacement. +// state. It prevents delayed events for an old state from removing a newer +// owner. func (t *regionTracker) RemoveIf( subscriptionID SubscriptionID, regionID uint64, diff --git a/logservice/logpuller/region_tracker_test.go b/logservice/logpuller/region_tracker_test.go index 5586c0789b..ad89844a3a 100644 --- a/logservice/logpuller/region_tracker_test.go +++ b/logservice/logpuller/region_tracker_test.go @@ -26,10 +26,10 @@ func TestRegionTrackerOperations(t *testing.T) { state3 := ®ionFeedState{} require.Nil(t, tracker.Get(1, 1)) - require.Nil(t, tracker.Replace(1, 1, state1)) + require.True(t, tracker.Add(1, 1, state1)) require.Same(t, state1, tracker.Get(1, 1)) - require.Nil(t, tracker.Replace(1, 2, state2)) - require.Nil(t, tracker.Replace(2, 3, state3)) + require.True(t, tracker.Add(1, 2, state2)) + require.True(t, tracker.Add(2, 3, state3)) require.ElementsMatch(t, []*regionFeedState{state1, state2}, tracker.TakeSubscription(1)) require.Nil(t, tracker.Get(1, 1)) @@ -44,17 +44,17 @@ func TestRegionTrackerOperations(t *testing.T) { require.Empty(t, tracker.Drain()) } -func TestRegionTrackerRemoveIfPreservesReplacement(t *testing.T) { +func TestRegionTrackerAddRejectsDuplicate(t *testing.T) { tracker := newRegionTracker() oldState := ®ionFeedState{} newState := ®ionFeedState{} - require.Nil(t, tracker.Replace(1, 1, oldState)) - require.Same(t, oldState, tracker.Replace(1, 1, newState)) + require.True(t, tracker.Add(1, 1, oldState)) + require.False(t, tracker.Add(1, 1, newState)) - require.False(t, tracker.RemoveIf(1, 1, oldState)) - require.Same(t, newState, tracker.Get(1, 1)) - require.True(t, tracker.RemoveIf(1, 1, newState)) - require.Nil(t, tracker.Get(1, 1)) require.False(t, tracker.RemoveIf(1, 1, newState)) + require.Same(t, oldState, tracker.Get(1, 1)) + require.True(t, tracker.RemoveIf(1, 1, oldState)) + require.Nil(t, tracker.Get(1, 1)) + require.False(t, tracker.RemoveIf(1, 1, oldState)) } From 635000c5e2f1a1f77dde688f0876ae9a3926f968 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Sat, 11 Jul 2026 20:17:46 +0800 Subject: [PATCH 08/71] introduce admission controller --- logservice/logpuller/priority_task.go | 11 +- .../logpuller/region_admission_controller.go | 285 ++++++++++++++++++ .../region_admission_controller_test.go | 196 ++++++++++++ .../logpuller/region_event_handler_test.go | 6 +- logservice/logpuller/region_req_cache.go | 242 --------------- logservice/logpuller/region_req_cache_test.go | 236 --------------- logservice/logpuller/region_request_worker.go | 52 ++-- .../logpuller/region_request_worker_test.go | 128 ++++---- logservice/logpuller/region_state.go | 4 +- logservice/logpuller/subscription_client.go | 61 +--- .../logpuller/subscription_client_test.go | 15 +- pkg/config/debug.go | 38 ++- pkg/config/debug_test.go | 35 +++ 13 files changed, 674 insertions(+), 635 deletions(-) create mode 100644 logservice/logpuller/region_admission_controller.go create mode 100644 logservice/logpuller/region_admission_controller_test.go delete mode 100644 logservice/logpuller/region_req_cache.go delete mode 100644 logservice/logpuller/region_req_cache_test.go create mode 100644 pkg/config/debug_test.go diff --git a/logservice/logpuller/priority_task.go b/logservice/logpuller/priority_task.go index 68ef9e885d..e57f860804 100644 --- a/logservice/logpuller/priority_task.go +++ b/logservice/logpuller/priority_task.go @@ -33,9 +33,8 @@ const ( ) const ( - highPriorityBase = 0 - lowPriorityBase = 60 * 60 * 24 // 1 day - forcedPriorityBase = 60 * 60 // 60 minutes + highPriorityBase = 0 + lowPriorityBase = 60 * 60 * 24 // 1 day ) func (t TaskType) String() string { @@ -50,6 +49,8 @@ type PriorityTask interface { // GetRegionInfo returns the underlying regionInfo GetRegionInfo() regionInfo + // GetTaskType returns whether this is a recovery or new-subscription task. + GetTaskType() TaskType // heap.Item interface methods SetHeapIndex(int) @@ -110,6 +111,10 @@ func (pt *regionPriorityTask) GetRegionInfo() regionInfo { return pt.regionInfo } +func (pt *regionPriorityTask) GetTaskType() TaskType { + return pt.taskType +} + // SetHeapIndex sets the heap index for heap.Item interface func (pt *regionPriorityTask) SetHeapIndex(index int) { pt.heapIndex = index diff --git a/logservice/logpuller/region_admission_controller.go b/logservice/logpuller/region_admission_controller.go new file mode 100644 index 0000000000..8f539f0e55 --- /dev/null +++ b/logservice/logpuller/region_admission_controller.go @@ -0,0 +1,285 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package logpuller + +import ( + "context" + "math" + "sync" + "sync/atomic" + "time" + + "github.com/pingcap/log" + "github.com/pingcap/ticdc/pkg/metrics" + "github.com/pingcap/ticdc/utils/heap" + "github.com/tikv/client-go/v2/oracle" + "go.uber.org/zap" +) + +const ( + fastRegionScanLag = 10 * time.Minute + abnormalRequestDurationInSec = 60 * 60 * 2 // 2 hours +) + +// pendingRegionRequest waits in the store-level admission queue. It does not +// own an admission slot until it is returned by pop or tryPop. +type pendingRegionRequest struct { + task PriorityTask + regionInfo regionInfo + fastScan bool + heapIndex int +} + +func (r *pendingRegionRequest) SetHeapIndex(index int) { + r.heapIndex = index +} + +func (r *pendingRegionRequest) GetHeapIndex() int { + return r.heapIndex +} + +func (r *pendingRegionRequest) LessThan(other *pendingRegionRequest) bool { + if r.task.GetTaskType() != other.task.GetTaskType() { + return r.task.GetTaskType() == TaskHighPrior + } + if r.canUseMaxWindow() != other.canUseMaxWindow() { + return r.canUseMaxWindow() + } + return r.task.LessThan(other.task) +} + +func (r *pendingRegionRequest) canUseMaxWindow() bool { + return r.fastScan || r.task.GetTaskType() == TaskHighPrior +} + +// regionReq is an admission lease for one sent-but-not-initialized region. +// finish and abort are idempotent and return the lease to the store controller. +type regionReq struct { + regionInfo regionInfo + createTime time.Time + controller *regionAdmissionController + released atomic.Bool +} + +func (r *regionReq) finish() bool { + if !r.release() { + return false + } + + cost := time.Since(r.createTime).Seconds() + if cost > 0 && cost < abnormalRequestDurationInSec { + log.Debug("cdc resolve region request", + zap.Uint64("subID", uint64(r.regionInfo.subscribedSpan.subID)), + zap.Uint64("regionID", r.regionInfo.verID.GetID()), + zap.Float64("cost", cost), + zap.Int("inflightCount", r.controller.inflightCount())) + metrics.RegionRequestFinishScanDuration.Observe(cost) + return true + } + log.Info("region request duration abnormal, skip metric", + zap.Float64("cost", cost), + zap.Uint64("regionID", r.regionInfo.verID.GetID())) + return true +} + +func (r *regionReq) abort() bool { + return r.release() +} + +func (r *regionReq) isActive() bool { + return r != nil && !r.released.Load() +} + +func (r *regionReq) release() bool { + if r == nil || !r.released.CompareAndSwap(false, true) { + return false + } + r.controller.release() + return true +} + +// regionAdmissionController owns the pending queue and initial-scan window for +// all request workers connected to one TiKV store. +type regionAdmissionController struct { + mu sync.Mutex + + currentWindow int + maxWindow int + inflight int + pending *heap.Heap[*pendingRegionRequest] + notify chan struct{} + closed bool +} + +func newRegionAdmissionController(currentWindow, maxWindowMultiplier int) *regionAdmissionController { + if currentWindow <= 0 { + currentWindow = 1 + } + if maxWindowMultiplier <= 0 { + maxWindowMultiplier = 1 + } + maxWindow := math.MaxInt + if currentWindow <= math.MaxInt/maxWindowMultiplier { + maxWindow = currentWindow * maxWindowMultiplier + } + return ®ionAdmissionController{ + currentWindow: currentWindow, + maxWindow: maxWindow, + pending: heap.NewHeap[*pendingRegionRequest](), + notify: make(chan struct{}, 1), + } +} + +func (c *regionAdmissionController) submit( + task PriorityTask, + region regionInfo, + currentTs uint64, +) bool { + request := &pendingRegionRequest{ + task: task, + regionInfo: region, + fastScan: regionScanLag(currentTs, region.resolvedTs()) < fastRegionScanLag, + } + + c.mu.Lock() + if c.closed { + c.mu.Unlock() + return false + } + c.pending.AddOrUpdate(request) + c.notifyOneLocked() + c.mu.Unlock() + return true +} + +func regionScanLag(currentTs, checkpointTs uint64) time.Duration { + currentTime := oracle.GetTimeFromTS(currentTs) + checkpointTime := oracle.GetTimeFromTS(checkpointTs) + if !currentTime.After(checkpointTime) { + return 0 + } + return currentTime.Sub(checkpointTime) +} + +func (c *regionAdmissionController) pop(ctx context.Context) (*regionReq, error) { + for { + request, closed := c.tryPop() + if request != nil { + return request, nil + } + if closed { + return nil, context.Canceled + } + + select { + case <-c.notify: + case <-ctx.Done(): + return nil, ctx.Err() + } + } +} + +// tryPop returns whether the controller has been closed as its second result. +func (c *regionAdmissionController) tryPop() (*regionReq, bool) { + c.mu.Lock() + defer c.mu.Unlock() + + if c.closed { + return nil, true + } + request := c.popEligibleLocked() + if request == nil { + return nil, false + } + c.inflight++ + if c.hasEligibleRequestLocked() { + c.notifyOneLocked() + } + return ®ionReq{ + regionInfo: request.regionInfo, + createTime: time.Now(), + controller: c, + }, false +} + +func (c *regionAdmissionController) popEligibleLocked() *pendingRegionRequest { + request, ok := c.pending.PeekTop() + if !ok { + return nil + } + if c.inflight >= c.windowFor(request) { + return nil + } + request, _ = c.pending.PopTop() + return request +} + +func (c *regionAdmissionController) hasEligibleRequestLocked() bool { + request, ok := c.pending.PeekTop() + if !ok { + return false + } + return c.inflight < c.windowFor(request) +} + +func (c *regionAdmissionController) windowFor(request *pendingRegionRequest) int { + if request.canUseMaxWindow() { + return c.maxWindow + } + return c.currentWindow +} + +func (c *regionAdmissionController) release() { + c.mu.Lock() + if c.inflight > 0 { + c.inflight-- + c.notifyOneLocked() + } + c.mu.Unlock() +} + +func (c *regionAdmissionController) close() { + c.mu.Lock() + if !c.closed { + c.closed = true + close(c.notify) + } + c.mu.Unlock() +} + +func (c *regionAdmissionController) ready() <-chan struct{} { + return c.notify +} + +func (c *regionAdmissionController) inflightCount() int { + c.mu.Lock() + defer c.mu.Unlock() + return c.inflight +} + +func (c *regionAdmissionController) pendingCount() int { + c.mu.Lock() + defer c.mu.Unlock() + return c.pending.Len() +} + +func (c *regionAdmissionController) notifyOneLocked() { + if c.closed { + return + } + select { + case c.notify <- struct{}{}: + default: + } +} diff --git a/logservice/logpuller/region_admission_controller_test.go b/logservice/logpuller/region_admission_controller_test.go new file mode 100644 index 0000000000..6c66451608 --- /dev/null +++ b/logservice/logpuller/region_admission_controller_test.go @@ -0,0 +1,196 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package logpuller + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/pingcap/ticdc/heartbeatpb" + "github.com/pingcap/ticdc/logservice/logpuller/regionlock" + "github.com/stretchr/testify/require" + "github.com/tikv/client-go/v2/oracle" + "github.com/tikv/client-go/v2/tikv" +) + +func createTestRegionInfo(subID SubscriptionID, regionID uint64) regionInfo { + span := heartbeatpb.TableSpan{ + TableID: 1, + StartKey: []byte("start"), + EndKey: []byte("end"), + } + return newRegionInfo( + tikv.NewRegionVerID(regionID, 1, 1), + span, + nil, + &subscribedSpan{subID: subID, startTs: 100, span: span}, + false, + ) +} + +func prepareRegionForAdmission(region regionInfo, checkpointTs uint64) regionInfo { + region.lockedRangeState = ®ionlock.LockedRangeState{} + region.lockedRangeState.ResolvedTs.Store(checkpointTs) + return region +} + +func submitRegionForAdmission( + t *testing.T, + controller *regionAdmissionController, + region regionInfo, + taskType TaskType, + currentTs uint64, +) { + t.Helper() + task := NewRegionPriorityTask(taskType, region, currentTs) + require.True(t, controller.submit(task, region, currentTs)) +} + +func TestRegionAdmissionControllerNormalWindow(t *testing.T) { + controller := newRegionAdmissionController(1, 2) + currentTs := oracle.GoTimeToTS(time.Now()) + checkpointTs := oracle.GoTimeToTS(time.Now().Add(-20 * time.Minute)) + region1 := prepareRegionForAdmission(createTestRegionInfo(1, 1), checkpointTs) + region2 := prepareRegionForAdmission(createTestRegionInfo(1, 2), checkpointTs) + submitRegionForAdmission(t, controller, region1, TaskLowPrior, currentTs) + submitRegionForAdmission(t, controller, region2, TaskLowPrior, currentTs) + + req1, err := controller.pop(t.Context()) + require.NoError(t, err) + require.Equal(t, 1, controller.inflightCount()) + req2, closed := controller.tryPop() + require.Nil(t, req2) + require.False(t, closed) + + require.True(t, req1.abort()) + req2, err = controller.pop(t.Context()) + require.NoError(t, err) + require.Equal(t, uint64(2), req2.regionInfo.verID.GetID()) + require.True(t, req2.abort()) +} + +func TestRegionAdmissionControllerFastScanUsesMaxWindow(t *testing.T) { + controller := newRegionAdmissionController(1, 2) + currentTs := oracle.GoTimeToTS(time.Now()) + slowCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-20 * time.Minute)) + fastCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Minute)) + + submitRegionForAdmission(t, controller, + prepareRegionForAdmission(createTestRegionInfo(1, 1), slowCheckpointTs), + TaskLowPrior, currentTs) + req1, err := controller.pop(t.Context()) + require.NoError(t, err) + + submitRegionForAdmission(t, controller, + prepareRegionForAdmission(createTestRegionInfo(1, 2), slowCheckpointTs), + TaskLowPrior, currentTs) + submitRegionForAdmission(t, controller, + prepareRegionForAdmission(createTestRegionInfo(1, 3), fastCheckpointTs), + TaskLowPrior, currentTs) + + req2, err := controller.pop(t.Context()) + require.NoError(t, err) + require.Equal(t, uint64(3), req2.regionInfo.verID.GetID()) + req3, closed := controller.tryPop() + require.Nil(t, req3) + require.False(t, closed) + require.Equal(t, 2, controller.inflightCount()) + + require.True(t, req1.abort()) + require.True(t, req2.abort()) + req3, err = controller.pop(t.Context()) + require.NoError(t, err) + require.Equal(t, uint64(2), req3.regionInfo.verID.GetID()) + require.True(t, req3.abort()) +} + +func TestRegionAdmissionControllerPrioritizesRecovery(t *testing.T) { + controller := newRegionAdmissionController(1, 2) + currentTs := oracle.GoTimeToTS(time.Now()) + slowCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-20 * time.Minute)) + fastCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Minute)) + + submitRegionForAdmission(t, controller, + prepareRegionForAdmission(createTestRegionInfo(1, 1), slowCheckpointTs), + TaskLowPrior, currentTs) + req1, err := controller.pop(t.Context()) + require.NoError(t, err) + + submitRegionForAdmission(t, controller, + prepareRegionForAdmission(createTestRegionInfo(1, 2), fastCheckpointTs), + TaskLowPrior, currentTs) + submitRegionForAdmission(t, controller, + prepareRegionForAdmission(createTestRegionInfo(1, 3), slowCheckpointTs), + TaskHighPrior, currentTs) + + req2, err := controller.pop(t.Context()) + require.NoError(t, err) + require.Equal(t, uint64(3), req2.regionInfo.verID.GetID()) + + require.True(t, req1.abort()) + require.True(t, req2.abort()) + req3, err := controller.pop(t.Context()) + require.NoError(t, err) + require.Equal(t, uint64(2), req3.regionInfo.verID.GetID()) + require.True(t, req3.abort()) +} + +func TestRegionAdmissionLeaseReleasedOnce(t *testing.T) { + controller := newRegionAdmissionController(1, 1) + currentTs := oracle.GoTimeToTS(time.Now()) + region := prepareRegionForAdmission(createTestRegionInfo(1, 1), currentTs) + submitRegionForAdmission(t, controller, region, TaskLowPrior, currentTs) + req, err := controller.pop(t.Context()) + require.NoError(t, err) + + start := make(chan struct{}) + results := make(chan bool, 2) + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + <-start + results <- req.finish() + }() + go func() { + defer wg.Done() + <-start + results <- req.abort() + }() + close(start) + wg.Wait() + close(results) + + successes := 0 + for result := range results { + if result { + successes++ + } + } + require.Equal(t, 1, successes) + require.Zero(t, controller.inflightCount()) +} + +func TestRegionAdmissionControllerClose(t *testing.T) { + controller := newRegionAdmissionController(1, 1) + controller.close() + region := prepareRegionForAdmission(createTestRegionInfo(1, 1), 1) + require.False(t, controller.submit(NewRegionPriorityTask(TaskLowPrior, region, 1), region, 1)) + + _, err := controller.pop(context.Background()) + require.ErrorIs(t, err, context.Canceled) +} diff --git a/logservice/logpuller/region_event_handler_test.go b/logservice/logpuller/region_event_handler_test.go index 219767a900..aef2ab63d4 100644 --- a/logservice/logpuller/region_event_handler_test.go +++ b/logservice/logpuller/region_event_handler_test.go @@ -78,8 +78,7 @@ func TestHandleEventEntryEventOutOfOrder(t *testing.T) { ds.AddPath(subID, subSpan, dynstream.AreaSettings{}) worker := ®ionRequestWorker{ - requestCache: &requestCache{}, - tracker: newRegionTracker(), + tracker: newRegionTracker(), } region := newRegionInfo( tikv.RegionVerID{}, @@ -217,8 +216,7 @@ func TestHandleResolvedTs(t *testing.T) { subID1 := SubscriptionID(1) worker := ®ionRequestWorker{ - requestCache: &requestCache{}, - tracker: newRegionTracker(), + tracker: newRegionTracker(), } state1 := newRegionFeedState(regionInfo{verID: tikv.NewRegionVerID(1, 1, 1)}, uint64(subID1), worker, nil) { diff --git a/logservice/logpuller/region_req_cache.go b/logservice/logpuller/region_req_cache.go deleted file mode 100644 index 0658847120..0000000000 --- a/logservice/logpuller/region_req_cache.go +++ /dev/null @@ -1,242 +0,0 @@ -// Copyright 2025 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package logpuller - -import ( - "context" - "sync" - "time" - - "github.com/pingcap/log" - "github.com/pingcap/ticdc/pkg/metrics" - "github.com/pingcap/ticdc/utils/notifyqueue" - "go.uber.org/zap" -) - -const ( - addReqRetryInterval = time.Millisecond - addReqRetryLimit = 3 - abnormalRequestDurationInSec = 60 * 60 * 2 // 2 hours -) - -// regionReq tracks one region request from admission to cleanup. -type regionReq struct { - regionInfo regionInfo - createTime time.Time -} - -func newRegionReq(region regionInfo) *regionReq { - return ®ionReq{ - regionInfo: region, - createTime: time.Now(), - } -} - -// requestCache owns flow-control slots from enqueue until the initial region -// scan finishes or the request is aborted. -type requestCache struct { - mu sync.Mutex - - maxPendingCount int - // requests is the source of truth for every live request. - requests map[*regionReq]struct{} - queue *notifyqueue.Queue[*regionReq] - - spaceAvailable chan struct{} -} - -func newRequestCache(maxPendingCount int) *requestCache { - return &requestCache{ - maxPendingCount: maxPendingCount, - requests: make(map[*regionReq]struct{}), - queue: notifyqueue.New[*regionReq](), - spaceAvailable: make(chan struct{}, 1), - } -} - -func (c *requestCache) add(ctx context.Context, region regionInfo, force bool) (bool, error) { - start := time.Now() - ticker := time.NewTicker(addReqRetryInterval) - defer ticker.Stop() - retries := addReqRetryLimit - - for { - if c.tryAdd(region, force) { - metrics.SubscriptionClientAddRegionRequestDuration.Observe(time.Since(start).Seconds()) - return true, nil - } - - select { - case <-ticker.C: - retries-- - if retries <= 0 { - return false, nil - } - case <-c.spaceAvailable: - case <-ctx.Done(): - return false, ctx.Err() - } - } -} - -func (c *requestCache) tryAdd(region regionInfo, force bool) bool { - c.mu.Lock() - defer c.mu.Unlock() - - if len(c.requests) >= c.maxPendingCount && !force { - return false - } - - req := newRegionReq(region) - c.requests[req] = struct{}{} - c.queue.Push(req) - return true -} - -func (c *requestCache) pop(ctx context.Context) (*regionReq, error) { - for { - if req := c.tryPop(); req != nil { - return req, nil - } - - select { - case <-c.queue.Ready(): - case <-ctx.Done(): - return nil, ctx.Err() - } - } -} - -func (c *requestCache) tryPop() *regionReq { - c.mu.Lock() - defer c.mu.Unlock() - - for { - req, ok := c.queue.TryPop() - if !ok { - return nil - } - if _, ok := c.requests[req]; ok { - return req - } - c.logRemovedRequest(req) - } -} - -func (c *requestCache) isPending(req *regionReq) bool { - c.mu.Lock() - defer c.mu.Unlock() - _, ok := c.requests[req] - return ok -} - -func (c *requestCache) resolve(req *regionReq) bool { - if !c.remove(req) { - return false - } - - cost := time.Since(req.createTime).Seconds() - if cost > 0 && cost < abnormalRequestDurationInSec { - log.Debug("cdc resolve region request", - zap.Uint64("subID", uint64(req.regionInfo.subscribedSpan.subID)), - zap.Uint64("regionID", req.regionInfo.verID.GetID()), - zap.Float64("cost", cost), - zap.Int("pendingCount", c.pendingCount())) - metrics.RegionRequestFinishScanDuration.Observe(cost) - return true - } - log.Info("region request duration abnormal, skip metric", - zap.Float64("cost", cost), - zap.Uint64("regionID", req.regionInfo.verID.GetID())) - return true -} - -func (c *requestCache) remove(req *regionReq) bool { - if req == nil { - return false - } - - c.mu.Lock() - removed := c.removeLocked(req) - c.mu.Unlock() - - if removed { - c.notifySpace() - } - return removed -} - -// drain removes all remaining requests. A worker must stop its tracked region -// states first, so only requests that were never sent remain here. -func (c *requestCache) drain() []regionInfo { - c.mu.Lock() - regions := make([]regionInfo, 0, len(c.requests)) - for req := range c.requests { - regions = append(regions, req.regionInfo) - } - clear(c.requests) - c.queue.Drain() - c.mu.Unlock() - - if len(regions) > 0 { - c.notifySpace() - } - return regions -} - -func (c *requestCache) close() { - c.mu.Lock() - removed := len(c.requests) > 0 - clear(c.requests) - c.queue.Drain() - c.mu.Unlock() - - if removed { - c.notifySpace() - } -} - -func (c *requestCache) pendingCount() int { - c.mu.Lock() - defer c.mu.Unlock() - return len(c.requests) -} - -func (c *requestCache) removeLocked(req *regionReq) bool { - if req == nil { - return false - } - if _, ok := c.requests[req]; !ok { - return false - } - delete(c.requests, req) - return true -} - -func (c *requestCache) logRemovedRequest(req *regionReq) { - log.Warn("request cache pops a removed request", - zap.Uint64("subID", uint64(req.regionInfo.subscribedSpan.subID)), - zap.Uint64("regionID", req.regionInfo.verID.GetID())) -} - -func (c *requestCache) ready() <-chan struct{} { - return c.queue.Ready() -} - -func (c *requestCache) notifySpace() { - select { - case c.spaceAvailable <- struct{}{}: - default: - } -} diff --git a/logservice/logpuller/region_req_cache_test.go b/logservice/logpuller/region_req_cache_test.go deleted file mode 100644 index c34d906957..0000000000 --- a/logservice/logpuller/region_req_cache_test.go +++ /dev/null @@ -1,236 +0,0 @@ -// Copyright 2025 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package logpuller - -import ( - "context" - "sync" - "testing" - "time" - - "github.com/pingcap/ticdc/heartbeatpb" - "github.com/stretchr/testify/require" - "github.com/tikv/client-go/v2/tikv" -) - -func createTestRegionInfo(subID SubscriptionID, regionID uint64) regionInfo { - span := heartbeatpb.TableSpan{ - TableID: 1, - StartKey: []byte("start"), - EndKey: []byte("end"), - } - return newRegionInfo( - tikv.NewRegionVerID(regionID, 1, 1), - span, - nil, - &subscribedSpan{subID: subID, startTs: 100, span: span}, - false, - ) -} - -func TestRequestCacheLifecycle(t *testing.T) { - cache := newRequestCache(1) - region := createTestRegionInfo(1, 1) - - ok, err := cache.add(t.Context(), region, false) - require.NoError(t, err) - require.True(t, ok) - require.Equal(t, 1, cache.pendingCount()) - - req, err := cache.pop(t.Context()) - require.NoError(t, err) - require.Equal(t, 1, cache.pendingCount()) - - require.True(t, cache.isPending(req)) - require.True(t, cache.resolve(req)) - require.False(t, cache.resolve(req)) - require.Equal(t, 0, cache.pendingCount()) -} - -func TestRequestCacheCapacityAndForceAdd(t *testing.T) { - cache := newRequestCache(1) - - ok, err := cache.add(t.Context(), createTestRegionInfo(1, 1), false) - require.NoError(t, err) - require.True(t, ok) - - ok, err = cache.add(t.Context(), createTestRegionInfo(1, 2), false) - require.NoError(t, err) - require.False(t, ok) - - ok, err = cache.add(t.Context(), createTestRegionInfo(1, 3), true) - require.NoError(t, err) - require.True(t, ok) - require.Equal(t, 2, cache.pendingCount()) -} - -func TestRequestCacheAddHonorsCancellationWhenFull(t *testing.T) { - cache := newRequestCache(1) - ok, err := cache.add(t.Context(), createTestRegionInfo(1, 1), false) - require.NoError(t, err) - require.True(t, ok) - - ctx, cancel := context.WithCancel(context.Background()) - cancel() - ok, err = cache.add(ctx, createTestRegionInfo(1, 2), false) - require.False(t, ok) - require.ErrorIs(t, err, context.Canceled) -} - -func TestRequestCacheTracksDuplicateRegionsIndependently(t *testing.T) { - cache := newRequestCache(2) - region := createTestRegionInfo(1, 1) - - ok, err := cache.add(t.Context(), region, false) - require.NoError(t, err) - require.True(t, ok) - ok, err = cache.add(t.Context(), region, false) - require.NoError(t, err) - require.True(t, ok) - - req1, err := cache.pop(t.Context()) - require.NoError(t, err) - req2, err := cache.pop(t.Context()) - require.NoError(t, err) - require.NotSame(t, req1, req2) - require.Equal(t, 2, cache.pendingCount()) -} - -func TestRequestCacheDrainRemainingRequests(t *testing.T) { - cache := newRequestCache(3) - for regionID := uint64(1); regionID <= 3; regionID++ { - ok, err := cache.add(t.Context(), createTestRegionInfo(1, regionID), false) - require.NoError(t, err) - require.True(t, ok) - } - - finishedReq, err := cache.pop(t.Context()) - require.NoError(t, err) - require.True(t, cache.remove(finishedReq)) - processingReq, err := cache.pop(t.Context()) - require.NoError(t, err) - require.NotNil(t, processingReq) - - regions := cache.drain() - require.Len(t, regions, 2) - require.Equal(t, 0, cache.pendingCount()) -} - -func TestRequestCacheCloseRemovesAllRequests(t *testing.T) { - cache := newRequestCache(2) - for regionID := uint64(1); regionID <= 2; regionID++ { - ok, err := cache.add(t.Context(), createTestRegionInfo(1, regionID), false) - require.NoError(t, err) - require.True(t, ok) - } - - req, err := cache.pop(t.Context()) - require.NoError(t, err) - cache.close() - - require.Equal(t, 0, cache.pendingCount()) - require.Nil(t, cache.tryPop()) - require.False(t, cache.remove(req)) -} - -func TestRequestCacheConcurrentAdds(t *testing.T) { - const requestCount = 16 - cache := newRequestCache(requestCount) - start := make(chan struct{}) - results := make(chan bool, requestCount) - - for regionID := uint64(1); regionID <= requestCount; regionID++ { - go func() { - <-start - ok, err := cache.add(t.Context(), createTestRegionInfo(1, regionID), false) - results <- ok && err == nil - }() - } - close(start) - - for range requestCount { - require.True(t, <-results) - } - require.Equal(t, requestCount, cache.pendingCount()) -} - -func TestRequestCacheSpaceAvailableWakesAdd(t *testing.T) { - cache := newRequestCache(1) - require.True(t, cache.tryAdd(createTestRegionInfo(1, 1), false)) - firstReq := cache.tryPop() - require.NotNil(t, firstReq) - - type addResult struct { - ok bool - err error - } - started := make(chan struct{}) - resultCh := make(chan addResult, 1) - go func() { - close(started) - ok, err := cache.add(t.Context(), createTestRegionInfo(1, 2), false) - resultCh <- addResult{ok: ok, err: err} - }() - <-started - require.True(t, cache.remove(firstReq)) - - select { - case result := <-resultCh: - require.NoError(t, result.err) - require.True(t, result.ok) - case <-time.After(time.Second): - t.Fatal("add was not woken after request cache space became available") - } - require.Equal(t, uint64(2), cache.tryPop().regionInfo.verID.GetID()) -} - -func TestRequestCacheTracksOnlyPendingRequests(t *testing.T) { - cache := newRequestCache(1) - require.True(t, cache.tryAdd(createTestRegionInfo(1, 1), false)) - req := cache.tryPop() - require.NotNil(t, req) - - require.True(t, cache.isPending(req)) - require.True(t, cache.remove(req)) - require.False(t, cache.isPending(req)) -} - -func TestRegionFeedStateCleansRequestOnce(t *testing.T) { - cache := newRequestCache(1) - require.True(t, cache.tryAdd(createTestRegionInfo(1, 1), false)) - req := cache.tryPop() - require.NotNil(t, req) - - worker := ®ionRequestWorker{requestCache: cache} - state := newRegionFeedState(req.regionInfo, uint64(req.regionInfo.subscribedSpan.subID), worker, req) - start := make(chan struct{}) - var wg sync.WaitGroup - wg.Add(2) - go func() { - defer wg.Done() - <-start - state.finishScan() - }() - go func() { - defer wg.Done() - <-start - state.abortScanIfNeeded() - }() - close(start) - wg.Wait() - - require.Nil(t, state.regionReq.Load()) - require.Zero(t, cache.pendingCount()) - require.False(t, cache.remove(req)) -} diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index edd068c269..336bc4dcf7 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -92,7 +92,7 @@ type regionRequestWorker struct { client *subscriptionClient store *requestedStore - requestCache *requestCache + admission *regionAdmissionController controlQueue *controlQueue tracker *regionTracker } @@ -100,14 +100,14 @@ type regionRequestWorker struct { func newRegionRequestWorker( client *subscriptionClient, store *requestedStore, - requestCache *requestCache, + admission *regionAdmissionController, ) *regionRequestWorker { workerID := workerIDGen.Add(1) return ®ionRequestWorker{ workerID: workerID, client: client, store: store, - requestCache: requestCache, + admission: admission, controlQueue: newControlQueue(), tracker: newRegionTracker(), } @@ -128,10 +128,16 @@ func (s *regionRequestWorker) Run(ctx context.Context) error { regionErr = err } if ctx.Err() != nil { + firstReq.abort() return ctx.Err() } + // Stop sent requests first so their states release the admission leases. + // firstReq still owns its lease only if the stream failed before Send. s.failStreamRegions(regionErr) + if firstReq.abort() { + s.client.onRegionFail(newRegionErrorInfo(firstReq.regionInfo, regionErr)) + } if err := util.Hang(ctx, storeReconnectBackoff); err != nil { return err @@ -139,18 +145,15 @@ func (s *regionRequestWorker) Run(ctx context.Context) error { } } -// failStreamRegions transfers every request owned by a failed stream to the -// recovery pipeline. Sent requests go through their state; unsent requests can -// be reported directly because no state exists for them yet. +// failStreamRegions transfers every request sent by a failed stream to the +// recovery pipeline. Requests still waiting in the store admission controller +// are not owned by this stream. func (s *regionRequestWorker) failStreamRegions(err error) { for _, states := range s.tracker.Drain() { for _, state := range states { s.notifyRegionError(state, err) } } - for _, region := range s.requestCache.drain() { - s.client.onRegionFail(newRegionErrorInfo(region, err)) - } // The failed stream no longer owns remote registrations. s.controlQueue.drain() } @@ -166,7 +169,7 @@ func (s *regionRequestWorker) notifyRegionError(state *regionFeedState, err erro func (s *regionRequestWorker) waitForRegionRequest(ctx context.Context) (*regionReq, error) { // Without a stream there are no remote registrations to deregister. s.controlQueue.drain() - req, err := s.requestCache.pop(ctx) + req, err := s.admission.pop(ctx) if err != nil { return nil, err } @@ -421,6 +424,9 @@ func (s *regionRequestWorker) drainControlQueue(conn *ConnAndClient) error { } func (s *regionRequestWorker) sendRegionRequest(conn *ConnAndClient, req *regionReq) error { + if !req.isActive() { + return &storeStreamErr{} + } region := req.regionInfo subID := region.subscribedSpan.subID log.Debug("region request worker gets a singleRegionInfo", @@ -431,21 +437,13 @@ func (s *regionRequestWorker) sendRegionRequest(conn *ConnAndClient, req *region zap.Bool("bdrMode", region.filterLoop)) if region.subscribedSpan.stopped.Load() { - s.requestCache.remove(req) + req.abort() s.client.onRegionFail(newRegionErrorInfo(region, &storeStreamErr{})) return nil } - // The request must still own a cache slot before it becomes visible to the - // receiver. Both are published before Send, so a fast response observes a - // complete state. - if !s.requestCache.isPending(req) { - log.Warn("region request transition to sent failed", - zap.Uint64("workerID", s.workerID), - zap.Uint64("subscriptionID", uint64(subID)), - zap.Uint64("regionID", region.verID.GetID())) - return &storeStreamErr{} - } + // Publish the state before Send so a fast response observes its owner and + // admission lease. state := newRegionFeedState(region, uint64(subID), s, req) if !s.tracker.Add(subID, region.verID.GetID(), state) { // RangeLock normally prevents duplicate active regions. Keep the existing @@ -487,12 +485,16 @@ func (s *regionRequestWorker) processRegionSendTask( if err := s.drainControlQueue(conn); err != nil { return err } - if regionReq = s.requestCache.tryPop(); regionReq != nil { + var closed bool + if regionReq, closed = s.admission.tryPop(); regionReq != nil { continue } + if closed { + return context.Canceled + } select { case <-s.controlQueue.ready(): - case <-s.requestCache.ready(): + case <-s.admission.ready(): case <-ctx.Done(): return ctx.Err() } @@ -512,7 +514,3 @@ func (s *regionRequestWorker) createRegionRequest(region regionInfo) *cdcpb.Chan FilterLoop: region.filterLoop, } } - -func (s *regionRequestWorker) add(ctx context.Context, region regionInfo, force bool) (bool, error) { - return s.requestCache.add(ctx, region, force) -} diff --git a/logservice/logpuller/region_request_worker_test.go b/logservice/logpuller/region_request_worker_test.go index 1faa60870d..9179b0b49b 100644 --- a/logservice/logpuller/region_request_worker_test.go +++ b/logservice/logpuller/region_request_worker_test.go @@ -26,6 +26,7 @@ import ( "github.com/pingcap/ticdc/utils/dynstream" "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/require" + "github.com/tikv/client-go/v2/oracle" "github.com/tikv/client-go/v2/tikv" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -64,36 +65,42 @@ func prepareRegionForSendTest(region regionInfo) regionInfo { return region } +func admitRegionRequest( + t *testing.T, + controller *regionAdmissionController, + region regionInfo, +) *regionReq { + t.Helper() + currentTs := oracle.GoTimeToTS(time.Now()) + submitRegionForAdmission(t, controller, region, TaskLowPrior, currentTs) + req, err := controller.pop(t.Context()) + require.NoError(t, err) + return req +} + func TestRegionRequestWorkerIgnoresDuplicateActiveRegion(t *testing.T) { + admission := newRegionAdmissionController(10, 1) worker := ®ionRequestWorker{ - requestCache: newRequestCache(10), - store: &requestedStore{storeAddr: "store-1"}, - client: &subscriptionClient{}, - tracker: newRegionTracker(), + admission: admission, + store: &requestedStore{storeAddr: "store-1"}, + client: &subscriptionClient{}, + tracker: newRegionTracker(), } region := prepareRegionForSendTest(createTestRegionInfo(1, 1)) - ok, err := worker.requestCache.add(t.Context(), region, false) - require.NoError(t, err) - require.True(t, ok) - req1, err := worker.requestCache.pop(t.Context()) - require.NoError(t, err) + req1 := admitRegionRequest(t, admission, region) state1 := newRegionFeedState(region, uint64(region.subscribedSpan.subID), worker, req1) require.True(t, worker.tracker.Add(region.subscribedSpan.subID, region.verID.GetID(), state1)) - ok, err = worker.requestCache.add(t.Context(), region, false) - require.NoError(t, err) - require.True(t, ok) - req2, err := worker.requestCache.pop(t.Context()) - require.NoError(t, err) + req2 := admitRegionRequest(t, admission, region) sendCh := make(chan *cdcpb.ChangeDataRequest, 1) - err = worker.sendRegionRequest(&ConnAndClient{ + err := worker.sendRegionRequest(&ConnAndClient{ Client: &mockEventFeedV2Client{sendCh: sendCh}, Conn: &grpc.ClientConn{}, }, req2) require.NoError(t, err) - require.Equal(t, 1, worker.requestCache.pendingCount()) + require.Equal(t, 1, admission.inflightCount()) require.Same(t, state1, worker.tracker.Get(region.subscribedSpan.subID, region.verID.GetID())) require.False(t, state1.isStale()) select { @@ -236,8 +243,9 @@ func benchmarkDispatchResolvedTsEvent(b *testing.B, regionCount int, useLegacy b } func TestWaitForRegionRequestDrainsIdleControlQueue(t *testing.T) { + admission := newRegionAdmissionController(1, 1) worker := ®ionRequestWorker{ - requestCache: newRequestCache(1), + admission: admission, controlQueue: newControlQueue(), } @@ -255,9 +263,8 @@ func TestWaitForRegionRequestDrainsIdleControlQueue(t *testing.T) { worker.controlQueue.push(deregisterRequest{subID: subID}) } - ok, err := worker.requestCache.add(t.Context(), createTestRegionInfo(1, 1), false) - require.NoError(t, err) - require.True(t, ok) + region := prepareRegionForAdmission(createTestRegionInfo(1, 1), 1) + submitRegionForAdmission(t, admission, region, TaskLowPrior, 1) select { case result := <-resultCh: @@ -318,30 +325,49 @@ func BenchmarkDispatchResolvedTsEventSmallBatchCurrent(b *testing.B) { } func TestStoppedStateRemovesSentRequest(t *testing.T) { + admission := newRegionAdmissionController(10, 1) worker := ®ionRequestWorker{ - requestCache: newRequestCache(10), - tracker: newRegionTracker(), + admission: admission, + tracker: newRegionTracker(), } - region := createTestRegionInfo(1, 1) - - ok, err := worker.requestCache.add(t.Context(), region, false) - require.NoError(t, err) - require.True(t, ok) - req, err := worker.requestCache.pop(t.Context()) - require.NoError(t, err) + region := prepareRegionForSendTest(createTestRegionInfo(1, 1)) + req := admitRegionRequest(t, admission, region) state := newRegionFeedState(req.regionInfo, uint64(req.regionInfo.subscribedSpan.subID), worker, req) require.True(t, worker.tracker.Add(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID(), state)) state.markStopped(errors.New("send request to store error")) worker.tracker.RemoveIf(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID(), state) - require.Equal(t, 0, worker.requestCache.pendingCount()) - require.Empty(t, worker.requestCache.drain()) + require.Equal(t, 0, admission.inflightCount()) +} + +func TestFailStreamRegionsReleasesSentAdmission(t *testing.T) { + admission := newRegionAdmissionController(1, 1) + ds := &mockRegionEventDynamicStream{} + worker := ®ionRequestWorker{ + admission: admission, + controlQueue: newControlQueue(), + client: &subscriptionClient{ + eventSink: ®ionEventSink{ds: ds}, + }, + tracker: newRegionTracker(), + } + region := prepareRegionForSendTest(createTestRegionInfo(1, 1)) + req := admitRegionRequest(t, admission, region) + state := newRegionFeedState(region, uint64(region.subscribedSpan.subID), worker, req) + require.True(t, worker.tracker.Add(region.subscribedSpan.subID, region.verID.GetID(), state)) + + worker.failStreamRegions(&storeStreamErr{}) + + require.Zero(t, admission.inflightCount()) + require.False(t, req.abort()) + require.Equal(t, 1, ds.pushCount) } func TestProcessRegionSendTaskSendFailureCleansSentRequest(t *testing.T) { + admission := newRegionAdmissionController(10, 1) worker := ®ionRequestWorker{ - requestCache: newRequestCache(10), + admission: admission, controlQueue: newControlQueue(), store: &requestedStore{storeAddr: "store-1"}, client: &subscriptionClient{}, @@ -350,13 +376,8 @@ func TestProcessRegionSendTaskSendFailureCleansSentRequest(t *testing.T) { region := prepareRegionForSendTest(createTestRegionInfo(1, 1)) - ok, err := worker.requestCache.add(t.Context(), region, false) - require.NoError(t, err) - require.True(t, ok) - require.Equal(t, 1, worker.requestCache.pendingCount()) - - req, err := worker.requestCache.pop(t.Context()) - require.NoError(t, err) + req := admitRegionRequest(t, admission, region) + require.Equal(t, 1, admission.inflightCount()) sendErr := errors.New("send failed") conn := &ConnAndClient{ @@ -364,9 +385,9 @@ func TestProcessRegionSendTaskSendFailureCleansSentRequest(t *testing.T) { Conn: &grpc.ClientConn{}, } - err = worker.processRegionSendTask(t.Context(), conn, req) + err := worker.processRegionSendTask(t.Context(), conn, req) require.ErrorIs(t, err, sendErr) - require.Equal(t, 0, worker.requestCache.pendingCount()) + require.Equal(t, 0, admission.inflightCount()) state := worker.tracker.Get(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID()) require.NotNil(t, state) require.True(t, state.isStale()) @@ -375,18 +396,17 @@ func TestProcessRegionSendTaskSendFailureCleansSentRequest(t *testing.T) { } func TestProcessRegionSendTaskDoesNotSendRemovedRequest(t *testing.T) { + admission := newRegionAdmissionController(1, 1) worker := ®ionRequestWorker{ - requestCache: newRequestCache(1), + admission: admission, controlQueue: newControlQueue(), store: &requestedStore{storeAddr: "store-1"}, client: &subscriptionClient{}, tracker: newRegionTracker(), } region := prepareRegionForSendTest(createTestRegionInfo(1, 1)) - require.True(t, worker.requestCache.tryAdd(region, false)) - req := worker.requestCache.tryPop() - require.NotNil(t, req) - require.True(t, worker.requestCache.remove(req)) + req := admitRegionRequest(t, admission, region) + require.True(t, req.abort()) sendCh := make(chan *cdcpb.ChangeDataRequest, 1) err := worker.processRegionSendTask(t.Context(), &ConnAndClient{ @@ -419,8 +439,9 @@ func TestProcessRegionSendTaskSendEOFIsRetriable(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { + admission := newRegionAdmissionController(10, 1) worker := ®ionRequestWorker{ - requestCache: newRequestCache(10), + admission: admission, controlQueue: newControlQueue(), store: &requestedStore{storeAddr: "store-1"}, client: &subscriptionClient{}, @@ -428,22 +449,17 @@ func TestProcessRegionSendTaskSendEOFIsRetriable(t *testing.T) { } region := prepareRegionForSendTest(createTestRegionInfo(1, 1)) - ok, err := worker.requestCache.add(t.Context(), region, false) - require.NoError(t, err) - require.True(t, ok) - - req, err := worker.requestCache.pop(t.Context()) - require.NoError(t, err) + req := admitRegionRequest(t, admission, region) conn := &ConnAndClient{ Client: &mockEventFeedV2Client{sendErr: tc.sendErr}, Conn: &grpc.ClientConn{}, } - err = worker.processRegionSendTask(t.Context(), conn, req) + err := worker.processRegionSendTask(t.Context(), conn, req) var streamErr *storeStreamErr require.ErrorAs(t, err, &streamErr) - require.Equal(t, 0, worker.requestCache.pendingCount()) + require.Equal(t, 0, admission.inflightCount()) state := worker.tracker.Get(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID()) require.NotNil(t, state) @@ -458,7 +474,7 @@ func TestProcessRegionSendTaskSendEOFIsRetriable(t *testing.T) { func TestProcessRegionSendTaskHandlesDeregisterFromControlQueue(t *testing.T) { ds := &mockRegionEventDynamicStream{} worker := ®ionRequestWorker{ - requestCache: newRequestCache(1), + admission: newRegionAdmissionController(1, 1), controlQueue: newControlQueue(), store: &requestedStore{storeAddr: "store-1"}, client: &subscriptionClient{ diff --git a/logservice/logpuller/region_state.go b/logservice/logpuller/region_state.go index 042da211aa..9e4fb52d58 100644 --- a/logservice/logpuller/region_state.go +++ b/logservice/logpuller/region_state.go @@ -169,14 +169,14 @@ func (s *regionFeedState) setInitialized() { func (s *regionFeedState) finishScan() { request := s.regionReq.Swap(nil) if request != nil { - s.worker.requestCache.resolve(request) + request.finish() } } func (s *regionFeedState) abortScanIfNeeded() { request := s.regionReq.Swap(nil) if request != nil { - s.worker.requestCache.remove(request) + request.abort() } } diff --git a/logservice/logpuller/subscription_client.go b/logservice/logpuller/subscription_client.go index cfaf2bcfa0..e8fa26ba43 100644 --- a/logservice/logpuller/subscription_client.go +++ b/logservice/logpuller/subscription_client.go @@ -212,11 +212,7 @@ func (s *subscriptionClient) updateMetrics(ctx context.Context) error { pendingRegionReqCount := 0 s.stores.Range(func(_, value any) bool { store := value.(*requestedStore) - store.requestWorkers.RLock() - for _, worker := range store.requestWorkers.s { - pendingRegionReqCount += worker.requestCache.pendingCount() - } - store.requestWorkers.RUnlock() + pendingRegionReqCount += store.admission.inflightCount() return true }) @@ -357,8 +353,7 @@ func (s *subscriptionClient) onRegionFail(errInfo regionErrorInfo) { // requestedStore represents a store that has been connected. type requestedStore struct { storeAddr string - // Use to select a worker to send request. - nextWorker atomic.Uint32 + admission *regionAdmissionController requestWorkers struct { sync.RWMutex @@ -366,19 +361,12 @@ type requestedStore struct { } } -func (rs *requestedStore) getRequestWorker() *regionRequestWorker { - rs.requestWorkers.RLock() - defer rs.requestWorkers.RUnlock() - - index := rs.nextWorker.Add(1) % uint32(len(rs.requestWorkers.s)) - return rs.requestWorkers.s[index] -} - // handleRegions receives regionInfo from regionTaskQueue and attach rpcCtx to them, // then send them to corresponding requestedStore. func (s *subscriptionClient) handleRegions(ctx context.Context, eg *errgroup.Group) error { cfg := config.GetGlobalServerConfig() - pendingRegionRequestQueueSize := cfg.Debug.Puller.PendingRegionRequestQueueSize + currentWindow := cfg.Debug.Puller.PendingRegionRequestQueueSize + maxWindowMultiplier := cfg.Debug.Puller.RegionRequestMaxWindowMultiplier getStore := func(storeAddr string) *requestedStore { var rs *requestedStore if v, ok := s.stores.Load(storeAddr); ok { @@ -386,20 +374,15 @@ func (s *subscriptionClient) handleRegions(ctx context.Context, eg *errgroup.Gro return rs } - rs = &requestedStore{storeAddr: storeAddr} - rs.requestWorkers.s = make([]*regionRequestWorker, 0, s.config.RegionRequestWorkerPerStore) - - perWorkerQueueSize := pendingRegionRequestQueueSize / int(s.config.RegionRequestWorkerPerStore) - if perWorkerQueueSize <= 0 { - log.Warn("pending region request queue size is smaller than the number of workers, adjust per worker queue size to 1", - zap.Int("pendingRegionRequestQueueSize", pendingRegionRequestQueueSize), - zap.Uint("regionRequestWorkerPerStore", s.config.RegionRequestWorkerPerStore)) - perWorkerQueueSize = 1 + rs = &requestedStore{ + storeAddr: storeAddr, + admission: newRegionAdmissionController(currentWindow, maxWindowMultiplier), } + rs.requestWorkers.s = make([]*regionRequestWorker, 0, s.config.RegionRequestWorkerPerStore) rs.requestWorkers.Lock() for i := uint(0); i < s.config.RegionRequestWorkerPerStore; i++ { - requestWorker := newRegionRequestWorker(s, rs, newRequestCache(perWorkerQueueSize)) + requestWorker := newRegionRequestWorker(s, rs, rs.admission) rs.requestWorkers.s = append(rs.requestWorkers.s, requestWorker) } rs.requestWorkers.Unlock() @@ -415,13 +398,7 @@ func (s *subscriptionClient) handleRegions(ctx context.Context, eg *errgroup.Gro defer func() { s.stores.Range(func(_, value any) bool { rs := value.(*requestedStore) - - rs.requestWorkers.RLock() - for _, w := range rs.requestWorkers.s { - w.requestCache.close() - } - rs.requestWorkers.RUnlock() - + rs.admission.close() return true }) }() @@ -449,25 +426,11 @@ func (s *subscriptionClient) handleRegions(ctx context.Context, eg *errgroup.Gro } store := getStore(region.rpcCtx.Addr) - worker := store.getRequestWorker() - force := regionTask.Priority() <= forcedPriorityBase - - ok, err = worker.add(ctx, region, force) - if err != nil { - log.Warn("subscription client add region request failed", - zap.Uint64("subscriptionID", uint64(region.subscribedSpan.subID)), - zap.Uint64("regionID", region.verID.GetID()), - zap.Error(err)) - return err - } - - if !ok { - s.regionTaskQueue.Push(regionTask) - continue + if !store.admission.submit(regionTask, region, s.pdClock.CurrentTS()) { + return context.Canceled } log.Debug("subscription client will request a region", - zap.Uint64("workID", worker.workerID), zap.Uint64("subscriptionID", uint64(region.subscribedSpan.subID)), zap.Uint64("regionID", region.verID.GetID()), zap.String("addr", store.storeAddr)) diff --git a/logservice/logpuller/subscription_client_test.go b/logservice/logpuller/subscription_client_test.go index 41dc761bd1..d60e13ff8b 100644 --- a/logservice/logpuller/subscription_client_test.go +++ b/logservice/logpuller/subscription_client_test.go @@ -104,8 +104,7 @@ func TestGenerateResolveLockTask(t *testing.T) { } worker := ®ionRequestWorker{ - requestCache: &requestCache{}, - tracker: newRegionTracker(), + tracker: newRegionTracker(), } // Lock another range, no task will be triggered before initialized. res = span.rangeLock.LockRange(context.Background(), []byte{'c'}, []byte{'d'}, 2, 100) @@ -453,12 +452,13 @@ func TestPushRegionEventToDSUnblocksOnClose(t *testing.T) { func TestBroadcastDeregisterUsesWorkerControlQueue(t *testing.T) { client := &subscriptionClient{} + admission := newRegionAdmissionController(1, 1) worker := ®ionRequestWorker{ - requestCache: newRequestCache(1), + admission: admission, controlQueue: newControlQueue(), } - store := &requestedStore{storeAddr: "store-1"} + store := &requestedStore{storeAddr: "store-1", admission: admission} store.requestWorkers.s = []*regionRequestWorker{worker} client.stores.Store(store.storeAddr, store) @@ -466,9 +466,8 @@ func TestBroadcastDeregisterUsesWorkerControlQueue(t *testing.T) { subscribedSpan: &subscribedSpan{subID: SubscriptionID(2)}, lockedRangeState: ®ionlock.LockedRangeState{}, } - ok, err := worker.add(t.Context(), dummyRegion, true) - require.NoError(t, err) - require.True(t, ok) + require.True(t, admission.submit( + NewRegionPriorityTask(TaskHighPrior, dummyRegion, 1), dummyRegion, 1)) client.broadcastDeregister(SubscriptionID(1), true) require.Equal(t, 1, worker.controlQueue.len()) @@ -476,7 +475,7 @@ func TestBroadcastDeregisterUsesWorkerControlQueue(t *testing.T) { require.True(t, ok) require.Equal(t, SubscriptionID(1), req.subID) require.True(t, req.filterLoop) - require.Equal(t, 1, worker.requestCache.pendingCount()) + require.Equal(t, 1, admission.pendingCount()) } func TestSubscriptionWithFailedTiKV(t *testing.T) { diff --git a/pkg/config/debug.go b/pkg/config/debug.go index 1aa6f9b924..b95be0f8ab 100644 --- a/pkg/config/debug.go +++ b/pkg/config/debug.go @@ -17,6 +17,8 @@ import ( "time" "github.com/pingcap/errors" + "github.com/pingcap/log" + "go.uber.org/zap" ) // DebugConfig represents config for ticdc unexposed feature configurations @@ -49,6 +51,7 @@ func (c *DebugConfig) ValidateAndAdjust() error { if err := c.Scheduler.ValidateAndAdjust(); err != nil { return errors.Trace(err) } + c.Puller.ValidateAndAdjust() return nil } @@ -62,20 +65,39 @@ type PullerConfig struct { // LogRegionDetails determines whether logs Region details or not in puller and kv-client. LogRegionDetails bool `toml:"log-region-details" json:"log_region_details"` - // PendingRegionRequestQueueSize is the total size of the pending region request queue shared across - // all puller workers connecting to a single TiKV store. This size is divided equally among all workers. - // For example, if PendingRegionRequestQueueSize is 32 and there are 8 workers connecting to the same store, - // each worker's queue size will be 32 / 8 = 4. + // PendingRegionRequestQueueSize is the normal initial-scan window shared by + // all puller workers connecting to one TiKV store. PendingRegionRequestQueueSize int `toml:"pending-region-request-queue-size" json:"pending_region_request_queue_size"` + // RegionRequestMaxWindowMultiplier controls the maximum window available to + // high-priority requests and regions whose scan lag is less than 10 minutes. + // The maximum window is PendingRegionRequestQueueSize multiplied by this value. + RegionRequestMaxWindowMultiplier int `toml:"region-request-max-window-multiplier" json:"region_request_max_window_multiplier"` } // NewDefaultPullerConfig return the default puller configuration func NewDefaultPullerConfig() *PullerConfig { return &PullerConfig{ - EnableResolvedTsStuckDetection: false, - ResolvedTsStuckInterval: TomlDuration(5 * time.Minute), - LogRegionDetails: false, - PendingRegionRequestQueueSize: 32, // This value is chosen to reduce the impact of new changefeeds on existing ones. + EnableResolvedTsStuckDetection: false, + ResolvedTsStuckInterval: TomlDuration(5 * time.Minute), + LogRegionDetails: false, + PendingRegionRequestQueueSize: 32, // This value is chosen to reduce the impact of new changefeeds on existing ones. + RegionRequestMaxWindowMultiplier: 4, + } +} + +func (c *PullerConfig) ValidateAndAdjust() { + defaultCfg := NewDefaultPullerConfig() + if c.PendingRegionRequestQueueSize <= 0 { + log.Warn("pending region request queue size must be positive, use default value", + zap.Int("value", c.PendingRegionRequestQueueSize), + zap.Int("default", defaultCfg.PendingRegionRequestQueueSize)) + c.PendingRegionRequestQueueSize = defaultCfg.PendingRegionRequestQueueSize + } + if c.RegionRequestMaxWindowMultiplier <= 0 { + log.Warn("region request max window multiplier must be positive, use default value", + zap.Int("value", c.RegionRequestMaxWindowMultiplier), + zap.Int("default", defaultCfg.RegionRequestMaxWindowMultiplier)) + c.RegionRequestMaxWindowMultiplier = defaultCfg.RegionRequestMaxWindowMultiplier } } diff --git a/pkg/config/debug_test.go b/pkg/config/debug_test.go new file mode 100644 index 0000000000..8f38e5d029 --- /dev/null +++ b/pkg/config/debug_test.go @@ -0,0 +1,35 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestPullerConfigValidateAndAdjustRegionRequestWindow(t *testing.T) { + defaultCfg := NewDefaultPullerConfig() + require.Equal(t, 32, defaultCfg.PendingRegionRequestQueueSize) + require.Equal(t, 4, defaultCfg.RegionRequestMaxWindowMultiplier) + + cfg := &PullerConfig{ + PendingRegionRequestQueueSize: -1, + RegionRequestMaxWindowMultiplier: 0, + } + cfg.ValidateAndAdjust() + require.Equal(t, defaultCfg.PendingRegionRequestQueueSize, cfg.PendingRegionRequestQueueSize) + require.Equal(t, defaultCfg.RegionRequestMaxWindowMultiplier, cfg.RegionRequestMaxWindowMultiplier) +} From 975ca6fd042447e4451c454570a0c44816f25516 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Sat, 11 Jul 2026 23:13:42 +0800 Subject: [PATCH 09/71] fix --- logservice/logpuller/priority_task.go | 117 +++----- logservice/logpuller/priority_task_test.go | 283 ++++++------------ .../logpuller/region_admission_controller.go | 111 +++---- .../region_admission_controller_test.go | 60 ++-- .../logpuller/region_failure_handler.go | 18 +- logservice/logpuller/region_request_worker.go | 17 +- .../logpuller/region_request_worker_test.go | 51 +++- logservice/logpuller/region_state.go | 6 + logservice/logpuller/subscription_client.go | 94 ++++-- .../logpuller/subscription_client_test.go | 26 +- pkg/config/debug.go | 9 +- 11 files changed, 379 insertions(+), 413 deletions(-) diff --git a/logservice/logpuller/priority_task.go b/logservice/logpuller/priority_task.go index e57f860804..c0ec14faa1 100644 --- a/logservice/logpuller/priority_task.go +++ b/logservice/logpuller/priority_task.go @@ -14,96 +14,51 @@ package logpuller import ( - "fmt" "time" "github.com/tikv/client-go/v2/oracle" ) -// TaskType represents the type of region task -type TaskType int - const ( - // TaskHighPrior represents region error or region change - // This type has the highest priority - TaskHighPrior TaskType = iota - // TaskLowPrior represents new subscription - // This type has the lowest priority - TaskLowPrior + lowLagRegionThreshold = 30 * time.Minute ) +type regionTaskPriority int + const ( - highPriorityBase = 0 - lowPriorityBase = 60 * 60 * 24 // 1 day + initializedRegionPriority regionTaskPriority = iota + lowLagRegionPriority + normalRegionPriority ) -func (t TaskType) String() string { - return fmt.Sprintf("%d", t) -} - -// PriorityTask is the interface for priority-based tasks -// It implements heap.Item interface -type PriorityTask interface { - // Priority returns the priority value, lower value means higher priority - Priority() int - - // GetRegionInfo returns the underlying regionInfo - GetRegionInfo() regionInfo - // GetTaskType returns whether this is a recovery or new-subscription task. - GetTaskType() TaskType - - // heap.Item interface methods - SetHeapIndex(int) - GetHeapIndex() int - LessThan(PriorityTask) bool -} - -// regionPriorityTask implements PriorityTask interface type regionPriorityTask struct { - taskType TaskType - createTime time.Time regionInfo regionInfo + sequence uint64 heapIndex int // for heap.Item interface - currentTs uint64 + priority regionTaskPriority } // NewRegionPriorityTask creates a new priority task for region -func NewRegionPriorityTask(taskType TaskType, regionInfo regionInfo, currentTs uint64) PriorityTask { - return ®ionPriorityTask{ - taskType: taskType, - createTime: time.Now(), - regionInfo: regionInfo, - heapIndex: 0, // 0 means not in heap - currentTs: currentTs, +func NewRegionPriorityTask(regionInfo regionInfo, currentTs, sequence uint64) *regionPriorityTask { + task := ®ionPriorityTask{ + sequence: sequence, + heapIndex: 0, // 0 means not in heap } + task.updateRegion(regionInfo, currentTs) + return task } -// Priority calculates the priority based on task type and wait time -// Lower value means higher priority -func (pt *regionPriorityTask) Priority() int { - // Base priority based on task type - basePriority := 0 - switch pt.taskType { - case TaskHighPrior: - basePriority = highPriorityBase // Highest priority - case TaskLowPrior: - basePriority = lowPriorityBase // Lowest priority +// updateRegion refreshes both the request data and its priority before the task +// enters another scheduling stage. +func (pt *regionPriorityTask) updateRegion(regionInfo regionInfo, currentTs uint64) { + priority := normalRegionPriority + if regionInfo.wasInitialized { + priority = initializedRegionPriority + } else if regionScanLag(currentTs, regionInfo.resolvedTs()) < lowLagRegionThreshold { + priority = lowLagRegionPriority } - - // Add time-based priority bonus - // Wait time in seconds, longer wait time means higher priority (lower value) - waitTime := time.Since(pt.createTime) - timeBonus := int(waitTime.Seconds()) - - // ResolvedTsLag in seconds, longer lag means lower priority (higher value) - resolvedTsLag := oracle.GetTimeFromTS(pt.currentTs).Sub(oracle.GetTimeFromTS(pt.regionInfo.subscribedSpan.resolvedTs.Load())) - resolvedTsLagPenalty := int(resolvedTsLag.Seconds()) - - priority := basePriority - timeBonus + resolvedTsLagPenalty - if priority < 0 { - priority = 0 - } - return priority + pt.regionInfo = regionInfo + pt.priority = priority } // GetRegionInfo returns the underlying regionInfo @@ -111,8 +66,8 @@ func (pt *regionPriorityTask) GetRegionInfo() regionInfo { return pt.regionInfo } -func (pt *regionPriorityTask) GetTaskType() TaskType { - return pt.taskType +func (pt *regionPriorityTask) canUseMaxWindow() bool { + return pt.priority != normalRegionPriority } // SetHeapIndex sets the heap index for heap.Item interface @@ -125,8 +80,20 @@ func (pt *regionPriorityTask) GetHeapIndex() int { return pt.heapIndex } -// LessThan implements heap.Item interface -// Returns true if this task has higher priority (lower priority value) than the other task -func (pt *regionPriorityTask) LessThan(other PriorityTask) bool { - return pt.Priority() < other.Priority() +// LessThan implements heap.Item interface. Tasks in the same priority class are +// processed in submission order. +func (pt *regionPriorityTask) LessThan(other *regionPriorityTask) bool { + if pt.priority != other.priority { + return pt.priority < other.priority + } + return pt.sequence < other.sequence +} + +func regionScanLag(currentTs, checkpointTs uint64) time.Duration { + currentTime := oracle.GetTimeFromTS(currentTs) + checkpointTime := oracle.GetTimeFromTS(checkpointTs) + if !currentTime.After(checkpointTime) { + return 0 + } + return currentTime.Sub(checkpointTime) } diff --git a/logservice/logpuller/priority_task_test.go b/logservice/logpuller/priority_task_test.go index d8b3f26d48..4f765ce88c 100644 --- a/logservice/logpuller/priority_task_test.go +++ b/logservice/logpuller/priority_task_test.go @@ -8,239 +8,126 @@ // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package logpuller import ( - "sync/atomic" "testing" "time" "github.com/pingcap/ticdc/heartbeatpb" + "github.com/pingcap/ticdc/logservice/logpuller/regionlock" "github.com/pingcap/ticdc/utils/priorityqueue" "github.com/stretchr/testify/require" "github.com/tikv/client-go/v2/oracle" "github.com/tikv/client-go/v2/tikv" ) -// TestPriorityCalculationLogic tests the priority calculation logic in isolation -func TestPriorityCalculationLogic(t *testing.T) { - currentTime := time.Now() - currentTs := oracle.GoTimeToTS(currentTime) - - // Test cases for priority calculation - tests := []struct { - name string - taskType TaskType - resolvedTsOffsetSeconds int64 // Offset relative to currentTs (negative means resolvedTs is older) - waitTimeSeconds int // Task wait time - description string - }{ - { - name: "high_priority_new_resolvedTs", - taskType: TaskHighPrior, - resolvedTsOffsetSeconds: -5, // resolvedTs is 5 seconds earlier than currentTs - waitTimeSeconds: 10, // Waited for 10 seconds - description: "High priority task with newer resolvedTs", - }, - { - name: "high_priority_old_resolvedTs", - taskType: TaskHighPrior, - resolvedTsOffsetSeconds: -30, // resolvedTs is 30 seconds earlier than currentTs - waitTimeSeconds: 10, // Waited for 10 seconds - description: "High priority task with older resolvedTs", - }, - { - name: "low_priority_new_resolvedTs", - taskType: TaskLowPrior, - resolvedTsOffsetSeconds: -5, // resolvedTs is 5 seconds earlier than currentTs - waitTimeSeconds: 10, // Waited for 10 seconds - description: "Low priority task with newer resolvedTs", - }, - { - name: "low_priority_old_resolvedTs", - taskType: TaskLowPrior, - resolvedTsOffsetSeconds: -30, // resolvedTs is 30 seconds earlier than currentTs - waitTimeSeconds: 10, // Waited for 10 seconds - description: "Low priority task with older resolvedTs", - }, - } - - var priorities []int - var taskDescriptions []string - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Calculate resolvedTs: currentTs + offset - resolvedTime := oracle.GetTimeFromTS(currentTs).Add(time.Duration(tt.resolvedTsOffsetSeconds) * time.Second) - resolvedTs := oracle.GoTimeToTS(resolvedTime) - - // Simulate priority calculation logic - priority := calculatePriorityDirectly(tt.taskType, currentTs, resolvedTs, tt.waitTimeSeconds) - - t.Logf("%s: Priority = %d", tt.description, priority) - priorities = append(priorities, priority) - taskDescriptions = append(taskDescriptions, tt.description) - }) +func newPriorityTestRegion( + regionID uint64, + checkpointTs uint64, + wasInitialized bool, +) regionInfo { + span := heartbeatpb.TableSpan{TableID: 1, StartKey: []byte("a"), EndKey: []byte("z")} + state := ®ionlock.LockedRangeState{} + state.ResolvedTs.Store(checkpointTs) + return regionInfo{ + verID: tikv.NewRegionVerID(regionID, 1, 1), + span: span, + subscribedSpan: &subscribedSpan{subID: 1, startTs: checkpointTs, span: span}, + lockedRangeState: state, + wasInitialized: wasInitialized, } - - // Verify priority order - t.Run("verify_priority_order", func(t *testing.T) { - require.Equal(t, 4, len(priorities)) - - highPriorNewResolvedTs := priorities[0] // High priority with new resolvedTs - highPriorOldResolvedTs := priorities[1] // High priority with old resolvedTs - lowPriorNewResolvedTs := priorities[2] // Low priority with new resolvedTs - lowPriorOldResolvedTs := priorities[3] // Low priority with old resolvedTs - - t.Logf("Priority comparison:") - for i, desc := range taskDescriptions { - t.Logf(" %s: %d", desc, priorities[i]) - } - - // Core verification: For the same task type, newer resolvedTs (smaller lag) should have higher priority (smaller value) - require.Less(t, highPriorNewResolvedTs, highPriorOldResolvedTs, - "For the same task type, tasks with newer resolvedTs should have higher priority") - require.Less(t, lowPriorNewResolvedTs, lowPriorOldResolvedTs, - "For the same task type, tasks with newer resolvedTs should have higher priority") - - // Verify: High priority tasks always have higher priority than low priority tasks - require.Less(t, highPriorNewResolvedTs, lowPriorNewResolvedTs, - "High priority tasks should have higher priority than low priority tasks") - require.Less(t, highPriorOldResolvedTs, lowPriorOldResolvedTs, - "Even with older resolvedTs, high priority tasks should still have higher priority than low priority tasks") - }) } -// calculatePriorityDirectly directly calculates priority for testing -// Copies the logic from regionPriorityTask.Priority() -func calculatePriorityDirectly(taskType TaskType, currentTs, resolvedTs uint64, waitTimeSeconds int) int { - // Base priority based on task type - basePriority := 0 - switch taskType { - case TaskHighPrior: - basePriority = highPriorityBase // 1200 - case TaskLowPrior: - basePriority = lowPriorityBase // 3600 - } - - // Add time-based priority bonus - // Wait time in seconds, longer wait time means higher priority (lower value) - timeBonus := waitTimeSeconds - - // Calculate resolvedTs lag - resolvedTsLag := oracle.GetTimeFromTS(currentTs).Sub(oracle.GetTimeFromTS(resolvedTs)) - resolvedTsLagBonus := int(resolvedTsLag.Seconds()) - - priority := basePriority - timeBonus + resolvedTsLagBonus +func TestRegionPriorityTaskQueueOrder(t *testing.T) { + queue := priorityqueue.New[*regionPriorityTask]() + currentTime := time.Now() + currentTs := oracle.GoTimeToTS(currentTime) - if priority < 0 { - priority = 0 + normalTask := NewRegionPriorityTask( + newPriorityTestRegion(1, oracle.GoTimeToTS(currentTime.Add(-time.Hour)), false), + currentTs, 3, + ) + lowLagTask := NewRegionPriorityTask( + newPriorityTestRegion(2, oracle.GoTimeToTS(currentTime.Add(-10*time.Minute)), false), + currentTs, 2, + ) + initializedTask := NewRegionPriorityTask( + newPriorityTestRegion(3, oracle.GoTimeToTS(currentTime.Add(-time.Hour)), true), + currentTs, 1, + ) + + require.True(t, queue.Push(normalTask)) + require.True(t, queue.Push(lowLagTask)) + require.True(t, queue.Push(initializedTask)) + + for _, expectedRegionID := range []uint64{3, 2, 1} { + task, err := queue.Pop(t.Context()) + require.NoError(t, err) + require.Equal(t, expectedRegionID, task.regionInfo.verID.GetID()) } - return priority } -func TestResolvedTsLagLogic(t *testing.T) { +func TestRegionPriorityTaskFIFOWithinPriority(t *testing.T) { + queue := priorityqueue.New[*regionPriorityTask]() currentTime := time.Now() currentTs := oracle.GoTimeToTS(currentTime) + checkpointTs := oracle.GoTimeToTS(currentTime.Add(-time.Hour)) - t.Run("test_resolvedTs_lag_calculation_logic", func(t *testing.T) { - // Scenario 1: resolvedTs is 10 seconds earlier than currentTs (resolvedTs is older) - resolvedTs1 := oracle.GoTimeToTS(currentTime.Add(-10 * time.Second)) - lag1 := oracle.GetTimeFromTS(currentTs).Sub(oracle.GetTimeFromTS(resolvedTs1)) - t.Logf("resolvedTs is 10 seconds earlier, lag = %v (%.0f seconds)", lag1, lag1.Seconds()) + first := NewRegionPriorityTask(newPriorityTestRegion(1, checkpointTs, false), currentTs, 1) + second := NewRegionPriorityTask(newPriorityTestRegion(2, checkpointTs, false), currentTs, 2) - // Scenario 2: resolvedTs is 1 second earlier than currentTs (resolvedTs is newer) - resolvedTs2 := oracle.GoTimeToTS(currentTime.Add(-1 * time.Second)) - lag2 := oracle.GetTimeFromTS(currentTs).Sub(oracle.GetTimeFromTS(resolvedTs2)) - t.Logf("resolvedTs is 1 second earlier, lag = %v (%.0f seconds)", lag2, lag2.Seconds()) + require.True(t, queue.Push(second)) + require.True(t, queue.Push(first)) - // Verify: newer resolvedTs should have smaller lag - require.Less(t, lag2, lag1, "newer resolvedTs should have smaller lag") - - // Calculate the impact on priority - priority1 := calculatePriorityDirectly(TaskHighPrior, currentTs, resolvedTs1, 5) - priority2 := calculatePriorityDirectly(TaskHighPrior, currentTs, resolvedTs2, 5) - - t.Logf("Priority with resolvedTs 10 seconds old: %d", priority1) - t.Logf("Priority with resolvedTs 1 second old: %d", priority2) - - // Verify: newer resolvedTs should have higher priority (smaller value) - require.Less(t, priority2, priority1, - "tasks with newer resolvedTs should have higher priority") - }) + task, err := queue.Pop(t.Context()) + require.NoError(t, err) + require.Equal(t, uint64(1), task.regionInfo.verID.GetID()) + task, err = queue.Pop(t.Context()) + require.NoError(t, err) + require.Equal(t, uint64(2), task.regionInfo.verID.GetID()) } -func TestEdgeCases(t *testing.T) { +func TestRegionPriorityTaskLowLagBoundary(t *testing.T) { currentTime := time.Now() currentTs := oracle.GoTimeToTS(currentTime) - t.Run("resolvedTs in the future", func(t *testing.T) { - resolvedTs := oracle.GoTimeToTS(currentTime.Add(5 * time.Second)) - - lag := oracle.GetTimeFromTS(currentTs).Sub(oracle.GetTimeFromTS(resolvedTs)) - t.Logf("resolvedTs in the future 5 seconds, lag = %v (%.0f seconds)", lag, lag.Seconds()) - - priority := calculatePriorityDirectly(TaskHighPrior, currentTs, resolvedTs, 5) - t.Logf("resolvedTs in the future priority: %d", priority) - - require.GreaterOrEqual(t, priority, 0, "priority should not be less than 0") - }) - - t.Run("different wait time impact", func(t *testing.T) { - resolvedTs := oracle.GoTimeToTS(currentTime.Add(-10 * time.Second)) - - priority1 := calculatePriorityDirectly(TaskHighPrior, currentTs, resolvedTs, 2) - priority2 := calculatePriorityDirectly(TaskHighPrior, currentTs, resolvedTs, 10) - - t.Logf("wait 2 seconds priority: %d", priority1) - t.Logf("wait 10 seconds priority: %d", priority2) - - // wait time longer task priority should be higher - require.Less(t, priority2, priority1, "wait time longer task priority should be higher") - }) + belowThreshold := NewRegionPriorityTask(newPriorityTestRegion( + 1, + oracle.GoTimeToTS(currentTime.Add(-lowLagRegionThreshold+time.Millisecond)), + false, + ), currentTs, 1) + atThreshold := NewRegionPriorityTask(newPriorityTestRegion( + 2, + oracle.GoTimeToTS(currentTime.Add(-lowLagRegionThreshold)), + false, + ), currentTs, 2) + futureCheckpoint := NewRegionPriorityTask(newPriorityTestRegion( + 3, + oracle.GoTimeToTS(currentTime.Add(time.Second)), + false, + ), currentTs, 3) + + require.Equal(t, lowLagRegionPriority, belowThreshold.priority) + require.Equal(t, normalRegionPriority, atThreshold.priority) + require.Equal(t, lowLagRegionPriority, futureCheckpoint.priority) } -func TestRegionPriorityTaskQueueOrder(t *testing.T) { - queue := priorityqueue.New[PriorityTask]() - ctx := t.Context() +func TestRegionPriorityTaskRefreshesPriorityBetweenStages(t *testing.T) { + checkpointTime := time.Now() + checkpointTs := oracle.GoTimeToTS(checkpointTime) + region := newPriorityTestRegion(1, checkpointTs, false) + task := NewRegionPriorityTask(region, oracle.GoTimeToTS(checkpointTime.Add(time.Minute)), 1) + require.Equal(t, lowLagRegionPriority, task.priority) - currentTs := oracle.GoTimeToTS(time.Now()) - verID := tikv.NewRegionVerID(1, 1, 1) - span := heartbeatpb.TableSpan{TableID: 1, StartKey: []byte("a"), EndKey: []byte("z")} - - subscribedSpan := &subscribedSpan{ - resolvedTs: atomic.Uint64{}, - } - subscribedSpan.resolvedTs.Store(oracle.GoTimeToTS(time.Now().Add(-time.Second))) - - regionInfo := regionInfo{ - verID: verID, - span: span, - subscribedSpan: subscribedSpan, - } - - errorTask := NewRegionPriorityTask(TaskHighPrior, regionInfo, currentTs+1) - highTask := NewRegionPriorityTask(TaskHighPrior, regionInfo, currentTs) - lowTask := NewRegionPriorityTask(TaskLowPrior, regionInfo, currentTs) - - require.True(t, queue.Push(lowTask)) - require.True(t, queue.Push(errorTask)) - require.True(t, queue.Push(highTask)) - - first, err := queue.Pop(ctx) - require.NoError(t, err) - require.Equal(t, TaskHighPrior, first.(*regionPriorityTask).taskType) - - second, err := queue.Pop(ctx) - require.NoError(t, err) - require.Equal(t, TaskHighPrior, second.(*regionPriorityTask).taskType) - - third, err := queue.Pop(ctx) - require.NoError(t, err) - require.Equal(t, TaskLowPrior, third.(*regionPriorityTask).taskType) + task.updateRegion(region, oracle.GoTimeToTS(checkpointTime.Add(time.Hour))) + require.Equal(t, normalRegionPriority, task.priority) - require.Equal(t, 0, queue.Len()) + region.wasInitialized = true + task.updateRegion(region, oracle.GoTimeToTS(checkpointTime.Add(time.Hour))) + require.Equal(t, initializedRegionPriority, task.priority) } diff --git a/logservice/logpuller/region_admission_controller.go b/logservice/logpuller/region_admission_controller.go index 8f539f0e55..851b1bb6ba 100644 --- a/logservice/logpuller/region_admission_controller.go +++ b/logservice/logpuller/region_admission_controller.go @@ -1,4 +1,4 @@ -// Copyright 2025 PingCAP, Inc. +// Copyright 2026 PingCAP, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -23,48 +23,15 @@ import ( "github.com/pingcap/log" "github.com/pingcap/ticdc/pkg/metrics" "github.com/pingcap/ticdc/utils/heap" - "github.com/tikv/client-go/v2/oracle" "go.uber.org/zap" ) const ( - fastRegionScanLag = 10 * time.Minute abnormalRequestDurationInSec = 60 * 60 * 2 // 2 hours ) -// pendingRegionRequest waits in the store-level admission queue. It does not -// own an admission slot until it is returned by pop or tryPop. -type pendingRegionRequest struct { - task PriorityTask - regionInfo regionInfo - fastScan bool - heapIndex int -} - -func (r *pendingRegionRequest) SetHeapIndex(index int) { - r.heapIndex = index -} - -func (r *pendingRegionRequest) GetHeapIndex() int { - return r.heapIndex -} - -func (r *pendingRegionRequest) LessThan(other *pendingRegionRequest) bool { - if r.task.GetTaskType() != other.task.GetTaskType() { - return r.task.GetTaskType() == TaskHighPrior - } - if r.canUseMaxWindow() != other.canUseMaxWindow() { - return r.canUseMaxWindow() - } - return r.task.LessThan(other.task) -} - -func (r *pendingRegionRequest) canUseMaxWindow() bool { - return r.fastScan || r.task.GetTaskType() == TaskHighPrior -} - // regionReq is an admission lease for one sent-but-not-initialized region. -// finish and abort are idempotent and return the lease to the store controller. +// finish and abort are idempotent and return the lease to its worker controller. type regionReq struct { regionInfo regionInfo createTime time.Time @@ -98,28 +65,41 @@ func (r *regionReq) abort() bool { } func (r *regionReq) isActive() bool { - return r != nil && !r.released.Load() + return !r.released.Load() } func (r *regionReq) release() bool { - if r == nil || !r.released.CompareAndSwap(false, true) { + if !r.released.CompareAndSwap(false, true) { return false } r.controller.release() return true } -// regionAdmissionController owns the pending queue and initial-scan window for -// all request workers connected to one TiKV store. +// regionAdmissionController owns one request worker's pending queue and +// initial-scan window. type regionAdmissionController struct { + // mu guards the window state, inflight count, pending queue and closed flag. mu sync.Mutex + // currentWindow limits ordinary region scans. currentWindow int - maxWindow int - inflight int - pending *heap.Heap[*pendingRegionRequest] - notify chan struct{} - closed bool + // maxWindow is the hard limit for previously initialized and low-lag regions. + maxWindow int + // inflight is the number of admitted regions that have not finished their + // initial scan. It is guarded by mu. + inflight int + // pending keeps requests that have not entered the initial-scan window. + // It is guarded by mu. + pending *heap.Heap[*regionPriorityTask] + // notify wakes workers when a request is submitted or an admission slot is + // released. The one-element buffer prevents a wakeup from being lost between + // checking the admission condition and waiting on this channel. Notifications + // are only signals to recheck state; they do not correspond one-to-one with + // pending requests or available slots. + notify chan struct{} + // closed prevents new submissions and makes waiting workers exit. + closed bool } func newRegionAdmissionController(currentWindow, maxWindowMultiplier int) *regionAdmissionController { @@ -136,42 +116,23 @@ func newRegionAdmissionController(currentWindow, maxWindowMultiplier int) *regio return ®ionAdmissionController{ currentWindow: currentWindow, maxWindow: maxWindow, - pending: heap.NewHeap[*pendingRegionRequest](), + pending: heap.NewHeap[*regionPriorityTask](), notify: make(chan struct{}, 1), } } -func (c *regionAdmissionController) submit( - task PriorityTask, - region regionInfo, - currentTs uint64, -) bool { - request := &pendingRegionRequest{ - task: task, - regionInfo: region, - fastScan: regionScanLag(currentTs, region.resolvedTs()) < fastRegionScanLag, - } - +func (c *regionAdmissionController) submit(task *regionPriorityTask) bool { c.mu.Lock() if c.closed { c.mu.Unlock() return false } - c.pending.AddOrUpdate(request) + c.pending.AddOrUpdate(task) c.notifyOneLocked() c.mu.Unlock() return true } -func regionScanLag(currentTs, checkpointTs uint64) time.Duration { - currentTime := oracle.GetTimeFromTS(currentTs) - checkpointTime := oracle.GetTimeFromTS(checkpointTs) - if !currentTime.After(checkpointTime) { - return 0 - } - return currentTime.Sub(checkpointTime) -} - func (c *regionAdmissionController) pop(ctx context.Context) (*regionReq, error) { for { request, closed := c.tryPop() @@ -213,7 +174,7 @@ func (c *regionAdmissionController) tryPop() (*regionReq, bool) { }, false } -func (c *regionAdmissionController) popEligibleLocked() *pendingRegionRequest { +func (c *regionAdmissionController) popEligibleLocked() *regionPriorityTask { request, ok := c.pending.PeekTop() if !ok { return nil @@ -233,7 +194,7 @@ func (c *regionAdmissionController) hasEligibleRequestLocked() bool { return c.inflight < c.windowFor(request) } -func (c *regionAdmissionController) windowFor(request *pendingRegionRequest) int { +func (c *regionAdmissionController) windowFor(request *regionPriorityTask) int { if request.canUseMaxWindow() { return c.maxWindow } @@ -274,6 +235,20 @@ func (c *regionAdmissionController) pendingCount() int { return c.pending.Len() } +func (c *regionAdmissionController) drainPending() []*regionPriorityTask { + c.mu.Lock() + defer c.mu.Unlock() + + requests := make([]*regionPriorityTask, 0, c.pending.Len()) + for { + request, ok := c.pending.PopTop() + if !ok { + return requests + } + requests = append(requests, request) + } +} + func (c *regionAdmissionController) notifyOneLocked() { if c.closed { return diff --git a/logservice/logpuller/region_admission_controller_test.go b/logservice/logpuller/region_admission_controller_test.go index 6c66451608..a8ab72f184 100644 --- a/logservice/logpuller/region_admission_controller_test.go +++ b/logservice/logpuller/region_admission_controller_test.go @@ -1,4 +1,4 @@ -// Copyright 2025 PingCAP, Inc. +// Copyright 2026 PingCAP, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -52,22 +52,21 @@ func submitRegionForAdmission( t *testing.T, controller *regionAdmissionController, region regionInfo, - taskType TaskType, currentTs uint64, ) { t.Helper() - task := NewRegionPriorityTask(taskType, region, currentTs) - require.True(t, controller.submit(task, region, currentTs)) + task := NewRegionPriorityTask(region, currentTs, region.verID.GetID()) + require.True(t, controller.submit(task)) } func TestRegionAdmissionControllerNormalWindow(t *testing.T) { controller := newRegionAdmissionController(1, 2) currentTs := oracle.GoTimeToTS(time.Now()) - checkpointTs := oracle.GoTimeToTS(time.Now().Add(-20 * time.Minute)) + checkpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Hour)) region1 := prepareRegionForAdmission(createTestRegionInfo(1, 1), checkpointTs) region2 := prepareRegionForAdmission(createTestRegionInfo(1, 2), checkpointTs) - submitRegionForAdmission(t, controller, region1, TaskLowPrior, currentTs) - submitRegionForAdmission(t, controller, region2, TaskLowPrior, currentTs) + submitRegionForAdmission(t, controller, region1, currentTs) + submitRegionForAdmission(t, controller, region2, currentTs) req1, err := controller.pop(t.Context()) require.NoError(t, err) @@ -83,24 +82,24 @@ func TestRegionAdmissionControllerNormalWindow(t *testing.T) { require.True(t, req2.abort()) } -func TestRegionAdmissionControllerFastScanUsesMaxWindow(t *testing.T) { +func TestRegionAdmissionControllerLowLagUsesMaxWindow(t *testing.T) { controller := newRegionAdmissionController(1, 2) currentTs := oracle.GoTimeToTS(time.Now()) - slowCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-20 * time.Minute)) - fastCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Minute)) + slowCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Hour)) + lowLagCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Minute)) submitRegionForAdmission(t, controller, prepareRegionForAdmission(createTestRegionInfo(1, 1), slowCheckpointTs), - TaskLowPrior, currentTs) + currentTs) req1, err := controller.pop(t.Context()) require.NoError(t, err) submitRegionForAdmission(t, controller, prepareRegionForAdmission(createTestRegionInfo(1, 2), slowCheckpointTs), - TaskLowPrior, currentTs) + currentTs) submitRegionForAdmission(t, controller, - prepareRegionForAdmission(createTestRegionInfo(1, 3), fastCheckpointTs), - TaskLowPrior, currentTs) + prepareRegionForAdmission(createTestRegionInfo(1, 3), lowLagCheckpointTs), + currentTs) req2, err := controller.pop(t.Context()) require.NoError(t, err) @@ -118,24 +117,25 @@ func TestRegionAdmissionControllerFastScanUsesMaxWindow(t *testing.T) { require.True(t, req3.abort()) } -func TestRegionAdmissionControllerPrioritizesRecovery(t *testing.T) { +func TestRegionAdmissionControllerPrioritizesInitializedRegion(t *testing.T) { controller := newRegionAdmissionController(1, 2) currentTs := oracle.GoTimeToTS(time.Now()) - slowCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-20 * time.Minute)) - fastCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Minute)) + slowCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Hour)) + lowLagCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Minute)) submitRegionForAdmission(t, controller, prepareRegionForAdmission(createTestRegionInfo(1, 1), slowCheckpointTs), - TaskLowPrior, currentTs) + currentTs) req1, err := controller.pop(t.Context()) require.NoError(t, err) submitRegionForAdmission(t, controller, - prepareRegionForAdmission(createTestRegionInfo(1, 2), fastCheckpointTs), - TaskLowPrior, currentTs) + prepareRegionForAdmission(createTestRegionInfo(1, 2), lowLagCheckpointTs), + currentTs) + initializedRegion := prepareRegionForAdmission(createTestRegionInfo(1, 3), slowCheckpointTs) + initializedRegion.wasInitialized = true submitRegionForAdmission(t, controller, - prepareRegionForAdmission(createTestRegionInfo(1, 3), slowCheckpointTs), - TaskHighPrior, currentTs) + initializedRegion, currentTs) req2, err := controller.pop(t.Context()) require.NoError(t, err) @@ -153,7 +153,7 @@ func TestRegionAdmissionLeaseReleasedOnce(t *testing.T) { controller := newRegionAdmissionController(1, 1) currentTs := oracle.GoTimeToTS(time.Now()) region := prepareRegionForAdmission(createTestRegionInfo(1, 1), currentTs) - submitRegionForAdmission(t, controller, region, TaskLowPrior, currentTs) + submitRegionForAdmission(t, controller, region, currentTs) req, err := controller.pop(t.Context()) require.NoError(t, err) @@ -189,8 +189,20 @@ func TestRegionAdmissionControllerClose(t *testing.T) { controller := newRegionAdmissionController(1, 1) controller.close() region := prepareRegionForAdmission(createTestRegionInfo(1, 1), 1) - require.False(t, controller.submit(NewRegionPriorityTask(TaskLowPrior, region, 1), region, 1)) + require.False(t, controller.submit(NewRegionPriorityTask(region, 1, 1))) _, err := controller.pop(context.Background()) require.ErrorIs(t, err, context.Canceled) } + +func TestRegionAdmissionControllerDrainPending(t *testing.T) { + controller := newRegionAdmissionController(1, 1) + region1 := prepareRegionForAdmission(createTestRegionInfo(1, 1), 1) + region2 := prepareRegionForAdmission(createTestRegionInfo(1, 2), 1) + submitRegionForAdmission(t, controller, region1, 1) + submitRegionForAdmission(t, controller, region2, 1) + + pending := controller.drainPending() + require.Len(t, pending, 2) + require.Zero(t, controller.pendingCount()) +} diff --git a/logservice/logpuller/region_failure_handler.go b/logservice/logpuller/region_failure_handler.go index ef86f55876..ee89e805df 100644 --- a/logservice/logpuller/region_failure_handler.go +++ b/logservice/logpuller/region_failure_handler.go @@ -101,27 +101,27 @@ func (r *regionFailureHandler) handleError(ctx context.Context, errInfo regionEr if notLeader := innerErr.GetNotLeader(); notLeader != nil { metricFeedNotLeaderCounter.Inc() r.client.regionCache.UpdateLeader(errInfo.verID, notLeader.GetLeader(), errInfo.rpcCtx.AccessIdx) - r.client.scheduleRegionRequest(ctx, errInfo.regionInfo, TaskHighPrior) + r.client.scheduleRegionRequest(ctx, errInfo.regionInfo) return nil } if innerErr.GetEpochNotMatch() != nil { metricFeedEpochNotMatchCounter.Inc() - r.client.scheduleRangeRequest(ctx, errInfo.span, errInfo.subscribedSpan, errInfo.filterLoop, TaskHighPrior) + r.client.scheduleRangeRequest(ctx, errInfo.span, errInfo.subscribedSpan, errInfo.filterLoop, errInfo.wasInitialized) return nil } if innerErr.GetRegionNotFound() != nil { metricFeedRegionNotFoundCounter.Inc() - r.client.scheduleRangeRequest(ctx, errInfo.span, errInfo.subscribedSpan, errInfo.filterLoop, TaskHighPrior) + r.client.scheduleRangeRequest(ctx, errInfo.span, errInfo.subscribedSpan, errInfo.filterLoop, errInfo.wasInitialized) return nil } if innerErr.GetCongested() != nil { metricKvCongestedCounter.Inc() - r.client.scheduleRegionRequest(ctx, errInfo.regionInfo, TaskLowPrior) + r.client.scheduleRegionRequest(ctx, errInfo.regionInfo) return nil } if innerErr.GetServerIsBusy() != nil { metricKvIsBusyCounter.Inc() - r.client.scheduleRegionRequest(ctx, errInfo.regionInfo, TaskLowPrior) + r.client.scheduleRegionRequest(ctx, errInfo.regionInfo) return nil } if duplicated := innerErr.GetDuplicateRequest(); duplicated != nil { @@ -140,24 +140,24 @@ func (r *regionFailureHandler) handleError(ctx context.Context, errInfo regionEr zap.Uint64("subscriptionID", uint64(errInfo.subscribedSpan.subID)), zap.Stringer("error", innerErr)) metricFeedUnknownErrorCounter.Inc() - r.client.scheduleRegionRequest(ctx, errInfo.regionInfo, TaskHighPrior) + r.client.scheduleRegionRequest(ctx, errInfo.regionInfo) return nil case *rpcCtxUnavailableErr: metricFeedRPCCtxUnavailable.Inc() - r.client.scheduleRangeRequest(ctx, errInfo.span, errInfo.subscribedSpan, errInfo.filterLoop, TaskHighPrior) + r.client.scheduleRangeRequest(ctx, errInfo.span, errInfo.subscribedSpan, errInfo.filterLoop, errInfo.wasInitialized) return nil case *getStoreErr: metricGetStoreErr.Inc() bo := tikv.NewBackoffer(ctx, tikvRequestMaxBackoff) // cannot get the store the region belongs to, so we need to reload the region. r.client.regionCache.OnSendFail(bo, errInfo.rpcCtx, true, err) - r.client.scheduleRangeRequest(ctx, errInfo.span, errInfo.subscribedSpan, errInfo.filterLoop, TaskHighPrior) + r.client.scheduleRangeRequest(ctx, errInfo.span, errInfo.subscribedSpan, errInfo.filterLoop, errInfo.wasInitialized) return nil case *storeStreamErr: metricStoreSendRequestErr.Inc() bo := tikv.NewBackoffer(ctx, tikvRequestMaxBackoff) r.client.regionCache.OnSendFail(bo, errInfo.rpcCtx, regionScheduleReload, err) - r.client.scheduleRegionRequest(ctx, errInfo.regionInfo, TaskHighPrior) + r.client.scheduleRegionRequest(ctx, errInfo.regionInfo) return nil case *requestCancelledErr: // the corresponding subscription has been unsubscribed, just ignore. diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index 336bc4dcf7..12dc5a9248 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -100,14 +100,15 @@ type regionRequestWorker struct { func newRegionRequestWorker( client *subscriptionClient, store *requestedStore, - admission *regionAdmissionController, + currentWindow int, + maxWindowMultiplier int, ) *regionRequestWorker { workerID := workerIDGen.Add(1) return ®ionRequestWorker{ workerID: workerID, client: client, store: store, - admission: admission, + admission: newRegionAdmissionController(currentWindow, maxWindowMultiplier), controlQueue: newControlQueue(), tracker: newRegionTracker(), } @@ -138,6 +139,7 @@ func (s *regionRequestWorker) Run(ctx context.Context) error { if firstReq.abort() { s.client.onRegionFail(newRegionErrorInfo(firstReq.regionInfo, regionErr)) } + s.failPendingRegions(regionErr) if err := util.Hang(ctx, storeReconnectBackoff); err != nil { return err @@ -146,8 +148,7 @@ func (s *regionRequestWorker) Run(ctx context.Context) error { } // failStreamRegions transfers every request sent by a failed stream to the -// recovery pipeline. Requests still waiting in the store admission controller -// are not owned by this stream. +// recovery pipeline. func (s *regionRequestWorker) failStreamRegions(err error) { for _, states := range s.tracker.Drain() { for _, state := range states { @@ -158,6 +159,14 @@ func (s *regionRequestWorker) failStreamRegions(err error) { s.controlQueue.drain() } +// failPendingRegions transfers requests owned by this worker but not yet sent +// to the recovery pipeline, so they can be resolved and routed again. +func (s *regionRequestWorker) failPendingRegions(err error) { + for _, task := range s.admission.drainPending() { + s.client.onRegionFail(newRegionErrorInfo(task.regionInfo, err)) + } +} + func (s *regionRequestWorker) notifyRegionError(state *regionFeedState, err error) { state.markStopped(err) s.client.eventSink.Push( diff --git a/logservice/logpuller/region_request_worker_test.go b/logservice/logpuller/region_request_worker_test.go index 9179b0b49b..ffd6416994 100644 --- a/logservice/logpuller/region_request_worker_test.go +++ b/logservice/logpuller/region_request_worker_test.go @@ -22,6 +22,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/cdcpb" "github.com/pingcap/kvproto/pkg/metapb" + "github.com/pingcap/ticdc/heartbeatpb" "github.com/pingcap/ticdc/logservice/logpuller/regionlock" "github.com/pingcap/ticdc/utils/dynstream" "github.com/prometheus/client_golang/prometheus" @@ -72,7 +73,7 @@ func admitRegionRequest( ) *regionReq { t.Helper() currentTs := oracle.GoTimeToTS(time.Now()) - submitRegionForAdmission(t, controller, region, TaskLowPrior, currentTs) + submitRegionForAdmission(t, controller, region, currentTs) req, err := controller.pop(t.Context()) require.NoError(t, err) return req @@ -264,7 +265,7 @@ func TestWaitForRegionRequestDrainsIdleControlQueue(t *testing.T) { } region := prepareRegionForAdmission(createTestRegionInfo(1, 1), 1) - submitRegionForAdmission(t, admission, region, TaskLowPrior, 1) + submitRegionForAdmission(t, admission, region, 1) select { case result := <-resultCh: @@ -364,6 +365,52 @@ func TestFailStreamRegionsReleasesSentAdmission(t *testing.T) { require.Equal(t, 1, ds.pushCount) } +func TestFailPendingRegionsReschedulesWorkerBuffer(t *testing.T) { + rawSpan := heartbeatpb.TableSpan{ + TableID: 1, + StartKey: []byte("a"), + EndKey: []byte("z"), + } + span := &subscribedSpan{ + subID: 1, + span: rawSpan, + rangeLock: regionlock.NewRangeLock(1, rawSpan.StartKey, rawSpan.EndKey, 100), + } + lock1 := span.rangeLock.LockRange(t.Context(), []byte("a"), []byte("m"), 1, 1) + lock2 := span.rangeLock.LockRange(t.Context(), []byte("m"), []byte("z"), 2, 1) + require.Equal(t, regionlock.LockRangeStatusSuccess, lock1.Status) + require.Equal(t, regionlock.LockRangeStatusSuccess, lock2.Status) + + admission := newRegionAdmissionController(1, 1) + client := &subscriptionClient{} + client.failureHandler = newRegionFailureHandler(client) + worker := ®ionRequestWorker{client: client, admission: admission} + regions := []regionInfo{ + { + verID: tikv.NewRegionVerID(1, 1, 1), + span: heartbeatpb.TableSpan{ + TableID: 1, StartKey: []byte("a"), EndKey: []byte("m"), + }, + subscribedSpan: span, lockedRangeState: lock1.LockedRangeState, + }, + { + verID: tikv.NewRegionVerID(2, 1, 1), + span: heartbeatpb.TableSpan{ + TableID: 1, StartKey: []byte("m"), EndKey: []byte("z"), + }, + subscribedSpan: span, lockedRangeState: lock2.LockedRangeState, + }, + } + for i, region := range regions { + require.True(t, admission.submit(NewRegionPriorityTask(region, 1, uint64(i+1)))) + } + + worker.failPendingRegions(&storeStreamErr{}) + + require.Zero(t, admission.pendingCount()) + require.Len(t, client.failureHandler.cache.cache, 2) +} + func TestProcessRegionSendTaskSendFailureCleansSentRequest(t *testing.T) { admission := newRegionAdmissionController(10, 1) worker := ®ionRequestWorker{ diff --git a/logservice/logpuller/region_state.go b/logservice/logpuller/region_state.go index 9e4fb52d58..0dc0d55bc0 100644 --- a/logservice/logpuller/region_state.go +++ b/logservice/logpuller/region_state.go @@ -44,6 +44,9 @@ type regionInfo struct { subscribedSpan *subscribedSpan // The state of the locked range of the region. lockedRangeState *regionlock.LockedRangeState + // wasInitialized preserves scheduling priority while a failed region is + // unlocked, resolved again, and subscribed with a new locked range state. + wasInitialized bool // Whether to filter out the value write by cdc itself. // It should be `true` in BDR mode filterLoop bool @@ -75,6 +78,9 @@ type regionErrorInfo struct { } func newRegionErrorInfo(info regionInfo, err error) regionErrorInfo { + if info.lockedRangeState != nil && info.lockedRangeState.Initialized.Load() { + info.wasInitialized = true + } return regionErrorInfo{ regionInfo: info, err: err, diff --git a/logservice/logpuller/subscription_client.go b/logservice/logpuller/subscription_client.go index e8fa26ba43..49a25cfb9a 100644 --- a/logservice/logpuller/subscription_client.go +++ b/logservice/logpuller/subscription_client.go @@ -87,7 +87,7 @@ type rangeTask struct { span heartbeatpb.TableSpan subscribedSpan *subscribedSpan filterLoop bool - priority TaskType + wasInitialized bool } type SubscriptionClientConfig struct { @@ -148,7 +148,10 @@ type subscriptionClient struct { rangeTaskCh chan rangeTask // regionTaskQueue is used to receive region tasks with priority. // The region will be handled in `handleRegions` goroutine. - regionTaskQueue *priorityqueue.PriorityQueue[PriorityTask] + regionTaskQueue *priorityqueue.PriorityQueue[*regionPriorityTask] + // regionTaskSequence provides a FIFO tie-breaker for tasks in the same + // priority class. + regionTaskSequence atomic.Uint64 // resolveLockTaskCh is used to receive resolve lock tasks. // The tasks will be handled in `handleResolveLockTasks` goroutine. resolveLockTaskCh chan resolveLockTask @@ -174,7 +177,7 @@ func NewSubscriptionClient( credential: credential, rangeTaskCh: make(chan rangeTask, 1024), - regionTaskQueue: priorityqueue.New[PriorityTask](), + regionTaskQueue: priorityqueue.New[*regionPriorityTask](), resolveLockTaskCh: make(chan resolveLockTask, 1024), resolveLockRateLimiter: newResolveLockRateLimiter(), } @@ -212,7 +215,7 @@ func (s *subscriptionClient) updateMetrics(ctx context.Context) error { pendingRegionReqCount := 0 s.stores.Range(func(_, value any) bool { store := value.(*requestedStore) - pendingRegionReqCount += store.admission.inflightCount() + pendingRegionReqCount += store.inflightCount() return true }) @@ -260,7 +263,7 @@ func (s *subscriptionClient) Subscribe( select { case <-s.ctx.Done(): log.Warn("subscribes span failed, the subscription client has closed") - case s.rangeTaskCh <- rangeTask{span: span, subscribedSpan: rt, filterLoop: rt.filterLoop, priority: TaskLowPrior}: + case s.rangeTaskCh <- rangeTask{span: span, subscribedSpan: rt, filterLoop: rt.filterLoop}: log.Info("subscribes span done", zap.Uint64("subscriptionID", uint64(subID)), zap.Int64("tableID", span.TableID), zap.Uint64("startTs", startTs), zap.String("startKey", spanz.HexKey(span.StartKey)), zap.String("endKey", spanz.HexKey(span.EndKey))) @@ -352,8 +355,8 @@ func (s *subscriptionClient) onRegionFail(errInfo regionErrorInfo) { // requestedStore represents a store that has been connected. type requestedStore struct { - storeAddr string - admission *regionAdmissionController + storeAddr string + nextWorker atomic.Uint64 requestWorkers struct { sync.RWMutex @@ -361,12 +364,47 @@ type requestedStore struct { } } +func (s *requestedStore) submit(task *regionPriorityTask) bool { + s.requestWorkers.RLock() + defer s.requestWorkers.RUnlock() + + workerCount := len(s.requestWorkers.s) + if workerCount == 0 { + return false + } + index := (s.nextWorker.Add(1) - 1) % uint64(workerCount) + return s.requestWorkers.s[index].admission.submit(task) +} + +func (s *requestedStore) close() { + s.requestWorkers.RLock() + defer s.requestWorkers.RUnlock() + for _, worker := range s.requestWorkers.s { + worker.admission.close() + } +} + +func (s *requestedStore) inflightCount() int { + s.requestWorkers.RLock() + defer s.requestWorkers.RUnlock() + count := 0 + for _, worker := range s.requestWorkers.s { + count += worker.admission.inflightCount() + } + return count +} + // handleRegions receives regionInfo from regionTaskQueue and attach rpcCtx to them, // then send them to corresponding requestedStore. func (s *subscriptionClient) handleRegions(ctx context.Context, eg *errgroup.Group) error { cfg := config.GetGlobalServerConfig() - currentWindow := cfg.Debug.Puller.PendingRegionRequestQueueSize + storeWindow := cfg.Debug.Puller.PendingRegionRequestQueueSize maxWindowMultiplier := cfg.Debug.Puller.RegionRequestMaxWindowMultiplier + workerCount := int(s.config.RegionRequestWorkerPerStore) + if workerCount <= 0 { + workerCount = 1 + } + workerWindow := (storeWindow + workerCount - 1) / workerCount getStore := func(storeAddr string) *requestedStore { var rs *requestedStore if v, ok := s.stores.Load(storeAddr); ok { @@ -374,15 +412,12 @@ func (s *subscriptionClient) handleRegions(ctx context.Context, eg *errgroup.Gro return rs } - rs = &requestedStore{ - storeAddr: storeAddr, - admission: newRegionAdmissionController(currentWindow, maxWindowMultiplier), - } - rs.requestWorkers.s = make([]*regionRequestWorker, 0, s.config.RegionRequestWorkerPerStore) + rs = &requestedStore{storeAddr: storeAddr} + rs.requestWorkers.s = make([]*regionRequestWorker, 0, workerCount) rs.requestWorkers.Lock() - for i := uint(0); i < s.config.RegionRequestWorkerPerStore; i++ { - requestWorker := newRegionRequestWorker(s, rs, rs.admission) + for i := 0; i < workerCount; i++ { + requestWorker := newRegionRequestWorker(s, rs, workerWindow, maxWindowMultiplier) rs.requestWorkers.s = append(rs.requestWorkers.s, requestWorker) } rs.requestWorkers.Unlock() @@ -398,7 +433,7 @@ func (s *subscriptionClient) handleRegions(ctx context.Context, eg *errgroup.Gro defer func() { s.stores.Range(func(_, value any) bool { rs := value.(*requestedStore) - rs.admission.close() + rs.close() return true }) }() @@ -426,7 +461,8 @@ func (s *subscriptionClient) handleRegions(ctx context.Context, eg *errgroup.Gro } store := getStore(region.rpcCtx.Addr) - if !store.admission.submit(regionTask, region, s.pdClock.CurrentTS()) { + regionTask.updateRegion(region, s.pdClock.CurrentTS()) + if !store.submit(regionTask) { return context.Canceled } @@ -476,7 +512,7 @@ func (s *subscriptionClient) handleRangeTasks(ctx context.Context) error { return ctx.Err() case task := <-s.rangeTaskCh: g.Go(func() error { - return s.divideSpanAndScheduleRegionRequests(ctx, task.span, task.subscribedSpan, task.filterLoop, task.priority) + return s.divideSpanAndScheduleRegionRequests(ctx, task.span, task.subscribedSpan, task.filterLoop, task.wasInitialized) }) } } @@ -492,7 +528,7 @@ func (s *subscriptionClient) divideSpanAndScheduleRegionRequests( span heartbeatpb.TableSpan, subscribedSpan *subscribedSpan, filterLoop bool, - taskType TaskType, + wasInitialized bool, ) error { // Limit the number of regions loaded at a time to make the load more stable. limit := 1024 @@ -554,9 +590,10 @@ func (s *subscriptionClient) divideSpanAndScheduleRegionRequests( verID := tikv.NewRegionVerID(regionMeta.Id, regionMeta.RegionEpoch.ConfVer, regionMeta.RegionEpoch.Version) regionInfo := newRegionInfo(verID, intersectSpan, nil, subscribedSpan, filterLoop) + regionInfo.wasInitialized = wasInitialized // Schedule a region request to subscribe the region. - s.scheduleRegionRequest(ctx, regionInfo, taskType) + s.scheduleRegionRequest(ctx, regionInfo) nextSpan.StartKey = regionMeta.EndKey // If the nextSpan.StartKey is larger than the subscribedSpan.span.EndKey, @@ -570,7 +607,10 @@ func (s *subscriptionClient) divideSpanAndScheduleRegionRequests( // scheduleRegionRequest locks the region's range and send the region to regionTaskQueue, // which will be handled by handleRegions. -func (s *subscriptionClient) scheduleRegionRequest(ctx context.Context, region regionInfo, priority TaskType) { +func (s *subscriptionClient) scheduleRegionRequest(ctx context.Context, region regionInfo) { + if region.lockedRangeState != nil && region.lockedRangeState.Initialized.Load() { + region.wasInitialized = true + } lockRangeResult := region.subscribedSpan.rangeLock.LockRange( ctx, region.span.StartKey, region.span.EndKey, region.verID.GetID(), region.verID.GetVer()) @@ -581,10 +621,11 @@ func (s *subscriptionClient) scheduleRegionRequest(ctx context.Context, region r switch lockRangeResult.Status { case regionlock.LockRangeStatusSuccess: region.lockedRangeState = lockRangeResult.LockedRangeState - s.regionTaskQueue.Push(NewRegionPriorityTask(priority, region, s.pdClock.CurrentTS())) + s.regionTaskQueue.Push(NewRegionPriorityTask( + region, s.pdClock.CurrentTS(), s.regionTaskSequence.Add(1))) case regionlock.LockRangeStatusStale: for _, r := range lockRangeResult.RetryRanges { - s.scheduleRangeRequest(ctx, r, region.subscribedSpan, region.filterLoop, priority) + s.scheduleRangeRequest(ctx, r, region.subscribedSpan, region.filterLoop, region.wasInitialized) } default: return @@ -595,11 +636,14 @@ func (s *subscriptionClient) scheduleRangeRequest( ctx context.Context, span heartbeatpb.TableSpan, subscribedSpan *subscribedSpan, filterLoop bool, - priority TaskType, + wasInitialized bool, ) { select { case <-ctx.Done(): - case s.rangeTaskCh <- rangeTask{span: span, subscribedSpan: subscribedSpan, filterLoop: filterLoop, priority: priority}: + case s.rangeTaskCh <- rangeTask{ + span: span, subscribedSpan: subscribedSpan, + filterLoop: filterLoop, wasInitialized: wasInitialized, + }: } } diff --git a/logservice/logpuller/subscription_client_test.go b/logservice/logpuller/subscription_client_test.go index d60e13ff8b..610be38cdd 100644 --- a/logservice/logpuller/subscription_client_test.go +++ b/logservice/logpuller/subscription_client_test.go @@ -423,7 +423,7 @@ func TestPushRegionEventToDSUnblocksOnClose(t *testing.T) { sink.cond = sync.NewCond(&sink.mu) client := &subscriptionClient{ eventSink: sink, - regionTaskQueue: priorityqueue.New[PriorityTask](), + regionTaskQueue: priorityqueue.New[*regionPriorityTask](), } client.ctx, client.cancel = context.WithCancel(context.Background()) @@ -458,7 +458,7 @@ func TestBroadcastDeregisterUsesWorkerControlQueue(t *testing.T) { admission: admission, controlQueue: newControlQueue(), } - store := &requestedStore{storeAddr: "store-1", admission: admission} + store := &requestedStore{storeAddr: "store-1"} store.requestWorkers.s = []*regionRequestWorker{worker} client.stores.Store(store.storeAddr, store) @@ -466,8 +466,7 @@ func TestBroadcastDeregisterUsesWorkerControlQueue(t *testing.T) { subscribedSpan: &subscribedSpan{subID: SubscriptionID(2)}, lockedRangeState: ®ionlock.LockedRangeState{}, } - require.True(t, admission.submit( - NewRegionPriorityTask(TaskHighPrior, dummyRegion, 1), dummyRegion, 1)) + require.True(t, admission.submit(NewRegionPriorityTask(dummyRegion, 1, 1))) client.broadcastDeregister(SubscriptionID(1), true) require.Equal(t, 1, worker.controlQueue.len()) @@ -478,6 +477,25 @@ func TestBroadcastDeregisterUsesWorkerControlQueue(t *testing.T) { require.Equal(t, 1, admission.pendingCount()) } +func TestRequestedStoreDistributesRegionsAcrossWorkerBuffers(t *testing.T) { + worker1 := ®ionRequestWorker{admission: newRegionAdmissionController(1, 1)} + worker2 := ®ionRequestWorker{admission: newRegionAdmissionController(1, 1)} + store := &requestedStore{storeAddr: "store-1"} + store.requestWorkers.s = []*regionRequestWorker{worker1, worker2} + + for i := uint64(1); i <= 4; i++ { + region := regionInfo{ + verID: tikv.NewRegionVerID(i, 1, 1), + subscribedSpan: &subscribedSpan{subID: 1}, + lockedRangeState: ®ionlock.LockedRangeState{}, + } + require.True(t, store.submit(NewRegionPriorityTask(region, 1, i))) + } + + require.Equal(t, 2, worker1.admission.pendingCount()) + require.Equal(t, 2, worker2.admission.pendingCount()) +} + func TestSubscriptionWithFailedTiKV(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) mockPDClock := pdutil.NewClock4Test() diff --git a/pkg/config/debug.go b/pkg/config/debug.go index b95be0f8ab..1858f54582 100644 --- a/pkg/config/debug.go +++ b/pkg/config/debug.go @@ -65,12 +65,13 @@ type PullerConfig struct { // LogRegionDetails determines whether logs Region details or not in puller and kv-client. LogRegionDetails bool `toml:"log-region-details" json:"log_region_details"` - // PendingRegionRequestQueueSize is the normal initial-scan window shared by - // all puller workers connecting to one TiKV store. + // PendingRegionRequestQueueSize is the approximate normal initial-scan window + // for one TiKV store. It is divided among the store's puller workers. PendingRegionRequestQueueSize int `toml:"pending-region-request-queue-size" json:"pending_region_request_queue_size"` // RegionRequestMaxWindowMultiplier controls the maximum window available to - // high-priority requests and regions whose scan lag is less than 10 minutes. - // The maximum window is PendingRegionRequestQueueSize multiplied by this value. + // previously initialized regions and regions whose scan lag is less than 30 minutes. + // The approximate maximum store window is PendingRegionRequestQueueSize + // multiplied by this value. RegionRequestMaxWindowMultiplier int `toml:"region-request-max-window-multiplier" json:"region_request_max_window_multiplier"` } From 06a40d34deba9bc82a9196f3c0cd7758b591e244 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Thu, 16 Jul 2026 21:10:47 +0800 Subject: [PATCH 10/71] refactor --- .../logpuller/region_admission_controller.go | 85 ++++++++----------- .../region_admission_controller_test.go | 42 ++++----- logservice/logpuller/region_request_worker.go | 20 ++--- .../logpuller/region_request_worker_test.go | 16 ++-- logservice/logpuller/subscription_client.go | 2 +- .../logpuller/subscription_client_test.go | 6 +- 6 files changed, 75 insertions(+), 96 deletions(-) diff --git a/logservice/logpuller/region_admission_controller.go b/logservice/logpuller/region_admission_controller.go index 851b1bb6ba..5f0b04fd46 100644 --- a/logservice/logpuller/region_admission_controller.go +++ b/logservice/logpuller/region_admission_controller.go @@ -50,7 +50,7 @@ func (r *regionReq) finish() bool { zap.Uint64("subID", uint64(r.regionInfo.subscribedSpan.subID)), zap.Uint64("regionID", r.regionInfo.verID.GetID()), zap.Float64("cost", cost), - zap.Int("inflightCount", r.controller.inflightCount())) + zap.Int("inflightCount", r.controller.stats().inflight)) metrics.RegionRequestFinishScanDuration.Observe(cost) return true } @@ -102,6 +102,11 @@ type regionAdmissionController struct { closed bool } +type regionAdmissionStats struct { + pending int + inflight int +} + func newRegionAdmissionController(currentWindow, maxWindowMultiplier int) *regionAdmissionController { if currentWindow <= 0 { currentWindow = 1 @@ -133,47 +138,40 @@ func (c *regionAdmissionController) submit(task *regionPriorityTask) bool { return true } -func (c *regionAdmissionController) pop(ctx context.Context) (*regionReq, error) { +// pop waits for an eligible request. If interrupt is signaled first, it returns +// nil without an error so the worker can handle its control queue. +func (c *regionAdmissionController) pop( + ctx context.Context, + interrupt <-chan struct{}, +) (*regionReq, error) { for { - request, closed := c.tryPop() - if request != nil { - return request, nil - } - if closed { + c.mu.Lock() + if c.closed { + c.mu.Unlock() return nil, context.Canceled } + request := c.popEligibleLocked() + if request != nil { + c.inflight++ + c.mu.Unlock() + return ®ionReq{ + regionInfo: request.regionInfo, + createTime: time.Now(), + controller: c, + }, nil + } + c.mu.Unlock() select { case <-c.notify: + case <-interrupt: + return nil, nil case <-ctx.Done(): return nil, ctx.Err() } } } -// tryPop returns whether the controller has been closed as its second result. -func (c *regionAdmissionController) tryPop() (*regionReq, bool) { - c.mu.Lock() - defer c.mu.Unlock() - - if c.closed { - return nil, true - } - request := c.popEligibleLocked() - if request == nil { - return nil, false - } - c.inflight++ - if c.hasEligibleRequestLocked() { - c.notifyOneLocked() - } - return ®ionReq{ - regionInfo: request.regionInfo, - createTime: time.Now(), - controller: c, - }, false -} - func (c *regionAdmissionController) popEligibleLocked() *regionPriorityTask { request, ok := c.pending.PeekTop() if !ok { @@ -186,14 +184,6 @@ func (c *regionAdmissionController) popEligibleLocked() *regionPriorityTask { return request } -func (c *regionAdmissionController) hasEligibleRequestLocked() bool { - request, ok := c.pending.PeekTop() - if !ok { - return false - } - return c.inflight < c.windowFor(request) -} - func (c *regionAdmissionController) windowFor(request *regionPriorityTask) int { if request.canUseMaxWindow() { return c.maxWindow @@ -219,23 +209,16 @@ func (c *regionAdmissionController) close() { c.mu.Unlock() } -func (c *regionAdmissionController) ready() <-chan struct{} { - return c.notify -} - -func (c *regionAdmissionController) inflightCount() int { +func (c *regionAdmissionController) stats() regionAdmissionStats { c.mu.Lock() defer c.mu.Unlock() - return c.inflight -} - -func (c *regionAdmissionController) pendingCount() int { - c.mu.Lock() - defer c.mu.Unlock() - return c.pending.Len() + return regionAdmissionStats{ + pending: c.pending.Len(), + inflight: c.inflight, + } } -func (c *regionAdmissionController) drainPending() []*regionPriorityTask { +func (c *regionAdmissionController) drain() []*regionPriorityTask { c.mu.Lock() defer c.mu.Unlock() diff --git a/logservice/logpuller/region_admission_controller_test.go b/logservice/logpuller/region_admission_controller_test.go index a8ab72f184..e8627aa98a 100644 --- a/logservice/logpuller/region_admission_controller_test.go +++ b/logservice/logpuller/region_admission_controller_test.go @@ -68,15 +68,17 @@ func TestRegionAdmissionControllerNormalWindow(t *testing.T) { submitRegionForAdmission(t, controller, region1, currentTs) submitRegionForAdmission(t, controller, region2, currentTs) - req1, err := controller.pop(t.Context()) + req1, err := controller.pop(t.Context(), nil) require.NoError(t, err) - require.Equal(t, 1, controller.inflightCount()) - req2, closed := controller.tryPop() + require.Equal(t, 1, controller.stats().inflight) + interrupt := make(chan struct{}) + close(interrupt) + req2, err := controller.pop(t.Context(), interrupt) require.Nil(t, req2) - require.False(t, closed) + require.NoError(t, err) require.True(t, req1.abort()) - req2, err = controller.pop(t.Context()) + req2, err = controller.pop(t.Context(), nil) require.NoError(t, err) require.Equal(t, uint64(2), req2.regionInfo.verID.GetID()) require.True(t, req2.abort()) @@ -91,7 +93,7 @@ func TestRegionAdmissionControllerLowLagUsesMaxWindow(t *testing.T) { submitRegionForAdmission(t, controller, prepareRegionForAdmission(createTestRegionInfo(1, 1), slowCheckpointTs), currentTs) - req1, err := controller.pop(t.Context()) + req1, err := controller.pop(t.Context(), nil) require.NoError(t, err) submitRegionForAdmission(t, controller, @@ -101,17 +103,19 @@ func TestRegionAdmissionControllerLowLagUsesMaxWindow(t *testing.T) { prepareRegionForAdmission(createTestRegionInfo(1, 3), lowLagCheckpointTs), currentTs) - req2, err := controller.pop(t.Context()) + req2, err := controller.pop(t.Context(), nil) require.NoError(t, err) require.Equal(t, uint64(3), req2.regionInfo.verID.GetID()) - req3, closed := controller.tryPop() + interrupt := make(chan struct{}) + close(interrupt) + req3, err := controller.pop(t.Context(), interrupt) require.Nil(t, req3) - require.False(t, closed) - require.Equal(t, 2, controller.inflightCount()) + require.NoError(t, err) + require.Equal(t, 2, controller.stats().inflight) require.True(t, req1.abort()) require.True(t, req2.abort()) - req3, err = controller.pop(t.Context()) + req3, err = controller.pop(t.Context(), nil) require.NoError(t, err) require.Equal(t, uint64(2), req3.regionInfo.verID.GetID()) require.True(t, req3.abort()) @@ -126,7 +130,7 @@ func TestRegionAdmissionControllerPrioritizesInitializedRegion(t *testing.T) { submitRegionForAdmission(t, controller, prepareRegionForAdmission(createTestRegionInfo(1, 1), slowCheckpointTs), currentTs) - req1, err := controller.pop(t.Context()) + req1, err := controller.pop(t.Context(), nil) require.NoError(t, err) submitRegionForAdmission(t, controller, @@ -137,13 +141,13 @@ func TestRegionAdmissionControllerPrioritizesInitializedRegion(t *testing.T) { submitRegionForAdmission(t, controller, initializedRegion, currentTs) - req2, err := controller.pop(t.Context()) + req2, err := controller.pop(t.Context(), nil) require.NoError(t, err) require.Equal(t, uint64(3), req2.regionInfo.verID.GetID()) require.True(t, req1.abort()) require.True(t, req2.abort()) - req3, err := controller.pop(t.Context()) + req3, err := controller.pop(t.Context(), nil) require.NoError(t, err) require.Equal(t, uint64(2), req3.regionInfo.verID.GetID()) require.True(t, req3.abort()) @@ -154,7 +158,7 @@ func TestRegionAdmissionLeaseReleasedOnce(t *testing.T) { currentTs := oracle.GoTimeToTS(time.Now()) region := prepareRegionForAdmission(createTestRegionInfo(1, 1), currentTs) submitRegionForAdmission(t, controller, region, currentTs) - req, err := controller.pop(t.Context()) + req, err := controller.pop(t.Context(), nil) require.NoError(t, err) start := make(chan struct{}) @@ -182,7 +186,7 @@ func TestRegionAdmissionLeaseReleasedOnce(t *testing.T) { } } require.Equal(t, 1, successes) - require.Zero(t, controller.inflightCount()) + require.Zero(t, controller.stats().inflight) } func TestRegionAdmissionControllerClose(t *testing.T) { @@ -191,7 +195,7 @@ func TestRegionAdmissionControllerClose(t *testing.T) { region := prepareRegionForAdmission(createTestRegionInfo(1, 1), 1) require.False(t, controller.submit(NewRegionPriorityTask(region, 1, 1))) - _, err := controller.pop(context.Background()) + _, err := controller.pop(context.Background(), nil) require.ErrorIs(t, err, context.Canceled) } @@ -202,7 +206,7 @@ func TestRegionAdmissionControllerDrainPending(t *testing.T) { submitRegionForAdmission(t, controller, region1, 1) submitRegionForAdmission(t, controller, region2, 1) - pending := controller.drainPending() + pending := controller.drain() require.Len(t, pending, 2) - require.Zero(t, controller.pendingCount()) + require.Zero(t, controller.stats().pending) } diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index 12dc5a9248..0e6a5507fb 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -162,7 +162,7 @@ func (s *regionRequestWorker) failStreamRegions(err error) { // failPendingRegions transfers requests owned by this worker but not yet sent // to the recovery pipeline, so they can be resolved and routed again. func (s *regionRequestWorker) failPendingRegions(err error) { - for _, task := range s.admission.drainPending() { + for _, task := range s.admission.drain() { s.client.onRegionFail(newRegionErrorInfo(task.regionInfo, err)) } } @@ -178,7 +178,7 @@ func (s *regionRequestWorker) notifyRegionError(state *regionFeedState, err erro func (s *regionRequestWorker) waitForRegionRequest(ctx context.Context) (*regionReq, error) { // Without a stream there are no remote registrations to deregister. s.controlQueue.drain() - req, err := s.admission.pop(ctx) + req, err := s.admission.pop(ctx, nil) if err != nil { return nil, err } @@ -494,18 +494,10 @@ func (s *regionRequestWorker) processRegionSendTask( if err := s.drainControlQueue(conn); err != nil { return err } - var closed bool - if regionReq, closed = s.admission.tryPop(); regionReq != nil { - continue - } - if closed { - return context.Canceled - } - select { - case <-s.controlQueue.ready(): - case <-s.admission.ready(): - case <-ctx.Done(): - return ctx.Err() + var err error + regionReq, err = s.admission.pop(ctx, s.controlQueue.ready()) + if err != nil { + return err } } } diff --git a/logservice/logpuller/region_request_worker_test.go b/logservice/logpuller/region_request_worker_test.go index ffd6416994..4259904e9c 100644 --- a/logservice/logpuller/region_request_worker_test.go +++ b/logservice/logpuller/region_request_worker_test.go @@ -74,7 +74,7 @@ func admitRegionRequest( t.Helper() currentTs := oracle.GoTimeToTS(time.Now()) submitRegionForAdmission(t, controller, region, currentTs) - req, err := controller.pop(t.Context()) + req, err := controller.pop(t.Context(), nil) require.NoError(t, err) return req } @@ -101,7 +101,7 @@ func TestRegionRequestWorkerIgnoresDuplicateActiveRegion(t *testing.T) { }, req2) require.NoError(t, err) - require.Equal(t, 1, admission.inflightCount()) + require.Equal(t, 1, admission.stats().inflight) require.Same(t, state1, worker.tracker.Get(region.subscribedSpan.subID, region.verID.GetID())) require.False(t, state1.isStale()) select { @@ -339,7 +339,7 @@ func TestStoppedStateRemovesSentRequest(t *testing.T) { state.markStopped(errors.New("send request to store error")) worker.tracker.RemoveIf(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID(), state) - require.Equal(t, 0, admission.inflightCount()) + require.Equal(t, 0, admission.stats().inflight) } func TestFailStreamRegionsReleasesSentAdmission(t *testing.T) { @@ -360,7 +360,7 @@ func TestFailStreamRegionsReleasesSentAdmission(t *testing.T) { worker.failStreamRegions(&storeStreamErr{}) - require.Zero(t, admission.inflightCount()) + require.Zero(t, admission.stats().inflight) require.False(t, req.abort()) require.Equal(t, 1, ds.pushCount) } @@ -407,7 +407,7 @@ func TestFailPendingRegionsReschedulesWorkerBuffer(t *testing.T) { worker.failPendingRegions(&storeStreamErr{}) - require.Zero(t, admission.pendingCount()) + require.Zero(t, admission.stats().pending) require.Len(t, client.failureHandler.cache.cache, 2) } @@ -424,7 +424,7 @@ func TestProcessRegionSendTaskSendFailureCleansSentRequest(t *testing.T) { region := prepareRegionForSendTest(createTestRegionInfo(1, 1)) req := admitRegionRequest(t, admission, region) - require.Equal(t, 1, admission.inflightCount()) + require.Equal(t, 1, admission.stats().inflight) sendErr := errors.New("send failed") conn := &ConnAndClient{ @@ -434,7 +434,7 @@ func TestProcessRegionSendTaskSendFailureCleansSentRequest(t *testing.T) { err := worker.processRegionSendTask(t.Context(), conn, req) require.ErrorIs(t, err, sendErr) - require.Equal(t, 0, admission.inflightCount()) + require.Equal(t, 0, admission.stats().inflight) state := worker.tracker.Get(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID()) require.NotNil(t, state) require.True(t, state.isStale()) @@ -506,7 +506,7 @@ func TestProcessRegionSendTaskSendEOFIsRetriable(t *testing.T) { err := worker.processRegionSendTask(t.Context(), conn, req) var streamErr *storeStreamErr require.ErrorAs(t, err, &streamErr) - require.Equal(t, 0, admission.inflightCount()) + require.Equal(t, 0, admission.stats().inflight) state := worker.tracker.Get(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID()) require.NotNil(t, state) diff --git a/logservice/logpuller/subscription_client.go b/logservice/logpuller/subscription_client.go index 49a25cfb9a..2542936b6a 100644 --- a/logservice/logpuller/subscription_client.go +++ b/logservice/logpuller/subscription_client.go @@ -389,7 +389,7 @@ func (s *requestedStore) inflightCount() int { defer s.requestWorkers.RUnlock() count := 0 for _, worker := range s.requestWorkers.s { - count += worker.admission.inflightCount() + count += worker.admission.stats().inflight } return count } diff --git a/logservice/logpuller/subscription_client_test.go b/logservice/logpuller/subscription_client_test.go index 610be38cdd..c3ef992a34 100644 --- a/logservice/logpuller/subscription_client_test.go +++ b/logservice/logpuller/subscription_client_test.go @@ -474,7 +474,7 @@ func TestBroadcastDeregisterUsesWorkerControlQueue(t *testing.T) { require.True(t, ok) require.Equal(t, SubscriptionID(1), req.subID) require.True(t, req.filterLoop) - require.Equal(t, 1, admission.pendingCount()) + require.Equal(t, 1, admission.stats().pending) } func TestRequestedStoreDistributesRegionsAcrossWorkerBuffers(t *testing.T) { @@ -492,8 +492,8 @@ func TestRequestedStoreDistributesRegionsAcrossWorkerBuffers(t *testing.T) { require.True(t, store.submit(NewRegionPriorityTask(region, 1, i))) } - require.Equal(t, 2, worker1.admission.pendingCount()) - require.Equal(t, 2, worker2.admission.pendingCount()) + require.Equal(t, 2, worker1.admission.stats().pending) + require.Equal(t, 2, worker2.admission.stats().pending) } func TestSubscriptionWithFailedTiKV(t *testing.T) { From e8496b1b485a30c9c6712810bda1b6aaf430f7dc Mon Sep 17 00:00:00 2001 From: lidezhu Date: Sun, 2 Aug 2026 15:24:30 +0800 Subject: [PATCH 11/71] small fix --- logservice/logpuller/priority_task.go | 21 +++++++++++++-------- logservice/logpuller/priority_task_test.go | 6 ++++-- pkg/config/debug.go | 3 ++- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/logservice/logpuller/priority_task.go b/logservice/logpuller/priority_task.go index 0798758d82..7b2287d47c 100644 --- a/logservice/logpuller/priority_task.go +++ b/logservice/logpuller/priority_task.go @@ -17,23 +17,24 @@ import ( "time" "github.com/pingcap/kvproto/pkg/cdcpb" + "github.com/pingcap/ticdc/pkg/config" "github.com/tikv/client-go/v2/oracle" ) -// TaskType represents the scan priority propagated to TiKV/CSE. +// TaskType represents the scan priority level associated with a region task. type TaskType int const ( - // TaskHighPrior represents retries or scans that should preempt older work. + // TaskHighPrior represents high scan priority. + // For example, a region task that is already close to caught up + // is typically tagged with this level. TaskHighPrior TaskType = iota - // TaskLowPrior represents background initial scans for new subscriptions. + // TaskLowPrior represents low scan priority. + // For example, a region task created for a subscription that starts from an + // older start-ts is typically tagged with this level before it catches up. TaskLowPrior ) -const ( - lowLagRegionThreshold = 30 * time.Minute -) - type regionTaskPriority int const ( @@ -100,7 +101,7 @@ func (pt *regionPriorityTask) updateRegion(regionInfo regionInfo, currentTs uint priority := normalRegionPriority if regionInfo.wasInitialized { priority = initializedRegionPriority - } else if regionScanLag(currentTs, regionInfo.resolvedTs()) < lowLagRegionThreshold { + } else if regionScanLag(currentTs, regionInfo.resolvedTs()) < lowLagRegionThreshold() { priority = lowLagRegionPriority } pt.regionInfo = regionInfo @@ -144,3 +145,7 @@ func regionScanLag(currentTs, checkpointTs uint64) time.Duration { } return currentTime.Sub(checkpointTime) } + +func lowLagRegionThreshold() time.Duration { + return time.Duration(config.GetGlobalServerConfig().Debug.Puller.OldStartTsScanLowPriorityThreshold) +} diff --git a/logservice/logpuller/priority_task_test.go b/logservice/logpuller/priority_task_test.go index d1afbd4d0c..081f78ba0a 100644 --- a/logservice/logpuller/priority_task_test.go +++ b/logservice/logpuller/priority_task_test.go @@ -21,6 +21,7 @@ import ( "github.com/pingcap/kvproto/pkg/cdcpb" "github.com/pingcap/ticdc/heartbeatpb" "github.com/pingcap/ticdc/logservice/logpuller/regionlock" + "github.com/pingcap/ticdc/pkg/config" "github.com/pingcap/ticdc/utils/priorityqueue" "github.com/stretchr/testify/require" "github.com/tikv/client-go/v2/oracle" @@ -105,15 +106,16 @@ func TestRegionPriorityTaskFIFOWithinPriority(t *testing.T) { func TestRegionPriorityTaskLowLagBoundary(t *testing.T) { currentTime := time.Now() currentTs := oracle.GoTimeToTS(currentTime) + threshold := time.Duration(config.GetGlobalServerConfig().Debug.Puller.OldStartTsScanLowPriorityThreshold) belowThreshold := NewRegionPriorityTask(newPriorityTestRegion( 1, - oracle.GoTimeToTS(currentTime.Add(-lowLagRegionThreshold+time.Millisecond)), + oracle.GoTimeToTS(currentTime.Add(-threshold+time.Millisecond)), false, ), currentTs, 1) atThreshold := NewRegionPriorityTask(newPriorityTestRegion( 2, - oracle.GoTimeToTS(currentTime.Add(-lowLagRegionThreshold)), + oracle.GoTimeToTS(currentTime.Add(-threshold)), false, ), currentTs, 2) futureCheckpoint := NewRegionPriorityTask(newPriorityTestRegion( diff --git a/pkg/config/debug.go b/pkg/config/debug.go index 9fd46364f5..ce5ee9ea4a 100644 --- a/pkg/config/debug.go +++ b/pkg/config/debug.go @@ -75,7 +75,8 @@ type PullerConfig struct { // for one TiKV store. It is divided among the store's puller workers. PendingRegionRequestQueueSize int `toml:"pending-region-request-queue-size" json:"pending_region_request_queue_size"` // RegionRequestMaxWindowMultiplier controls the maximum window available to - // previously initialized regions and regions whose scan lag is less than 30 minutes. + // previously initialized regions and regions whose scan lag is below the + // configured low-priority threshold. // The approximate maximum store window is PendingRegionRequestQueueSize // multiplied by this value. RegionRequestMaxWindowMultiplier int `toml:"region-request-max-window-multiplier" json:"region_request_max_window_multiplier"` From 8916817c4f43ea23962ad9f6a71e3b90c24cf4df Mon Sep 17 00:00:00 2001 From: lidezhu Date: Sun, 2 Aug 2026 15:27:09 +0800 Subject: [PATCH 12/71] small fix --- logservice/logpuller/priority_task.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/logservice/logpuller/priority_task.go b/logservice/logpuller/priority_task.go index 7b2287d47c..06630ea38b 100644 --- a/logservice/logpuller/priority_task.go +++ b/logservice/logpuller/priority_task.go @@ -26,8 +26,8 @@ type TaskType int const ( // TaskHighPrior represents high scan priority. - // For example, a region task that is already close to caught up - // is typically tagged with this level. + // For example, a region task that is already close to caught up or from an + // initialized span is typically tagged with this level. TaskHighPrior TaskType = iota // TaskLowPrior represents low scan priority. // For example, a region task created for a subscription that starts from an From 7c91def55f6665cec40a42e1f2ed02d7cc808b17 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Sun, 2 Aug 2026 15:45:54 +0800 Subject: [PATCH 13/71] remove unnecessary priority --- logservice/logpuller/priority_task.go | 53 ++--------- logservice/logpuller/priority_task_test.go | 90 +++++++++---------- .../logpuller/region_admission_controller.go | 2 +- .../region_admission_controller_test.go | 21 ++--- 4 files changed, 61 insertions(+), 105 deletions(-) diff --git a/logservice/logpuller/priority_task.go b/logservice/logpuller/priority_task.go index 06630ea38b..dfb66315fd 100644 --- a/logservice/logpuller/priority_task.go +++ b/logservice/logpuller/priority_task.go @@ -13,21 +13,15 @@ package logpuller -import ( - "time" - - "github.com/pingcap/kvproto/pkg/cdcpb" - "github.com/pingcap/ticdc/pkg/config" - "github.com/tikv/client-go/v2/oracle" -) +import "github.com/pingcap/kvproto/pkg/cdcpb" // TaskType represents the scan priority level associated with a region task. type TaskType int const ( // TaskHighPrior represents high scan priority. - // For example, a region task that is already close to caught up or from an - // initialized span is typically tagged with this level. + // For example, a region task for a region that is already close to caught up + // is typically tagged with this level. TaskHighPrior TaskType = iota // TaskLowPrior represents low scan priority. // For example, a region task created for a subscription that starts from an @@ -35,14 +29,6 @@ const ( TaskLowPrior ) -type regionTaskPriority int - -const ( - initializedRegionPriority regionTaskPriority = iota - lowLagRegionPriority - normalRegionPriority -) - func (t TaskType) String() string { switch t { case TaskHighPrior: @@ -81,32 +67,24 @@ type regionPriorityTask struct { regionInfo regionInfo sequence uint64 heapIndex int // for heap.Item interface - priority regionTaskPriority } // NewRegionPriorityTask creates a new priority task for region -func NewRegionPriorityTask(regionInfo regionInfo, currentTs, sequence uint64) *regionPriorityTask { +func NewRegionPriorityTask(regionInfo regionInfo, _ uint64, sequence uint64) *regionPriorityTask { task := ®ionPriorityTask{ taskType: taskTypeFromScanPriority(regionInfo.scanPriority), sequence: sequence, heapIndex: 0, // 0 means not in heap } - task.updateRegion(regionInfo, currentTs) + task.updateRegion(regionInfo, 0) return task } // updateRegion refreshes both the request data and its priority before the task // enters another scheduling stage. -func (pt *regionPriorityTask) updateRegion(regionInfo regionInfo, currentTs uint64) { - priority := normalRegionPriority - if regionInfo.wasInitialized { - priority = initializedRegionPriority - } else if regionScanLag(currentTs, regionInfo.resolvedTs()) < lowLagRegionThreshold() { - priority = lowLagRegionPriority - } +func (pt *regionPriorityTask) updateRegion(regionInfo regionInfo, _ uint64) { pt.regionInfo = regionInfo pt.taskType = taskTypeFromScanPriority(regionInfo.scanPriority) - pt.priority = priority } // GetRegionInfo returns the underlying regionInfo @@ -115,7 +93,7 @@ func (pt *regionPriorityTask) GetRegionInfo() regionInfo { } func (pt *regionPriorityTask) canUseMaxWindow() bool { - return pt.priority != normalRegionPriority + return pt.taskType == TaskHighPrior } // SetHeapIndex sets the heap index for heap.Item interface @@ -131,21 +109,8 @@ func (pt *regionPriorityTask) GetHeapIndex() int { // LessThan implements heap.Item interface. Tasks in the same priority class are // processed in submission order. func (pt *regionPriorityTask) LessThan(other *regionPriorityTask) bool { - if pt.priority != other.priority { - return pt.priority < other.priority + if pt.taskType != other.taskType { + return pt.taskType < other.taskType } return pt.sequence < other.sequence } - -func regionScanLag(currentTs, checkpointTs uint64) time.Duration { - currentTime := oracle.GetTimeFromTS(currentTs) - checkpointTime := oracle.GetTimeFromTS(checkpointTs) - if !currentTime.After(checkpointTime) { - return 0 - } - return currentTime.Sub(checkpointTime) -} - -func lowLagRegionThreshold() time.Duration { - return time.Duration(config.GetGlobalServerConfig().Debug.Puller.OldStartTsScanLowPriorityThreshold) -} diff --git a/logservice/logpuller/priority_task_test.go b/logservice/logpuller/priority_task_test.go index 081f78ba0a..e0c1c99fd4 100644 --- a/logservice/logpuller/priority_task_test.go +++ b/logservice/logpuller/priority_task_test.go @@ -21,7 +21,6 @@ import ( "github.com/pingcap/kvproto/pkg/cdcpb" "github.com/pingcap/ticdc/heartbeatpb" "github.com/pingcap/ticdc/logservice/logpuller/regionlock" - "github.com/pingcap/ticdc/pkg/config" "github.com/pingcap/ticdc/utils/priorityqueue" "github.com/stretchr/testify/require" "github.com/tikv/client-go/v2/oracle" @@ -45,6 +44,11 @@ func newPriorityTestRegion( } } +func withScanPriority(region regionInfo, priority TaskType) regionInfo { + region.scanPriority = priority.scanPriority() + return region +} + func TestTaskTypeScanPriorityMapping(t *testing.T) { require.Equal(t, cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, TaskHighPrior.scanPriority()) require.Equal(t, cdcpb.ScanPriority_SCAN_PRIORITY_LOW, TaskLowPrior.scanPriority()) @@ -59,22 +63,31 @@ func TestRegionPriorityTaskQueueOrder(t *testing.T) { currentTime := time.Now() currentTs := oracle.GoTimeToTS(currentTime) - normalTask := NewRegionPriorityTask( - newPriorityTestRegion(1, oracle.GoTimeToTS(currentTime.Add(-time.Hour)), false), + lowTask := NewRegionPriorityTask( + withScanPriority( + newPriorityTestRegion(1, oracle.GoTimeToTS(currentTime.Add(-time.Hour)), false), + TaskLowPrior, + ), currentTs, 3, ) - lowLagTask := NewRegionPriorityTask( - newPriorityTestRegion(2, oracle.GoTimeToTS(currentTime.Add(-10*time.Minute)), false), + highTask1 := NewRegionPriorityTask( + withScanPriority( + newPriorityTestRegion(2, oracle.GoTimeToTS(currentTime.Add(-10*time.Minute)), false), + TaskHighPrior, + ), currentTs, 2, ) - initializedTask := NewRegionPriorityTask( - newPriorityTestRegion(3, oracle.GoTimeToTS(currentTime.Add(-time.Hour)), true), + highTask2 := NewRegionPriorityTask( + withScanPriority( + newPriorityTestRegion(3, oracle.GoTimeToTS(currentTime.Add(-time.Hour)), true), + TaskHighPrior, + ), currentTs, 1, ) - require.True(t, queue.Push(normalTask)) - require.True(t, queue.Push(lowLagTask)) - require.True(t, queue.Push(initializedTask)) + require.True(t, queue.Push(lowTask)) + require.True(t, queue.Push(highTask1)) + require.True(t, queue.Push(highTask2)) for _, expectedRegionID := range []uint64{3, 2, 1} { task, err := queue.Pop(t.Context()) @@ -89,8 +102,10 @@ func TestRegionPriorityTaskFIFOWithinPriority(t *testing.T) { currentTs := oracle.GoTimeToTS(currentTime) checkpointTs := oracle.GoTimeToTS(currentTime.Add(-time.Hour)) - first := NewRegionPriorityTask(newPriorityTestRegion(1, checkpointTs, false), currentTs, 1) - second := NewRegionPriorityTask(newPriorityTestRegion(2, checkpointTs, false), currentTs, 2) + first := NewRegionPriorityTask( + withScanPriority(newPriorityTestRegion(1, checkpointTs, false), TaskHighPrior), currentTs, 1) + second := NewRegionPriorityTask( + withScanPriority(newPriorityTestRegion(2, checkpointTs, false), TaskHighPrior), currentTs, 2) require.True(t, queue.Push(second)) require.True(t, queue.Push(first)) @@ -103,43 +118,22 @@ func TestRegionPriorityTaskFIFOWithinPriority(t *testing.T) { require.Equal(t, uint64(2), task.regionInfo.verID.GetID()) } -func TestRegionPriorityTaskLowLagBoundary(t *testing.T) { - currentTime := time.Now() - currentTs := oracle.GoTimeToTS(currentTime) - threshold := time.Duration(config.GetGlobalServerConfig().Debug.Puller.OldStartTsScanLowPriorityThreshold) - - belowThreshold := NewRegionPriorityTask(newPriorityTestRegion( - 1, - oracle.GoTimeToTS(currentTime.Add(-threshold+time.Millisecond)), - false, - ), currentTs, 1) - atThreshold := NewRegionPriorityTask(newPriorityTestRegion( - 2, - oracle.GoTimeToTS(currentTime.Add(-threshold)), - false, - ), currentTs, 2) - futureCheckpoint := NewRegionPriorityTask(newPriorityTestRegion( - 3, - oracle.GoTimeToTS(currentTime.Add(time.Second)), - false, - ), currentTs, 3) - - require.Equal(t, lowLagRegionPriority, belowThreshold.priority) - require.Equal(t, normalRegionPriority, atThreshold.priority) - require.Equal(t, lowLagRegionPriority, futureCheckpoint.priority) +func TestRegionPriorityTaskUsesHighPriorityWindow(t *testing.T) { + highTask := NewRegionPriorityTask( + withScanPriority(newPriorityTestRegion(1, 1, false), TaskHighPrior), 0, 1) + lowTask := NewRegionPriorityTask( + withScanPriority(newPriorityTestRegion(2, 1, true), TaskLowPrior), 0, 2) + + require.True(t, highTask.canUseMaxWindow()) + require.False(t, lowTask.canUseMaxWindow()) } func TestRegionPriorityTaskRefreshesPriorityBetweenStages(t *testing.T) { - checkpointTime := time.Now() - checkpointTs := oracle.GoTimeToTS(checkpointTime) - region := newPriorityTestRegion(1, checkpointTs, false) - task := NewRegionPriorityTask(region, oracle.GoTimeToTS(checkpointTime.Add(time.Minute)), 1) - require.Equal(t, lowLagRegionPriority, task.priority) - - task.updateRegion(region, oracle.GoTimeToTS(checkpointTime.Add(time.Hour))) - require.Equal(t, normalRegionPriority, task.priority) - - region.wasInitialized = true - task.updateRegion(region, oracle.GoTimeToTS(checkpointTime.Add(time.Hour))) - require.Equal(t, initializedRegionPriority, task.priority) + region := withScanPriority(newPriorityTestRegion(1, 1, false), TaskLowPrior) + task := NewRegionPriorityTask(region, 0, 1) + require.Equal(t, TaskLowPrior, task.taskType) + + region.scanPriority = TaskHighPrior.scanPriority() + task.updateRegion(region, 0) + require.Equal(t, TaskHighPrior, task.taskType) } diff --git a/logservice/logpuller/region_admission_controller.go b/logservice/logpuller/region_admission_controller.go index 5f0b04fd46..6994e6e52c 100644 --- a/logservice/logpuller/region_admission_controller.go +++ b/logservice/logpuller/region_admission_controller.go @@ -84,7 +84,7 @@ type regionAdmissionController struct { // currentWindow limits ordinary region scans. currentWindow int - // maxWindow is the hard limit for previously initialized and low-lag regions. + // maxWindow is the hard limit for high-priority region scans. maxWindow int // inflight is the number of admitted regions that have not finished their // initial scan. It is guarded by mu. diff --git a/logservice/logpuller/region_admission_controller_test.go b/logservice/logpuller/region_admission_controller_test.go index e8627aa98a..9e7c7104da 100644 --- a/logservice/logpuller/region_admission_controller_test.go +++ b/logservice/logpuller/region_admission_controller_test.go @@ -84,11 +84,10 @@ func TestRegionAdmissionControllerNormalWindow(t *testing.T) { require.True(t, req2.abort()) } -func TestRegionAdmissionControllerLowLagUsesMaxWindow(t *testing.T) { +func TestRegionAdmissionControllerHighPriorityUsesMaxWindow(t *testing.T) { controller := newRegionAdmissionController(1, 2) currentTs := oracle.GoTimeToTS(time.Now()) slowCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Hour)) - lowLagCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Minute)) submitRegionForAdmission(t, controller, prepareRegionForAdmission(createTestRegionInfo(1, 1), slowCheckpointTs), @@ -99,9 +98,9 @@ func TestRegionAdmissionControllerLowLagUsesMaxWindow(t *testing.T) { submitRegionForAdmission(t, controller, prepareRegionForAdmission(createTestRegionInfo(1, 2), slowCheckpointTs), currentTs) - submitRegionForAdmission(t, controller, - prepareRegionForAdmission(createTestRegionInfo(1, 3), lowLagCheckpointTs), - currentTs) + highPriorityRegion := prepareRegionForAdmission(createTestRegionInfo(1, 3), slowCheckpointTs) + highPriorityRegion.scanPriority = TaskHighPrior.scanPriority() + submitRegionForAdmission(t, controller, highPriorityRegion, currentTs) req2, err := controller.pop(t.Context(), nil) require.NoError(t, err) @@ -121,11 +120,10 @@ func TestRegionAdmissionControllerLowLagUsesMaxWindow(t *testing.T) { require.True(t, req3.abort()) } -func TestRegionAdmissionControllerPrioritizesInitializedRegion(t *testing.T) { +func TestRegionAdmissionControllerPrioritizesHighPriorityRegion(t *testing.T) { controller := newRegionAdmissionController(1, 2) currentTs := oracle.GoTimeToTS(time.Now()) slowCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Hour)) - lowLagCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Minute)) submitRegionForAdmission(t, controller, prepareRegionForAdmission(createTestRegionInfo(1, 1), slowCheckpointTs), @@ -134,12 +132,11 @@ func TestRegionAdmissionControllerPrioritizesInitializedRegion(t *testing.T) { require.NoError(t, err) submitRegionForAdmission(t, controller, - prepareRegionForAdmission(createTestRegionInfo(1, 2), lowLagCheckpointTs), + prepareRegionForAdmission(createTestRegionInfo(1, 2), slowCheckpointTs), currentTs) - initializedRegion := prepareRegionForAdmission(createTestRegionInfo(1, 3), slowCheckpointTs) - initializedRegion.wasInitialized = true - submitRegionForAdmission(t, controller, - initializedRegion, currentTs) + highPriorityRegion := prepareRegionForAdmission(createTestRegionInfo(1, 3), slowCheckpointTs) + highPriorityRegion.scanPriority = TaskHighPrior.scanPriority() + submitRegionForAdmission(t, controller, highPriorityRegion, currentTs) req2, err := controller.pop(t.Context(), nil) require.NoError(t, err) From 77e7c17f34d0c8444a4c5c2330e935e237ab17eb Mon Sep 17 00:00:00 2001 From: lidezhu Date: Sun, 2 Aug 2026 16:43:40 +0800 Subject: [PATCH 14/71] more refactor --- logservice/logpuller/priority_task.go | 73 ++++--------------- logservice/logpuller/priority_task_test.go | 52 +++++++------ .../region_admission_controller_test.go | 9 ++- .../logpuller/region_failure_handler.go | 2 +- .../logpuller/region_request_worker_test.go | 2 +- logservice/logpuller/region_state.go | 2 +- logservice/logpuller/scan_priority.go | 11 +-- logservice/logpuller/scan_priority_test.go | 40 +++++----- logservice/logpuller/subscription_client.go | 24 +++--- .../logpuller/subscription_client_test.go | 38 +++++----- 10 files changed, 109 insertions(+), 144 deletions(-) diff --git a/logservice/logpuller/priority_task.go b/logservice/logpuller/priority_task.go index dfb66315fd..5511006534 100644 --- a/logservice/logpuller/priority_task.go +++ b/logservice/logpuller/priority_task.go @@ -15,76 +15,35 @@ package logpuller import "github.com/pingcap/kvproto/pkg/cdcpb" -// TaskType represents the scan priority level associated with a region task. -type TaskType int - -const ( - // TaskHighPrior represents high scan priority. - // For example, a region task for a region that is already close to caught up - // is typically tagged with this level. - TaskHighPrior TaskType = iota - // TaskLowPrior represents low scan priority. - // For example, a region task created for a subscription that starts from an - // older start-ts is typically tagged with this level before it catches up. - TaskLowPrior -) - -func (t TaskType) String() string { - switch t { - case TaskHighPrior: - return "high" - case TaskLowPrior: - return "low" - default: - return "unknown" - } -} - -func (t TaskType) scanPriority() cdcpb.ScanPriority { - switch t { - case TaskHighPrior: - return cdcpb.ScanPriority_SCAN_PRIORITY_HIGH - case TaskLowPrior: - return cdcpb.ScanPriority_SCAN_PRIORITY_LOW - default: - return cdcpb.ScanPriority_SCAN_PRIORITY_LOW - } -} - -func taskTypeFromScanPriority(priority cdcpb.ScanPriority) TaskType { +func normalizeScanPriority(priority cdcpb.ScanPriority) cdcpb.ScanPriority { if priority == cdcpb.ScanPriority_SCAN_PRIORITY_HIGH { - return TaskHighPrior + return cdcpb.ScanPriority_SCAN_PRIORITY_HIGH } - return TaskLowPrior + return cdcpb.ScanPriority_SCAN_PRIORITY_LOW } -func normalizeScanPriority(priority cdcpb.ScanPriority) cdcpb.ScanPriority { - return taskTypeFromScanPriority(priority).scanPriority() +func isHighScanPriority(priority cdcpb.ScanPriority) bool { + return normalizeScanPriority(priority) == cdcpb.ScanPriority_SCAN_PRIORITY_HIGH } type regionPriorityTask struct { - taskType TaskType regionInfo regionInfo sequence uint64 heapIndex int // for heap.Item interface } // NewRegionPriorityTask creates a new priority task for region -func NewRegionPriorityTask(regionInfo regionInfo, _ uint64, sequence uint64) *regionPriorityTask { - task := ®ionPriorityTask{ - taskType: taskTypeFromScanPriority(regionInfo.scanPriority), - sequence: sequence, - heapIndex: 0, // 0 means not in heap +func NewRegionPriorityTask(regionInfo regionInfo, sequence uint64) *regionPriorityTask { + regionInfo.scanPriority = normalizeScanPriority(regionInfo.scanPriority) + return ®ionPriorityTask{ + regionInfo: regionInfo, + sequence: sequence, + heapIndex: 0, // 0 means not in heap } - task.updateRegion(regionInfo, 0) - return task } -// updateRegion refreshes both the request data and its priority before the task -// enters another scheduling stage. -func (pt *regionPriorityTask) updateRegion(regionInfo regionInfo, _ uint64) { - pt.regionInfo = regionInfo - pt.taskType = taskTypeFromScanPriority(regionInfo.scanPriority) +func (pt *regionPriorityTask) priority() cdcpb.ScanPriority { + return normalizeScanPriority(pt.regionInfo.scanPriority) } // GetRegionInfo returns the underlying regionInfo @@ -93,7 +52,7 @@ func (pt *regionPriorityTask) GetRegionInfo() regionInfo { } func (pt *regionPriorityTask) canUseMaxWindow() bool { - return pt.taskType == TaskHighPrior + return isHighScanPriority(pt.regionInfo.scanPriority) } // SetHeapIndex sets the heap index for heap.Item interface @@ -109,8 +68,8 @@ func (pt *regionPriorityTask) GetHeapIndex() int { // LessThan implements heap.Item interface. Tasks in the same priority class are // processed in submission order. func (pt *regionPriorityTask) LessThan(other *regionPriorityTask) bool { - if pt.taskType != other.taskType { - return pt.taskType < other.taskType + if isHighScanPriority(pt.regionInfo.scanPriority) != isHighScanPriority(other.regionInfo.scanPriority) { + return isHighScanPriority(pt.regionInfo.scanPriority) } return pt.sequence < other.sequence } diff --git a/logservice/logpuller/priority_task_test.go b/logservice/logpuller/priority_task_test.go index e0c1c99fd4..27ef0983ba 100644 --- a/logservice/logpuller/priority_task_test.go +++ b/logservice/logpuller/priority_task_test.go @@ -44,45 +44,44 @@ func newPriorityTestRegion( } } -func withScanPriority(region regionInfo, priority TaskType) regionInfo { - region.scanPriority = priority.scanPriority() +func withScanPriority(region regionInfo, priority cdcpb.ScanPriority) regionInfo { + region.scanPriority = priority return region } -func TestTaskTypeScanPriorityMapping(t *testing.T) { - require.Equal(t, cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, TaskHighPrior.scanPriority()) - require.Equal(t, cdcpb.ScanPriority_SCAN_PRIORITY_LOW, TaskLowPrior.scanPriority()) - require.Equal(t, TaskHighPrior, taskTypeFromScanPriority(cdcpb.ScanPriority_SCAN_PRIORITY_HIGH)) - require.Equal(t, TaskLowPrior, taskTypeFromScanPriority(cdcpb.ScanPriority_SCAN_PRIORITY_LOW)) - require.Equal(t, TaskLowPrior, taskTypeFromScanPriority(cdcpb.ScanPriority_SCAN_PRIORITY_UNKNOWN)) +func TestNormalizeScanPriority(t *testing.T) { + require.Equal(t, cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, normalizeScanPriority(cdcpb.ScanPriority_SCAN_PRIORITY_HIGH)) + require.Equal(t, cdcpb.ScanPriority_SCAN_PRIORITY_LOW, normalizeScanPriority(cdcpb.ScanPriority_SCAN_PRIORITY_LOW)) require.Equal(t, cdcpb.ScanPriority_SCAN_PRIORITY_LOW, normalizeScanPriority(cdcpb.ScanPriority_SCAN_PRIORITY_UNKNOWN)) + require.True(t, isHighScanPriority(cdcpb.ScanPriority_SCAN_PRIORITY_HIGH)) + require.False(t, isHighScanPriority(cdcpb.ScanPriority_SCAN_PRIORITY_LOW)) + require.False(t, isHighScanPriority(cdcpb.ScanPriority_SCAN_PRIORITY_UNKNOWN)) } func TestRegionPriorityTaskQueueOrder(t *testing.T) { queue := priorityqueue.New[*regionPriorityTask]() currentTime := time.Now() - currentTs := oracle.GoTimeToTS(currentTime) lowTask := NewRegionPriorityTask( withScanPriority( newPriorityTestRegion(1, oracle.GoTimeToTS(currentTime.Add(-time.Hour)), false), - TaskLowPrior, + cdcpb.ScanPriority_SCAN_PRIORITY_LOW, ), - currentTs, 3, + 3, ) highTask1 := NewRegionPriorityTask( withScanPriority( newPriorityTestRegion(2, oracle.GoTimeToTS(currentTime.Add(-10*time.Minute)), false), - TaskHighPrior, + cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, ), - currentTs, 2, + 2, ) highTask2 := NewRegionPriorityTask( withScanPriority( newPriorityTestRegion(3, oracle.GoTimeToTS(currentTime.Add(-time.Hour)), true), - TaskHighPrior, + cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, ), - currentTs, 1, + 1, ) require.True(t, queue.Push(lowTask)) @@ -99,13 +98,12 @@ func TestRegionPriorityTaskQueueOrder(t *testing.T) { func TestRegionPriorityTaskFIFOWithinPriority(t *testing.T) { queue := priorityqueue.New[*regionPriorityTask]() currentTime := time.Now() - currentTs := oracle.GoTimeToTS(currentTime) checkpointTs := oracle.GoTimeToTS(currentTime.Add(-time.Hour)) first := NewRegionPriorityTask( - withScanPriority(newPriorityTestRegion(1, checkpointTs, false), TaskHighPrior), currentTs, 1) + withScanPriority(newPriorityTestRegion(1, checkpointTs, false), cdcpb.ScanPriority_SCAN_PRIORITY_HIGH), 1) second := NewRegionPriorityTask( - withScanPriority(newPriorityTestRegion(2, checkpointTs, false), TaskHighPrior), currentTs, 2) + withScanPriority(newPriorityTestRegion(2, checkpointTs, false), cdcpb.ScanPriority_SCAN_PRIORITY_HIGH), 2) require.True(t, queue.Push(second)) require.True(t, queue.Push(first)) @@ -120,20 +118,20 @@ func TestRegionPriorityTaskFIFOWithinPriority(t *testing.T) { func TestRegionPriorityTaskUsesHighPriorityWindow(t *testing.T) { highTask := NewRegionPriorityTask( - withScanPriority(newPriorityTestRegion(1, 1, false), TaskHighPrior), 0, 1) + withScanPriority(newPriorityTestRegion(1, 1, false), cdcpb.ScanPriority_SCAN_PRIORITY_HIGH), 1) lowTask := NewRegionPriorityTask( - withScanPriority(newPriorityTestRegion(2, 1, true), TaskLowPrior), 0, 2) + withScanPriority(newPriorityTestRegion(2, 1, true), cdcpb.ScanPriority_SCAN_PRIORITY_LOW), 2) require.True(t, highTask.canUseMaxWindow()) require.False(t, lowTask.canUseMaxWindow()) } -func TestRegionPriorityTaskRefreshesPriorityBetweenStages(t *testing.T) { - region := withScanPriority(newPriorityTestRegion(1, 1, false), TaskLowPrior) - task := NewRegionPriorityTask(region, 0, 1) - require.Equal(t, TaskLowPrior, task.taskType) +func TestRegionPriorityTaskRefreshesRegionInfoBetweenStages(t *testing.T) { + region := withScanPriority(newPriorityTestRegion(1, 1, false), cdcpb.ScanPriority_SCAN_PRIORITY_LOW) + task := NewRegionPriorityTask(region, 1) + require.Equal(t, cdcpb.ScanPriority_SCAN_PRIORITY_LOW, task.priority()) - region.scanPriority = TaskHighPrior.scanPriority() - task.updateRegion(region, 0) - require.Equal(t, TaskHighPrior, task.taskType) + region.scanPriority = cdcpb.ScanPriority_SCAN_PRIORITY_HIGH + task.regionInfo = region + require.Equal(t, cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, task.priority()) } diff --git a/logservice/logpuller/region_admission_controller_test.go b/logservice/logpuller/region_admission_controller_test.go index 9e7c7104da..476e0637fe 100644 --- a/logservice/logpuller/region_admission_controller_test.go +++ b/logservice/logpuller/region_admission_controller_test.go @@ -20,6 +20,7 @@ import ( "testing" "time" + "github.com/pingcap/kvproto/pkg/cdcpb" "github.com/pingcap/ticdc/heartbeatpb" "github.com/pingcap/ticdc/logservice/logpuller/regionlock" "github.com/stretchr/testify/require" @@ -55,7 +56,7 @@ func submitRegionForAdmission( currentTs uint64, ) { t.Helper() - task := NewRegionPriorityTask(region, currentTs, region.verID.GetID()) + task := NewRegionPriorityTask(region, region.verID.GetID()) require.True(t, controller.submit(task)) } @@ -99,7 +100,7 @@ func TestRegionAdmissionControllerHighPriorityUsesMaxWindow(t *testing.T) { prepareRegionForAdmission(createTestRegionInfo(1, 2), slowCheckpointTs), currentTs) highPriorityRegion := prepareRegionForAdmission(createTestRegionInfo(1, 3), slowCheckpointTs) - highPriorityRegion.scanPriority = TaskHighPrior.scanPriority() + highPriorityRegion.scanPriority = cdcpb.ScanPriority_SCAN_PRIORITY_HIGH submitRegionForAdmission(t, controller, highPriorityRegion, currentTs) req2, err := controller.pop(t.Context(), nil) @@ -135,7 +136,7 @@ func TestRegionAdmissionControllerPrioritizesHighPriorityRegion(t *testing.T) { prepareRegionForAdmission(createTestRegionInfo(1, 2), slowCheckpointTs), currentTs) highPriorityRegion := prepareRegionForAdmission(createTestRegionInfo(1, 3), slowCheckpointTs) - highPriorityRegion.scanPriority = TaskHighPrior.scanPriority() + highPriorityRegion.scanPriority = cdcpb.ScanPriority_SCAN_PRIORITY_HIGH submitRegionForAdmission(t, controller, highPriorityRegion, currentTs) req2, err := controller.pop(t.Context(), nil) @@ -190,7 +191,7 @@ func TestRegionAdmissionControllerClose(t *testing.T) { controller := newRegionAdmissionController(1, 1) controller.close() region := prepareRegionForAdmission(createTestRegionInfo(1, 1), 1) - require.False(t, controller.submit(NewRegionPriorityTask(region, 1, 1))) + require.False(t, controller.submit(NewRegionPriorityTask(region, 1))) _, err := controller.pop(context.Background(), nil) require.ErrorIs(t, err, context.Canceled) diff --git a/logservice/logpuller/region_failure_handler.go b/logservice/logpuller/region_failure_handler.go index abb681d538..d97f12f202 100644 --- a/logservice/logpuller/region_failure_handler.go +++ b/logservice/logpuller/region_failure_handler.go @@ -107,7 +107,7 @@ func (r *regionFailureHandler) Run(ctx context.Context) error { func (r *regionFailureHandler) handleError(ctx context.Context, errInfo regionErrorInfo) error { err := errors.Cause(errInfo.err) - retryPriority := taskTypeFromScanPriority(errInfo.scanPriority) + retryPriority := normalizeScanPriority(errInfo.scanPriority) //nolint:errorlint // converting large type switch to errors.As is a significant refactor if _, requestCancelled := err.(*requestCancelledErr); !requestCancelled { log.Debug("cdc region error", diff --git a/logservice/logpuller/region_request_worker_test.go b/logservice/logpuller/region_request_worker_test.go index 3b41270f09..a7a145c178 100644 --- a/logservice/logpuller/region_request_worker_test.go +++ b/logservice/logpuller/region_request_worker_test.go @@ -438,7 +438,7 @@ func TestFailPendingRegionsReschedulesWorkerBuffer(t *testing.T) { }, } for i, region := range regions { - require.True(t, admission.submit(NewRegionPriorityTask(region, 1, uint64(i+1)))) + require.True(t, admission.submit(NewRegionPriorityTask(region, uint64(i+1)))) } worker.failPendingRegions(&storeStreamErr{}) diff --git a/logservice/logpuller/region_state.go b/logservice/logpuller/region_state.go index abf142c7fc..6cd014a1d2 100644 --- a/logservice/logpuller/region_state.go +++ b/logservice/logpuller/region_state.go @@ -69,7 +69,7 @@ func newRegionInfo( rpcCtx: rpcCtx, subscribedSpan: subscribedSpan, filterLoop: filterLoop, - scanPriority: TaskLowPrior.scanPriority(), + scanPriority: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, } } diff --git a/logservice/logpuller/scan_priority.go b/logservice/logpuller/scan_priority.go index 7f8480265a..176b9122d5 100644 --- a/logservice/logpuller/scan_priority.go +++ b/logservice/logpuller/scan_priority.go @@ -18,6 +18,7 @@ import ( "sync/atomic" "time" + "github.com/pingcap/kvproto/pkg/cdcpb" "github.com/pingcap/ticdc/pkg/pdutil" "github.com/tikv/client-go/v2/oracle" ) @@ -48,14 +49,14 @@ func (p *scanPriorityPolicy) observeSpanResolved(resolvedTs uint64) bool { // resolve returns the effective priority after combining inherited, span, and // region progress. A high priority decision is never downgraded. func (p *scanPriorityPolicy) resolve( - inherited TaskType, + inherited cdcpb.ScanPriority, regionResolvedTs uint64, currentTime time.Time, -) TaskType { - if inherited == TaskHighPrior || p.everCaughtUp.Load() || p.isTsClose(regionResolvedTs, currentTime) { - return TaskHighPrior +) cdcpb.ScanPriority { + if isHighScanPriority(inherited) || p.everCaughtUp.Load() || p.isTsClose(regionResolvedTs, currentTime) { + return cdcpb.ScanPriority_SCAN_PRIORITY_HIGH } - return TaskLowPrior + return cdcpb.ScanPriority_SCAN_PRIORITY_LOW } func (p *scanPriorityPolicy) isTsClose(ts uint64, currentTime time.Time) bool { diff --git a/logservice/logpuller/scan_priority_test.go b/logservice/logpuller/scan_priority_test.go index 676ef363dc..e42d0b9a9c 100644 --- a/logservice/logpuller/scan_priority_test.go +++ b/logservice/logpuller/scan_priority_test.go @@ -37,45 +37,45 @@ func TestScanPriorityPolicyResolve(t *testing.T) { for _, tc := range []struct { name string - inherited TaskType + inherited cdcpb.ScanPriority regionResolvedTs uint64 - expected TaskType + expected cdcpb.ScanPriority }{ { name: "zero resolved ts", - inherited: TaskLowPrior, + inherited: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, regionResolvedTs: 0, - expected: TaskLowPrior, + expected: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, }, { name: "recent region", - inherited: TaskLowPrior, + inherited: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, regionResolvedTs: oracle.GoTimeToTS(currentTime.Add(-29 * time.Minute)), - expected: TaskHighPrior, + expected: cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, }, { name: "threshold boundary", - inherited: TaskLowPrior, + inherited: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, regionResolvedTs: oracle.GoTimeToTS(currentTime.Add(-30 * time.Minute)), - expected: TaskHighPrior, + expected: cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, }, { name: "old region", - inherited: TaskLowPrior, + inherited: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, regionResolvedTs: oracle.GoTimeToTS(currentTime.Add(-31 * time.Minute)), - expected: TaskLowPrior, + expected: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, }, { name: "future region", - inherited: TaskLowPrior, + inherited: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, regionResolvedTs: oracle.GoTimeToTS(currentTime.Add(time.Minute)), - expected: TaskHighPrior, + expected: cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, }, { name: "inherited high", - inherited: TaskHighPrior, + inherited: cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, regionResolvedTs: oracle.GoTimeToTS(currentTime.Add(-time.Hour)), - expected: TaskHighPrior, + expected: cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, }, } { t.Run(tc.name, func(t *testing.T) { @@ -93,8 +93,8 @@ func TestScanPriorityPolicyRemainsHighAfterCatchUp(t *testing.T) { require.False(t, policy.observeSpanResolved(oracle.GoTimeToTS(currentTime.Add(-31*time.Minute)))) require.True(t, policy.observeSpanResolved(oracle.GoTimeToTS(currentTime.Add(-time.Minute)))) require.False(t, policy.observeSpanResolved(oracle.GoTimeToTS(currentTime))) - require.Equal(t, TaskHighPrior, policy.resolve( - TaskLowPrior, + require.Equal(t, cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, policy.resolve( + cdcpb.ScanPriority_SCAN_PRIORITY_LOW, oracle.GoTimeToTS(currentTime.Add(-time.Hour)), currentTime, )) @@ -125,9 +125,9 @@ func TestScanPriorityUsesRestoredRegionProgress(t *testing.T) { } region := newRegionInfo(tikv.NewRegionVerID(1, 1, 1), rawSpan, nil, span, false) - client.scheduleRegionRequest(context.Background(), region, TaskLowPrior) + client.scheduleRegionRequest(context.Background(), region, cdcpb.ScanPriority_SCAN_PRIORITY_LOW) firstTask := popRegionPriorityTask(t, client.regionTaskQueue) - require.Equal(t, TaskLowPrior, firstTask.taskType) + require.Equal(t, cdcpb.ScanPriority_SCAN_PRIORITY_LOW, firstTask.priority()) firstRegion := firstTask.GetRegionInfo() firstRegion.lockedRangeState.ResolvedTs.Store(oracle.GoTimeToTS(currentTime.Add(-time.Minute))) @@ -139,9 +139,9 @@ func TestScanPriorityUsesRestoredRegionProgress(t *testing.T) { ) retryRegion := newRegionInfo(tikv.NewRegionVerID(1, 1, 2), rawSpan, nil, span, false) - client.scheduleRegionRequest(context.Background(), retryRegion, TaskLowPrior) + client.scheduleRegionRequest(context.Background(), retryRegion, cdcpb.ScanPriority_SCAN_PRIORITY_LOW) retryTask := popRegionPriorityTask(t, client.regionTaskQueue) - require.Equal(t, TaskHighPrior, retryTask.taskType) + require.Equal(t, cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, retryTask.priority()) require.Equal(t, cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, retryTask.GetRegionInfo().scanPriority) require.False(t, span.priorityPolicy.everCaughtUp.Load()) } diff --git a/logservice/logpuller/subscription_client.go b/logservice/logpuller/subscription_client.go index 6f25c89716..b45c83cc08 100644 --- a/logservice/logpuller/subscription_client.go +++ b/logservice/logpuller/subscription_client.go @@ -19,6 +19,7 @@ import ( "sync/atomic" "time" + "github.com/pingcap/kvproto/pkg/cdcpb" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/log" "github.com/pingcap/ticdc/heartbeatpb" @@ -89,7 +90,7 @@ type rangeTask struct { span heartbeatpb.TableSpan subscribedSpan *subscribedSpan filterLoop bool - priority TaskType + priority cdcpb.ScanPriority wasInitialized bool } @@ -268,7 +269,12 @@ func (s *subscriptionClient) Subscribe( select { case <-s.ctx.Done(): log.Warn("subscribes span failed, the subscription client has closed") - case s.rangeTaskCh <- rangeTask{span: span, subscribedSpan: rt, filterLoop: rt.filterLoop, priority: TaskLowPrior}: + case s.rangeTaskCh <- rangeTask{ + span: span, + subscribedSpan: rt, + filterLoop: rt.filterLoop, + priority: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, + }: log.Info("subscribes span done", zap.Uint64("subscriptionID", uint64(subID)), zap.Int64("tableID", span.TableID), zap.Uint64("startTs", startTs), @@ -467,7 +473,7 @@ func (s *subscriptionClient) handleRegions(ctx context.Context, eg *errgroup.Gro } store := getStore(region.rpcCtx.Addr) - regionTask.updateRegion(region, s.pdClock.CurrentTS()) + regionTask.regionInfo = region if !store.submit(regionTask) { return context.Canceled } @@ -535,7 +541,7 @@ func (s *subscriptionClient) divideSpanAndScheduleRegionRequests( span heartbeatpb.TableSpan, subscribedSpan *subscribedSpan, filterLoop bool, - inheritedPriority TaskType, + inheritedPriority cdcpb.ScanPriority, wasInitialized bool, ) error { // Limit the number of regions loaded at a time to make the load more stable. @@ -618,7 +624,7 @@ func (s *subscriptionClient) divideSpanAndScheduleRegionRequests( func (s *subscriptionClient) scheduleRegionRequest( ctx context.Context, region regionInfo, - inheritedPriority TaskType, + inheritedPriority cdcpb.ScanPriority, ) { if region.lockedRangeState != nil && region.lockedRangeState.Initialized.Load() { region.wasInitialized = true @@ -639,8 +645,8 @@ func (s *subscriptionClient) scheduleRegionRequest( region.resolvedTs(), oracle.GetTimeFromTS(currentTs), ) - region.scanPriority = priority.scanPriority() - s.regionTaskQueue.Push(NewRegionPriorityTask(region, currentTs, s.regionTaskSequence.Add(1))) + region.scanPriority = priority + s.regionTaskQueue.Push(NewRegionPriorityTask(region, s.regionTaskSequence.Add(1))) if log.GetLevel() <= zapcore.DebugLevel { log.Debug("cdc region scan task enqueued", zap.Uint64("subscriptionID", uint64(region.subscribedSpan.subID)), @@ -649,7 +655,7 @@ func (s *subscriptionClient) scheduleRegionRequest( zap.Uint64("regionID", region.verID.GetID()), zap.Uint64("regionEpochVersion", region.verID.GetVer()), zap.Uint64("regionEpochConfVer", region.verID.GetConfVer()), - zap.String("priority", priority.String()), + zap.String("priority", normalizeScanPriority(priority).String()), zap.String("scanPriority", region.scanPriority.String()), zap.String("span", common.FormatTableSpan(®ion.span))) } @@ -668,7 +674,7 @@ func (s *subscriptionClient) scheduleRangeRequest( subscribedSpan *subscribedSpan, filterLoop bool, wasInitialized bool, - inheritedPriority TaskType, + inheritedPriority cdcpb.ScanPriority, ) { select { case <-ctx.Done(): diff --git a/logservice/logpuller/subscription_client_test.go b/logservice/logpuller/subscription_client_test.go index 7ed9a556b5..3ab9819db6 100644 --- a/logservice/logpuller/subscription_client_test.go +++ b/logservice/logpuller/subscription_client_test.go @@ -401,50 +401,50 @@ func TestRegionRetryScanPriority(t *testing.T) { priority cdcpb.ScanPriority cdcErr *cdcpb.Error everCaughtUp bool - expected TaskType + expected cdcpb.ScanPriority }{ { name: "server is busy high", priority: cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, cdcErr: &cdcpb.Error{ServerIsBusy: &errorpb.ServerIsBusy{}}, - expected: TaskHighPrior, + expected: cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, }, { name: "server is busy low", priority: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, cdcErr: &cdcpb.Error{ServerIsBusy: &errorpb.ServerIsBusy{}}, - expected: TaskLowPrior, + expected: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, }, { name: "server is busy low after catch up", priority: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, cdcErr: &cdcpb.Error{ServerIsBusy: &errorpb.ServerIsBusy{}}, everCaughtUp: true, - expected: TaskHighPrior, + expected: cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, }, { name: "congested high", priority: cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, cdcErr: &cdcpb.Error{Congested: &cdcpb.Congested{}}, - expected: TaskHighPrior, + expected: cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, }, { name: "congested low", priority: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, cdcErr: &cdcpb.Error{Congested: &cdcpb.Congested{}}, - expected: TaskLowPrior, + expected: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, }, { name: "unknown retry high", priority: cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, cdcErr: &cdcpb.Error{}, - expected: TaskHighPrior, + expected: cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, }, { name: "unknown retry low", priority: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, cdcErr: &cdcpb.Error{}, - expected: TaskLowPrior, + expected: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, }, } { t.Run(tc.name, func(t *testing.T) { @@ -466,8 +466,8 @@ func TestRegionRetryScanPriority(t *testing.T) { defer cancel() task, err := client.regionTaskQueue.Pop(ctx) require.NoError(t, err) - require.Equal(t, tc.expected, task.taskType) - require.Equal(t, tc.expected.scanPriority(), task.GetRegionInfo().scanPriority) + require.Equal(t, tc.expected, task.priority()) + require.Equal(t, tc.expected, task.GetRegionInfo().scanPriority) }) } } @@ -477,43 +477,43 @@ func TestRangeRetryPreservesScanPriority(t *testing.T) { name string priority cdcpb.ScanPriority err error - expected TaskType + expected cdcpb.ScanPriority }{ { name: "epoch not match high", priority: cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, err: &eventError{err: &cdcpb.Error{EpochNotMatch: &errorpb.EpochNotMatch{}}}, - expected: TaskHighPrior, + expected: cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, }, { name: "epoch not match low", priority: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, err: &eventError{err: &cdcpb.Error{EpochNotMatch: &errorpb.EpochNotMatch{}}}, - expected: TaskLowPrior, + expected: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, }, { name: "region not found high", priority: cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, err: &eventError{err: &cdcpb.Error{RegionNotFound: &errorpb.RegionNotFound{}}}, - expected: TaskHighPrior, + expected: cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, }, { name: "region not found low", priority: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, err: &eventError{err: &cdcpb.Error{RegionNotFound: &errorpb.RegionNotFound{}}}, - expected: TaskLowPrior, + expected: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, }, { name: "rpc context unavailable high", priority: cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, err: &rpcCtxUnavailableErr{verID: tikv.NewRegionVerID(1, 1, 1)}, - expected: TaskHighPrior, + expected: cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, }, { name: "rpc context unavailable low", priority: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, err: &rpcCtxUnavailableErr{verID: tikv.NewRegionVerID(1, 1, 1)}, - expected: TaskLowPrior, + expected: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, }, } { t.Run(tc.name, func(t *testing.T) { @@ -639,7 +639,7 @@ func TestBroadcastDeregisterUsesWorkerControlQueue(t *testing.T) { subscribedSpan: &subscribedSpan{subID: SubscriptionID(2)}, lockedRangeState: ®ionlock.LockedRangeState{}, } - require.True(t, admission.submit(NewRegionPriorityTask(dummyRegion, 1, 1))) + require.True(t, admission.submit(NewRegionPriorityTask(dummyRegion, 1))) client.broadcastDeregister(SubscriptionID(1), true) require.Equal(t, 1, worker.controlQueue.len()) @@ -662,7 +662,7 @@ func TestRequestedStoreDistributesRegionsAcrossWorkerBuffers(t *testing.T) { subscribedSpan: &subscribedSpan{subID: 1}, lockedRangeState: ®ionlock.LockedRangeState{}, } - require.True(t, store.submit(NewRegionPriorityTask(region, 1, i))) + require.True(t, store.submit(NewRegionPriorityTask(region, i))) } require.Equal(t, 2, worker1.admission.stats().pending) From d46539f0916f477b685c1bd56f7aa415caa6cc9a Mon Sep 17 00:00:00 2001 From: lidezhu Date: Sun, 2 Aug 2026 18:23:06 +0800 Subject: [PATCH 15/71] more refactor --- logservice/logpuller/priority_task_test.go | 18 +++-- .../logpuller/region_failure_handler.go | 8 +-- logservice/logpuller/region_request_worker.go | 70 +++++++++---------- .../logpuller/region_request_worker_test.go | 13 ++-- logservice/logpuller/region_state.go | 6 -- logservice/logpuller/region_tracker.go | 10 +-- logservice/logpuller/region_tracker_test.go | 4 +- logservice/logpuller/subscription_client.go | 12 +--- 8 files changed, 64 insertions(+), 77 deletions(-) diff --git a/logservice/logpuller/priority_task_test.go b/logservice/logpuller/priority_task_test.go index 27ef0983ba..71ea50f1ce 100644 --- a/logservice/logpuller/priority_task_test.go +++ b/logservice/logpuller/priority_task_test.go @@ -30,7 +30,6 @@ import ( func newPriorityTestRegion( regionID uint64, checkpointTs uint64, - wasInitialized bool, ) regionInfo { span := heartbeatpb.TableSpan{TableID: 1, StartKey: []byte("a"), EndKey: []byte("z")} state := ®ionlock.LockedRangeState{} @@ -40,7 +39,6 @@ func newPriorityTestRegion( span: span, subscribedSpan: &subscribedSpan{subID: 1, startTs: checkpointTs, span: span}, lockedRangeState: state, - wasInitialized: wasInitialized, } } @@ -64,21 +62,21 @@ func TestRegionPriorityTaskQueueOrder(t *testing.T) { lowTask := NewRegionPriorityTask( withScanPriority( - newPriorityTestRegion(1, oracle.GoTimeToTS(currentTime.Add(-time.Hour)), false), + newPriorityTestRegion(1, oracle.GoTimeToTS(currentTime.Add(-time.Hour))), cdcpb.ScanPriority_SCAN_PRIORITY_LOW, ), 3, ) highTask1 := NewRegionPriorityTask( withScanPriority( - newPriorityTestRegion(2, oracle.GoTimeToTS(currentTime.Add(-10*time.Minute)), false), + newPriorityTestRegion(2, oracle.GoTimeToTS(currentTime.Add(-10*time.Minute))), cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, ), 2, ) highTask2 := NewRegionPriorityTask( withScanPriority( - newPriorityTestRegion(3, oracle.GoTimeToTS(currentTime.Add(-time.Hour)), true), + newPriorityTestRegion(3, oracle.GoTimeToTS(currentTime.Add(-time.Hour))), cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, ), 1, @@ -101,9 +99,9 @@ func TestRegionPriorityTaskFIFOWithinPriority(t *testing.T) { checkpointTs := oracle.GoTimeToTS(currentTime.Add(-time.Hour)) first := NewRegionPriorityTask( - withScanPriority(newPriorityTestRegion(1, checkpointTs, false), cdcpb.ScanPriority_SCAN_PRIORITY_HIGH), 1) + withScanPriority(newPriorityTestRegion(1, checkpointTs), cdcpb.ScanPriority_SCAN_PRIORITY_HIGH), 1) second := NewRegionPriorityTask( - withScanPriority(newPriorityTestRegion(2, checkpointTs, false), cdcpb.ScanPriority_SCAN_PRIORITY_HIGH), 2) + withScanPriority(newPriorityTestRegion(2, checkpointTs), cdcpb.ScanPriority_SCAN_PRIORITY_HIGH), 2) require.True(t, queue.Push(second)) require.True(t, queue.Push(first)) @@ -118,16 +116,16 @@ func TestRegionPriorityTaskFIFOWithinPriority(t *testing.T) { func TestRegionPriorityTaskUsesHighPriorityWindow(t *testing.T) { highTask := NewRegionPriorityTask( - withScanPriority(newPriorityTestRegion(1, 1, false), cdcpb.ScanPriority_SCAN_PRIORITY_HIGH), 1) + withScanPriority(newPriorityTestRegion(1, 1), cdcpb.ScanPriority_SCAN_PRIORITY_HIGH), 1) lowTask := NewRegionPriorityTask( - withScanPriority(newPriorityTestRegion(2, 1, true), cdcpb.ScanPriority_SCAN_PRIORITY_LOW), 2) + withScanPriority(newPriorityTestRegion(2, 1), cdcpb.ScanPriority_SCAN_PRIORITY_LOW), 2) require.True(t, highTask.canUseMaxWindow()) require.False(t, lowTask.canUseMaxWindow()) } func TestRegionPriorityTaskRefreshesRegionInfoBetweenStages(t *testing.T) { - region := withScanPriority(newPriorityTestRegion(1, 1, false), cdcpb.ScanPriority_SCAN_PRIORITY_LOW) + region := withScanPriority(newPriorityTestRegion(1, 1), cdcpb.ScanPriority_SCAN_PRIORITY_LOW) task := NewRegionPriorityTask(region, 1) require.Equal(t, cdcpb.ScanPriority_SCAN_PRIORITY_LOW, task.priority()) diff --git a/logservice/logpuller/region_failure_handler.go b/logservice/logpuller/region_failure_handler.go index d97f12f202..879fe0a090 100644 --- a/logservice/logpuller/region_failure_handler.go +++ b/logservice/logpuller/region_failure_handler.go @@ -129,13 +129,13 @@ func (r *regionFailureHandler) handleError(ctx context.Context, errInfo regionEr if innerErr.GetEpochNotMatch() != nil { metricFeedEpochNotMatchCounter.Inc() r.client.scheduleRangeRequest( - ctx, errInfo.span, errInfo.subscribedSpan, errInfo.filterLoop, errInfo.wasInitialized, retryPriority) + ctx, errInfo.span, errInfo.subscribedSpan, errInfo.filterLoop, retryPriority) return nil } if innerErr.GetRegionNotFound() != nil { metricFeedRegionNotFoundCounter.Inc() r.client.scheduleRangeRequest( - ctx, errInfo.span, errInfo.subscribedSpan, errInfo.filterLoop, errInfo.wasInitialized, retryPriority) + ctx, errInfo.span, errInfo.subscribedSpan, errInfo.filterLoop, retryPriority) return nil } if innerErr.GetCongested() != nil { @@ -169,7 +169,7 @@ func (r *regionFailureHandler) handleError(ctx context.Context, errInfo regionEr case *rpcCtxUnavailableErr: metricFeedRPCCtxUnavailable.Inc() r.client.scheduleRangeRequest( - ctx, errInfo.span, errInfo.subscribedSpan, errInfo.filterLoop, errInfo.wasInitialized, retryPriority) + ctx, errInfo.span, errInfo.subscribedSpan, errInfo.filterLoop, retryPriority) return nil case *getStoreErr: metricGetStoreErr.Inc() @@ -177,7 +177,7 @@ func (r *regionFailureHandler) handleError(ctx context.Context, errInfo regionEr // cannot get the store the region belongs to, so we need to reload the region. r.client.regionCache.OnSendFail(bo, errInfo.rpcCtx, true, err) r.client.scheduleRangeRequest( - ctx, errInfo.span, errInfo.subscribedSpan, errInfo.filterLoop, errInfo.wasInitialized, retryPriority) + ctx, errInfo.span, errInfo.subscribedSpan, errInfo.filterLoop, retryPriority) return nil case *storeStreamErr: metricStoreSendRequestErr.Inc() diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index eac17cc648..dec0c61c93 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -115,6 +115,27 @@ func newRegionRequestWorker( } func (s *regionRequestWorker) Run(ctx context.Context) error { + handleStreamFailure := func(firstReq *regionReq, regionErr error) error { + // Stream failure handle cases: + // - tracker: requests already sent to this stream. + // - firstReq: popped from admission for this stream, but not necessarily + // added to tracker yet if the stream fails before sendRegionRequest calls + // tracker.Add. + // - admission: requests owned by this worker but not sent yet. + for _, state := range s.tracker.Drain() { + s.notifyRegionError(state, regionErr) + } + // The failed stream no longer owns remote registrations. + s.controlQueue.drain() + if firstReq.abort() { + s.client.onRegionFail(newRegionErrorInfo(firstReq.regionInfo, regionErr)) + } + for _, task := range s.admission.drain() { + s.client.onRegionFail(newRegionErrorInfo(task.regionInfo, regionErr)) + } + return util.Hang(ctx, storeReconnectBackoff) + } + for { // Do not connect an idle worker to an unavailable store indefinitely. firstReq, err := s.waitForRegionRequest(ctx) @@ -122,57 +143,36 @@ func (s *regionRequestWorker) Run(ctx context.Context) error { return err } - regionErr := error(&storeStreamErr{}) if err := s.checkStoreVersion(ctx); err != nil { - regionErr = err - } else if err := s.runStream(ctx, firstReq); err != nil { - regionErr = err + if ctx.Err() != nil { + firstReq.abort() + return ctx.Err() + } + if err := handleStreamFailure(firstReq, err); err != nil { + return err + } + continue } + + regionErr := s.runStream(ctx, firstReq) if ctx.Err() != nil { firstReq.abort() return ctx.Err() } - - // Stop sent requests first so their states release the admission leases. - // firstReq still owns its lease only if the stream failed before Send. - s.failStreamRegions(regionErr) - if firstReq.abort() { - s.client.onRegionFail(newRegionErrorInfo(firstReq.regionInfo, regionErr)) + if regionErr == nil { + regionErr = &storeStreamErr{} } - s.failPendingRegions(regionErr) - - if err := util.Hang(ctx, storeReconnectBackoff); err != nil { + if err := handleStreamFailure(firstReq, regionErr); err != nil { return err } } } -// failStreamRegions transfers every request sent by a failed stream to the -// recovery pipeline. -func (s *regionRequestWorker) failStreamRegions(err error) { - for _, states := range s.tracker.Drain() { - for _, state := range states { - s.notifyRegionError(state, err) - } - } - // The failed stream no longer owns remote registrations. - s.controlQueue.drain() -} - -// failPendingRegions transfers requests owned by this worker but not yet sent -// to the recovery pipeline, so they can be resolved and routed again. -func (s *regionRequestWorker) failPendingRegions(err error) { - for _, task := range s.admission.drain() { - s.client.onRegionFail(newRegionErrorInfo(task.regionInfo, err)) - } -} - func (s *regionRequestWorker) notifyRegionError(state *regionFeedState, err error) { state.markStopped(err) s.client.eventSink.Push( SubscriptionID(state.requestID), - regionEvent{states: []*regionFeedState{state}}, - ) + regionEvent{states: []*regionFeedState{state}}) } func (s *regionRequestWorker) waitForRegionRequest(ctx context.Context) (*regionReq, error) { diff --git a/logservice/logpuller/region_request_worker_test.go b/logservice/logpuller/region_request_worker_test.go index a7a145c178..c7d6268248 100644 --- a/logservice/logpuller/region_request_worker_test.go +++ b/logservice/logpuller/region_request_worker_test.go @@ -378,7 +378,7 @@ func TestStoppedStateRemovesSentRequest(t *testing.T) { require.Equal(t, 0, admission.stats().inflight) } -func TestFailStreamRegionsReleasesSentAdmission(t *testing.T) { +func TestStreamRecoveryReleasesSentAdmission(t *testing.T) { admission := newRegionAdmissionController(1, 1) ds := &mockRegionEventDynamicStream{} worker := ®ionRequestWorker{ @@ -394,14 +394,17 @@ func TestFailStreamRegionsReleasesSentAdmission(t *testing.T) { state := newRegionFeedState(region, uint64(region.subscribedSpan.subID), worker, req) require.True(t, worker.tracker.Add(region.subscribedSpan.subID, region.verID.GetID(), state)) - worker.failStreamRegions(&storeStreamErr{}) + for _, state := range worker.tracker.Drain() { + worker.notifyRegionError(state, &storeStreamErr{}) + } + worker.controlQueue.drain() require.Zero(t, admission.stats().inflight) require.False(t, req.abort()) require.Equal(t, 1, ds.pushCount) } -func TestFailPendingRegionsReschedulesWorkerBuffer(t *testing.T) { +func TestStreamRecoveryReschedulesWorkerBuffer(t *testing.T) { rawSpan := heartbeatpb.TableSpan{ TableID: 1, StartKey: []byte("a"), @@ -441,7 +444,9 @@ func TestFailPendingRegionsReschedulesWorkerBuffer(t *testing.T) { require.True(t, admission.submit(NewRegionPriorityTask(region, uint64(i+1)))) } - worker.failPendingRegions(&storeStreamErr{}) + for _, task := range worker.admission.drain() { + worker.client.onRegionFail(newRegionErrorInfo(task.regionInfo, &storeStreamErr{})) + } require.Zero(t, admission.stats().pending) require.Len(t, client.failureHandler.cache.cache, 2) diff --git a/logservice/logpuller/region_state.go b/logservice/logpuller/region_state.go index 6cd014a1d2..0f2f84ce3e 100644 --- a/logservice/logpuller/region_state.go +++ b/logservice/logpuller/region_state.go @@ -45,9 +45,6 @@ type regionInfo struct { subscribedSpan *subscribedSpan // The state of the locked range of the region. lockedRangeState *regionlock.LockedRangeState - // wasInitialized preserves scheduling priority while a failed region is - // unlocked, resolved again, and subscribed with a new locked range state. - wasInitialized bool // Whether to filter out the value write by cdc itself. // It should be `true` in BDR mode filterLoop bool @@ -83,9 +80,6 @@ type regionErrorInfo struct { } func newRegionErrorInfo(info regionInfo, err error) regionErrorInfo { - if info.lockedRangeState != nil && info.lockedRangeState.Initialized.Load() { - info.wasInitialized = true - } return regionErrorInfo{ regionInfo: info, err: err, diff --git a/logservice/logpuller/region_tracker.go b/logservice/logpuller/region_tracker.go index 866ee49f73..dad8e6df69 100644 --- a/logservice/logpuller/region_tracker.go +++ b/logservice/logpuller/region_tracker.go @@ -105,16 +105,16 @@ func (t *regionTracker) TakeSubscription(subscriptionID SubscriptionID) []*regio return slices.Collect(maps.Values(states)) } -// Drain removes and returns all tracked states grouped by subscription. -func (t *regionTracker) Drain() map[SubscriptionID][]*regionFeedState { +// Drain removes and returns all tracked states. +func (t *regionTracker) Drain() []*regionFeedState { t.mu.Lock() statesBySubscription := t.statesBySubscription t.statesBySubscription = make(map[SubscriptionID]regionStatesByID) t.mu.Unlock() - drainedStates := make(map[SubscriptionID][]*regionFeedState, len(statesBySubscription)) - for subID, states := range statesBySubscription { - drainedStates[subID] = slices.Collect(maps.Values(states)) + drainedStates := make([]*regionFeedState, 0) + for _, states := range statesBySubscription { + drainedStates = append(drainedStates, slices.Collect(maps.Values(states))...) } return drainedStates } diff --git a/logservice/logpuller/region_tracker_test.go b/logservice/logpuller/region_tracker_test.go index ad89844a3a..8a08ce0d94 100644 --- a/logservice/logpuller/region_tracker_test.go +++ b/logservice/logpuller/region_tracker_test.go @@ -37,9 +37,7 @@ func TestRegionTrackerOperations(t *testing.T) { require.Empty(t, tracker.TakeSubscription(1)) drained := tracker.Drain() - require.Equal(t, map[SubscriptionID][]*regionFeedState{ - 2: {state3}, - }, drained) + require.ElementsMatch(t, []*regionFeedState{state3}, drained) require.Nil(t, tracker.Get(2, 3)) require.Empty(t, tracker.Drain()) } diff --git a/logservice/logpuller/subscription_client.go b/logservice/logpuller/subscription_client.go index b45c83cc08..7b99c4cb96 100644 --- a/logservice/logpuller/subscription_client.go +++ b/logservice/logpuller/subscription_client.go @@ -91,7 +91,6 @@ type rangeTask struct { subscribedSpan *subscribedSpan filterLoop bool priority cdcpb.ScanPriority - wasInitialized bool } type SubscriptionClientConfig struct { @@ -525,7 +524,7 @@ func (s *subscriptionClient) handleRangeTasks(ctx context.Context) error { case task := <-s.rangeTaskCh: g.Go(func() error { return s.divideSpanAndScheduleRegionRequests( - ctx, task.span, task.subscribedSpan, task.filterLoop, task.priority, task.wasInitialized) + ctx, task.span, task.subscribedSpan, task.filterLoop, task.priority) }) } } @@ -542,7 +541,6 @@ func (s *subscriptionClient) divideSpanAndScheduleRegionRequests( subscribedSpan *subscribedSpan, filterLoop bool, inheritedPriority cdcpb.ScanPriority, - wasInitialized bool, ) error { // Limit the number of regions loaded at a time to make the load more stable. limit := 1024 @@ -604,7 +602,6 @@ func (s *subscriptionClient) divideSpanAndScheduleRegionRequests( verID := tikv.NewRegionVerID(regionMeta.Id, regionMeta.RegionEpoch.ConfVer, regionMeta.RegionEpoch.Version) regionInfo := newRegionInfo(verID, intersectSpan, nil, subscribedSpan, filterLoop) - regionInfo.wasInitialized = wasInitialized // Schedule a region request to subscribe the region. s.scheduleRegionRequest(ctx, regionInfo, inheritedPriority) @@ -626,9 +623,6 @@ func (s *subscriptionClient) scheduleRegionRequest( region regionInfo, inheritedPriority cdcpb.ScanPriority, ) { - if region.lockedRangeState != nil && region.lockedRangeState.Initialized.Load() { - region.wasInitialized = true - } lockRangeResult := region.subscribedSpan.rangeLock.LockRange( ctx, region.span.StartKey, region.span.EndKey, region.verID.GetID(), region.verID.GetVer()) @@ -662,7 +656,7 @@ func (s *subscriptionClient) scheduleRegionRequest( case regionlock.LockRangeStatusStale: for _, r := range lockRangeResult.RetryRanges { s.scheduleRangeRequest( - ctx, r, region.subscribedSpan, region.filterLoop, region.wasInitialized, inheritedPriority) + ctx, r, region.subscribedSpan, region.filterLoop, inheritedPriority) } default: return @@ -673,7 +667,6 @@ func (s *subscriptionClient) scheduleRangeRequest( ctx context.Context, span heartbeatpb.TableSpan, subscribedSpan *subscribedSpan, filterLoop bool, - wasInitialized bool, inheritedPriority cdcpb.ScanPriority, ) { select { @@ -683,7 +676,6 @@ func (s *subscriptionClient) scheduleRangeRequest( subscribedSpan: subscribedSpan, filterLoop: filterLoop, priority: inheritedPriority, - wasInitialized: wasInitialized, }: } } From f87c6a45daa9fe2751871554bcf08c12ab0cf51e Mon Sep 17 00:00:00 2001 From: lidezhu Date: Sun, 2 Aug 2026 18:28:22 +0800 Subject: [PATCH 16/71] small fix --- logservice/logpuller/region_request_worker.go | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index dec0c61c93..3c9b903611 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -319,20 +319,19 @@ func (s *regionRequestWorker) dispatchRegionChangeEvents(events []*cdcpb.Event) log.Panic("unknown event type", zap.Any("event", event)) } s.client.eventSink.Push(subscriptionID, regionEvent) - continue - } - - switch event.Event.(type) { - case *cdcpb.Event_Error: - log.Debug("region request worker receives an error for a stale region, ignore it", - zap.Uint64("workerID", s.workerID), - zap.Uint64("subscriptionID", uint64(subscriptionID)), - zap.Uint64("regionID", event.RegionId)) - default: - log.Warn("region request worker receives a region event for an untracked region", - zap.Uint64("workerID", s.workerID), - zap.Uint64("subscriptionID", uint64(subscriptionID)), - zap.Uint64("regionID", event.RegionId)) + } else { + switch event.Event.(type) { + case *cdcpb.Event_Error: + log.Debug("region request worker receives an error for a stale region, ignore it", + zap.Uint64("workerID", s.workerID), + zap.Uint64("subscriptionID", uint64(subscriptionID)), + zap.Uint64("regionID", event.RegionId)) + default: + log.Warn("region request worker receives a region event for an untracked region", + zap.Uint64("workerID", s.workerID), + zap.Uint64("subscriptionID", uint64(subscriptionID)), + zap.Uint64("regionID", event.RegionId)) + } } } } From 76727427d526ce867f2d3a5e6b9dae9b6134203e Mon Sep 17 00:00:00 2001 From: lidezhu Date: Sun, 2 Aug 2026 18:36:48 +0800 Subject: [PATCH 17/71] small fix --- logservice/logpuller/region_request_worker.go | 23 ++++++++++--------- .../logpuller/region_request_worker_test.go | 6 ++++- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index 3c9b903611..3d6963a87c 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -123,7 +123,11 @@ func (s *regionRequestWorker) Run(ctx context.Context) error { // tracker.Add. // - admission: requests owned by this worker but not sent yet. for _, state := range s.tracker.Drain() { - s.notifyRegionError(state, regionErr) + state.markStopped(regionErr) + s.client.eventSink.Push( + SubscriptionID(state.requestID), + regionEvent{states: []*regionFeedState{state}}, + ) } // The failed stream no longer owns remote registrations. s.controlQueue.drain() @@ -168,13 +172,6 @@ func (s *regionRequestWorker) Run(ctx context.Context) error { } } -func (s *regionRequestWorker) notifyRegionError(state *regionFeedState, err error) { - state.markStopped(err) - s.client.eventSink.Push( - SubscriptionID(state.requestID), - regionEvent{states: []*regionFeedState{state}}) -} - func (s *regionRequestWorker) waitForRegionRequest(ctx context.Context) (*regionReq, error) { // Without a stream there are no remote registrations to deregister. s.controlQueue.drain() @@ -309,11 +306,11 @@ func (s *regionRequestWorker) dispatchRegionChangeEvents(events []*cdcpb.Event) zap.Uint64("subscriptionID", uint64(subscriptionID)), zap.Uint64("regionID", event.RegionId), zap.Any("error", eventData.Error)) - s.notifyRegionError(state, &eventError{err: eventData.Error}) - continue + state.markStopped(&eventError{err: eventData.Error}) case *cdcpb.Event_ResolvedTs: regionEvent.resolvedTs = eventData.ResolvedTs case *cdcpb.Event_LongTxn_: + // ignore continue default: log.Panic("unknown event type", zap.Any("event", event)) @@ -322,6 +319,7 @@ func (s *regionRequestWorker) dispatchRegionChangeEvents(events []*cdcpb.Event) } else { switch event.Event.(type) { case *cdcpb.Event_Error: + // it is normal to receive region error after deregister a subscription log.Debug("region request worker receives an error for a stale region, ignore it", zap.Uint64("workerID", s.workerID), zap.Uint64("subscriptionID", uint64(subscriptionID)), @@ -349,6 +347,8 @@ func (s *regionRequestWorker) dispatchResolvedTsEvent(resolvedTsEvent *cdcpb.Res } const resolvedTsStateBatchSize = 1024 + // Avoid allocating a huge states slice when resolvedTsEvent.Regions is large. + // Push resolved-ts events in batches to reduce peak memory usage and improve GC behavior. capHint := min(len(resolvedTsEvent.Regions), resolvedTsStateBatchSize) resolvedStates := make([]*regionFeedState, 0, capHint) flush := func() { @@ -414,7 +414,8 @@ func (s *regionRequestWorker) sendDeregisterRequest( return err } for _, state := range s.tracker.TakeSubscription(req.subID) { - s.notifyRegionError(state, &requestCancelledErr{}) + state.markStopped(&requestCancelledErr{}) + s.client.eventSink.Push(req.subID, regionEvent{states: []*regionFeedState{state}}) } return nil } diff --git a/logservice/logpuller/region_request_worker_test.go b/logservice/logpuller/region_request_worker_test.go index c7d6268248..29d1bc063b 100644 --- a/logservice/logpuller/region_request_worker_test.go +++ b/logservice/logpuller/region_request_worker_test.go @@ -395,7 +395,11 @@ func TestStreamRecoveryReleasesSentAdmission(t *testing.T) { require.True(t, worker.tracker.Add(region.subscribedSpan.subID, region.verID.GetID(), state)) for _, state := range worker.tracker.Drain() { - worker.notifyRegionError(state, &storeStreamErr{}) + state.markStopped(&storeStreamErr{}) + worker.client.eventSink.Push( + SubscriptionID(state.requestID), + regionEvent{states: []*regionFeedState{state}}, + ) } worker.controlQueue.drain() From ad90640e319fd46c3c1822eaf5f09552f634cf46 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Sun, 2 Aug 2026 18:49:13 +0800 Subject: [PATCH 18/71] small fix --- logservice/logpuller/region_request_worker.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index 3d6963a87c..acaeafccad 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -338,6 +338,7 @@ func (s *regionRequestWorker) dispatchResolvedTsEvent(resolvedTsEvent *cdcpb.Res subscriptionID := SubscriptionID(resolvedTsEvent.RequestId) metricsResolvedTsCount.Add(float64(len(resolvedTsEvent.Regions))) s.client.metrics.batchResolvedSize.Observe(float64(len(resolvedTsEvent.Regions))) + // TODO: resolvedTsEvent.Ts be 0 is impossible, we need find the root cause. if resolvedTsEvent.Ts == 0 { log.Warn("region request worker receives a resolved ts event with zero value, ignore it", zap.Uint64("workerID", s.workerID), @@ -420,7 +421,7 @@ func (s *regionRequestWorker) sendDeregisterRequest( return nil } -func (s *regionRequestWorker) drainControlQueue(conn *ConnAndClient) error { +func (s *regionRequestWorker) sendPendingDeregisterRequests(conn *ConnAndClient) error { for { req, ok := s.controlQueue.tryPop() if !ok { @@ -491,7 +492,7 @@ func (s *regionRequestWorker) processRegionSendTask( continue } - if err := s.drainControlQueue(conn); err != nil { + if err := s.sendPendingDeregisterRequests(conn); err != nil { return err } var err error From 23b085bf6cb1b82257f06abfe4ed79c74535786e Mon Sep 17 00:00:00 2001 From: lidezhu Date: Sun, 2 Aug 2026 19:05:07 +0800 Subject: [PATCH 19/71] small fix --- logservice/logpuller/priority_task.go | 5 ----- logservice/logpuller/scan_priority_test.go | 4 ++-- logservice/logpuller/subscription_client.go | 2 +- logservice/logpuller/subscription_client_test.go | 2 +- 4 files changed, 4 insertions(+), 9 deletions(-) diff --git a/logservice/logpuller/priority_task.go b/logservice/logpuller/priority_task.go index 5511006534..d576e72cf3 100644 --- a/logservice/logpuller/priority_task.go +++ b/logservice/logpuller/priority_task.go @@ -46,11 +46,6 @@ func (pt *regionPriorityTask) priority() cdcpb.ScanPriority { return normalizeScanPriority(pt.regionInfo.scanPriority) } -// GetRegionInfo returns the underlying regionInfo -func (pt *regionPriorityTask) GetRegionInfo() regionInfo { - return pt.regionInfo -} - func (pt *regionPriorityTask) canUseMaxWindow() bool { return isHighScanPriority(pt.regionInfo.scanPriority) } diff --git a/logservice/logpuller/scan_priority_test.go b/logservice/logpuller/scan_priority_test.go index e42d0b9a9c..42806ca52a 100644 --- a/logservice/logpuller/scan_priority_test.go +++ b/logservice/logpuller/scan_priority_test.go @@ -129,7 +129,7 @@ func TestScanPriorityUsesRestoredRegionProgress(t *testing.T) { firstTask := popRegionPriorityTask(t, client.regionTaskQueue) require.Equal(t, cdcpb.ScanPriority_SCAN_PRIORITY_LOW, firstTask.priority()) - firstRegion := firstTask.GetRegionInfo() + firstRegion := firstTask.regionInfo firstRegion.lockedRangeState.ResolvedTs.Store(oracle.GoTimeToTS(currentTime.Add(-time.Minute))) span.rangeLock.UnlockRange( firstRegion.span.StartKey, @@ -142,7 +142,7 @@ func TestScanPriorityUsesRestoredRegionProgress(t *testing.T) { client.scheduleRegionRequest(context.Background(), retryRegion, cdcpb.ScanPriority_SCAN_PRIORITY_LOW) retryTask := popRegionPriorityTask(t, client.regionTaskQueue) require.Equal(t, cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, retryTask.priority()) - require.Equal(t, cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, retryTask.GetRegionInfo().scanPriority) + require.Equal(t, cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, retryTask.regionInfo.scanPriority) require.False(t, span.priorityPolicy.everCaughtUp.Load()) } diff --git a/logservice/logpuller/subscription_client.go b/logservice/logpuller/subscription_client.go index 7b99c4cb96..9beab5c8c5 100644 --- a/logservice/logpuller/subscription_client.go +++ b/logservice/logpuller/subscription_client.go @@ -464,7 +464,7 @@ func (s *subscriptionClient) handleRegions(ctx context.Context, eg *errgroup.Gro return err } - region := regionTask.GetRegionInfo() + region := regionTask.regionInfo region, ok := s.attachRPCContextForRegion(ctx, region) // If attachRPCContextForRegion fails, the region will be re-scheduled. if !ok { diff --git a/logservice/logpuller/subscription_client_test.go b/logservice/logpuller/subscription_client_test.go index 3ab9819db6..a14c4a07fb 100644 --- a/logservice/logpuller/subscription_client_test.go +++ b/logservice/logpuller/subscription_client_test.go @@ -467,7 +467,7 @@ func TestRegionRetryScanPriority(t *testing.T) { task, err := client.regionTaskQueue.Pop(ctx) require.NoError(t, err) require.Equal(t, tc.expected, task.priority()) - require.Equal(t, tc.expected, task.GetRegionInfo().scanPriority) + require.Equal(t, tc.expected, task.regionInfo.scanPriority) }) } } From ce00cc29c0e8023f2b4aeaada7f3e87372eb64f9 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Sun, 2 Aug 2026 19:32:37 +0800 Subject: [PATCH 20/71] address comment --- logservice/logpuller/priority_task.go | 4 +-- logservice/logpuller/priority_task_test.go | 16 ++++----- .../region_admission_controller_test.go | 4 +-- .../logpuller/region_request_worker_test.go | 2 +- logservice/logpuller/subscription_client.go | 9 ++--- .../logpuller/subscription_client_test.go | 36 +++++++++++++++++-- 6 files changed, 52 insertions(+), 19 deletions(-) diff --git a/logservice/logpuller/priority_task.go b/logservice/logpuller/priority_task.go index d576e72cf3..38cd813c8b 100644 --- a/logservice/logpuller/priority_task.go +++ b/logservice/logpuller/priority_task.go @@ -32,8 +32,8 @@ type regionPriorityTask struct { heapIndex int // for heap.Item interface } -// NewRegionPriorityTask creates a new priority task for region -func NewRegionPriorityTask(regionInfo regionInfo, sequence uint64) *regionPriorityTask { +// newRegionPriorityTask creates a new priority task for region. +func newRegionPriorityTask(regionInfo regionInfo, sequence uint64) *regionPriorityTask { regionInfo.scanPriority = normalizeScanPriority(regionInfo.scanPriority) return ®ionPriorityTask{ regionInfo: regionInfo, diff --git a/logservice/logpuller/priority_task_test.go b/logservice/logpuller/priority_task_test.go index 71ea50f1ce..41501181fb 100644 --- a/logservice/logpuller/priority_task_test.go +++ b/logservice/logpuller/priority_task_test.go @@ -60,21 +60,21 @@ func TestRegionPriorityTaskQueueOrder(t *testing.T) { queue := priorityqueue.New[*regionPriorityTask]() currentTime := time.Now() - lowTask := NewRegionPriorityTask( + lowTask := newRegionPriorityTask( withScanPriority( newPriorityTestRegion(1, oracle.GoTimeToTS(currentTime.Add(-time.Hour))), cdcpb.ScanPriority_SCAN_PRIORITY_LOW, ), 3, ) - highTask1 := NewRegionPriorityTask( + highTask1 := newRegionPriorityTask( withScanPriority( newPriorityTestRegion(2, oracle.GoTimeToTS(currentTime.Add(-10*time.Minute))), cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, ), 2, ) - highTask2 := NewRegionPriorityTask( + highTask2 := newRegionPriorityTask( withScanPriority( newPriorityTestRegion(3, oracle.GoTimeToTS(currentTime.Add(-time.Hour))), cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, @@ -98,9 +98,9 @@ func TestRegionPriorityTaskFIFOWithinPriority(t *testing.T) { currentTime := time.Now() checkpointTs := oracle.GoTimeToTS(currentTime.Add(-time.Hour)) - first := NewRegionPriorityTask( + first := newRegionPriorityTask( withScanPriority(newPriorityTestRegion(1, checkpointTs), cdcpb.ScanPriority_SCAN_PRIORITY_HIGH), 1) - second := NewRegionPriorityTask( + second := newRegionPriorityTask( withScanPriority(newPriorityTestRegion(2, checkpointTs), cdcpb.ScanPriority_SCAN_PRIORITY_HIGH), 2) require.True(t, queue.Push(second)) @@ -115,9 +115,9 @@ func TestRegionPriorityTaskFIFOWithinPriority(t *testing.T) { } func TestRegionPriorityTaskUsesHighPriorityWindow(t *testing.T) { - highTask := NewRegionPriorityTask( + highTask := newRegionPriorityTask( withScanPriority(newPriorityTestRegion(1, 1), cdcpb.ScanPriority_SCAN_PRIORITY_HIGH), 1) - lowTask := NewRegionPriorityTask( + lowTask := newRegionPriorityTask( withScanPriority(newPriorityTestRegion(2, 1), cdcpb.ScanPriority_SCAN_PRIORITY_LOW), 2) require.True(t, highTask.canUseMaxWindow()) @@ -126,7 +126,7 @@ func TestRegionPriorityTaskUsesHighPriorityWindow(t *testing.T) { func TestRegionPriorityTaskRefreshesRegionInfoBetweenStages(t *testing.T) { region := withScanPriority(newPriorityTestRegion(1, 1), cdcpb.ScanPriority_SCAN_PRIORITY_LOW) - task := NewRegionPriorityTask(region, 1) + task := newRegionPriorityTask(region, 1) require.Equal(t, cdcpb.ScanPriority_SCAN_PRIORITY_LOW, task.priority()) region.scanPriority = cdcpb.ScanPriority_SCAN_PRIORITY_HIGH diff --git a/logservice/logpuller/region_admission_controller_test.go b/logservice/logpuller/region_admission_controller_test.go index 476e0637fe..13fc2ed2bc 100644 --- a/logservice/logpuller/region_admission_controller_test.go +++ b/logservice/logpuller/region_admission_controller_test.go @@ -56,7 +56,7 @@ func submitRegionForAdmission( currentTs uint64, ) { t.Helper() - task := NewRegionPriorityTask(region, region.verID.GetID()) + task := newRegionPriorityTask(region, region.verID.GetID()) require.True(t, controller.submit(task)) } @@ -191,7 +191,7 @@ func TestRegionAdmissionControllerClose(t *testing.T) { controller := newRegionAdmissionController(1, 1) controller.close() region := prepareRegionForAdmission(createTestRegionInfo(1, 1), 1) - require.False(t, controller.submit(NewRegionPriorityTask(region, 1))) + require.False(t, controller.submit(newRegionPriorityTask(region, 1))) _, err := controller.pop(context.Background(), nil) require.ErrorIs(t, err, context.Canceled) diff --git a/logservice/logpuller/region_request_worker_test.go b/logservice/logpuller/region_request_worker_test.go index 29d1bc063b..ca04097d83 100644 --- a/logservice/logpuller/region_request_worker_test.go +++ b/logservice/logpuller/region_request_worker_test.go @@ -445,7 +445,7 @@ func TestStreamRecoveryReschedulesWorkerBuffer(t *testing.T) { }, } for i, region := range regions { - require.True(t, admission.submit(NewRegionPriorityTask(region, uint64(i+1)))) + require.True(t, admission.submit(newRegionPriorityTask(region, uint64(i+1)))) } for _, task := range worker.admission.drain() { diff --git a/logservice/logpuller/subscription_client.go b/logservice/logpuller/subscription_client.go index 9beab5c8c5..29772aa6bc 100644 --- a/logservice/logpuller/subscription_client.go +++ b/logservice/logpuller/subscription_client.go @@ -218,7 +218,7 @@ func (s *subscriptionClient) updateMetrics(ctx context.Context) error { pendingRegionReqCount := 0 s.stores.Range(func(_, value any) bool { store := value.(*requestedStore) - pendingRegionReqCount += store.inflightCount() + pendingRegionReqCount += store.requestedRegionCount() return true }) @@ -395,12 +395,13 @@ func (s *requestedStore) close() { } } -func (s *requestedStore) inflightCount() int { +func (s *requestedStore) requestedRegionCount() int { s.requestWorkers.RLock() defer s.requestWorkers.RUnlock() count := 0 for _, worker := range s.requestWorkers.s { - count += worker.admission.stats().inflight + stats := worker.admission.stats() + count += stats.pending + stats.inflight } return count } @@ -640,7 +641,7 @@ func (s *subscriptionClient) scheduleRegionRequest( oracle.GetTimeFromTS(currentTs), ) region.scanPriority = priority - s.regionTaskQueue.Push(NewRegionPriorityTask(region, s.regionTaskSequence.Add(1))) + s.regionTaskQueue.Push(newRegionPriorityTask(region, s.regionTaskSequence.Add(1))) if log.GetLevel() <= zapcore.DebugLevel { log.Debug("cdc region scan task enqueued", zap.Uint64("subscriptionID", uint64(region.subscribedSpan.subID)), diff --git a/logservice/logpuller/subscription_client_test.go b/logservice/logpuller/subscription_client_test.go index a14c4a07fb..d6ab9c85cb 100644 --- a/logservice/logpuller/subscription_client_test.go +++ b/logservice/logpuller/subscription_client_test.go @@ -639,7 +639,7 @@ func TestBroadcastDeregisterUsesWorkerControlQueue(t *testing.T) { subscribedSpan: &subscribedSpan{subID: SubscriptionID(2)}, lockedRangeState: ®ionlock.LockedRangeState{}, } - require.True(t, admission.submit(NewRegionPriorityTask(dummyRegion, 1))) + require.True(t, admission.submit(newRegionPriorityTask(dummyRegion, 1))) client.broadcastDeregister(SubscriptionID(1), true) require.Equal(t, 1, worker.controlQueue.len()) @@ -662,13 +662,45 @@ func TestRequestedStoreDistributesRegionsAcrossWorkerBuffers(t *testing.T) { subscribedSpan: &subscribedSpan{subID: 1}, lockedRangeState: ®ionlock.LockedRangeState{}, } - require.True(t, store.submit(NewRegionPriorityTask(region, i))) + require.True(t, store.submit(newRegionPriorityTask(region, i))) } require.Equal(t, 2, worker1.admission.stats().pending) require.Equal(t, 2, worker2.admission.stats().pending) } +func TestRequestedStoreRequestedRegionCountIncludesPendingAndInflight(t *testing.T) { + worker1 := ®ionRequestWorker{admission: newRegionAdmissionController(1, 1)} + worker2 := ®ionRequestWorker{admission: newRegionAdmissionController(1, 1)} + store := &requestedStore{storeAddr: "store-1"} + store.requestWorkers.s = []*regionRequestWorker{worker1, worker2} + + region1 := regionInfo{ + verID: tikv.NewRegionVerID(1, 1, 1), + subscribedSpan: &subscribedSpan{subID: 1}, + lockedRangeState: ®ionlock.LockedRangeState{}, + } + region2 := regionInfo{ + verID: tikv.NewRegionVerID(2, 1, 1), + subscribedSpan: &subscribedSpan{subID: 1}, + lockedRangeState: ®ionlock.LockedRangeState{}, + } + region3 := regionInfo{ + verID: tikv.NewRegionVerID(3, 1, 1), + subscribedSpan: &subscribedSpan{subID: 1}, + lockedRangeState: ®ionlock.LockedRangeState{}, + } + + require.True(t, worker1.admission.submit(newRegionPriorityTask(region1, 1))) + req, err := worker1.admission.pop(t.Context(), nil) + require.NoError(t, err) + require.True(t, worker2.admission.submit(newRegionPriorityTask(region2, 2))) + require.True(t, worker2.admission.submit(newRegionPriorityTask(region3, 3))) + + require.Equal(t, 3, store.requestedRegionCount()) + require.True(t, req.abort()) +} + func TestSubscriptionWithFailedTiKV(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) mockPDClock := pdutil.NewClock4Test() From 4b86ed764f444a311c49f2df43f6ebeb63e9ac87 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Sun, 2 Aug 2026 20:02:16 +0800 Subject: [PATCH 21/71] address comment --- logservice/logpuller/region_request_worker.go | 6 +- .../logpuller/region_request_worker_test.go | 36 ++-- logservice/logpuller/region_tracker.go | 6 +- logservice/logpuller/subscription_client.go | 10 +- .../logpuller/subscription_client_test.go | 164 ++++++++++++++++++ pkg/config/debug.go | 2 +- utils/notifyqueue/notify_queue.go | 1 + utils/notifyqueue/notify_queue_test.go | 1 + utils/priorityqueue/priority_queue_test.go | 52 ++++-- 9 files changed, 247 insertions(+), 31 deletions(-) diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index acaeafccad..908f87b071 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -214,8 +214,7 @@ func (s *regionRequestWorker) runStream(ctx context.Context, firstReq *regionReq zap.Error(err)) }() - g, gctx := errgroup.WithContext(ctx) - conn, err := Connect(gctx, s.client.credential, s.store.storeAddr) + conn, err := Connect(ctx, s.client.credential, s.store.storeAddr) if err != nil { log.Warn("region request worker create grpc stream failed", zap.Uint64("workerID", s.workerID), @@ -231,6 +230,7 @@ func (s *regionRequestWorker) runStream(ctx context.Context, firstReq *regionReq } defer func() { _ = conn.Conn.Close() }() + g, gctx := errgroup.WithContext(ctx) g.Go(func() error { return s.receiveAndDispatchChangeEvents(conn) }) g.Go(func() error { return s.processRegionSendTask(gctx, conn, firstReq) }) @@ -435,7 +435,7 @@ func (s *regionRequestWorker) sendPendingDeregisterRequests(conn *ConnAndClient) func (s *regionRequestWorker) sendRegionRequest(conn *ConnAndClient, req *regionReq) error { if !req.isActive() { - return &storeStreamErr{} + return nil } region := req.regionInfo subID := region.subscribedSpan.subID diff --git a/logservice/logpuller/region_request_worker_test.go b/logservice/logpuller/region_request_worker_test.go index ca04097d83..92c3023b55 100644 --- a/logservice/logpuller/region_request_worker_test.go +++ b/logservice/logpuller/region_request_worker_test.go @@ -487,7 +487,7 @@ func TestProcessRegionSendTaskSendFailureCleansSentRequest(t *testing.T) { require.ErrorAs(t, state.takeError(), &streamErr) } -func TestProcessRegionSendTaskDoesNotSendRemovedRequest(t *testing.T) { +func TestProcessRegionSendTaskSkipsRemovedRequest(t *testing.T) { admission := newRegionAdmissionController(1, 1) worker := ®ionRequestWorker{ admission: admission, @@ -496,22 +496,28 @@ func TestProcessRegionSendTaskDoesNotSendRemovedRequest(t *testing.T) { client: &subscriptionClient{}, tracker: newRegionTracker(), } - region := prepareRegionForSendTest(createTestRegionInfo(1, 1)) - req := admitRegionRequest(t, admission, region) - require.True(t, req.abort()) + firstRegion := prepareRegionForSendTest(createTestRegionInfo(1, 1)) + firstReq := admitRegionRequest(t, admission, firstRegion) + require.True(t, firstReq.abort()) + + secondRegion := prepareRegionForSendTest(createTestRegionInfo(1, 2)) + require.True(t, admission.submit(newRegionPriorityTask(secondRegion, 2))) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() sendCh := make(chan *cdcpb.ChangeDataRequest, 1) - err := worker.processRegionSendTask(t.Context(), &ConnAndClient{ - Client: &mockEventFeedV2Client{sendCh: sendCh}, - Conn: &grpc.ClientConn{}, - }, req) - var streamErr *storeStreamErr - require.ErrorAs(t, err, &streamErr) - select { - case sentReq := <-sendCh: - t.Fatalf("removed request was sent: %+v", sentReq) - default: - } + done := make(chan error, 1) + go func() { + done <- worker.processRegionSendTask(ctx, &ConnAndClient{ + Client: &mockEventFeedV2Client{sendCh: sendCh}, + Conn: &grpc.ClientConn{}, + }, firstReq) + }() + + sentReq := <-sendCh + require.Equal(t, secondRegion.verID.GetID(), sentReq.RegionId) + cancel() + require.ErrorIs(t, <-done, context.Canceled) } func TestProcessRegionSendTaskSendEOFIsRetriable(t *testing.T) { diff --git a/logservice/logpuller/region_tracker.go b/logservice/logpuller/region_tracker.go index dad8e6df69..81a27dc86c 100644 --- a/logservice/logpuller/region_tracker.go +++ b/logservice/logpuller/region_tracker.go @@ -112,7 +112,11 @@ func (t *regionTracker) Drain() []*regionFeedState { t.statesBySubscription = make(map[SubscriptionID]regionStatesByID) t.mu.Unlock() - drainedStates := make([]*regionFeedState, 0) + totalStates := 0 + for _, states := range statesBySubscription { + totalStates += len(states) + } + drainedStates := make([]*regionFeedState, 0, totalStates) for _, states := range statesBySubscription { drainedStates = append(drainedStates, slices.Collect(maps.Values(states))...) } diff --git a/logservice/logpuller/subscription_client.go b/logservice/logpuller/subscription_client.go index 29772aa6bc..3dcb7098d2 100644 --- a/logservice/logpuller/subscription_client.go +++ b/logservice/logpuller/subscription_client.go @@ -471,11 +471,19 @@ func (s *subscriptionClient) handleRegions(ctx context.Context, eg *errgroup.Gro if !ok { continue } + if region.subscribedSpan.stopped.Load() { + s.onRegionFail(newRegionErrorInfo(region, &requestCancelledErr{})) + continue + } store := getStore(region.rpcCtx.Addr) regionTask.regionInfo = region if !store.submit(regionTask) { - return context.Canceled + if ctx.Err() != nil { + return ctx.Err() + } + s.onRegionFail(newRegionErrorInfo(region, &storeStreamErr{})) + continue } log.Debug("subscription client will request a region", diff --git a/logservice/logpuller/subscription_client_test.go b/logservice/logpuller/subscription_client_test.go index d6ab9c85cb..89900a4df3 100644 --- a/logservice/logpuller/subscription_client_test.go +++ b/logservice/logpuller/subscription_client_test.go @@ -38,6 +38,7 @@ import ( "github.com/tikv/client-go/v2/oracle" "github.com/tikv/client-go/v2/testutils" "github.com/tikv/client-go/v2/tikv" + "golang.org/x/sync/errgroup" ) type mockLockResolver struct { @@ -395,6 +396,169 @@ func TestOnRegionFailQueuesCanceledErrorCache(t *testing.T) { require.Nil(t, client.spanRegistry.Get(span.subID)) } +func TestHandleRegionsSkipsStoppedSubscriptionBeforeCreatingStore(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + _, cluster, pdClient, _ := testutils.NewMockTiKV("", mockcopr.NewCoprRPCHandler()) + pdClient = &mockPDClient{Client: pdClient, versionGen: defaultVersionGen} + defer pdClient.Close() + + const storeAddr = "store-1" + cluster.AddStore(1, storeAddr) + cluster.Bootstrap(11, []uint64{1}, []uint64{2}, 2) + + regionCache := tikv.NewRegionCache(pdClient) + defer regionCache.Close() + + bo := tikv.NewBackoffer(ctx, tikvRequestMaxBackoff) + location, err := regionCache.LocateKey(bo, []byte("a")) + require.NoError(t, err) + + rawSpan := heartbeatpb.TableSpan{ + TableID: 1, + StartKey: []byte("a"), + EndKey: []byte("b"), + } + span := &subscribedSpan{ + subID: SubscriptionID(1), + span: rawSpan, + rangeLock: regionlock.NewRangeLock(1, rawSpan.StartKey, rawSpan.EndKey, 100), + } + + lockRes := span.rangeLock.LockRange( + context.Background(), rawSpan.StartKey, rawSpan.EndKey, location.Region.GetID(), location.Region.GetVer()) + require.Equal(t, regionlock.LockRangeStatusSuccess, lockRes.Status) + + client := &subscriptionClient{ + ctx: ctx, + config: &SubscriptionClientConfig{RegionRequestWorkerPerStore: 1}, + pd: pdClient, + pdClock: pdutil.NewClock4Test(), + regionCache: regionCache, + credential: &security.Credential{}, + eventSink: newTestRegionEventSink(&mockDynamicStream{}), + spanRegistry: newSpanRegistry(nil, nil), + regionTaskQueue: priorityqueue.New[*regionPriorityTask](), + } + client.failureHandler = newRegionFailureHandler(client) + client.spanRegistry.Add(span) + + region := newRegionInfo(location.Region, rawSpan, nil, span, false) + region.lockedRangeState = lockRes.LockedRangeState + client.regionTaskQueue.Push(newRegionPriorityTask(region, 1)) + client.setTableStopped(span) + + var eg errgroup.Group + errCh := make(chan error, 1) + go func() { + errCh <- client.handleRegions(ctx, &eg) + }() + + require.Eventually(t, func() bool { + return client.spanRegistry.Get(span.subID) == nil + }, time.Second, 20*time.Millisecond) + + _, ok := client.stores.Load(storeAddr) + require.False(t, ok) + + cancel() + err = <-errCh + require.ErrorIs(t, err, context.Canceled) +} + +func TestHandleRegionsReschedulesRegionWhenStoreSubmitFails(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + _, cluster, pdClient, _ := testutils.NewMockTiKV("", mockcopr.NewCoprRPCHandler()) + pdClient = &mockPDClient{Client: pdClient, versionGen: defaultVersionGen} + defer pdClient.Close() + + const storeAddr = "store-1" + cluster.AddStore(1, storeAddr) + cluster.Bootstrap(11, []uint64{1}, []uint64{2}, 2) + + regionCache := tikv.NewRegionCache(pdClient) + defer regionCache.Close() + + bo := tikv.NewBackoffer(ctx, tikvRequestMaxBackoff) + location, err := regionCache.LocateKey(bo, []byte("a")) + require.NoError(t, err) + + rawSpan := heartbeatpb.TableSpan{ + TableID: 1, + StartKey: []byte("a"), + EndKey: []byte("b"), + } + span := &subscribedSpan{ + subID: SubscriptionID(1), + span: rawSpan, + rangeLock: regionlock.NewRangeLock(1, rawSpan.StartKey, rawSpan.EndKey, 100), + priorityPolicy: newTestScanPriorityPolicy(), + } + lockRes := span.rangeLock.LockRange( + context.Background(), rawSpan.StartKey, rawSpan.EndKey, location.Region.GetID(), location.Region.GetVer()) + require.Equal(t, regionlock.LockRangeStatusSuccess, lockRes.Status) + + admission := newRegionAdmissionController(1, 1) + admission.close() + store := &requestedStore{storeAddr: storeAddr} + store.requestWorkers.s = []*regionRequestWorker{{admission: admission}} + + client := &subscriptionClient{ + ctx: ctx, + config: &SubscriptionClientConfig{RegionRequestWorkerPerStore: 1}, + pd: pdClient, + pdClock: pdutil.NewClock4Test(), + regionCache: regionCache, + credential: &security.Credential{}, + eventSink: newTestRegionEventSink(&mockDynamicStream{}), + spanRegistry: newSpanRegistry(nil, nil), + regionTaskQueue: priorityqueue.New[*regionPriorityTask](), + } + client.failureHandler = newRegionFailureHandler(client) + client.stores.Store(storeAddr, store) + + region := newRegionInfo(location.Region, rawSpan, nil, span, false) + region.lockedRangeState = lockRes.LockedRangeState + client.regionTaskQueue.Push(newRegionPriorityTask(region, 1)) + + var eg errgroup.Group + errCh := make(chan error, 1) + go func() { + errCh <- client.handleRegions(ctx, &eg) + }() + + require.Eventually(t, func() bool { + client.failureHandler.cache.Lock() + defer client.failureHandler.cache.Unlock() + return len(client.failureHandler.cache.cache) == 1 + }, time.Second, 20*time.Millisecond) + + select { + case err := <-errCh: + t.Fatalf("handleRegions exited unexpectedly: %v", err) + default: + } + + cancel() + err = <-errCh + require.ErrorIs(t, err, context.Canceled) + + batch := client.failureHandler.cache.popBatch(1) + require.Len(t, batch, 1) + require.IsType(t, &storeStreamErr{}, batch[0].err) + require.NoError(t, client.failureHandler.handleError(context.Background(), batch[0])) + + popCtx, popCancel := context.WithTimeout(context.Background(), time.Second) + defer popCancel() + retriedTask, err := client.regionTaskQueue.Pop(popCtx) + require.NoError(t, err) + require.Equal(t, region.verID, retriedTask.regionInfo.verID) + require.Equal(t, rawSpan, retriedTask.regionInfo.span) +} + func TestRegionRetryScanPriority(t *testing.T) { for _, tc := range []struct { name string diff --git a/pkg/config/debug.go b/pkg/config/debug.go index ce5ee9ea4a..6df9c767a7 100644 --- a/pkg/config/debug.go +++ b/pkg/config/debug.go @@ -24,7 +24,7 @@ import ( const ( // DefaultOldStartTsScanLowPriorityThreshold is the default lag threshold for // classifying scan tasks as low priority. - DefaultOldStartTsScanLowPriorityThreshold = 30 * time.Minute + DefaultOldStartTsScanLowPriorityThreshold = 10 * time.Minute ) // DebugConfig represents config for ticdc unexposed feature configurations diff --git a/utils/notifyqueue/notify_queue.go b/utils/notifyqueue/notify_queue.go index 331aaf4b02..9b2fea8d70 100644 --- a/utils/notifyqueue/notify_queue.go +++ b/utils/notifyqueue/notify_queue.go @@ -8,6 +8,7 @@ // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. diff --git a/utils/notifyqueue/notify_queue_test.go b/utils/notifyqueue/notify_queue_test.go index 1532f23d3d..228e9e563f 100644 --- a/utils/notifyqueue/notify_queue_test.go +++ b/utils/notifyqueue/notify_queue_test.go @@ -8,6 +8,7 @@ // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. diff --git a/utils/priorityqueue/priority_queue_test.go b/utils/priorityqueue/priority_queue_test.go index 7e9dba6d1e..119fe23488 100644 --- a/utils/priorityqueue/priority_queue_test.go +++ b/utils/priorityqueue/priority_queue_test.go @@ -83,22 +83,54 @@ func TestQueuePopBlocking(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) defer cancel() - start := time.Now() - task, err := q.Pop(ctx) - require.ErrorIs(t, err, context.DeadlineExceeded) - require.Nil(t, task) - require.GreaterOrEqual(t, time.Since(start), 50*time.Millisecond) + type popResult struct { + task *mockItem + err error + } + resultCh := make(chan popResult, 1) + go func() { + task, err := q.Pop(ctx) + resultCh <- popResult{task: task, err: err} + }() + + select { + case result := <-resultCh: + t.Fatalf("Pop returned before context timeout: task=%v err=%v", result.task, result.err) + case <-time.After(10 * time.Millisecond): + } + + select { + case result := <-resultCh: + require.ErrorIs(t, result.err, context.DeadlineExceeded) + require.Nil(t, result.task) + case <-time.After(time.Second): + t.Fatal("Pop did not return after context timeout") + } + resultCh = make(chan popResult, 1) go func() { time.Sleep(50 * time.Millisecond) q.Push(newMockItem(10, "task1")) }() - start = time.Now() - task, err = q.Pop(context.Background()) - require.NoError(t, err) - require.Equal(t, "task1", task.description) - require.GreaterOrEqual(t, time.Since(start), 50*time.Millisecond) + go func() { + task, err := q.Pop(context.Background()) + resultCh <- popResult{task: task, err: err} + }() + + select { + case result := <-resultCh: + t.Fatalf("Pop returned before Push: task=%v err=%v", result.task, result.err) + case <-time.After(10 * time.Millisecond): + } + + select { + case result := <-resultCh: + require.NoError(t, result.err) + require.Equal(t, "task1", result.task.description) + case <-time.After(time.Second): + t.Fatal("Pop did not return after Push") + } } func TestQueueTryPopAndUpdateExistingItem(t *testing.T) { From 79d2f9aed24d3751f5a5919bb9a23e6ac4a35db7 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Mon, 3 Aug 2026 13:02:26 +0800 Subject: [PATCH 22/71] small refactor --- logservice/logpuller/region_request_worker.go | 20 +++++++------------ 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index 908f87b071..bf060a5d63 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -147,22 +147,12 @@ func (s *regionRequestWorker) Run(ctx context.Context) error { return err } - if err := s.checkStoreVersion(ctx); err != nil { - if ctx.Err() != nil { - firstReq.abort() - return ctx.Err() - } - if err := handleStreamFailure(firstReq, err); err != nil { - return err - } - continue - } - regionErr := s.runStream(ctx, firstReq) if ctx.Err() != nil { firstReq.abort() return ctx.Err() } + // Treat an unexpected clean stream exit as a recoverable store-stream failure. if regionErr == nil { regionErr = &storeStreamErr{} } @@ -204,6 +194,10 @@ func (s *regionRequestWorker) checkStoreVersion(ctx context.Context) error { } func (s *regionRequestWorker) runStream(ctx context.Context, firstReq *regionReq) (err error) { + if err := s.checkStoreVersion(ctx); err != nil { + return err + } + log.Info("region request worker going to create grpc stream", zap.Uint64("workerID", s.workerID), zap.String("addr", s.store.storeAddr)) @@ -439,11 +433,11 @@ func (s *regionRequestWorker) sendRegionRequest(conn *ConnAndClient, req *region } region := req.regionInfo subID := region.subscribedSpan.subID - log.Debug("region request worker gets a singleRegionInfo", + log.Debug("region request worker sends region request", zap.Uint64("workerID", s.workerID), zap.Uint64("subscriptionID", uint64(subID)), zap.Uint64("regionID", region.verID.GetID()), - zap.String("addr", s.store.storeAddr), + zap.String("storeAddr", s.store.storeAddr), zap.Bool("bdrMode", region.filterLoop)) if region.subscribedSpan.stopped.Load() { From 268966b661d0b7d0c89457b87f5ae1eae80c9b34 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Mon, 3 Aug 2026 14:10:41 +0800 Subject: [PATCH 23/71] small refactor --- logservice/logpuller/region_request_worker.go | 36 +++++++++---------- .../logpuller/region_request_worker_test.go | 2 +- 2 files changed, 18 insertions(+), 20 deletions(-) diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index bf060a5d63..9745dd5854 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -415,18 +415,6 @@ func (s *regionRequestWorker) sendDeregisterRequest( return nil } -func (s *regionRequestWorker) sendPendingDeregisterRequests(conn *ConnAndClient) error { - for { - req, ok := s.controlQueue.tryPop() - if !ok { - return nil - } - if err := s.sendDeregisterRequest(conn, req); err != nil { - return err - } - } -} - func (s *regionRequestWorker) sendRegionRequest(conn *ConnAndClient, req *regionReq) error { if !req.isActive() { return nil @@ -461,7 +449,7 @@ func (s *regionRequestWorker) sendRegionRequest(conn *ConnAndClient, req *region zap.Uint64("regionID", region.verID.GetID())) return nil } - if err := s.sendChangeDataRequest(conn, s.createRegionRequest(region)); err != nil { + if err := s.sendChangeDataRequest(conn, createRegionRequest(s.client.clusterID, region)); err != nil { // Transport failures are always recoverable at the region level. Preserve // the stream error as the function result, but classify the region for // rescheduling instead of exposing an arbitrary gRPC error downstream. @@ -478,17 +466,27 @@ func (s *regionRequestWorker) processRegionSendTask( ) error { regionReq := firstReq for { + // Send the current region request before handling anything newly queued. if regionReq != nil { if err := s.sendRegionRequest(conn, regionReq); err != nil { return err } regionReq = nil - continue } - - if err := s.sendPendingDeregisterRequests(conn); err != nil { - return err + // Flush pending deregisters before admitting the next region request. + // Admission may still contain stale tasks from a stopped subscription, but + // sendRegionRequest re-checks subscription liveness before tracker.Add/Send, + // so those tasks are dropped locally instead of recreating remote registrations. + for { + req, ok := s.controlQueue.tryPop() + if !ok { + break + } + if err := s.sendDeregisterRequest(conn, req); err != nil { + return err + } } + // Block for the next request, but wake early when deregisters arrive. var err error regionReq, err = s.admission.pop(ctx, s.controlQueue.ready()) if err != nil { @@ -497,9 +495,9 @@ func (s *regionRequestWorker) processRegionSendTask( } } -func (s *regionRequestWorker) createRegionRequest(region regionInfo) *cdcpb.ChangeDataRequest { +func createRegionRequest(clusterID uint64, region regionInfo) *cdcpb.ChangeDataRequest { return &cdcpb.ChangeDataRequest{ - Header: &cdcpb.Header{ClusterId: s.client.clusterID, TicdcVersion: version.ReleaseSemver()}, + Header: &cdcpb.Header{ClusterId: clusterID, TicdcVersion: version.ReleaseSemver()}, RegionId: region.verID.GetID(), RequestId: uint64(region.subscribedSpan.subID), RegionEpoch: region.rpcCtx.Meta.RegionEpoch, diff --git a/logservice/logpuller/region_request_worker_test.go b/logservice/logpuller/region_request_worker_test.go index 92c3023b55..52c44b5531 100644 --- a/logservice/logpuller/region_request_worker_test.go +++ b/logservice/logpuller/region_request_worker_test.go @@ -109,7 +109,7 @@ func TestCreateRegionRequestScanPriority(t *testing.T) { region := prepareRegionForSendTest(createTestRegionInfo(1, 1)) region.scanPriority = tc.priority - req := worker.createRegionRequest(region) + req := createRegionRequest(worker.client.clusterID, region) require.Equal(t, tc.expected, req.GetScanPriority()) }) } From 2973fcce1174044b1a6f2de61f89fcc0e47829ca Mon Sep 17 00:00:00 2001 From: lidezhu Date: Mon, 3 Aug 2026 14:15:30 +0800 Subject: [PATCH 24/71] fix test --- utils/priorityqueue/priority_queue_test.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/utils/priorityqueue/priority_queue_test.go b/utils/priorityqueue/priority_queue_test.go index 119fe23488..b501c456d2 100644 --- a/utils/priorityqueue/priority_queue_test.go +++ b/utils/priorityqueue/priority_queue_test.go @@ -93,12 +93,6 @@ func TestQueuePopBlocking(t *testing.T) { resultCh <- popResult{task: task, err: err} }() - select { - case result := <-resultCh: - t.Fatalf("Pop returned before context timeout: task=%v err=%v", result.task, result.err) - case <-time.After(10 * time.Millisecond): - } - select { case result := <-resultCh: require.ErrorIs(t, result.err, context.DeadlineExceeded) @@ -118,12 +112,6 @@ func TestQueuePopBlocking(t *testing.T) { resultCh <- popResult{task: task, err: err} }() - select { - case result := <-resultCh: - t.Fatalf("Pop returned before Push: task=%v err=%v", result.task, result.err) - case <-time.After(10 * time.Millisecond): - } - select { case result := <-resultCh: require.NoError(t, result.err) From 057576303d779709d15aaeff2e743065a87cb584 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Mon, 3 Aug 2026 17:00:54 +0800 Subject: [PATCH 25/71] fix lint --- logservice/logpuller/region_request_worker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index 9745dd5854..5bd48fbe3d 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -471,7 +471,6 @@ func (s *regionRequestWorker) processRegionSendTask( if err := s.sendRegionRequest(conn, regionReq); err != nil { return err } - regionReq = nil } // Flush pending deregisters before admitting the next region request. // Admission may still contain stale tasks from a stopped subscription, but @@ -487,6 +486,7 @@ func (s *regionRequestWorker) processRegionSendTask( } } // Block for the next request, but wake early when deregisters arrive. + // regionReq above is already consumed and will be replaced by the next pop. var err error regionReq, err = s.admission.pop(ctx, s.controlQueue.ready()) if err != nil { From 4b0768b1bc92ec9ae4767244a7e6f1bed5818fdd Mon Sep 17 00:00:00 2001 From: lidezhu Date: Tue, 4 Aug 2026 00:25:48 +0800 Subject: [PATCH 26/71] small refactor --- logservice/logpuller/region_event_handler.go | 3 +-- logservice/logpuller/region_request_worker.go | 16 +++++++++++----- .../logpuller/region_request_worker_test.go | 4 ---- logservice/logpuller/subscription_client.go | 12 ------------ 4 files changed, 12 insertions(+), 23 deletions(-) diff --git a/logservice/logpuller/region_event_handler.go b/logservice/logpuller/region_event_handler.go index 7390d9c9cd..e13b91cbd0 100644 --- a/logservice/logpuller/region_event_handler.go +++ b/logservice/logpuller/region_event_handler.go @@ -29,8 +29,7 @@ import ( ) var ( - metricsResolvedTsCount = metrics.PullerEventCounter.WithLabelValues("resolved_ts") - metricsEventCount = metrics.PullerEventCounter.WithLabelValues("event") + metricsEventCount = metrics.PullerEventCounter.WithLabelValues("event") metricRegionEventHandleDurationEntries = metrics.SubscriptionClientRegionEventHandleDuration.WithLabelValues("entries") metricRegionEventHandleDurationResolved = metrics.SubscriptionClientRegionEventHandleDuration.WithLabelValues("resolved") diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index 5bd48fbe3d..1fb76d1e23 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -25,6 +25,7 @@ import ( "github.com/pingcap/kvproto/pkg/kvrpcpb" "github.com/pingcap/log" cerror "github.com/pingcap/ticdc/pkg/errors" + "github.com/pingcap/ticdc/pkg/metrics" "github.com/pingcap/ticdc/pkg/util" "github.com/pingcap/ticdc/pkg/version" "github.com/pingcap/ticdc/utils/notifyqueue" @@ -38,6 +39,11 @@ const storeReconnectBackoff = time.Second // To generate a workerID in `newRegionRequestWorker`. var workerIDGen atomic.Uint64 +var ( + metricsResolvedTsCount = metrics.PullerEventCounter.WithLabelValues("resolved_ts") + metricBatchResolvedSize = metrics.BatchResolvedEventSize.WithLabelValues("event-store") +) + type deregisterRequest struct { subID SubscriptionID filterLoop bool @@ -115,7 +121,7 @@ func newRegionRequestWorker( } func (s *regionRequestWorker) Run(ctx context.Context) error { - handleStreamFailure := func(firstReq *regionReq, regionErr error) error { + handleStreamFailure := func(firstReq *regionReq, regionErr error) { // Stream failure handle cases: // - tracker: requests already sent to this stream. // - firstReq: popped from admission for this stream, but not necessarily @@ -131,13 +137,12 @@ func (s *regionRequestWorker) Run(ctx context.Context) error { } // The failed stream no longer owns remote registrations. s.controlQueue.drain() - if firstReq.abort() { + if firstReq != nil && firstReq.abort() { s.client.onRegionFail(newRegionErrorInfo(firstReq.regionInfo, regionErr)) } for _, task := range s.admission.drain() { s.client.onRegionFail(newRegionErrorInfo(task.regionInfo, regionErr)) } - return util.Hang(ctx, storeReconnectBackoff) } for { @@ -156,7 +161,8 @@ func (s *regionRequestWorker) Run(ctx context.Context) error { if regionErr == nil { regionErr = &storeStreamErr{} } - if err := handleStreamFailure(firstReq, regionErr); err != nil { + handleStreamFailure(firstReq, regionErr) + if err := util.Hang(ctx, storeReconnectBackoff); err != nil { return err } } @@ -331,7 +337,7 @@ func (s *regionRequestWorker) dispatchRegionChangeEvents(events []*cdcpb.Event) func (s *regionRequestWorker) dispatchResolvedTsEvent(resolvedTsEvent *cdcpb.ResolvedTs) { subscriptionID := SubscriptionID(resolvedTsEvent.RequestId) metricsResolvedTsCount.Add(float64(len(resolvedTsEvent.Regions))) - s.client.metrics.batchResolvedSize.Observe(float64(len(resolvedTsEvent.Regions))) + metricBatchResolvedSize.Observe(float64(len(resolvedTsEvent.Regions))) // TODO: resolvedTsEvent.Ts be 0 is impossible, we need find the root cause. if resolvedTsEvent.Ts == 0 { log.Warn("region request worker receives a resolved ts event with zero value, ignore it", diff --git a/logservice/logpuller/region_request_worker_test.go b/logservice/logpuller/region_request_worker_test.go index 52c44b5531..2d67926fa5 100644 --- a/logservice/logpuller/region_request_worker_test.go +++ b/logservice/logpuller/region_request_worker_test.go @@ -25,7 +25,6 @@ import ( "github.com/pingcap/ticdc/heartbeatpb" "github.com/pingcap/ticdc/logservice/logpuller/regionlock" "github.com/pingcap/ticdc/utils/dynstream" - "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/require" "github.com/tikv/client-go/v2/oracle" "github.com/tikv/client-go/v2/tikv" @@ -211,9 +210,6 @@ func newDispatchResolvedTsTestWorker(regionCount int) (*regionRequestWorker, *mo ds := &mockRegionEventDynamicStream{} worker := ®ionRequestWorker{ client: &subscriptionClient{ - metrics: sharedClientMetrics{ - batchResolvedSize: prometheus.ObserverFunc(func(float64) {}), - }, eventSink: newTestRegionEventSink(ds), }, tracker: newRegionTracker(), diff --git a/logservice/logpuller/subscription_client.go b/logservice/logpuller/subscription_client.go index 3dcb7098d2..7112797014 100644 --- a/logservice/logpuller/subscription_client.go +++ b/logservice/logpuller/subscription_client.go @@ -35,7 +35,6 @@ import ( "github.com/pingcap/ticdc/pkg/spanz" "github.com/pingcap/ticdc/pkg/util" "github.com/pingcap/ticdc/utils/priorityqueue" - "github.com/prometheus/client_golang/prometheus" kvclientv2 "github.com/tikv/client-go/v2/kv" "github.com/tikv/client-go/v2/oracle" "github.com/tikv/client-go/v2/tikv" @@ -98,10 +97,6 @@ type SubscriptionClientConfig struct { RegionRequestWorkerPerStore uint } -type sharedClientMetrics struct { - batchResolvedSize prometheus.Observer -} - // subscriptionClient is used to subscribe events of table ranges from TiKV. // All exported Methods are thread-safe. type SubscriptionClient interface { @@ -126,7 +121,6 @@ type subscriptionClient struct { ctx context.Context cancel context.CancelFunc config *SubscriptionClientConfig - metrics sharedClientMetrics clusterID uint64 pd pd.Client @@ -189,7 +183,6 @@ func NewSubscriptionClient( subClient.eventSink = newRegionEventSink(subClient.failureHandler) subClient.spanRegistry = newSpanRegistry(subClient.pd, subClient.pdClock) - subClient.initMetrics() return subClient } @@ -202,11 +195,6 @@ func (s *subscriptionClient) AllocSubscriptionID() SubscriptionID { return SubscriptionID(subscriptionIDGen.Add(1)) } -func (s *subscriptionClient) initMetrics() { - // TODO: fix metrics - s.metrics.batchResolvedSize = metrics.BatchResolvedEventSize.WithLabelValues("event-store") -} - func (s *subscriptionClient) updateMetrics(ctx context.Context) error { ticker := time.NewTicker(10 * time.Second) defer ticker.Stop() From 3311a1197d676c867beaa723f810c0b8364ad7af Mon Sep 17 00:00:00 2001 From: lidezhu Date: Tue, 4 Aug 2026 15:55:45 +0800 Subject: [PATCH 27/71] small fix --- logservice/logpuller/region_request_worker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index 1fb76d1e23..df9027e5f8 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -436,7 +436,7 @@ func (s *regionRequestWorker) sendRegionRequest(conn *ConnAndClient, req *region if region.subscribedSpan.stopped.Load() { req.abort() - s.client.onRegionFail(newRegionErrorInfo(region, &storeStreamErr{})) + s.client.onRegionFail(newRegionErrorInfo(region, &requestCancelledErr{})) return nil } From 1878c44f12864572385e0ae20d44e79e25b4528f Mon Sep 17 00:00:00 2001 From: lidezhu Date: Fri, 17 Jul 2026 09:54:45 +0800 Subject: [PATCH 28/71] logpuller: extract region request scheduler from subscription client --- .../logpuller/region_request_scheduler.go | 178 ++++++ logservice/logpuller/requested_store.go | 83 +++ logservice/logpuller/subscription_client.go | 273 ++------- .../logpuller/subscription_client_test.go | 566 ++++++------------ 4 files changed, 471 insertions(+), 629 deletions(-) create mode 100644 logservice/logpuller/region_request_scheduler.go create mode 100644 logservice/logpuller/requested_store.go diff --git a/logservice/logpuller/region_request_scheduler.go b/logservice/logpuller/region_request_scheduler.go new file mode 100644 index 0000000000..e81d169e9f --- /dev/null +++ b/logservice/logpuller/region_request_scheduler.go @@ -0,0 +1,178 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package logpuller + +import ( + "context" + "sync" + "sync/atomic" + + "github.com/pingcap/log" + "github.com/pingcap/ticdc/pkg/config" + "github.com/pingcap/ticdc/pkg/errors" + "github.com/pingcap/ticdc/utils/priorityqueue" + kvclientv2 "github.com/tikv/client-go/v2/kv" + "github.com/tikv/client-go/v2/tikv" + "go.uber.org/zap" + "golang.org/x/sync/errgroup" +) + +// regionRequestScheduler routes locked Region requests through the global +// priority queue to a worker connected to the Region's TiKV store. Range +// resolution and retry policy remain owned by subscriptionClient and +// regionFailureHandler respectively. +type regionRequestScheduler struct { + client *subscriptionClient + + // taskQueue orders all Regions before they are assigned to a TiKV store. + taskQueue *priorityqueue.PriorityQueue[*regionPriorityTask] + // sequence is the FIFO tie-breaker for Regions in the same priority class. + sequence atomic.Uint64 + // stores maps TiKV addresses to requestedStore. Stores are created only by + // run, but are also read by metrics and deregistration goroutines. + stores sync.Map + + // workerCount is the configured number of request workers per store. + workerCount int + // workerWindow is each worker's share of the configured store window. + workerWindow int + // maxWindowMultiplier is passed to each worker's admission controller. + maxWindowMultiplier int +} + +func newRegionRequestScheduler(client *subscriptionClient) *regionRequestScheduler { + pullerConfig := config.GetGlobalServerConfig().Debug.Puller + workerCount := int(client.config.RegionRequestWorkerPerStore) + if workerCount <= 0 { + workerCount = 1 + } + workerWindow := (pullerConfig.PendingRegionRequestQueueSize + workerCount - 1) / workerCount + return ®ionRequestScheduler{ + client: client, + taskQueue: priorityqueue.New[*regionPriorityTask](), + workerCount: workerCount, + workerWindow: workerWindow, + maxWindowMultiplier: pullerConfig.RegionRequestMaxWindowMultiplier, + } +} + +func (s *regionRequestScheduler) submit(region regionInfo) { + s.taskQueue.Push(NewRegionPriorityTask( + region, s.client.pdClock.CurrentTS(), s.sequence.Add(1))) +} + +func (s *regionRequestScheduler) run(ctx context.Context, group *errgroup.Group) error { + defer s.closeStores() + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + task, err := s.taskQueue.Pop(ctx) + if err != nil { + if errors.Is(err, priorityqueue.ErrClosed) { + return nil + } + return err + } + + region, ok := s.attachRPCContext(ctx, task.GetRegionInfo()) + if !ok { + continue + } + + store := s.getOrCreateStore(ctx, group, region.rpcCtx.Addr) + task.updateRegion(region, s.client.pdClock.CurrentTS()) + if !store.submit(task) { + return context.Canceled + } + + log.Debug("subscription client will request a region", + zap.Uint64("subscriptionID", uint64(region.subscribedSpan.subID)), + zap.Uint64("regionID", region.verID.GetID()), + zap.String("addr", store.storeAddr)) + } +} + +func (s *regionRequestScheduler) attachRPCContext( + ctx context.Context, + region regionInfo, +) (regionInfo, bool) { + bo := tikv.NewBackoffer(ctx, tikvRequestMaxBackoff) + rpcCtx, err := s.client.regionCache.GetTiKVRPCContext( + bo, region.verID, kvclientv2.ReplicaReadLeader, 0) + if rpcCtx != nil { + region.rpcCtx = rpcCtx + return region, true + } + if err != nil { + log.Debug("subscription client get rpc context fail", + zap.Uint64("subscriptionID", uint64(region.subscribedSpan.subID)), + zap.Uint64("regionID", region.verID.GetID()), + zap.Error(err)) + } + s.client.onRegionFail(newRegionErrorInfo(region, &rpcCtxUnavailableErr{verID: region.verID})) + return region, false +} + +func (s *regionRequestScheduler) getOrCreateStore( + ctx context.Context, + group *errgroup.Group, + storeAddr string, +) *requestedStore { + if value, ok := s.stores.Load(storeAddr); ok { + return value.(*requestedStore) + } + + store := newRequestedStore( + s.client, storeAddr, s.workerCount, s.workerWindow, s.maxWindowMultiplier) + // run is the only writer. Publish the store after its immutable worker list + // is complete, then start its workers. + s.stores.Store(storeAddr, store) + store.run(ctx, group) + return store +} + +func (s *regionRequestScheduler) broadcastDeregister( + subID SubscriptionID, + filterLoop bool, +) { + s.stores.Range(func(_, value any) bool { + value.(*requestedStore).broadcastDeregister(subID, filterLoop) + return true + }) +} + +func (s *regionRequestScheduler) inflightCount() int { + count := 0 + s.stores.Range(func(_, value any) bool { + count += value.(*requestedStore).inflightCount() + return true + }) + return count +} + +func (s *regionRequestScheduler) close() { + s.taskQueue.Close() +} + +func (s *regionRequestScheduler) closeStores() { + s.stores.Range(func(_, value any) bool { + value.(*requestedStore).close() + return true + }) +} diff --git a/logservice/logpuller/requested_store.go b/logservice/logpuller/requested_store.go new file mode 100644 index 0000000000..f98b59b679 --- /dev/null +++ b/logservice/logpuller/requested_store.go @@ -0,0 +1,83 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package logpuller + +import ( + "context" + "sync/atomic" + + "golang.org/x/sync/errgroup" +) + +// requestedStore owns the request workers connected to one TiKV store. The +// worker slice is complete before the store is published and is immutable +// afterwards, so task submission only needs an atomic round-robin counter. +type requestedStore struct { + storeAddr string + workers []*regionRequestWorker + nextWorker atomic.Uint64 +} + +func newRequestedStore( + client *subscriptionClient, + storeAddr string, + workerCount int, + workerWindow int, + maxWindowMultiplier int, +) *requestedStore { + store := &requestedStore{ + storeAddr: storeAddr, + workers: make([]*regionRequestWorker, 0, workerCount), + } + for i := 0; i < workerCount; i++ { + store.workers = append(store.workers, newRegionRequestWorker( + client, store, workerWindow, maxWindowMultiplier)) + } + return store +} + +func (s *requestedStore) run(ctx context.Context, group *errgroup.Group) { + for _, worker := range s.workers { + group.Go(func() error { return worker.Run(ctx) }) + } +} + +func (s *requestedStore) submit(task *regionPriorityTask) bool { + if len(s.workers) == 0 { + return false + } + index := (s.nextWorker.Add(1) - 1) % uint64(len(s.workers)) + return s.workers[index].admission.submit(task) +} + +func (s *requestedStore) broadcastDeregister(subID SubscriptionID, filterLoop bool) { + for _, worker := range s.workers { + worker.controlQueue.push(deregisterRequest{subID: subID, filterLoop: filterLoop}) + } +} + +func (s *requestedStore) close() { + for _, worker := range s.workers { + worker.admission.close() + } +} + +func (s *requestedStore) inflightCount() int { + count := 0 + for _, worker := range s.workers { + count += worker.admission.stats().inflight + } + return count +} diff --git a/logservice/logpuller/subscription_client.go b/logservice/logpuller/subscription_client.go index 7112797014..a7edab9ac5 100644 --- a/logservice/logpuller/subscription_client.go +++ b/logservice/logpuller/subscription_client.go @@ -15,11 +15,9 @@ package logpuller import ( "context" - "sync" "sync/atomic" "time" - "github.com/pingcap/kvproto/pkg/cdcpb" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/log" "github.com/pingcap/ticdc/heartbeatpb" @@ -27,20 +25,15 @@ import ( "github.com/pingcap/ticdc/logservice/txnutil" "github.com/pingcap/ticdc/pkg/common" appcontext "github.com/pingcap/ticdc/pkg/common/context" - "github.com/pingcap/ticdc/pkg/config" - "github.com/pingcap/ticdc/pkg/errors" "github.com/pingcap/ticdc/pkg/metrics" "github.com/pingcap/ticdc/pkg/pdutil" "github.com/pingcap/ticdc/pkg/security" "github.com/pingcap/ticdc/pkg/spanz" "github.com/pingcap/ticdc/pkg/util" - "github.com/pingcap/ticdc/utils/priorityqueue" - kvclientv2 "github.com/tikv/client-go/v2/kv" - "github.com/tikv/client-go/v2/oracle" + "github.com/prometheus/client_golang/prometheus" "github.com/tikv/client-go/v2/tikv" pd "github.com/tikv/pd/client" "go.uber.org/zap" - "go.uber.org/zap/zapcore" "golang.org/x/sync/errgroup" ) @@ -89,7 +82,7 @@ type rangeTask struct { span heartbeatpb.TableSpan subscribedSpan *subscribedSpan filterLoop bool - priority cdcpb.ScanPriority + wasInitialized bool } type SubscriptionClientConfig struct { @@ -97,6 +90,10 @@ type SubscriptionClientConfig struct { RegionRequestWorkerPerStore uint } +type sharedClientMetrics struct { + batchResolvedSize prometheus.Observer +} + // subscriptionClient is used to subscribe events of table ranges from TiKV. // All exported Methods are thread-safe. type SubscriptionClient interface { @@ -121,6 +118,7 @@ type subscriptionClient struct { ctx context.Context cancel context.CancelFunc config *SubscriptionClientConfig + metrics sharedClientMetrics clusterID uint64 pd pd.Client @@ -128,8 +126,6 @@ type subscriptionClient struct { pdClock pdutil.Clock lockResolver txnutil.LockResolver - stores sync.Map - // the credential to connect tikv credential *security.Credential @@ -139,16 +135,12 @@ type subscriptionClient struct { eventSink *regionEventSink // spanRegistry tracks subscribed spans and owns span-level background tasks. spanRegistry *spanRegistry + // regionScheduler assigns locked Region requests to per-store workers. + regionScheduler *regionRequestScheduler // rangeTaskCh is used to receive range tasks. // The tasks will be handled in `handleRangeTask` goroutine. rangeTaskCh chan rangeTask - // regionTaskQueue is used to receive region tasks with priority. - // The region will be handled in `handleRegions` goroutine. - regionTaskQueue *priorityqueue.PriorityQueue[*regionPriorityTask] - // regionTaskSequence provides a FIFO tie-breaker for tasks in the same - // priority class. - regionTaskSequence atomic.Uint64 // resolveLockTaskCh is used to receive resolve lock tasks. // The tasks will be handled in `handleResolveLockTasks` goroutine. resolveLockTaskCh chan resolveLockTask @@ -165,7 +157,6 @@ func NewSubscriptionClient( subClient := &subscriptionClient{ config: config, - stores: sync.Map{}, pd: pd, regionCache: appcontext.GetService[*tikv.RegionCache](appcontext.RegionCache), pdClock: appcontext.GetService[pdutil.Clock](appcontext.DefaultPDClock), @@ -174,15 +165,16 @@ func NewSubscriptionClient( credential: credential, rangeTaskCh: make(chan rangeTask, 1024), - regionTaskQueue: priorityqueue.New[*regionPriorityTask](), resolveLockTaskCh: make(chan resolveLockTask, 1024), resolveLockRateLimiter: newResolveLockRateLimiter(), } subClient.ctx, subClient.cancel = context.WithCancel(context.Background()) subClient.failureHandler = newRegionFailureHandler(subClient) - subClient.eventSink = newRegionEventSink(subClient.failureHandler) + subClient.eventSink = newRegionEventSink(subClient.ctx, subClient.failureHandler) subClient.spanRegistry = newSpanRegistry(subClient.pd, subClient.pdClock) + subClient.regionScheduler = newRegionRequestScheduler(subClient) + subClient.initMetrics() return subClient } @@ -195,6 +187,11 @@ func (s *subscriptionClient) AllocSubscriptionID() SubscriptionID { return SubscriptionID(subscriptionIDGen.Add(1)) } +func (s *subscriptionClient) initMetrics() { + // TODO: fix metrics + s.metrics.batchResolvedSize = metrics.BatchResolvedEventSize.WithLabelValues("event-store") +} + func (s *subscriptionClient) updateMetrics(ctx context.Context) error { ticker := time.NewTicker(10 * time.Second) defer ticker.Stop() @@ -203,12 +200,7 @@ func (s *subscriptionClient) updateMetrics(ctx context.Context) error { case <-ctx.Done(): return ctx.Err() case <-ticker.C: - pendingRegionReqCount := 0 - s.stores.Range(func(_, value any) bool { - store := value.(*requestedStore) - pendingRegionReqCount += store.requestedRegionCount() - return true - }) + pendingRegionReqCount := s.regionScheduler.inflightCount() metrics.SubscriptionClientRequestedRegionCount.WithLabelValues("pending").Set(float64(pendingRegionReqCount)) s.eventSink.UpdateMetrics() @@ -247,8 +239,6 @@ func (s *subscriptionClient) Subscribe( advanceResolvedTs, advanceInterval, bdrMode, - s.pdClock, - time.Duration(config.GetGlobalServerConfig().Debug.Puller.OldStartTsScanLowPriorityThreshold), ) s.spanRegistry.Add(rt) s.eventSink.AddPath(rt) @@ -256,14 +246,8 @@ func (s *subscriptionClient) Subscribe( select { case <-s.ctx.Done(): log.Warn("subscribes span failed, the subscription client has closed") - case s.rangeTaskCh <- rangeTask{ - span: span, - subscribedSpan: rt, - filterLoop: rt.filterLoop, - priority: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, - }: - log.Info("subscribes span done", - zap.Uint64("subscriptionID", uint64(subID)), + case s.rangeTaskCh <- rangeTask{span: span, subscribedSpan: rt, filterLoop: rt.filterLoop}: + log.Info("subscribes span done", zap.Uint64("subscriptionID", uint64(subID)), zap.Int64("tableID", span.TableID), zap.Uint64("startTs", startTs), zap.String("startKey", spanz.HexKey(span.StartKey)), zap.String("endKey", spanz.HexKey(span.EndKey))) } @@ -302,7 +286,7 @@ func (s *subscriptionClient) Run(ctx context.Context) error { g.Go(func() error { return s.updateMetrics(ctx) }) g.Go(func() error { return s.eventSink.Run(ctx) }) g.Go(func() error { return s.handleRangeTasks(ctx) }) - g.Go(func() error { return s.handleRegions(ctx, g) }) + g.Go(func() error { return s.regionScheduler.run(ctx, g) }) g.Go(func() error { return s.failureHandler.Run(ctx) }) g.Go(func() error { return s.handleResolveLockTasks(ctx) }) g.Go(func() error { return s.spanRegistry.Run(ctx) }) @@ -316,7 +300,7 @@ func (s *subscriptionClient) Run(ctx context.Context) error { func (s *subscriptionClient) Close(ctx context.Context) error { s.cancel() s.eventSink.Close() - s.regionTaskQueue.Close() + s.regionScheduler.close() return nil } @@ -327,7 +311,7 @@ func (s *subscriptionClient) setTableStopped(rt *subscribedSpan) { // Set stopped to true so we can stop handling region events from the table, // then notify every existing worker to deregister the subscription. if rt.stopped.CompareAndSwap(false, true) { - s.broadcastDeregister(rt.subID, rt.filterLoop) + s.regionScheduler.broadcastDeregister(rt.subID, rt.filterLoop) if rt.rangeLock.Stop() { s.onTableDrained(rt) } @@ -352,164 +336,6 @@ func (s *subscriptionClient) onRegionFail(errInfo regionErrorInfo) { s.failureHandler.Report(errInfo) } -// requestedStore represents a store that has been connected. -type requestedStore struct { - storeAddr string - nextWorker atomic.Uint64 - - requestWorkers struct { - sync.RWMutex - s []*regionRequestWorker - } -} - -func (s *requestedStore) submit(task *regionPriorityTask) bool { - s.requestWorkers.RLock() - defer s.requestWorkers.RUnlock() - - workerCount := len(s.requestWorkers.s) - if workerCount == 0 { - return false - } - index := (s.nextWorker.Add(1) - 1) % uint64(workerCount) - return s.requestWorkers.s[index].admission.submit(task) -} - -func (s *requestedStore) close() { - s.requestWorkers.RLock() - defer s.requestWorkers.RUnlock() - for _, worker := range s.requestWorkers.s { - worker.admission.close() - } -} - -func (s *requestedStore) requestedRegionCount() int { - s.requestWorkers.RLock() - defer s.requestWorkers.RUnlock() - count := 0 - for _, worker := range s.requestWorkers.s { - stats := worker.admission.stats() - count += stats.pending + stats.inflight - } - return count -} - -// handleRegions receives regionInfo from regionTaskQueue and attach rpcCtx to them, -// then send them to corresponding requestedStore. -func (s *subscriptionClient) handleRegions(ctx context.Context, eg *errgroup.Group) error { - cfg := config.GetGlobalServerConfig() - storeWindow := cfg.Debug.Puller.PendingRegionRequestQueueSize - maxWindowMultiplier := cfg.Debug.Puller.RegionRequestMaxWindowMultiplier - workerCount := int(s.config.RegionRequestWorkerPerStore) - if workerCount <= 0 { - workerCount = 1 - } - workerWindow := (storeWindow + workerCount - 1) / workerCount - getStore := func(storeAddr string) *requestedStore { - var rs *requestedStore - if v, ok := s.stores.Load(storeAddr); ok { - rs = v.(*requestedStore) - return rs - } - - rs = &requestedStore{storeAddr: storeAddr} - rs.requestWorkers.s = make([]*regionRequestWorker, 0, workerCount) - - rs.requestWorkers.Lock() - for i := 0; i < workerCount; i++ { - requestWorker := newRegionRequestWorker(s, rs, workerWindow, maxWindowMultiplier) - rs.requestWorkers.s = append(rs.requestWorkers.s, requestWorker) - } - rs.requestWorkers.Unlock() - - // Publish the store only after its immutable worker list is complete. - s.stores.Store(storeAddr, rs) - for _, requestWorker := range rs.requestWorkers.s { - eg.Go(func() error { return requestWorker.Run(ctx) }) - } - return rs - } - - defer func() { - s.stores.Range(func(_, value any) bool { - rs := value.(*requestedStore) - rs.close() - return true - }) - }() - - for { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - // Use blocking Pop to wait for tasks - regionTask, err := s.regionTaskQueue.Pop(ctx) - if err != nil { - if errors.Is(err, priorityqueue.ErrClosed) { - return nil - } - return err - } - - region := regionTask.regionInfo - region, ok := s.attachRPCContextForRegion(ctx, region) - // If attachRPCContextForRegion fails, the region will be re-scheduled. - if !ok { - continue - } - if region.subscribedSpan.stopped.Load() { - s.onRegionFail(newRegionErrorInfo(region, &requestCancelledErr{})) - continue - } - - store := getStore(region.rpcCtx.Addr) - regionTask.regionInfo = region - if !store.submit(regionTask) { - if ctx.Err() != nil { - return ctx.Err() - } - s.onRegionFail(newRegionErrorInfo(region, &storeStreamErr{})) - continue - } - - log.Debug("subscription client will request a region", - zap.Uint64("subscriptionID", uint64(region.subscribedSpan.subID)), - zap.Uint64("regionID", region.verID.GetID()), - zap.String("addr", store.storeAddr)) - } -} - -func (s *subscriptionClient) broadcastDeregister(subID SubscriptionID, filterLoop bool) { - s.stores.Range(func(_ any, value any) bool { - rs := value.(*requestedStore) - rs.requestWorkers.RLock() - for _, worker := range rs.requestWorkers.s { - worker.controlQueue.push(deregisterRequest{subID: subID, filterLoop: filterLoop}) - } - rs.requestWorkers.RUnlock() - return true - }) -} - -func (s *subscriptionClient) attachRPCContextForRegion(ctx context.Context, region regionInfo) (regionInfo, bool) { - bo := tikv.NewBackoffer(ctx, tikvRequestMaxBackoff) - rpcCtx, err := s.regionCache.GetTiKVRPCContext(bo, region.verID, kvclientv2.ReplicaReadLeader, 0) - if rpcCtx != nil { - region.rpcCtx = rpcCtx - return region, true - } - if err != nil { - log.Debug("subscription client get rpc context fail", - zap.Uint64("subscriptionID", uint64(region.subscribedSpan.subID)), - zap.Uint64("regionID", region.verID.GetID()), - zap.Error(err)) - } - s.onRegionFail(newRegionErrorInfo(region, &rpcCtxUnavailableErr{verID: region.verID})) - return region, false -} - func (s *subscriptionClient) handleRangeTasks(ctx context.Context) error { g, ctx := errgroup.WithContext(ctx) // Limit the concurrent number of goroutines to convert range tasks to region tasks. @@ -520,8 +346,7 @@ func (s *subscriptionClient) handleRangeTasks(ctx context.Context) error { return ctx.Err() case task := <-s.rangeTaskCh: g.Go(func() error { - return s.divideSpanAndScheduleRegionRequests( - ctx, task.span, task.subscribedSpan, task.filterLoop, task.priority) + return s.divideSpanAndScheduleRegionRequests(ctx, task.span, task.subscribedSpan, task.filterLoop, task.wasInitialized) }) } } @@ -537,7 +362,7 @@ func (s *subscriptionClient) divideSpanAndScheduleRegionRequests( span heartbeatpb.TableSpan, subscribedSpan *subscribedSpan, filterLoop bool, - inheritedPriority cdcpb.ScanPriority, + wasInitialized bool, ) error { // Limit the number of regions loaded at a time to make the load more stable. limit := 1024 @@ -599,9 +424,10 @@ func (s *subscriptionClient) divideSpanAndScheduleRegionRequests( verID := tikv.NewRegionVerID(regionMeta.Id, regionMeta.RegionEpoch.ConfVer, regionMeta.RegionEpoch.Version) regionInfo := newRegionInfo(verID, intersectSpan, nil, subscribedSpan, filterLoop) + regionInfo.wasInitialized = wasInitialized // Schedule a region request to subscribe the region. - s.scheduleRegionRequest(ctx, regionInfo, inheritedPriority) + s.scheduleRegionRequest(ctx, regionInfo) nextSpan.StartKey = regionMeta.EndKey // If the nextSpan.StartKey is larger than the subscribedSpan.span.EndKey, @@ -613,13 +439,12 @@ func (s *subscriptionClient) divideSpanAndScheduleRegionRequests( } } -// scheduleRegionRequest locks the region's range and send the region to regionTaskQueue, -// which will be handled by handleRegions. -func (s *subscriptionClient) scheduleRegionRequest( - ctx context.Context, - region regionInfo, - inheritedPriority cdcpb.ScanPriority, -) { +// scheduleRegionRequest locks the Region's range before submitting it to the +// request scheduler. +func (s *subscriptionClient) scheduleRegionRequest(ctx context.Context, region regionInfo) { + if region.lockedRangeState != nil && region.lockedRangeState.Initialized.Load() { + region.wasInitialized = true + } lockRangeResult := region.subscribedSpan.rangeLock.LockRange( ctx, region.span.StartKey, region.span.EndKey, region.verID.GetID(), region.verID.GetVer()) @@ -630,30 +455,10 @@ func (s *subscriptionClient) scheduleRegionRequest( switch lockRangeResult.Status { case regionlock.LockRangeStatusSuccess: region.lockedRangeState = lockRangeResult.LockedRangeState - currentTs := s.pdClock.CurrentTS() - priority := region.subscribedSpan.priorityPolicy.resolve( - inheritedPriority, - region.resolvedTs(), - oracle.GetTimeFromTS(currentTs), - ) - region.scanPriority = priority - s.regionTaskQueue.Push(newRegionPriorityTask(region, s.regionTaskSequence.Add(1))) - if log.GetLevel() <= zapcore.DebugLevel { - log.Debug("cdc region scan task enqueued", - zap.Uint64("subscriptionID", uint64(region.subscribedSpan.subID)), - zap.Int64("tableID", region.subscribedSpan.span.TableID), - zap.Uint64("startTs", region.subscribedSpan.startTs), - zap.Uint64("regionID", region.verID.GetID()), - zap.Uint64("regionEpochVersion", region.verID.GetVer()), - zap.Uint64("regionEpochConfVer", region.verID.GetConfVer()), - zap.String("priority", normalizeScanPriority(priority).String()), - zap.String("scanPriority", region.scanPriority.String()), - zap.String("span", common.FormatTableSpan(®ion.span))) - } + s.regionScheduler.submit(region) case regionlock.LockRangeStatusStale: for _, r := range lockRangeResult.RetryRanges { - s.scheduleRangeRequest( - ctx, r, region.subscribedSpan, region.filterLoop, inheritedPriority) + s.scheduleRangeRequest(ctx, r, region.subscribedSpan, region.filterLoop, region.wasInitialized) } default: return @@ -664,15 +469,13 @@ func (s *subscriptionClient) scheduleRangeRequest( ctx context.Context, span heartbeatpb.TableSpan, subscribedSpan *subscribedSpan, filterLoop bool, - inheritedPriority cdcpb.ScanPriority, + wasInitialized bool, ) { select { case <-ctx.Done(): case s.rangeTaskCh <- rangeTask{ - span: span, - subscribedSpan: subscribedSpan, - filterLoop: filterLoop, - priority: inheritedPriority, + span: span, subscribedSpan: subscribedSpan, + filterLoop: filterLoop, wasInitialized: wasInitialized, }: } } diff --git a/logservice/logpuller/subscription_client_test.go b/logservice/logpuller/subscription_client_test.go index 89900a4df3..4fb76df372 100644 --- a/logservice/logpuller/subscription_client_test.go +++ b/logservice/logpuller/subscription_client_test.go @@ -22,7 +22,6 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/cdcpb" - "github.com/pingcap/kvproto/pkg/errorpb" "github.com/pingcap/ticdc/heartbeatpb" "github.com/pingcap/ticdc/logservice/logpuller/regionlock" "github.com/pingcap/ticdc/pkg/common" @@ -38,7 +37,6 @@ import ( "github.com/tikv/client-go/v2/oracle" "github.com/tikv/client-go/v2/testutils" "github.com/tikv/client-go/v2/tikv" - "golang.org/x/sync/errgroup" ) type mockLockResolver struct { @@ -81,8 +79,6 @@ func TestGenerateResolveLockTask(t *testing.T) { advanceResolvedTs, 0, false, - pdutil.NewClock4Test(), - 30*time.Minute, ) client.spanRegistry.Add(span) @@ -155,12 +151,12 @@ func TestResolveLockTaskDeduplicatedAcrossSubscribedSpans(t *testing.T) { TableID: 1, StartKey: []byte{'a'}, EndKey: []byte{'z'}, - }, 100, consumeKVEvents, advanceResolvedTs, 0, false, pdutil.NewClock4Test(), 30*time.Minute) + }, 100, consumeKVEvents, advanceResolvedTs, 0, false) span2 := newSubscribedSpan(client.ctx, client.resolveLockRateLimiter, client.resolveLockTaskCh, SubscriptionID(2), heartbeatpb.TableSpan{ TableID: 2, StartKey: []byte{'a'}, EndKey: []byte{'z'}, - }, 100, consumeKVEvents, advanceResolvedTs, 0, false, pdutil.NewClock4Test(), 30*time.Minute) + }, 100, consumeKVEvents, advanceResolvedTs, 0, false) res := span1.rangeLock.LockRange(context.Background(), []byte{'b'}, []byte{'c'}, 1, 100) require.Equal(t, regionlock.LockRangeStatusSuccess, res.Status) @@ -186,7 +182,7 @@ func TestResolveLockTaskDeduplicatedAcrossSubscribedSpans(t *testing.T) { } func TestHandleResolveLockTasksMetrics(t *testing.T) { - ctx, cancel := context.WithCancel(t.Context()) + ctx, cancel := context.WithCancel(context.Background()) defer cancel() resolver := &mockLockResolver{} @@ -201,10 +197,7 @@ func TestHandleResolveLockTasksMetrics(t *testing.T) { errCh <- client.handleResolveLockTasks(ctx) }() - rangeLock := regionlock.NewRangeLock(1, []byte{'a'}, []byte{'b'}, 100) - lockResult := rangeLock.LockRange(context.Background(), []byte{'a'}, []byte{'b'}, 1, 1) - require.Equal(t, regionlock.LockRangeStatusSuccess, lockResult.Status) - state := lockResult.LockedRangeState + state := ®ionlock.LockedRangeState{} state.Initialized.Store(true) state.ResolvedTs.Store(100) @@ -273,8 +266,6 @@ func TestResolveLockTaskDroppedWhenChannelFull(t *testing.T) { advanceResolvedTs, 0, false, - pdutil.NewClock4Test(), - 30*time.Minute, ) res := span.rangeLock.LockRange(context.Background(), []byte{'b'}, []byte{'c'}, 1, 100) @@ -332,16 +323,14 @@ func TestStopTaskUsesSubscribedSpanFilterLoop(t *testing.T) { advanceResolvedTs, 0, true, - pdutil.NewClock4Test(), - 30*time.Minute, ) res := span.rangeLock.LockRange(context.Background(), rawSpan.StartKey, rawSpan.EndKey, 1, 1) require.Equal(t, regionlock.LockRangeStatusSuccess, res.Status) worker := ®ionRequestWorker{controlQueue: newControlQueue()} - store := &requestedStore{storeAddr: "store-1"} - store.requestWorkers.s = []*regionRequestWorker{worker} - client.stores.Store(store.storeAddr, store) + store := &requestedStore{storeAddr: "store-1", workers: []*regionRequestWorker{worker}} + client.regionScheduler = ®ionRequestScheduler{client: client} + client.regionScheduler.stores.Store(store.storeAddr, store) client.setTableStopped(span) @@ -353,7 +342,7 @@ func TestStopTaskUsesSubscribedSpanFilterLoop(t *testing.T) { func TestOnRegionFailQueuesCanceledErrorCache(t *testing.T) { client := &subscriptionClient{ - eventSink: newTestRegionEventSink(&mockDynamicStream{}), + eventSink: ®ionEventSink{ds: &mockDynamicStream{}}, } client.spanRegistry = newSpanRegistry(nil, nil) client.failureHandler = newRegionFailureHandler(client) @@ -396,313 +385,6 @@ func TestOnRegionFailQueuesCanceledErrorCache(t *testing.T) { require.Nil(t, client.spanRegistry.Get(span.subID)) } -func TestHandleRegionsSkipsStoppedSubscriptionBeforeCreatingStore(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - _, cluster, pdClient, _ := testutils.NewMockTiKV("", mockcopr.NewCoprRPCHandler()) - pdClient = &mockPDClient{Client: pdClient, versionGen: defaultVersionGen} - defer pdClient.Close() - - const storeAddr = "store-1" - cluster.AddStore(1, storeAddr) - cluster.Bootstrap(11, []uint64{1}, []uint64{2}, 2) - - regionCache := tikv.NewRegionCache(pdClient) - defer regionCache.Close() - - bo := tikv.NewBackoffer(ctx, tikvRequestMaxBackoff) - location, err := regionCache.LocateKey(bo, []byte("a")) - require.NoError(t, err) - - rawSpan := heartbeatpb.TableSpan{ - TableID: 1, - StartKey: []byte("a"), - EndKey: []byte("b"), - } - span := &subscribedSpan{ - subID: SubscriptionID(1), - span: rawSpan, - rangeLock: regionlock.NewRangeLock(1, rawSpan.StartKey, rawSpan.EndKey, 100), - } - - lockRes := span.rangeLock.LockRange( - context.Background(), rawSpan.StartKey, rawSpan.EndKey, location.Region.GetID(), location.Region.GetVer()) - require.Equal(t, regionlock.LockRangeStatusSuccess, lockRes.Status) - - client := &subscriptionClient{ - ctx: ctx, - config: &SubscriptionClientConfig{RegionRequestWorkerPerStore: 1}, - pd: pdClient, - pdClock: pdutil.NewClock4Test(), - regionCache: regionCache, - credential: &security.Credential{}, - eventSink: newTestRegionEventSink(&mockDynamicStream{}), - spanRegistry: newSpanRegistry(nil, nil), - regionTaskQueue: priorityqueue.New[*regionPriorityTask](), - } - client.failureHandler = newRegionFailureHandler(client) - client.spanRegistry.Add(span) - - region := newRegionInfo(location.Region, rawSpan, nil, span, false) - region.lockedRangeState = lockRes.LockedRangeState - client.regionTaskQueue.Push(newRegionPriorityTask(region, 1)) - client.setTableStopped(span) - - var eg errgroup.Group - errCh := make(chan error, 1) - go func() { - errCh <- client.handleRegions(ctx, &eg) - }() - - require.Eventually(t, func() bool { - return client.spanRegistry.Get(span.subID) == nil - }, time.Second, 20*time.Millisecond) - - _, ok := client.stores.Load(storeAddr) - require.False(t, ok) - - cancel() - err = <-errCh - require.ErrorIs(t, err, context.Canceled) -} - -func TestHandleRegionsReschedulesRegionWhenStoreSubmitFails(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - _, cluster, pdClient, _ := testutils.NewMockTiKV("", mockcopr.NewCoprRPCHandler()) - pdClient = &mockPDClient{Client: pdClient, versionGen: defaultVersionGen} - defer pdClient.Close() - - const storeAddr = "store-1" - cluster.AddStore(1, storeAddr) - cluster.Bootstrap(11, []uint64{1}, []uint64{2}, 2) - - regionCache := tikv.NewRegionCache(pdClient) - defer regionCache.Close() - - bo := tikv.NewBackoffer(ctx, tikvRequestMaxBackoff) - location, err := regionCache.LocateKey(bo, []byte("a")) - require.NoError(t, err) - - rawSpan := heartbeatpb.TableSpan{ - TableID: 1, - StartKey: []byte("a"), - EndKey: []byte("b"), - } - span := &subscribedSpan{ - subID: SubscriptionID(1), - span: rawSpan, - rangeLock: regionlock.NewRangeLock(1, rawSpan.StartKey, rawSpan.EndKey, 100), - priorityPolicy: newTestScanPriorityPolicy(), - } - lockRes := span.rangeLock.LockRange( - context.Background(), rawSpan.StartKey, rawSpan.EndKey, location.Region.GetID(), location.Region.GetVer()) - require.Equal(t, regionlock.LockRangeStatusSuccess, lockRes.Status) - - admission := newRegionAdmissionController(1, 1) - admission.close() - store := &requestedStore{storeAddr: storeAddr} - store.requestWorkers.s = []*regionRequestWorker{{admission: admission}} - - client := &subscriptionClient{ - ctx: ctx, - config: &SubscriptionClientConfig{RegionRequestWorkerPerStore: 1}, - pd: pdClient, - pdClock: pdutil.NewClock4Test(), - regionCache: regionCache, - credential: &security.Credential{}, - eventSink: newTestRegionEventSink(&mockDynamicStream{}), - spanRegistry: newSpanRegistry(nil, nil), - regionTaskQueue: priorityqueue.New[*regionPriorityTask](), - } - client.failureHandler = newRegionFailureHandler(client) - client.stores.Store(storeAddr, store) - - region := newRegionInfo(location.Region, rawSpan, nil, span, false) - region.lockedRangeState = lockRes.LockedRangeState - client.regionTaskQueue.Push(newRegionPriorityTask(region, 1)) - - var eg errgroup.Group - errCh := make(chan error, 1) - go func() { - errCh <- client.handleRegions(ctx, &eg) - }() - - require.Eventually(t, func() bool { - client.failureHandler.cache.Lock() - defer client.failureHandler.cache.Unlock() - return len(client.failureHandler.cache.cache) == 1 - }, time.Second, 20*time.Millisecond) - - select { - case err := <-errCh: - t.Fatalf("handleRegions exited unexpectedly: %v", err) - default: - } - - cancel() - err = <-errCh - require.ErrorIs(t, err, context.Canceled) - - batch := client.failureHandler.cache.popBatch(1) - require.Len(t, batch, 1) - require.IsType(t, &storeStreamErr{}, batch[0].err) - require.NoError(t, client.failureHandler.handleError(context.Background(), batch[0])) - - popCtx, popCancel := context.WithTimeout(context.Background(), time.Second) - defer popCancel() - retriedTask, err := client.regionTaskQueue.Pop(popCtx) - require.NoError(t, err) - require.Equal(t, region.verID, retriedTask.regionInfo.verID) - require.Equal(t, rawSpan, retriedTask.regionInfo.span) -} - -func TestRegionRetryScanPriority(t *testing.T) { - for _, tc := range []struct { - name string - priority cdcpb.ScanPriority - cdcErr *cdcpb.Error - everCaughtUp bool - expected cdcpb.ScanPriority - }{ - { - name: "server is busy high", - priority: cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, - cdcErr: &cdcpb.Error{ServerIsBusy: &errorpb.ServerIsBusy{}}, - expected: cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, - }, - { - name: "server is busy low", - priority: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, - cdcErr: &cdcpb.Error{ServerIsBusy: &errorpb.ServerIsBusy{}}, - expected: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, - }, - { - name: "server is busy low after catch up", - priority: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, - cdcErr: &cdcpb.Error{ServerIsBusy: &errorpb.ServerIsBusy{}}, - everCaughtUp: true, - expected: cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, - }, - { - name: "congested high", - priority: cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, - cdcErr: &cdcpb.Error{Congested: &cdcpb.Congested{}}, - expected: cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, - }, - { - name: "congested low", - priority: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, - cdcErr: &cdcpb.Error{Congested: &cdcpb.Congested{}}, - expected: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, - }, - { - name: "unknown retry high", - priority: cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, - cdcErr: &cdcpb.Error{}, - expected: cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, - }, - { - name: "unknown retry low", - priority: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, - cdcErr: &cdcpb.Error{}, - expected: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, - }, - } { - t.Run(tc.name, func(t *testing.T) { - client := &subscriptionClient{ - regionTaskQueue: priorityqueue.New[*regionPriorityTask](), - } - client.pdClock = pdutil.NewClock4Test() - client.pdClock.(*pdutil.Clock4Test).SetTS(oracle.GoTimeToTS(time.Now())) - client.failureHandler = newRegionFailureHandler(client) - _, span := newScanPriorityTestSpan() - span.priorityPolicy.everCaughtUp.Store(tc.everCaughtUp) - region := newScanPriorityTestRegion(span) - region.scanPriority = tc.priority - - err := client.failureHandler.handleError(context.Background(), newRegionErrorInfo(region, &eventError{err: tc.cdcErr})) - require.NoError(t, err) - - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - task, err := client.regionTaskQueue.Pop(ctx) - require.NoError(t, err) - require.Equal(t, tc.expected, task.priority()) - require.Equal(t, tc.expected, task.regionInfo.scanPriority) - }) - } -} - -func TestRangeRetryPreservesScanPriority(t *testing.T) { - for _, tc := range []struct { - name string - priority cdcpb.ScanPriority - err error - expected cdcpb.ScanPriority - }{ - { - name: "epoch not match high", - priority: cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, - err: &eventError{err: &cdcpb.Error{EpochNotMatch: &errorpb.EpochNotMatch{}}}, - expected: cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, - }, - { - name: "epoch not match low", - priority: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, - err: &eventError{err: &cdcpb.Error{EpochNotMatch: &errorpb.EpochNotMatch{}}}, - expected: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, - }, - { - name: "region not found high", - priority: cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, - err: &eventError{err: &cdcpb.Error{RegionNotFound: &errorpb.RegionNotFound{}}}, - expected: cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, - }, - { - name: "region not found low", - priority: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, - err: &eventError{err: &cdcpb.Error{RegionNotFound: &errorpb.RegionNotFound{}}}, - expected: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, - }, - { - name: "rpc context unavailable high", - priority: cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, - err: &rpcCtxUnavailableErr{verID: tikv.NewRegionVerID(1, 1, 1)}, - expected: cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, - }, - { - name: "rpc context unavailable low", - priority: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, - err: &rpcCtxUnavailableErr{verID: tikv.NewRegionVerID(1, 1, 1)}, - expected: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, - }, - } { - t.Run(tc.name, func(t *testing.T) { - client := &subscriptionClient{ - rangeTaskCh: make(chan rangeTask, 1), - } - client.failureHandler = newRegionFailureHandler(client) - rawSpan, span := newScanPriorityTestSpan() - region := newScanPriorityTestRegion(span) - region.scanPriority = tc.priority - - err := client.failureHandler.handleError(context.Background(), newRegionErrorInfo(region, tc.err)) - require.NoError(t, err) - - select { - case task := <-client.rangeTaskCh: - require.Equal(t, tc.expected, task.priority) - require.Equal(t, rawSpan, task.span) - case <-time.After(time.Second): - require.Fail(t, "expected range retry task") - } - }) - } -} - type mockDynamicStream struct{} func (s *mockDynamicStream) Start() {} @@ -733,34 +415,16 @@ func (s *mockDynamicStream) GetMetrics() dynstream.Metrics[int, SubscriptionID] return dynstream.Metrics[int, SubscriptionID]{} } -func newScanPriorityTestSpan() (heartbeatpb.TableSpan, *subscribedSpan) { - rawSpan := heartbeatpb.TableSpan{ - TableID: 1, - StartKey: []byte("a"), - EndKey: []byte("z"), - } - span := &subscribedSpan{ - subID: SubscriptionID(1), - span: rawSpan, - rangeLock: regionlock.NewRangeLock(1, rawSpan.StartKey, rawSpan.EndKey, 100), - priorityPolicy: newTestScanPriorityPolicy(), - } - return rawSpan, span -} - -func newScanPriorityTestRegion(span *subscribedSpan) regionInfo { - return newRegionInfo(tikv.NewRegionVerID(1, 1, 1), span.span, nil, span, false) -} - -func newTestScanPriorityPolicy() scanPriorityPolicy { - return newScanPriorityPolicy(pdutil.NewClock4Test(), 30*time.Minute) -} - func TestPushRegionEventToDSUnblocksOnClose(t *testing.T) { - sink := newTestRegionEventSink(&mockDynamicStream{}) - client := &subscriptionClient{ - eventSink: sink, - regionTaskQueue: priorityqueue.New[*regionPriorityTask](), + sink := ®ionEventSink{ + ctx: context.Background(), + ds: &mockDynamicStream{}, + } + sink.cond = sync.NewCond(&sink.mu) + client := &subscriptionClient{eventSink: sink} + client.regionScheduler = ®ionRequestScheduler{ + client: client, + taskQueue: priorityqueue.New[*regionPriorityTask](), } client.ctx, client.cancel = context.WithCancel(context.Background()) @@ -789,23 +453,23 @@ func TestPushRegionEventToDSUnblocksOnClose(t *testing.T) { func TestBroadcastDeregisterUsesWorkerControlQueue(t *testing.T) { client := &subscriptionClient{} + scheduler := ®ionRequestScheduler{client: client} admission := newRegionAdmissionController(1, 1) worker := ®ionRequestWorker{ admission: admission, controlQueue: newControlQueue(), } - store := &requestedStore{storeAddr: "store-1"} - store.requestWorkers.s = []*regionRequestWorker{worker} - client.stores.Store(store.storeAddr, store) + store := &requestedStore{storeAddr: "store-1", workers: []*regionRequestWorker{worker}} + scheduler.stores.Store(store.storeAddr, store) dummyRegion := regionInfo{ subscribedSpan: &subscribedSpan{subID: SubscriptionID(2)}, lockedRangeState: ®ionlock.LockedRangeState{}, } - require.True(t, admission.submit(newRegionPriorityTask(dummyRegion, 1))) + require.True(t, admission.submit(NewRegionPriorityTask(dummyRegion, 1, 1))) - client.broadcastDeregister(SubscriptionID(1), true) + scheduler.broadcastDeregister(SubscriptionID(1), true) require.Equal(t, 1, worker.controlQueue.len()) req, ok := worker.controlQueue.tryPop() require.True(t, ok) @@ -817,8 +481,10 @@ func TestBroadcastDeregisterUsesWorkerControlQueue(t *testing.T) { func TestRequestedStoreDistributesRegionsAcrossWorkerBuffers(t *testing.T) { worker1 := ®ionRequestWorker{admission: newRegionAdmissionController(1, 1)} worker2 := ®ionRequestWorker{admission: newRegionAdmissionController(1, 1)} - store := &requestedStore{storeAddr: "store-1"} - store.requestWorkers.s = []*regionRequestWorker{worker1, worker2} + store := &requestedStore{ + storeAddr: "store-1", + workers: []*regionRequestWorker{worker1, worker2}, + } for i := uint64(1); i <= 4; i++ { region := regionInfo{ @@ -826,45 +492,13 @@ func TestRequestedStoreDistributesRegionsAcrossWorkerBuffers(t *testing.T) { subscribedSpan: &subscribedSpan{subID: 1}, lockedRangeState: ®ionlock.LockedRangeState{}, } - require.True(t, store.submit(newRegionPriorityTask(region, i))) + require.True(t, store.submit(NewRegionPriorityTask(region, 1, i))) } require.Equal(t, 2, worker1.admission.stats().pending) require.Equal(t, 2, worker2.admission.stats().pending) } -func TestRequestedStoreRequestedRegionCountIncludesPendingAndInflight(t *testing.T) { - worker1 := ®ionRequestWorker{admission: newRegionAdmissionController(1, 1)} - worker2 := ®ionRequestWorker{admission: newRegionAdmissionController(1, 1)} - store := &requestedStore{storeAddr: "store-1"} - store.requestWorkers.s = []*regionRequestWorker{worker1, worker2} - - region1 := regionInfo{ - verID: tikv.NewRegionVerID(1, 1, 1), - subscribedSpan: &subscribedSpan{subID: 1}, - lockedRangeState: ®ionlock.LockedRangeState{}, - } - region2 := regionInfo{ - verID: tikv.NewRegionVerID(2, 1, 1), - subscribedSpan: &subscribedSpan{subID: 1}, - lockedRangeState: ®ionlock.LockedRangeState{}, - } - region3 := regionInfo{ - verID: tikv.NewRegionVerID(3, 1, 1), - subscribedSpan: &subscribedSpan{subID: 1}, - lockedRangeState: ®ionlock.LockedRangeState{}, - } - - require.True(t, worker1.admission.submit(newRegionPriorityTask(region1, 1))) - req, err := worker1.admission.pop(t.Context(), nil) - require.NoError(t, err) - require.True(t, worker2.admission.submit(newRegionPriorityTask(region2, 2))) - require.True(t, worker2.admission.submit(newRegionPriorityTask(region3, 3))) - - require.Equal(t, 3, store.requestedRegionCount()) - require.True(t, req.abort()) -} - func TestSubscriptionWithFailedTiKV(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) mockPDClock := pdutil.NewClock4Test() @@ -968,6 +602,150 @@ func TestSubscriptionWithFailedTiKV(t *testing.T) { } } +// TestErrCacheDispatchWithFullChannelAndCanceledContext tests that when errCh is full +// and context is canceled, the dispatch method doesn't get stuck. +func TestErrCacheDispatchWithFullChannelAndCanceledContext(t *testing.T) { + // Create errCache with a small errCh to easily fill it up + errCache := &errCache{ + cache: make([]regionErrorInfo, 0, 10), + errCh: make(chan regionErrorInfo, 2), // Small buffer to easily fill + notify: make(chan struct{}, 10), + } + + // Create a mock regionErrorInfo + mockErrInfo := regionErrorInfo{ + regionInfo: regionInfo{ + verID: tikv.NewRegionVerID(1, 1, 1), + span: heartbeatpb.TableSpan{TableID: 1, StartKey: []byte("a"), EndKey: []byte("b")}, + }, + err: errors.New("test error"), + } + + // Fill up the errCh channel to make it full + errCache.errCh <- mockErrInfo + errCache.errCh <- mockErrInfo + + // Add some errors to the cache + for i := 0; i < 5; i++ { + errCache.add(mockErrInfo) + } + + // Create a context that will be canceled + ctx, cancel := context.WithCancel(context.Background()) + + // Channel to signal when dispatch returns + dispatchDone := make(chan error, 1) + + // Start dispatch in a goroutine + go func() { + err := errCache.dispatch(ctx) + dispatchDone <- err + }() + + // Give dispatch some time to start and potentially get stuck + time.Sleep(50 * time.Millisecond) + + // Cancel the context + cancel() + + // Wait for dispatch to return with a timeout + select { + case err := <-dispatchDone: + // Verify that dispatch returned with context.Canceled error + require.Equal(t, context.Canceled, err) + case <-time.After(5 * time.Second): + // If we timeout here, it means dispatch is stuck + t.Fatal("dispatch method is stuck and didn't return after context cancellation") + } +} + +func TestErrCacheDispatchBatch(t *testing.T) { + mockErrInfo := regionErrorInfo{ + regionInfo: regionInfo{ + verID: tikv.NewRegionVerID(1, 1, 1), + span: heartbeatpb.TableSpan{TableID: 1, StartKey: []byte("a"), EndKey: []byte("b")}, + }, + err: errors.New("test error"), + } + + tests := []struct { + name string + cacheLen int + limit int + expectedN int + expectedCache int + expectedErrCh int + }{ + { + name: "dispatch all when limit equals cache length", + cacheLen: 5, + limit: 5, + expectedN: 5, + expectedCache: 0, + expectedErrCh: 5, + }, + { + name: "keep remaining cache when limit is smaller", + cacheLen: 5, + limit: 2, + expectedN: 2, + expectedCache: 3, + expectedErrCh: 2, + }, + { + name: "dispatch all when limit is larger", + cacheLen: 5, + limit: 10, + expectedN: 5, + expectedCache: 0, + expectedErrCh: 5, + }, + { + name: "dispatch all when limit is zero", + cacheLen: 5, + limit: 0, + expectedN: 5, + expectedCache: 0, + expectedErrCh: 5, + }, + { + name: "dispatch all when limit is negative", + cacheLen: 5, + limit: -1, + expectedN: 5, + expectedCache: 0, + expectedErrCh: 5, + }, + { + name: "empty cache", + cacheLen: 0, + limit: 5, + expectedN: 0, + expectedCache: 0, + expectedErrCh: 0, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + errCache := &errCache{ + cache: make([]regionErrorInfo, 0, 10), + errCh: make(chan regionErrorInfo, 10), + notify: make(chan struct{}, 1), + } + for i := 0; i < tc.cacheLen; i++ { + errCache.add(mockErrInfo) + } + + n, err := errCache.dispatchBatch(context.Background(), tc.limit) + require.NoError(t, err) + require.Equal(t, tc.expectedN, n) + require.Len(t, errCache.cache, tc.expectedCache) + require.Len(t, errCache.errCh, tc.expectedErrCh) + }) + } +} + func TestGetResolvedTargetTs(t *testing.T) { client := &subscriptionClient{ resolveLockTaskCh: make(chan resolveLockTask, 10), @@ -981,7 +759,7 @@ func TestGetResolvedTargetTs(t *testing.T) { TableID: 1, StartKey: []byte{'a'}, EndKey: []byte{'z'}, - }, 100, consumeKVEvents, advanceResolvedTs, 0, false, pdutil.NewClock4Test(), 30*time.Minute) + }, 100, consumeKVEvents, advanceResolvedTs, 0, false) span.initialized.Store(true) // Replicate the getResolvedTargetTs closure from runResolveLockChecker From 89cf4c6637c37bacc00015bc59ff164432af3ef9 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Fri, 17 Jul 2026 11:07:32 +0800 Subject: [PATCH 29/71] refactor --- .../logpuller/region_failure_handler.go | 146 ++++++++----- .../logpuller/region_request_scheduler.go | 34 ++- logservice/logpuller/region_request_worker.go | 201 +++++++++--------- .../logpuller/region_request_worker_test.go | 130 ++++------- logservice/logpuller/requested_store.go | 6 +- logservice/logpuller/subscription_client.go | 86 ++++---- .../logpuller/subscription_client_test.go | 22 +- 7 files changed, 311 insertions(+), 314 deletions(-) diff --git a/logservice/logpuller/region_failure_handler.go b/logservice/logpuller/region_failure_handler.go index 879fe0a090..34434bedde 100644 --- a/logservice/logpuller/region_failure_handler.go +++ b/logservice/logpuller/region_failure_handler.go @@ -19,10 +19,12 @@ import ( "time" "github.com/pingcap/log" + "github.com/pingcap/ticdc/heartbeatpb" "github.com/pingcap/ticdc/pkg/errors" "github.com/pingcap/ticdc/pkg/metrics" "github.com/tikv/client-go/v2/tikv" "go.uber.org/zap" + "golang.org/x/sync/errgroup" ) var ( @@ -40,14 +42,26 @@ var ( // regionFailureHandler handles failed regions and owns retry and reschedule decisions. type regionFailureHandler struct { - cache *errCache - client *subscriptionClient + cache *errCache + regionCache *tikv.RegionCache + + onTableDrained func(*subscribedSpan) + scheduleRegionRequest func(context.Context, regionInfo) + scheduleRangeRequest func(context.Context, heartbeatpb.TableSpan, *subscribedSpan, bool, bool) } -func newRegionFailureHandler(client *subscriptionClient) *regionFailureHandler { +func newRegionFailureHandler( + regionCache *tikv.RegionCache, + onTableDrained func(*subscribedSpan), + scheduleRegionRequest func(context.Context, regionInfo), + scheduleRangeRequest func(context.Context, heartbeatpb.TableSpan, *subscribedSpan, bool, bool), +) *regionFailureHandler { return ®ionFailureHandler{ - cache: newErrCache(), - client: client, + cache: newErrCache(), + regionCache: regionCache, + onTableDrained: onTableDrained, + scheduleRegionRequest: scheduleRegionRequest, + scheduleRangeRequest: scheduleRangeRequest, } } @@ -58,56 +72,33 @@ func (r *regionFailureHandler) Report(errInfo regionErrorInfo) { if errInfo.subscribedSpan.rangeLock.UnlockRange( errInfo.span.StartKey, errInfo.span.EndKey, errInfo.verID.GetID(), errInfo.verID.GetVer(), errInfo.resolvedTs()) { - r.client.onTableDrained(errInfo.subscribedSpan) + r.onTableDrained(errInfo.subscribedSpan) return } r.cache.add(errInfo) } func (r *regionFailureHandler) Run(ctx context.Context) error { - handleCachedErrors := func() error { + g, ctx := errgroup.WithContext(ctx) + g.Go(func() error { return r.cache.dispatch(ctx) }) + g.Go(func() error { for { - batch := r.cache.popBatch(errCacheBatchSize) - for _, errInfo := range batch { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } + select { + case <-ctx.Done(): + log.Info("subscription client handle errors and exit") + return ctx.Err() + case errInfo := <-r.cache.errCh: if err := r.handleError(ctx, errInfo); err != nil { return err } } - if len(batch) < errCacheBatchSize { - return nil - } - } - } - - // r.cache.ready() should handle failures promptly in normal flow. The ticker is only a - // fallback scan and is not expected to be needed in practice. - ticker := time.NewTicker(200 * time.Millisecond) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - log.Info("subscription client handle errors and exit") - return ctx.Err() - case <-ticker.C: - if err := handleCachedErrors(); err != nil { - return err - } - case <-r.cache.ready(): - if err := handleCachedErrors(); err != nil { - return err - } } - } + }) + return g.Wait() } func (r *regionFailureHandler) handleError(ctx context.Context, errInfo regionErrorInfo) error { err := errors.Cause(errInfo.err) - retryPriority := normalizeScanPriority(errInfo.scanPriority) //nolint:errorlint // converting large type switch to errors.As is a significant refactor if _, requestCancelled := err.(*requestCancelledErr); !requestCancelled { log.Debug("cdc region error", @@ -122,30 +113,28 @@ func (r *regionFailureHandler) handleError(ctx context.Context, errInfo regionEr innerErr := eerr.err if notLeader := innerErr.GetNotLeader(); notLeader != nil { metricFeedNotLeaderCounter.Inc() - r.client.regionCache.UpdateLeader(errInfo.verID, notLeader.GetLeader(), errInfo.rpcCtx.AccessIdx) - r.client.scheduleRegionRequest(ctx, errInfo.regionInfo, retryPriority) + r.regionCache.UpdateLeader(errInfo.verID, notLeader.GetLeader(), errInfo.rpcCtx.AccessIdx) + r.scheduleRegionRequest(ctx, errInfo.regionInfo) return nil } if innerErr.GetEpochNotMatch() != nil { metricFeedEpochNotMatchCounter.Inc() - r.client.scheduleRangeRequest( - ctx, errInfo.span, errInfo.subscribedSpan, errInfo.filterLoop, retryPriority) + r.scheduleRangeRequest(ctx, errInfo.span, errInfo.subscribedSpan, errInfo.filterLoop, errInfo.wasInitialized) return nil } if innerErr.GetRegionNotFound() != nil { metricFeedRegionNotFoundCounter.Inc() - r.client.scheduleRangeRequest( - ctx, errInfo.span, errInfo.subscribedSpan, errInfo.filterLoop, retryPriority) + r.scheduleRangeRequest(ctx, errInfo.span, errInfo.subscribedSpan, errInfo.filterLoop, errInfo.wasInitialized) return nil } if innerErr.GetCongested() != nil { metricKvCongestedCounter.Inc() - r.client.scheduleRegionRequest(ctx, errInfo.regionInfo, retryPriority) + r.scheduleRegionRequest(ctx, errInfo.regionInfo) return nil } if innerErr.GetServerIsBusy() != nil { metricKvIsBusyCounter.Inc() - r.client.scheduleRegionRequest(ctx, errInfo.regionInfo, retryPriority) + r.scheduleRegionRequest(ctx, errInfo.regionInfo) return nil } if duplicated := innerErr.GetDuplicateRequest(); duplicated != nil { @@ -164,26 +153,24 @@ func (r *regionFailureHandler) handleError(ctx context.Context, errInfo regionEr zap.Uint64("subscriptionID", uint64(errInfo.subscribedSpan.subID)), zap.Stringer("error", innerErr)) metricFeedUnknownErrorCounter.Inc() - r.client.scheduleRegionRequest(ctx, errInfo.regionInfo, retryPriority) + r.scheduleRegionRequest(ctx, errInfo.regionInfo) return nil case *rpcCtxUnavailableErr: metricFeedRPCCtxUnavailable.Inc() - r.client.scheduleRangeRequest( - ctx, errInfo.span, errInfo.subscribedSpan, errInfo.filterLoop, retryPriority) + r.scheduleRangeRequest(ctx, errInfo.span, errInfo.subscribedSpan, errInfo.filterLoop, errInfo.wasInitialized) return nil case *getStoreErr: metricGetStoreErr.Inc() bo := tikv.NewBackoffer(ctx, tikvRequestMaxBackoff) // cannot get the store the region belongs to, so we need to reload the region. - r.client.regionCache.OnSendFail(bo, errInfo.rpcCtx, true, err) - r.client.scheduleRangeRequest( - ctx, errInfo.span, errInfo.subscribedSpan, errInfo.filterLoop, retryPriority) + r.regionCache.OnSendFail(bo, errInfo.rpcCtx, true, err) + r.scheduleRangeRequest(ctx, errInfo.span, errInfo.subscribedSpan, errInfo.filterLoop, errInfo.wasInitialized) return nil case *storeStreamErr: metricStoreSendRequestErr.Inc() bo := tikv.NewBackoffer(ctx, tikvRequestMaxBackoff) - r.client.regionCache.OnSendFail(bo, errInfo.rpcCtx, regionScheduleReload, err) - r.client.scheduleRegionRequest(ctx, errInfo.regionInfo, retryPriority) + r.regionCache.OnSendFail(bo, errInfo.rpcCtx, regionScheduleReload, err) + r.scheduleRegionRequest(ctx, errInfo.regionInfo) return nil case *requestCancelledErr: // the corresponding subscription has been unsubscribed, just ignore. @@ -200,15 +187,17 @@ func (r *regionFailureHandler) handleError(ctx context.Context, errInfo regionEr type errCache struct { sync.Mutex cache []regionErrorInfo + errCh chan regionErrorInfo notify chan struct{} } -const errCacheBatchSize = 1024 +const errCacheDispatchBatchSize = 1024 func newErrCache() *errCache { return &errCache{ cache: make([]regionErrorInfo, 0, 1024), - notify: make(chan struct{}, 1), + errCh: make(chan regionErrorInfo, 4096), + notify: make(chan struct{}, 1024), } } @@ -242,6 +231,45 @@ func (e *errCache) popBatch(limit int) []regionErrorInfo { return batch } -func (e *errCache) ready() <-chan struct{} { - return e.notify +func (e *errCache) dispatchBatch(ctx context.Context, limit int) (int, error) { + batch := e.popBatch(limit) + for _, errInfo := range batch { + select { + case <-ctx.Done(): + log.Info("subscription client dispatch err cache done") + return 0, ctx.Err() + case e.errCh <- errInfo: + } + } + return len(batch), nil +} + +func (e *errCache) dispatch(ctx context.Context) error { + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() + sendToErrCh := func() error { + for { + n, err := e.dispatchBatch(ctx, errCacheDispatchBatchSize) + if err != nil { + return err + } + if n < errCacheDispatchBatchSize { + return nil + } + } + } + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + if err := sendToErrCh(); err != nil { + return err + } + case <-e.notify: + if err := sendToErrCh(); err != nil { + return err + } + } + } } diff --git a/logservice/logpuller/region_request_scheduler.go b/logservice/logpuller/region_request_scheduler.go index e81d169e9f..0d457d7080 100644 --- a/logservice/logpuller/region_request_scheduler.go +++ b/logservice/logpuller/region_request_scheduler.go @@ -34,7 +34,9 @@ import ( // resolution and retry policy remain owned by subscriptionClient and // regionFailureHandler respectively. type regionRequestScheduler struct { - client *subscriptionClient + upstream *upstreamHandle + eventSink *regionEventSink + failureHandler *regionFailureHandler // taskQueue orders all Regions before they are assigned to a TiKV store. taskQueue *priorityqueue.PriorityQueue[*regionPriorityTask] @@ -52,15 +54,22 @@ type regionRequestScheduler struct { maxWindowMultiplier int } -func newRegionRequestScheduler(client *subscriptionClient) *regionRequestScheduler { +func newRegionRequestScheduler( + clientConfig *SubscriptionClientConfig, + upstream *upstreamHandle, + eventSink *regionEventSink, + failureHandler *regionFailureHandler, +) *regionRequestScheduler { pullerConfig := config.GetGlobalServerConfig().Debug.Puller - workerCount := int(client.config.RegionRequestWorkerPerStore) + workerCount := int(clientConfig.RegionRequestWorkerPerStore) if workerCount <= 0 { workerCount = 1 } workerWindow := (pullerConfig.PendingRegionRequestQueueSize + workerCount - 1) / workerCount return ®ionRequestScheduler{ - client: client, + upstream: upstream, + eventSink: eventSink, + failureHandler: failureHandler, taskQueue: priorityqueue.New[*regionPriorityTask](), workerCount: workerCount, workerWindow: workerWindow, @@ -70,7 +79,7 @@ func newRegionRequestScheduler(client *subscriptionClient) *regionRequestSchedul func (s *regionRequestScheduler) submit(region regionInfo) { s.taskQueue.Push(NewRegionPriorityTask( - region, s.client.pdClock.CurrentTS(), s.sequence.Add(1))) + region, s.upstream.pdClock.CurrentTS(), s.sequence.Add(1))) } func (s *regionRequestScheduler) run(ctx context.Context, group *errgroup.Group) error { @@ -96,7 +105,7 @@ func (s *regionRequestScheduler) run(ctx context.Context, group *errgroup.Group) } store := s.getOrCreateStore(ctx, group, region.rpcCtx.Addr) - task.updateRegion(region, s.client.pdClock.CurrentTS()) + task.updateRegion(region, s.upstream.pdClock.CurrentTS()) if !store.submit(task) { return context.Canceled } @@ -113,7 +122,7 @@ func (s *regionRequestScheduler) attachRPCContext( region regionInfo, ) (regionInfo, bool) { bo := tikv.NewBackoffer(ctx, tikvRequestMaxBackoff) - rpcCtx, err := s.client.regionCache.GetTiKVRPCContext( + rpcCtx, err := s.upstream.regionCache.GetTiKVRPCContext( bo, region.verID, kvclientv2.ReplicaReadLeader, 0) if rpcCtx != nil { region.rpcCtx = rpcCtx @@ -125,7 +134,7 @@ func (s *regionRequestScheduler) attachRPCContext( zap.Uint64("regionID", region.verID.GetID()), zap.Error(err)) } - s.client.onRegionFail(newRegionErrorInfo(region, &rpcCtxUnavailableErr{verID: region.verID})) + s.failureHandler.Report(newRegionErrorInfo(region, &rpcCtxUnavailableErr{verID: region.verID})) return region, false } @@ -139,7 +148,14 @@ func (s *regionRequestScheduler) getOrCreateStore( } store := newRequestedStore( - s.client, storeAddr, s.workerCount, s.workerWindow, s.maxWindowMultiplier) + s.upstream, + s.eventSink, + s.failureHandler, + storeAddr, + s.workerCount, + s.workerWindow, + s.maxWindowMultiplier, + ) // run is the only writer. Publish the store after its immutable worker list // is complete, then start its workers. s.stores.Store(storeAddr, store) diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index df9027e5f8..6c8e9826aa 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -39,10 +39,7 @@ const storeReconnectBackoff = time.Second // To generate a workerID in `newRegionRequestWorker`. var workerIDGen atomic.Uint64 -var ( - metricsResolvedTsCount = metrics.PullerEventCounter.WithLabelValues("resolved_ts") - metricBatchResolvedSize = metrics.BatchResolvedEventSize.WithLabelValues("event-store") -) +var metricBatchResolvedSize = metrics.BatchResolvedEventSize.WithLabelValues("event-store") type deregisterRequest struct { subID SubscriptionID @@ -95,8 +92,10 @@ func (q *controlQueue) ready() <-chan struct{} { type regionRequestWorker struct { workerID uint64 - client *subscriptionClient - store *requestedStore + upstream *upstreamHandle + eventSink *regionEventSink + failureHandler *regionFailureHandler + store *requestedStore admission *regionAdmissionController controlQueue *controlQueue @@ -104,47 +103,27 @@ type regionRequestWorker struct { } func newRegionRequestWorker( - client *subscriptionClient, + upstream *upstreamHandle, + eventSink *regionEventSink, + failureHandler *regionFailureHandler, store *requestedStore, currentWindow int, maxWindowMultiplier int, ) *regionRequestWorker { workerID := workerIDGen.Add(1) return ®ionRequestWorker{ - workerID: workerID, - client: client, - store: store, - admission: newRegionAdmissionController(currentWindow, maxWindowMultiplier), - controlQueue: newControlQueue(), - tracker: newRegionTracker(), + workerID: workerID, + upstream: upstream, + eventSink: eventSink, + failureHandler: failureHandler, + store: store, + admission: newRegionAdmissionController(currentWindow, maxWindowMultiplier), + controlQueue: newControlQueue(), + tracker: newRegionTracker(), } } func (s *regionRequestWorker) Run(ctx context.Context) error { - handleStreamFailure := func(firstReq *regionReq, regionErr error) { - // Stream failure handle cases: - // - tracker: requests already sent to this stream. - // - firstReq: popped from admission for this stream, but not necessarily - // added to tracker yet if the stream fails before sendRegionRequest calls - // tracker.Add. - // - admission: requests owned by this worker but not sent yet. - for _, state := range s.tracker.Drain() { - state.markStopped(regionErr) - s.client.eventSink.Push( - SubscriptionID(state.requestID), - regionEvent{states: []*regionFeedState{state}}, - ) - } - // The failed stream no longer owns remote registrations. - s.controlQueue.drain() - if firstReq != nil && firstReq.abort() { - s.client.onRegionFail(newRegionErrorInfo(firstReq.regionInfo, regionErr)) - } - for _, task := range s.admission.drain() { - s.client.onRegionFail(newRegionErrorInfo(task.regionInfo, regionErr)) - } - } - for { // Do not connect an idle worker to an unavailable store indefinitely. firstReq, err := s.waitForRegionRequest(ctx) @@ -152,22 +131,59 @@ func (s *regionRequestWorker) Run(ctx context.Context) error { return err } - regionErr := s.runStream(ctx, firstReq) + regionErr := error(&storeStreamErr{}) + if err := s.checkStoreVersion(ctx); err != nil { + regionErr = err + } else if err := s.runStream(ctx, firstReq); err != nil { + regionErr = err + } if ctx.Err() != nil { firstReq.abort() return ctx.Err() } - // Treat an unexpected clean stream exit as a recoverable store-stream failure. - if regionErr == nil { - regionErr = &storeStreamErr{} + + // Stop sent requests first so their states release the admission leases. + // firstReq still owns its lease only if the stream failed before Send. + s.failStreamRegions(regionErr) + if firstReq.abort() { + s.failureHandler.Report(newRegionErrorInfo(firstReq.regionInfo, regionErr)) } - handleStreamFailure(firstReq, regionErr) + s.failPendingRegions(regionErr) + if err := util.Hang(ctx, storeReconnectBackoff); err != nil { return err } } } +// failStreamRegions transfers every request sent by a failed stream to the +// recovery pipeline. +func (s *regionRequestWorker) failStreamRegions(err error) { + for _, states := range s.tracker.Drain() { + for _, state := range states { + s.notifyRegionError(state, err) + } + } + // The failed stream no longer owns remote registrations. + s.controlQueue.drain() +} + +// failPendingRegions transfers requests owned by this worker but not yet sent +// to the recovery pipeline, so they can be resolved and routed again. +func (s *regionRequestWorker) failPendingRegions(err error) { + for _, task := range s.admission.drain() { + s.failureHandler.Report(newRegionErrorInfo(task.regionInfo, err)) + } +} + +func (s *regionRequestWorker) notifyRegionError(state *regionFeedState, err error) { + state.markStopped(err) + s.eventSink.Push( + SubscriptionID(state.requestID), + regionEvent{states: []*regionFeedState{state}}, + ) +} + func (s *regionRequestWorker) waitForRegionRequest(ctx context.Context) (*regionReq, error) { // Without a stream there are no remote registrations to deregister. s.controlQueue.drain() @@ -182,7 +198,7 @@ func (s *regionRequestWorker) waitForRegionRequest(ctx context.Context) (*region } func (s *regionRequestWorker) checkStoreVersion(ctx context.Context) error { - err := version.CheckStoreVersion(ctx, s.client.pd) + err := version.CheckStoreVersion(ctx, s.upstream.pd) if err == nil { return nil } @@ -200,10 +216,6 @@ func (s *regionRequestWorker) checkStoreVersion(ctx context.Context) error { } func (s *regionRequestWorker) runStream(ctx context.Context, firstReq *regionReq) (err error) { - if err := s.checkStoreVersion(ctx); err != nil { - return err - } - log.Info("region request worker going to create grpc stream", zap.Uint64("workerID", s.workerID), zap.String("addr", s.store.storeAddr)) @@ -214,7 +226,8 @@ func (s *regionRequestWorker) runStream(ctx context.Context, firstReq *regionReq zap.Error(err)) }() - conn, err := Connect(ctx, s.client.credential, s.store.storeAddr) + g, gctx := errgroup.WithContext(ctx) + conn, err := Connect(gctx, s.upstream.credential, s.store.storeAddr) if err != nil { log.Warn("region request worker create grpc stream failed", zap.Uint64("workerID", s.workerID), @@ -230,7 +243,6 @@ func (s *regionRequestWorker) runStream(ctx context.Context, firstReq *regionReq } defer func() { _ = conn.Conn.Close() }() - g, gctx := errgroup.WithContext(ctx) g.Go(func() error { return s.receiveAndDispatchChangeEvents(conn) }) g.Go(func() error { return s.processRegionSendTask(gctx, conn, firstReq) }) @@ -306,30 +318,30 @@ func (s *regionRequestWorker) dispatchRegionChangeEvents(events []*cdcpb.Event) zap.Uint64("subscriptionID", uint64(subscriptionID)), zap.Uint64("regionID", event.RegionId), zap.Any("error", eventData.Error)) - state.markStopped(&eventError{err: eventData.Error}) + s.notifyRegionError(state, &eventError{err: eventData.Error}) + continue case *cdcpb.Event_ResolvedTs: regionEvent.resolvedTs = eventData.ResolvedTs case *cdcpb.Event_LongTxn_: - // ignore continue default: log.Panic("unknown event type", zap.Any("event", event)) } - s.client.eventSink.Push(subscriptionID, regionEvent) - } else { - switch event.Event.(type) { - case *cdcpb.Event_Error: - // it is normal to receive region error after deregister a subscription - log.Debug("region request worker receives an error for a stale region, ignore it", - zap.Uint64("workerID", s.workerID), - zap.Uint64("subscriptionID", uint64(subscriptionID)), - zap.Uint64("regionID", event.RegionId)) - default: - log.Warn("region request worker receives a region event for an untracked region", - zap.Uint64("workerID", s.workerID), - zap.Uint64("subscriptionID", uint64(subscriptionID)), - zap.Uint64("regionID", event.RegionId)) - } + s.eventSink.Push(subscriptionID, regionEvent) + continue + } + + switch event.Event.(type) { + case *cdcpb.Event_Error: + log.Debug("region request worker receives an error for a stale region, ignore it", + zap.Uint64("workerID", s.workerID), + zap.Uint64("subscriptionID", uint64(subscriptionID)), + zap.Uint64("regionID", event.RegionId)) + default: + log.Warn("region request worker receives a region event for an untracked region", + zap.Uint64("workerID", s.workerID), + zap.Uint64("subscriptionID", uint64(subscriptionID)), + zap.Uint64("regionID", event.RegionId)) } } } @@ -338,7 +350,6 @@ func (s *regionRequestWorker) dispatchResolvedTsEvent(resolvedTsEvent *cdcpb.Res subscriptionID := SubscriptionID(resolvedTsEvent.RequestId) metricsResolvedTsCount.Add(float64(len(resolvedTsEvent.Regions))) metricBatchResolvedSize.Observe(float64(len(resolvedTsEvent.Regions))) - // TODO: resolvedTsEvent.Ts be 0 is impossible, we need find the root cause. if resolvedTsEvent.Ts == 0 { log.Warn("region request worker receives a resolved ts event with zero value, ignore it", zap.Uint64("workerID", s.workerID), @@ -348,15 +359,13 @@ func (s *regionRequestWorker) dispatchResolvedTsEvent(resolvedTsEvent *cdcpb.Res } const resolvedTsStateBatchSize = 1024 - // Avoid allocating a huge states slice when resolvedTsEvent.Regions is large. - // Push resolved-ts events in batches to reduce peak memory usage and improve GC behavior. capHint := min(len(resolvedTsEvent.Regions), resolvedTsStateBatchSize) resolvedStates := make([]*regionFeedState, 0, capHint) flush := func() { if len(resolvedStates) == 0 { return } - s.client.eventSink.Push(subscriptionID, regionEvent{ + s.eventSink.Push(subscriptionID, regionEvent{ resolvedTs: resolvedTsEvent.Ts, states: resolvedStates, }) @@ -404,7 +413,7 @@ func (s *regionRequestWorker) sendDeregisterRequest( req deregisterRequest, ) error { changeDataReq := &cdcpb.ChangeDataRequest{ - Header: &cdcpb.Header{ClusterId: s.client.clusterID, TicdcVersion: version.ReleaseSemver()}, + Header: &cdcpb.Header{ClusterId: s.upstream.clusterID, TicdcVersion: version.ReleaseSemver()}, RequestId: uint64(req.subID), Request: &cdcpb.ChangeDataRequest_Deregister_{ Deregister: &cdcpb.ChangeDataRequest_Deregister{}, @@ -415,28 +424,39 @@ func (s *regionRequestWorker) sendDeregisterRequest( return err } for _, state := range s.tracker.TakeSubscription(req.subID) { - state.markStopped(&requestCancelledErr{}) - s.client.eventSink.Push(req.subID, regionEvent{states: []*regionFeedState{state}}) + s.notifyRegionError(state, &requestCancelledErr{}) } return nil } +func (s *regionRequestWorker) drainControlQueue(conn *ConnAndClient) error { + for { + req, ok := s.controlQueue.tryPop() + if !ok { + return nil + } + if err := s.sendDeregisterRequest(conn, req); err != nil { + return err + } + } +} + func (s *regionRequestWorker) sendRegionRequest(conn *ConnAndClient, req *regionReq) error { if !req.isActive() { - return nil + return &storeStreamErr{} } region := req.regionInfo subID := region.subscribedSpan.subID - log.Debug("region request worker sends region request", + log.Debug("region request worker gets a singleRegionInfo", zap.Uint64("workerID", s.workerID), zap.Uint64("subscriptionID", uint64(subID)), zap.Uint64("regionID", region.verID.GetID()), - zap.String("storeAddr", s.store.storeAddr), + zap.String("addr", s.store.storeAddr), zap.Bool("bdrMode", region.filterLoop)) if region.subscribedSpan.stopped.Load() { req.abort() - s.client.onRegionFail(newRegionErrorInfo(region, &requestCancelledErr{})) + s.failureHandler.Report(newRegionErrorInfo(region, &requestCancelledErr{})) return nil } @@ -455,7 +475,7 @@ func (s *regionRequestWorker) sendRegionRequest(conn *ConnAndClient, req *region zap.Uint64("regionID", region.verID.GetID())) return nil } - if err := s.sendChangeDataRequest(conn, createRegionRequest(s.client.clusterID, region)); err != nil { + if err := s.sendChangeDataRequest(conn, s.createRegionRequest(region)); err != nil { // Transport failures are always recoverable at the region level. Preserve // the stream error as the function result, but classify the region for // rescheduling instead of exposing an arbitrary gRPC error downstream. @@ -472,27 +492,17 @@ func (s *regionRequestWorker) processRegionSendTask( ) error { regionReq := firstReq for { - // Send the current region request before handling anything newly queued. if regionReq != nil { if err := s.sendRegionRequest(conn, regionReq); err != nil { return err } + regionReq = nil + continue } - // Flush pending deregisters before admitting the next region request. - // Admission may still contain stale tasks from a stopped subscription, but - // sendRegionRequest re-checks subscription liveness before tracker.Add/Send, - // so those tasks are dropped locally instead of recreating remote registrations. - for { - req, ok := s.controlQueue.tryPop() - if !ok { - break - } - if err := s.sendDeregisterRequest(conn, req); err != nil { - return err - } + + if err := s.drainControlQueue(conn); err != nil { + return err } - // Block for the next request, but wake early when deregisters arrive. - // regionReq above is already consumed and will be replaced by the next pop. var err error regionReq, err = s.admission.pop(ctx, s.controlQueue.ready()) if err != nil { @@ -501,9 +511,9 @@ func (s *regionRequestWorker) processRegionSendTask( } } -func createRegionRequest(clusterID uint64, region regionInfo) *cdcpb.ChangeDataRequest { +func (s *regionRequestWorker) createRegionRequest(region regionInfo) *cdcpb.ChangeDataRequest { return &cdcpb.ChangeDataRequest{ - Header: &cdcpb.Header{ClusterId: clusterID, TicdcVersion: version.ReleaseSemver()}, + Header: &cdcpb.Header{ClusterId: s.upstream.clusterID, TicdcVersion: version.ReleaseSemver()}, RegionId: region.verID.GetID(), RequestId: uint64(region.subscribedSpan.subID), RegionEpoch: region.rpcCtx.Meta.RegionEpoch, @@ -512,6 +522,5 @@ func createRegionRequest(clusterID uint64, region regionInfo) *cdcpb.ChangeDataR EndKey: region.span.EndKey, ExtraOp: kvrpcpb.ExtraOp_ReadOldValue, FilterLoop: region.filterLoop, - ScanPriority: normalizeScanPriority(region.scanPriority), } } diff --git a/logservice/logpuller/region_request_worker_test.go b/logservice/logpuller/region_request_worker_test.go index 2d67926fa5..70affb2d0a 100644 --- a/logservice/logpuller/region_request_worker_test.go +++ b/logservice/logpuller/region_request_worker_test.go @@ -78,48 +78,12 @@ func admitRegionRequest( return req } -func TestCreateRegionRequestScanPriority(t *testing.T) { - worker := ®ionRequestWorker{ - client: &subscriptionClient{clusterID: 1}, - } - - for _, tc := range []struct { - name string - priority cdcpb.ScanPriority - expected cdcpb.ScanPriority - }{ - { - name: "high", - priority: cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, - expected: cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, - }, - { - name: "low", - priority: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, - expected: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, - }, - { - name: "unknown defaults to low", - priority: cdcpb.ScanPriority_SCAN_PRIORITY_UNKNOWN, - expected: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, - }, - } { - t.Run(tc.name, func(t *testing.T) { - region := prepareRegionForSendTest(createTestRegionInfo(1, 1)) - region.scanPriority = tc.priority - - req := createRegionRequest(worker.client.clusterID, region) - require.Equal(t, tc.expected, req.GetScanPriority()) - }) - } -} - func TestRegionRequestWorkerIgnoresDuplicateActiveRegion(t *testing.T) { admission := newRegionAdmissionController(10, 1) worker := ®ionRequestWorker{ admission: admission, store: &requestedStore{storeAddr: "store-1"}, - client: &subscriptionClient{}, + upstream: &upstreamHandle{}, tracker: newRegionTracker(), } region := prepareRegionForSendTest(createTestRegionInfo(1, 1)) @@ -209,10 +173,8 @@ func (m *mockRegionEventDynamicStream) GetMetrics() dynstream.Metrics[int, Subsc func newDispatchResolvedTsTestWorker(regionCount int) (*regionRequestWorker, *mockRegionEventDynamicStream, *cdcpb.ResolvedTs) { ds := &mockRegionEventDynamicStream{} worker := ®ionRequestWorker{ - client: &subscriptionClient{ - eventSink: newTestRegionEventSink(ds), - }, - tracker: newRegionTracker(), + eventSink: ®ionEventSink{ds: ds}, + tracker: newRegionTracker(), } regions := make([]uint64, regionCount) for i := 0; i < regionCount; i++ { @@ -239,7 +201,7 @@ func dispatchResolvedTsEventLegacyForBenchmark(s *regionRequestWorker, resolvedT return } states := resolvedStates - s.client.pushRegionEventToDS(subscriptionID, regionEvent{ + s.eventSink.Push(subscriptionID, regionEvent{ resolvedTs: resolvedTsEvent.Ts, states: states, }) @@ -259,7 +221,7 @@ func dispatchResolvedTsEventLegacyForBenchmark(s *regionRequestWorker, resolvedT func benchmarkDispatchResolvedTsEvent(b *testing.B, regionCount int, useLegacy bool) { worker, _, event := newDispatchResolvedTsTestWorker(regionCount) ds := &countingRegionEventDynamicStream{} - worker.client.eventSink.ds = ds + worker.eventSink.ds = ds b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { @@ -374,37 +336,28 @@ func TestStoppedStateRemovesSentRequest(t *testing.T) { require.Equal(t, 0, admission.stats().inflight) } -func TestStreamRecoveryReleasesSentAdmission(t *testing.T) { +func TestFailStreamRegionsReleasesSentAdmission(t *testing.T) { admission := newRegionAdmissionController(1, 1) ds := &mockRegionEventDynamicStream{} worker := ®ionRequestWorker{ admission: admission, controlQueue: newControlQueue(), - client: &subscriptionClient{ - eventSink: ®ionEventSink{ds: ds}, - }, - tracker: newRegionTracker(), + eventSink: ®ionEventSink{ds: ds}, + tracker: newRegionTracker(), } region := prepareRegionForSendTest(createTestRegionInfo(1, 1)) req := admitRegionRequest(t, admission, region) state := newRegionFeedState(region, uint64(region.subscribedSpan.subID), worker, req) require.True(t, worker.tracker.Add(region.subscribedSpan.subID, region.verID.GetID(), state)) - for _, state := range worker.tracker.Drain() { - state.markStopped(&storeStreamErr{}) - worker.client.eventSink.Push( - SubscriptionID(state.requestID), - regionEvent{states: []*regionFeedState{state}}, - ) - } - worker.controlQueue.drain() + worker.failStreamRegions(&storeStreamErr{}) require.Zero(t, admission.stats().inflight) require.False(t, req.abort()) require.Equal(t, 1, ds.pushCount) } -func TestStreamRecoveryReschedulesWorkerBuffer(t *testing.T) { +func TestFailPendingRegionsReschedulesWorkerBuffer(t *testing.T) { rawSpan := heartbeatpb.TableSpan{ TableID: 1, StartKey: []byte("a"), @@ -421,9 +374,8 @@ func TestStreamRecoveryReschedulesWorkerBuffer(t *testing.T) { require.Equal(t, regionlock.LockRangeStatusSuccess, lock2.Status) admission := newRegionAdmissionController(1, 1) - client := &subscriptionClient{} - client.failureHandler = newRegionFailureHandler(client) - worker := ®ionRequestWorker{client: client, admission: admission} + failureHandler := ®ionFailureHandler{cache: newErrCache()} + worker := ®ionRequestWorker{failureHandler: failureHandler, admission: admission} regions := []regionInfo{ { verID: tikv.NewRegionVerID(1, 1, 1), @@ -441,15 +393,13 @@ func TestStreamRecoveryReschedulesWorkerBuffer(t *testing.T) { }, } for i, region := range regions { - require.True(t, admission.submit(newRegionPriorityTask(region, uint64(i+1)))) + require.True(t, admission.submit(NewRegionPriorityTask(region, 1, uint64(i+1)))) } - for _, task := range worker.admission.drain() { - worker.client.onRegionFail(newRegionErrorInfo(task.regionInfo, &storeStreamErr{})) - } + worker.failPendingRegions(&storeStreamErr{}) require.Zero(t, admission.stats().pending) - require.Len(t, client.failureHandler.cache.cache, 2) + require.Len(t, failureHandler.cache.cache, 2) } func TestProcessRegionSendTaskSendFailureCleansSentRequest(t *testing.T) { @@ -458,7 +408,7 @@ func TestProcessRegionSendTaskSendFailureCleansSentRequest(t *testing.T) { admission: admission, controlQueue: newControlQueue(), store: &requestedStore{storeAddr: "store-1"}, - client: &subscriptionClient{}, + upstream: &upstreamHandle{}, tracker: newRegionTracker(), } @@ -483,37 +433,31 @@ func TestProcessRegionSendTaskSendFailureCleansSentRequest(t *testing.T) { require.ErrorAs(t, state.takeError(), &streamErr) } -func TestProcessRegionSendTaskSkipsRemovedRequest(t *testing.T) { +func TestProcessRegionSendTaskDoesNotSendRemovedRequest(t *testing.T) { admission := newRegionAdmissionController(1, 1) worker := ®ionRequestWorker{ admission: admission, controlQueue: newControlQueue(), store: &requestedStore{storeAddr: "store-1"}, - client: &subscriptionClient{}, + upstream: &upstreamHandle{}, tracker: newRegionTracker(), } - firstRegion := prepareRegionForSendTest(createTestRegionInfo(1, 1)) - firstReq := admitRegionRequest(t, admission, firstRegion) - require.True(t, firstReq.abort()) - - secondRegion := prepareRegionForSendTest(createTestRegionInfo(1, 2)) - require.True(t, admission.submit(newRegionPriorityTask(secondRegion, 2))) + region := prepareRegionForSendTest(createTestRegionInfo(1, 1)) + req := admitRegionRequest(t, admission, region) + require.True(t, req.abort()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() sendCh := make(chan *cdcpb.ChangeDataRequest, 1) - done := make(chan error, 1) - go func() { - done <- worker.processRegionSendTask(ctx, &ConnAndClient{ - Client: &mockEventFeedV2Client{sendCh: sendCh}, - Conn: &grpc.ClientConn{}, - }, firstReq) - }() - - sentReq := <-sendCh - require.Equal(t, secondRegion.verID.GetID(), sentReq.RegionId) - cancel() - require.ErrorIs(t, <-done, context.Canceled) + err := worker.processRegionSendTask(t.Context(), &ConnAndClient{ + Client: &mockEventFeedV2Client{sendCh: sendCh}, + Conn: &grpc.ClientConn{}, + }, req) + var streamErr *storeStreamErr + require.ErrorAs(t, err, &streamErr) + select { + case sentReq := <-sendCh: + t.Fatalf("removed request was sent: %+v", sentReq) + default: + } } func TestProcessRegionSendTaskSendEOFIsRetriable(t *testing.T) { @@ -538,7 +482,7 @@ func TestProcessRegionSendTaskSendEOFIsRetriable(t *testing.T) { admission: admission, controlQueue: newControlQueue(), store: &requestedStore{storeAddr: "store-1"}, - client: &subscriptionClient{}, + upstream: &upstreamHandle{}, tracker: newRegionTracker(), } region := prepareRegionForSendTest(createTestRegionInfo(1, 1)) @@ -571,10 +515,9 @@ func TestProcessRegionSendTaskHandlesDeregisterFromControlQueue(t *testing.T) { admission: newRegionAdmissionController(1, 1), controlQueue: newControlQueue(), store: &requestedStore{storeAddr: "store-1"}, - client: &subscriptionClient{ - eventSink: ®ionEventSink{ds: ds}, - }, - tracker: newRegionTracker(), + upstream: &upstreamHandle{clusterID: 42}, + eventSink: ®ionEventSink{ds: ds}, + tracker: newRegionTracker(), } state := ®ionFeedState{worker: worker} require.True(t, worker.tracker.Add(1, 1, state)) @@ -591,6 +534,7 @@ func TestProcessRegionSendTaskHandlesDeregisterFromControlQueue(t *testing.T) { }() req := <-sendCh + require.Equal(t, uint64(42), req.Header.ClusterId) require.Equal(t, uint64(1), req.RequestId) require.True(t, req.FilterLoop) require.NotNil(t, req.GetDeregister()) diff --git a/logservice/logpuller/requested_store.go b/logservice/logpuller/requested_store.go index f98b59b679..97294a8e33 100644 --- a/logservice/logpuller/requested_store.go +++ b/logservice/logpuller/requested_store.go @@ -31,7 +31,9 @@ type requestedStore struct { } func newRequestedStore( - client *subscriptionClient, + upstream *upstreamHandle, + eventSink *regionEventSink, + failureHandler *regionFailureHandler, storeAddr string, workerCount int, workerWindow int, @@ -43,7 +45,7 @@ func newRequestedStore( } for i := 0; i < workerCount; i++ { store.workers = append(store.workers, newRegionRequestWorker( - client, store, workerWindow, maxWindowMultiplier)) + upstream, eventSink, failureHandler, store, workerWindow, maxWindowMultiplier)) } return store } diff --git a/logservice/logpuller/subscription_client.go b/logservice/logpuller/subscription_client.go index a7edab9ac5..1266af764a 100644 --- a/logservice/logpuller/subscription_client.go +++ b/logservice/logpuller/subscription_client.go @@ -30,7 +30,6 @@ import ( "github.com/pingcap/ticdc/pkg/security" "github.com/pingcap/ticdc/pkg/spanz" "github.com/pingcap/ticdc/pkg/util" - "github.com/prometheus/client_golang/prometheus" "github.com/tikv/client-go/v2/tikv" pd "github.com/tikv/pd/client" "go.uber.org/zap" @@ -90,8 +89,21 @@ type SubscriptionClientConfig struct { RegionRequestWorkerPerStore uint } -type sharedClientMetrics struct { - batchResolvedSize prometheus.Observer +// upstreamHandle contains the stable TiKV and PD dependencies shared by the +// region request pipeline. Runtime components keep their own event and error +// dependencies instead of using this as a general service container. +type upstreamHandle struct { + pd pd.Client + regionCache *tikv.RegionCache + pdClock pdutil.Clock + credential *security.Credential + clusterID uint64 +} + +// initialize loads the cluster metadata needed by Region request workers. It +// must run before the scheduler starts any workers. +func (u *upstreamHandle) initialize(ctx context.Context) { + u.clusterID = u.pd.GetClusterID(ctx) } // subscriptionClient is used to subscribe events of table ranges from TiKV. @@ -115,19 +127,12 @@ type SubscriptionClient interface { } type subscriptionClient struct { - ctx context.Context - cancel context.CancelFunc - config *SubscriptionClientConfig - metrics sharedClientMetrics - clusterID uint64 - - pd pd.Client - regionCache *tikv.RegionCache - pdClock pdutil.Clock - lockResolver txnutil.LockResolver + ctx context.Context + cancel context.CancelFunc + config *SubscriptionClientConfig + upstream *upstreamHandle - // the credential to connect tikv - credential *security.Credential + lockResolver txnutil.LockResolver // failureHandler handles failed regions and owns reschedule/retry decisions. failureHandler *regionFailureHandler @@ -156,25 +161,33 @@ func NewSubscriptionClient( ) SubscriptionClient { subClient := &subscriptionClient{ config: config, - - pd: pd, - regionCache: appcontext.GetService[*tikv.RegionCache](appcontext.RegionCache), - pdClock: appcontext.GetService[pdutil.Clock](appcontext.DefaultPDClock), + upstream: &upstreamHandle{ + pd: pd, + regionCache: appcontext.GetService[*tikv.RegionCache](appcontext.RegionCache), + pdClock: appcontext.GetService[pdutil.Clock](appcontext.DefaultPDClock), + credential: credential, + }, lockResolver: lockResolver, - credential: credential, - rangeTaskCh: make(chan rangeTask, 1024), resolveLockTaskCh: make(chan resolveLockTask, 1024), resolveLockRateLimiter: newResolveLockRateLimiter(), } subClient.ctx, subClient.cancel = context.WithCancel(context.Background()) - subClient.failureHandler = newRegionFailureHandler(subClient) + subClient.failureHandler = newRegionFailureHandler( + subClient.upstream.regionCache, + subClient.onTableDrained, + subClient.scheduleRegionRequest, + subClient.scheduleRangeRequest, + ) subClient.eventSink = newRegionEventSink(subClient.ctx, subClient.failureHandler) - subClient.spanRegistry = newSpanRegistry(subClient.pd, subClient.pdClock) - subClient.regionScheduler = newRegionRequestScheduler(subClient) - - subClient.initMetrics() + subClient.spanRegistry = newSpanRegistry(subClient.upstream.pd, subClient.upstream.pdClock) + subClient.regionScheduler = newRegionRequestScheduler( + subClient.config, + subClient.upstream, + subClient.eventSink, + subClient.failureHandler, + ) return subClient } @@ -187,11 +200,6 @@ func (s *subscriptionClient) AllocSubscriptionID() SubscriptionID { return SubscriptionID(subscriptionIDGen.Add(1)) } -func (s *subscriptionClient) initMetrics() { - // TODO: fix metrics - s.metrics.batchResolvedSize = metrics.BatchResolvedEventSize.WithLabelValues("event-store") -} - func (s *subscriptionClient) updateMetrics(ctx context.Context) error { ticker := time.NewTicker(10 * time.Second) defer ticker.Stop() @@ -269,17 +277,13 @@ func (s *subscriptionClient) Unsubscribe(subID SubscriptionID) { zap.Bool("exists", rt != nil)) } -func (s *subscriptionClient) pushRegionEventToDS(subID SubscriptionID, event regionEvent) { - s.eventSink.Push(subID, event) -} - func (s *subscriptionClient) Run(ctx context.Context) error { // s.consume = consume - if s.pd == nil { + if s.upstream == nil || s.upstream.pd == nil { log.Warn("subscription client should be in test mode, skip run") return nil } - s.clusterID = s.pd.GetClusterID(ctx) + s.upstream.initialize(ctx) g, ctx := errgroup.WithContext(ctx) @@ -331,11 +335,6 @@ func (s *subscriptionClient) onTableDrained(rt *subscribedSpan) { s.spanRegistry.Remove(rt.subID) } -// Note: don't block the caller, otherwise there may be deadlock -func (s *subscriptionClient) onRegionFail(errInfo regionErrorInfo) { - s.failureHandler.Report(errInfo) -} - func (s *subscriptionClient) handleRangeTasks(ctx context.Context) error { g, ctx := errgroup.WithContext(ctx) // Limit the concurrent number of goroutines to convert range tasks to region tasks. @@ -380,7 +379,8 @@ func (s *subscriptionClient) divideSpanAndScheduleRegionRequests( zap.Any("span", common.FormatTableSpan(&nextSpan))) backoff := tikv.NewBackoffer(ctx, tikvRequestMaxBackoff) - regions, err := s.regionCache.BatchLoadRegionsWithKeyRange(backoff, nextSpan.StartKey, nextSpan.EndKey, limit) + regions, err := s.upstream.regionCache.BatchLoadRegionsWithKeyRange( + backoff, nextSpan.StartKey, nextSpan.EndKey, limit) if err != nil { log.Warn("subscription client load regions failed", zap.Uint64("subscriptionID", uint64(subscribedSpan.subID)), diff --git a/logservice/logpuller/subscription_client_test.go b/logservice/logpuller/subscription_client_test.go index 4fb76df372..4387fa2054 100644 --- a/logservice/logpuller/subscription_client_test.go +++ b/logservice/logpuller/subscription_client_test.go @@ -66,8 +66,8 @@ func TestGenerateResolveLockTask(t *testing.T) { } consumeKVEvents := func(_ []common.RawKVEntry, _ func()) bool { return false } advanceResolvedTs := func(ts uint64) {} - client.pdClock = pdutil.NewClock4Test() - client.spanRegistry = newSpanRegistry(nil, client.pdClock) + client.upstream = &upstreamHandle{pdClock: pdutil.NewClock4Test()} + client.spanRegistry = newSpanRegistry(nil, client.upstream.pdClock) span := newSubscribedSpan( client.ctx, client.resolveLockRateLimiter, @@ -329,7 +329,7 @@ func TestStopTaskUsesSubscribedSpanFilterLoop(t *testing.T) { require.Equal(t, regionlock.LockRangeStatusSuccess, res.Status) worker := ®ionRequestWorker{controlQueue: newControlQueue()} store := &requestedStore{storeAddr: "store-1", workers: []*regionRequestWorker{worker}} - client.regionScheduler = ®ionRequestScheduler{client: client} + client.regionScheduler = ®ionRequestScheduler{} client.regionScheduler.stores.Store(store.storeAddr, store) client.setTableStopped(span) @@ -340,12 +340,12 @@ func TestStopTaskUsesSubscribedSpanFilterLoop(t *testing.T) { require.True(t, req.filterLoop) } -func TestOnRegionFailQueuesCanceledErrorCache(t *testing.T) { +func TestRegionFailureHandlerQueuesCanceledError(t *testing.T) { client := &subscriptionClient{ eventSink: ®ionEventSink{ds: &mockDynamicStream{}}, } client.spanRegistry = newSpanRegistry(nil, nil) - client.failureHandler = newRegionFailureHandler(client) + client.failureHandler = newRegionFailureHandler(nil, client.onTableDrained, nil, nil) rawSpan := heartbeatpb.TableSpan{ TableID: 1, StartKey: []byte("a"), @@ -364,7 +364,7 @@ func TestOnRegionFailQueuesCanceledErrorCache(t *testing.T) { require.Equal(t, regionlock.LockRangeStatusSuccess, res2.Status) require.False(t, span.rangeLock.Stop()) - client.onRegionFail(newRegionErrorInfo(regionInfo{ + client.failureHandler.Report(newRegionErrorInfo(regionInfo{ verID: tikv.NewRegionVerID(1, 1, 1), span: heartbeatpb.TableSpan{TableID: 1, StartKey: []byte("a"), EndKey: []byte("m")}, subscribedSpan: span, @@ -374,7 +374,7 @@ func TestOnRegionFailQueuesCanceledErrorCache(t *testing.T) { require.Len(t, client.failureHandler.cache.cache, 1) require.Len(t, span.rangeLock.IterAll(nil).UnLockedRanges, 1) - client.onRegionFail(newRegionErrorInfo(regionInfo{ + client.failureHandler.Report(newRegionErrorInfo(regionInfo{ verID: tikv.NewRegionVerID(2, 1, 1), span: heartbeatpb.TableSpan{TableID: 1, StartKey: []byte("m"), EndKey: []byte("z")}, subscribedSpan: span, @@ -415,7 +415,7 @@ func (s *mockDynamicStream) GetMetrics() dynstream.Metrics[int, SubscriptionID] return dynstream.Metrics[int, SubscriptionID]{} } -func TestPushRegionEventToDSUnblocksOnClose(t *testing.T) { +func TestRegionEventSinkPushUnblocksOnClientClose(t *testing.T) { sink := ®ionEventSink{ ctx: context.Background(), ds: &mockDynamicStream{}, @@ -423,7 +423,6 @@ func TestPushRegionEventToDSUnblocksOnClose(t *testing.T) { sink.cond = sync.NewCond(&sink.mu) client := &subscriptionClient{eventSink: sink} client.regionScheduler = ®ionRequestScheduler{ - client: client, taskQueue: priorityqueue.New[*regionPriorityTask](), } client.ctx, client.cancel = context.WithCancel(context.Background()) @@ -432,7 +431,7 @@ func TestPushRegionEventToDSUnblocksOnClose(t *testing.T) { done := make(chan struct{}) go func() { - client.pushRegionEventToDS(SubscriptionID(1), regionEvent{}) + sink.Push(SubscriptionID(1), regionEvent{}) close(done) }() @@ -452,8 +451,7 @@ func TestPushRegionEventToDSUnblocksOnClose(t *testing.T) { } func TestBroadcastDeregisterUsesWorkerControlQueue(t *testing.T) { - client := &subscriptionClient{} - scheduler := ®ionRequestScheduler{client: client} + scheduler := ®ionRequestScheduler{} admission := newRegionAdmissionController(1, 1) worker := ®ionRequestWorker{ From 85804e85b844048c9e1d99e62dcef5bae9b108f7 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Fri, 17 Jul 2026 14:20:20 +0800 Subject: [PATCH 30/71] refactor --- .../logpuller/region_failure_handler.go | 4 +- .../logpuller/region_request_scheduler.go | 77 +++++++++---------- logservice/logpuller/region_request_worker.go | 22 +++--- .../logpuller/region_request_worker_test.go | 14 ++-- ..._store.go => store_request_worker_pool.go} | 30 ++++---- logservice/logpuller/subscription_client.go | 16 +--- .../logpuller/subscription_client_test.go | 18 ++--- pkg/metrics/log_puller.go | 2 +- server/server.go | 4 +- 9 files changed, 80 insertions(+), 107 deletions(-) rename logservice/logpuller/{requested_store.go => store_request_worker_pool.go} (64%) diff --git a/logservice/logpuller/region_failure_handler.go b/logservice/logpuller/region_failure_handler.go index 34434bedde..f90483b740 100644 --- a/logservice/logpuller/region_failure_handler.go +++ b/logservice/logpuller/region_failure_handler.go @@ -85,7 +85,6 @@ func (r *regionFailureHandler) Run(ctx context.Context) error { for { select { case <-ctx.Done(): - log.Info("subscription client handle errors and exit") return ctx.Err() case errInfo := <-r.cache.errCh: if err := r.handleError(ctx, errInfo); err != nil { @@ -177,7 +176,7 @@ func (r *regionFailureHandler) handleError(ctx context.Context, errInfo regionEr return nil default: // TODO(qupeng): for some errors it's better to just deregister the region from TiKVs. - log.Warn("subscription client meets an internal error, fail the changefeed", + log.Warn("region failure cannot be recovered, fail the changefeed", zap.Uint64("subscriptionID", uint64(errInfo.subscribedSpan.subID)), zap.Error(err)) return err @@ -236,7 +235,6 @@ func (e *errCache) dispatchBatch(ctx context.Context, limit int) (int, error) { for _, errInfo := range batch { select { case <-ctx.Done(): - log.Info("subscription client dispatch err cache done") return 0, ctx.Err() case e.errCh <- errInfo: } diff --git a/logservice/logpuller/region_request_scheduler.go b/logservice/logpuller/region_request_scheduler.go index 0d457d7080..f6bb45e21d 100644 --- a/logservice/logpuller/region_request_scheduler.go +++ b/logservice/logpuller/region_request_scheduler.go @@ -29,6 +29,8 @@ import ( "golang.org/x/sync/errgroup" ) +const regionRequestWorkerPerStore = 8 + // regionRequestScheduler routes locked Region requests through the global // priority queue to a worker connected to the Region's TiKV store. Range // resolution and retry policy remain owned by subscriptionClient and @@ -42,9 +44,9 @@ type regionRequestScheduler struct { taskQueue *priorityqueue.PriorityQueue[*regionPriorityTask] // sequence is the FIFO tie-breaker for Regions in the same priority class. sequence atomic.Uint64 - // stores maps TiKV addresses to requestedStore. Stores are created only by - // run, but are also read by metrics and deregistration goroutines. - stores sync.Map + // workerPools maps TiKV addresses to storeRequestWorkerPool. Pools are + // created only by run, but are also read by metrics and deregistration goroutines. + workerPools sync.Map // workerCount is the configured number of request workers per store. workerCount int @@ -55,16 +57,12 @@ type regionRequestScheduler struct { } func newRegionRequestScheduler( - clientConfig *SubscriptionClientConfig, upstream *upstreamHandle, eventSink *regionEventSink, failureHandler *regionFailureHandler, ) *regionRequestScheduler { pullerConfig := config.GetGlobalServerConfig().Debug.Puller - workerCount := int(clientConfig.RegionRequestWorkerPerStore) - if workerCount <= 0 { - workerCount = 1 - } + workerCount := regionRequestWorkerPerStore workerWindow := (pullerConfig.PendingRegionRequestQueueSize + workerCount - 1) / workerCount return ®ionRequestScheduler{ upstream: upstream, @@ -82,8 +80,8 @@ func (s *regionRequestScheduler) submit(region regionInfo) { region, s.upstream.pdClock.CurrentTS(), s.sequence.Add(1))) } -func (s *regionRequestScheduler) run(ctx context.Context, group *errgroup.Group) error { - defer s.closeStores() +func (s *regionRequestScheduler) run(ctx context.Context, workerGroup *errgroup.Group) error { + defer s.closeWorkerPools() for { select { case <-ctx.Done(): @@ -99,55 +97,50 @@ func (s *regionRequestScheduler) run(ctx context.Context, group *errgroup.Group) return err } - region, ok := s.attachRPCContext(ctx, task.GetRegionInfo()) - if !ok { + region, err := s.attachRPCContext(ctx, task.GetRegionInfo()) + if err != nil { + s.failureHandler.Report(newRegionErrorInfo(region, err)) continue } - store := s.getOrCreateStore(ctx, group, region.rpcCtx.Addr) + workerPool := s.getOrCreateWorkerPool(ctx, workerGroup, region.rpcCtx.Addr) task.updateRegion(region, s.upstream.pdClock.CurrentTS()) - if !store.submit(task) { + if !workerPool.submit(task) { return context.Canceled } - - log.Debug("subscription client will request a region", - zap.Uint64("subscriptionID", uint64(region.subscribedSpan.subID)), - zap.Uint64("regionID", region.verID.GetID()), - zap.String("addr", store.storeAddr)) } } func (s *regionRequestScheduler) attachRPCContext( ctx context.Context, region regionInfo, -) (regionInfo, bool) { +) (regionInfo, error) { bo := tikv.NewBackoffer(ctx, tikvRequestMaxBackoff) rpcCtx, err := s.upstream.regionCache.GetTiKVRPCContext( bo, region.verID, kvclientv2.ReplicaReadLeader, 0) if rpcCtx != nil { region.rpcCtx = rpcCtx - return region, true + return region, nil } if err != nil { - log.Debug("subscription client get rpc context fail", + log.Debug("region request scheduler failed to get RPC context", zap.Uint64("subscriptionID", uint64(region.subscribedSpan.subID)), zap.Uint64("regionID", region.verID.GetID()), zap.Error(err)) } - s.failureHandler.Report(newRegionErrorInfo(region, &rpcCtxUnavailableErr{verID: region.verID})) - return region, false + return region, &rpcCtxUnavailableErr{verID: region.verID} } -func (s *regionRequestScheduler) getOrCreateStore( +func (s *regionRequestScheduler) getOrCreateWorkerPool( ctx context.Context, - group *errgroup.Group, + workerGroup *errgroup.Group, storeAddr string, -) *requestedStore { - if value, ok := s.stores.Load(storeAddr); ok { - return value.(*requestedStore) +) *storeRequestWorkerPool { + if value, ok := s.workerPools.Load(storeAddr); ok { + return value.(*storeRequestWorkerPool) } - store := newRequestedStore( + workerPool := newStoreRequestWorkerPool( s.upstream, s.eventSink, s.failureHandler, @@ -156,27 +149,27 @@ func (s *regionRequestScheduler) getOrCreateStore( s.workerWindow, s.maxWindowMultiplier, ) - // run is the only writer. Publish the store after its immutable worker list - // is complete, then start its workers. - s.stores.Store(storeAddr, store) - store.run(ctx, group) - return store + // The scheduler run loop is the only writer. Publish the pool after its + // immutable worker list is complete, then start its workers. + s.workerPools.Store(storeAddr, workerPool) + workerPool.startWorkers(ctx, workerGroup) + return workerPool } func (s *regionRequestScheduler) broadcastDeregister( subID SubscriptionID, filterLoop bool, ) { - s.stores.Range(func(_, value any) bool { - value.(*requestedStore).broadcastDeregister(subID, filterLoop) + s.workerPools.Range(func(_, value any) bool { + value.(*storeRequestWorkerPool).broadcastDeregister(subID, filterLoop) return true }) } func (s *regionRequestScheduler) inflightCount() int { count := 0 - s.stores.Range(func(_, value any) bool { - count += value.(*requestedStore).inflightCount() + s.workerPools.Range(func(_, value any) bool { + count += value.(*storeRequestWorkerPool).inflightCount() return true }) return count @@ -186,9 +179,9 @@ func (s *regionRequestScheduler) close() { s.taskQueue.Close() } -func (s *regionRequestScheduler) closeStores() { - s.stores.Range(func(_, value any) bool { - value.(*requestedStore).close() +func (s *regionRequestScheduler) closeWorkerPools() { + s.workerPools.Range(func(_, value any) bool { + value.(*storeRequestWorkerPool).close() return true }) } diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index 6c8e9826aa..b22541bb49 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -95,7 +95,7 @@ type regionRequestWorker struct { upstream *upstreamHandle eventSink *regionEventSink failureHandler *regionFailureHandler - store *requestedStore + storeAddr string admission *regionAdmissionController controlQueue *controlQueue @@ -106,7 +106,7 @@ func newRegionRequestWorker( upstream *upstreamHandle, eventSink *regionEventSink, failureHandler *regionFailureHandler, - store *requestedStore, + storeAddr string, currentWindow int, maxWindowMultiplier int, ) *regionRequestWorker { @@ -116,7 +116,7 @@ func newRegionRequestWorker( upstream: upstream, eventSink: eventSink, failureHandler: failureHandler, - store: store, + storeAddr: storeAddr, admission: newRegionAdmissionController(currentWindow, maxWindowMultiplier), controlQueue: newControlQueue(), tracker: newRegionTracker(), @@ -207,7 +207,7 @@ func (s *regionRequestWorker) checkStoreVersion(ctx context.Context) error { } log.Error("event feed check store version fails", zap.Uint64("workerID", s.workerID), - zap.String("addr", s.store.storeAddr), + zap.String("addr", s.storeAddr), zap.Error(err)) if cerror.Is(err, cerror.ErrGetAllStoresFailed) { return &getStoreErr{} @@ -218,20 +218,20 @@ func (s *regionRequestWorker) checkStoreVersion(ctx context.Context) error { func (s *regionRequestWorker) runStream(ctx context.Context, firstReq *regionReq) (err error) { log.Info("region request worker going to create grpc stream", zap.Uint64("workerID", s.workerID), - zap.String("addr", s.store.storeAddr)) + zap.String("addr", s.storeAddr)) defer func() { log.Info("region request worker exits", zap.Uint64("workerID", s.workerID), - zap.String("addr", s.store.storeAddr), + zap.String("addr", s.storeAddr), zap.Error(err)) }() g, gctx := errgroup.WithContext(ctx) - conn, err := Connect(gctx, s.upstream.credential, s.store.storeAddr) + conn, err := Connect(gctx, s.upstream.credential, s.storeAddr) if err != nil { log.Warn("region request worker create grpc stream failed", zap.Uint64("workerID", s.workerID), - zap.String("addr", s.store.storeAddr), + zap.String("addr", s.storeAddr), zap.Error(err)) if conn != nil && conn.Conn != nil { _ = conn.Conn.Close() @@ -279,7 +279,7 @@ func (s *regionRequestWorker) receiveAndDispatchChangeEvents(conn *ConnAndClient if err != nil { log.Info("region request worker receive from grpc stream failed", zap.Uint64("workerID", s.workerID), - zap.String("addr", s.store.storeAddr), + zap.String("addr", s.storeAddr), zap.String("code", grpcstatus.Code(err).String()), zap.Error(err)) return normalizeStreamError(err) @@ -401,7 +401,7 @@ func (s *regionRequestWorker) sendChangeDataRequest( zap.Uint64("workerID", s.workerID), zap.Uint64("subscriptionID", req.RequestId), zap.Uint64("regionID", req.RegionId), - zap.String("addr", s.store.storeAddr), + zap.String("addr", s.storeAddr), zap.Error(err)) return normalizeStreamError(err) } @@ -451,7 +451,7 @@ func (s *regionRequestWorker) sendRegionRequest(conn *ConnAndClient, req *region zap.Uint64("workerID", s.workerID), zap.Uint64("subscriptionID", uint64(subID)), zap.Uint64("regionID", region.verID.GetID()), - zap.String("addr", s.store.storeAddr), + zap.String("addr", s.storeAddr), zap.Bool("bdrMode", region.filterLoop)) if region.subscribedSpan.stopped.Load() { diff --git a/logservice/logpuller/region_request_worker_test.go b/logservice/logpuller/region_request_worker_test.go index 70affb2d0a..9b75667040 100644 --- a/logservice/logpuller/region_request_worker_test.go +++ b/logservice/logpuller/region_request_worker_test.go @@ -82,7 +82,7 @@ func TestRegionRequestWorkerIgnoresDuplicateActiveRegion(t *testing.T) { admission := newRegionAdmissionController(10, 1) worker := ®ionRequestWorker{ admission: admission, - store: &requestedStore{storeAddr: "store-1"}, + storeAddr: "store-1", upstream: &upstreamHandle{}, tracker: newRegionTracker(), } @@ -407,7 +407,7 @@ func TestProcessRegionSendTaskSendFailureCleansSentRequest(t *testing.T) { worker := ®ionRequestWorker{ admission: admission, controlQueue: newControlQueue(), - store: &requestedStore{storeAddr: "store-1"}, + storeAddr: "store-1", upstream: &upstreamHandle{}, tracker: newRegionTracker(), } @@ -438,7 +438,7 @@ func TestProcessRegionSendTaskDoesNotSendRemovedRequest(t *testing.T) { worker := ®ionRequestWorker{ admission: admission, controlQueue: newControlQueue(), - store: &requestedStore{storeAddr: "store-1"}, + storeAddr: "store-1", upstream: &upstreamHandle{}, tracker: newRegionTracker(), } @@ -481,7 +481,7 @@ func TestProcessRegionSendTaskSendEOFIsRetriable(t *testing.T) { worker := ®ionRequestWorker{ admission: admission, controlQueue: newControlQueue(), - store: &requestedStore{storeAddr: "store-1"}, + storeAddr: "store-1", upstream: &upstreamHandle{}, tracker: newRegionTracker(), } @@ -514,7 +514,7 @@ func TestProcessRegionSendTaskHandlesDeregisterFromControlQueue(t *testing.T) { worker := ®ionRequestWorker{ admission: newRegionAdmissionController(1, 1), controlQueue: newControlQueue(), - store: &requestedStore{storeAddr: "store-1"}, + storeAddr: "store-1", upstream: &upstreamHandle{clusterID: 42}, eventSink: ®ionEventSink{ds: ds}, tracker: newRegionTracker(), @@ -562,9 +562,7 @@ func TestReceiveAndDispatchChangeEventsEOFIsRetriable(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - worker := ®ionRequestWorker{ - store: &requestedStore{storeAddr: "store-1"}, - } + worker := ®ionRequestWorker{storeAddr: "store-1"} conn := &ConnAndClient{ Client: &mockEventFeedV2Client{recvErr: tc.recvErr}, Conn: &grpc.ClientConn{}, diff --git a/logservice/logpuller/requested_store.go b/logservice/logpuller/store_request_worker_pool.go similarity index 64% rename from logservice/logpuller/requested_store.go rename to logservice/logpuller/store_request_worker_pool.go index 97294a8e33..6f84e0019a 100644 --- a/logservice/logpuller/requested_store.go +++ b/logservice/logpuller/store_request_worker_pool.go @@ -21,16 +21,16 @@ import ( "golang.org/x/sync/errgroup" ) -// requestedStore owns the request workers connected to one TiKV store. The -// worker slice is complete before the store is published and is immutable +// storeRequestWorkerPool owns the request workers connected to one TiKV store. +// The worker slice is complete before the pool is published and is immutable // afterwards, so task submission only needs an atomic round-robin counter. -type requestedStore struct { +type storeRequestWorkerPool struct { storeAddr string workers []*regionRequestWorker nextWorker atomic.Uint64 } -func newRequestedStore( +func newStoreRequestWorkerPool( upstream *upstreamHandle, eventSink *regionEventSink, failureHandler *regionFailureHandler, @@ -38,25 +38,25 @@ func newRequestedStore( workerCount int, workerWindow int, maxWindowMultiplier int, -) *requestedStore { - store := &requestedStore{ +) *storeRequestWorkerPool { + workerPool := &storeRequestWorkerPool{ storeAddr: storeAddr, workers: make([]*regionRequestWorker, 0, workerCount), } for i := 0; i < workerCount; i++ { - store.workers = append(store.workers, newRegionRequestWorker( - upstream, eventSink, failureHandler, store, workerWindow, maxWindowMultiplier)) + workerPool.workers = append(workerPool.workers, newRegionRequestWorker( + upstream, eventSink, failureHandler, storeAddr, workerWindow, maxWindowMultiplier)) } - return store + return workerPool } -func (s *requestedStore) run(ctx context.Context, group *errgroup.Group) { +func (s *storeRequestWorkerPool) startWorkers(ctx context.Context, workerGroup *errgroup.Group) { for _, worker := range s.workers { - group.Go(func() error { return worker.Run(ctx) }) + workerGroup.Go(func() error { return worker.Run(ctx) }) } } -func (s *requestedStore) submit(task *regionPriorityTask) bool { +func (s *storeRequestWorkerPool) submit(task *regionPriorityTask) bool { if len(s.workers) == 0 { return false } @@ -64,19 +64,19 @@ func (s *requestedStore) submit(task *regionPriorityTask) bool { return s.workers[index].admission.submit(task) } -func (s *requestedStore) broadcastDeregister(subID SubscriptionID, filterLoop bool) { +func (s *storeRequestWorkerPool) broadcastDeregister(subID SubscriptionID, filterLoop bool) { for _, worker := range s.workers { worker.controlQueue.push(deregisterRequest{subID: subID, filterLoop: filterLoop}) } } -func (s *requestedStore) close() { +func (s *storeRequestWorkerPool) close() { for _, worker := range s.workers { worker.admission.close() } } -func (s *requestedStore) inflightCount() int { +func (s *storeRequestWorkerPool) inflightCount() int { count := 0 for _, worker := range s.workers { count += worker.admission.stats().inflight diff --git a/logservice/logpuller/subscription_client.go b/logservice/logpuller/subscription_client.go index 1266af764a..0b629d2fa7 100644 --- a/logservice/logpuller/subscription_client.go +++ b/logservice/logpuller/subscription_client.go @@ -84,14 +84,8 @@ type rangeTask struct { wasInitialized bool } -type SubscriptionClientConfig struct { - // The number of region request workers to send region task for every tikv store - RegionRequestWorkerPerStore uint -} - // upstreamHandle contains the stable TiKV and PD dependencies shared by the -// region request pipeline. Runtime components keep their own event and error -// dependencies instead of using this as a general service container. +// region request pipeline. type upstreamHandle struct { pd pd.Client regionCache *tikv.RegionCache @@ -129,7 +123,6 @@ type SubscriptionClient interface { type subscriptionClient struct { ctx context.Context cancel context.CancelFunc - config *SubscriptionClientConfig upstream *upstreamHandle lockResolver txnutil.LockResolver @@ -154,13 +147,11 @@ type subscriptionClient struct { // NewSubscriptionClient creates a client. func NewSubscriptionClient( - config *SubscriptionClientConfig, pd pd.Client, lockResolver txnutil.LockResolver, credential *security.Credential, ) SubscriptionClient { subClient := &subscriptionClient{ - config: config, upstream: &upstreamHandle{ pd: pd, regionCache: appcontext.GetService[*tikv.RegionCache](appcontext.RegionCache), @@ -183,7 +174,6 @@ func NewSubscriptionClient( subClient.eventSink = newRegionEventSink(subClient.ctx, subClient.failureHandler) subClient.spanRegistry = newSpanRegistry(subClient.upstream.pd, subClient.upstream.pdClock) subClient.regionScheduler = newRegionRequestScheduler( - subClient.config, subClient.upstream, subClient.eventSink, subClient.failureHandler, @@ -208,9 +198,9 @@ func (s *subscriptionClient) updateMetrics(ctx context.Context) error { case <-ctx.Done(): return ctx.Err() case <-ticker.C: - pendingRegionReqCount := s.regionScheduler.inflightCount() + inflightRegionRequestCount := s.regionScheduler.inflightCount() - metrics.SubscriptionClientRequestedRegionCount.WithLabelValues("pending").Set(float64(pendingRegionReqCount)) + metrics.SubscriptionClientRequestedRegionCount.WithLabelValues("inflight").Set(float64(inflightRegionRequestCount)) s.eventSink.UpdateMetrics() s.spanRegistry.UpdateMetrics() } diff --git a/logservice/logpuller/subscription_client_test.go b/logservice/logpuller/subscription_client_test.go index 4387fa2054..1a939d50d8 100644 --- a/logservice/logpuller/subscription_client_test.go +++ b/logservice/logpuller/subscription_client_test.go @@ -328,9 +328,9 @@ func TestStopTaskUsesSubscribedSpanFilterLoop(t *testing.T) { res := span.rangeLock.LockRange(context.Background(), rawSpan.StartKey, rawSpan.EndKey, 1, 1) require.Equal(t, regionlock.LockRangeStatusSuccess, res.Status) worker := ®ionRequestWorker{controlQueue: newControlQueue()} - store := &requestedStore{storeAddr: "store-1", workers: []*regionRequestWorker{worker}} + workerPool := &storeRequestWorkerPool{storeAddr: "store-1", workers: []*regionRequestWorker{worker}} client.regionScheduler = ®ionRequestScheduler{} - client.regionScheduler.stores.Store(store.storeAddr, store) + client.regionScheduler.workerPools.Store(workerPool.storeAddr, workerPool) client.setTableStopped(span) @@ -458,8 +458,8 @@ func TestBroadcastDeregisterUsesWorkerControlQueue(t *testing.T) { admission: admission, controlQueue: newControlQueue(), } - store := &requestedStore{storeAddr: "store-1", workers: []*regionRequestWorker{worker}} - scheduler.stores.Store(store.storeAddr, store) + workerPool := &storeRequestWorkerPool{storeAddr: "store-1", workers: []*regionRequestWorker{worker}} + scheduler.workerPools.Store(workerPool.storeAddr, workerPool) dummyRegion := regionInfo{ subscribedSpan: &subscribedSpan{subID: SubscriptionID(2)}, @@ -476,10 +476,10 @@ func TestBroadcastDeregisterUsesWorkerControlQueue(t *testing.T) { require.Equal(t, 1, admission.stats().pending) } -func TestRequestedStoreDistributesRegionsAcrossWorkerBuffers(t *testing.T) { +func TestStoreRequestWorkerPoolDistributesRegionsAcrossWorkers(t *testing.T) { worker1 := ®ionRequestWorker{admission: newRegionAdmissionController(1, 1)} worker2 := ®ionRequestWorker{admission: newRegionAdmissionController(1, 1)} - store := &requestedStore{ + workerPool := &storeRequestWorkerPool{ storeAddr: "store-1", workers: []*regionRequestWorker{worker1, worker2}, } @@ -490,7 +490,7 @@ func TestRequestedStoreDistributesRegionsAcrossWorkerBuffers(t *testing.T) { subscribedSpan: &subscribedSpan{subID: 1}, lockedRangeState: ®ionlock.LockedRangeState{}, } - require.True(t, store.submit(NewRegionPriorityTask(region, 1, i))) + require.True(t, workerPool.submit(NewRegionPriorityTask(region, 1, i))) } require.Equal(t, 2, worker1.admission.stats().pending) @@ -526,11 +526,7 @@ func TestSubscriptionWithFailedTiKV(t *testing.T) { // bootstrap cluster with a region which leader is in invalid store. cluster.Bootstrap(11, []uint64{1, 2, 3}, []uint64{4, 5, 6}, 6) - clientConfig := &SubscriptionClientConfig{ - RegionRequestWorkerPerStore: 2, - } client := NewSubscriptionClient( - clientConfig, pdClient, nil, // we don't need it in this unittest, so we can pass nil &security.Credential{}, diff --git a/pkg/metrics/log_puller.go b/pkg/metrics/log_puller.go index e874aa4428..af94bfe89a 100644 --- a/pkg/metrics/log_puller.go +++ b/pkg/metrics/log_puller.go @@ -78,7 +78,7 @@ var ( Namespace: "ticdc", Subsystem: "subscription_client", Name: "requested_region_count", - Help: "The number of requested regions", + Help: "The number of Region requests by state.", }, []string{"state"}) RegionRequestFinishScanDuration = prometheus.NewHistogram( prometheus.HistogramOpts{ diff --git a/server/server.go b/server/server.go index b9dc5edad8..8a604d35cc 100644 --- a/server/server.go +++ b/server/server.go @@ -198,9 +198,7 @@ func (c *server) initialize(ctx context.Context) error { conf := config.GetGlobalServerConfig() schemaStore := schemastore.New(conf.DataDir, c.pdClient) subscriptionClient := logpuller.NewSubscriptionClient( - &logpuller.SubscriptionClientConfig{ - RegionRequestWorkerPerStore: 8, - }, c.pdClient, + c.pdClient, txnutil.NewLockerResolver(), c.security, ) From 651897ee76b347dcb90142cb89346451117e62fb Mon Sep 17 00:00:00 2001 From: lidezhu Date: Fri, 17 Jul 2026 14:30:10 +0800 Subject: [PATCH 31/71] refactor --- .../logpuller/region_request_scheduler.go | 44 +++++++++---------- ...worker_pool.go => region_request_store.go} | 28 ++++++------ .../logpuller/subscription_client_test.go | 14 +++--- 3 files changed, 43 insertions(+), 43 deletions(-) rename logservice/logpuller/{store_request_worker_pool.go => region_request_store.go} (67%) diff --git a/logservice/logpuller/region_request_scheduler.go b/logservice/logpuller/region_request_scheduler.go index f6bb45e21d..2de2322128 100644 --- a/logservice/logpuller/region_request_scheduler.go +++ b/logservice/logpuller/region_request_scheduler.go @@ -44,9 +44,9 @@ type regionRequestScheduler struct { taskQueue *priorityqueue.PriorityQueue[*regionPriorityTask] // sequence is the FIFO tie-breaker for Regions in the same priority class. sequence atomic.Uint64 - // workerPools maps TiKV addresses to storeRequestWorkerPool. Pools are - // created only by run, but are also read by metrics and deregistration goroutines. - workerPools sync.Map + // stores maps TiKV addresses to regionRequestStore. Stores are created only + // by run, but are also read by metrics and deregistration goroutines. + stores sync.Map // workerCount is the configured number of request workers per store. workerCount int @@ -81,7 +81,7 @@ func (s *regionRequestScheduler) submit(region regionInfo) { } func (s *regionRequestScheduler) run(ctx context.Context, workerGroup *errgroup.Group) error { - defer s.closeWorkerPools() + defer s.closeStores() for { select { case <-ctx.Done(): @@ -103,9 +103,9 @@ func (s *regionRequestScheduler) run(ctx context.Context, workerGroup *errgroup. continue } - workerPool := s.getOrCreateWorkerPool(ctx, workerGroup, region.rpcCtx.Addr) + store := s.getOrCreateStore(ctx, workerGroup, region.rpcCtx.Addr) task.updateRegion(region, s.upstream.pdClock.CurrentTS()) - if !workerPool.submit(task) { + if !store.submit(task) { return context.Canceled } } @@ -131,16 +131,16 @@ func (s *regionRequestScheduler) attachRPCContext( return region, &rpcCtxUnavailableErr{verID: region.verID} } -func (s *regionRequestScheduler) getOrCreateWorkerPool( +func (s *regionRequestScheduler) getOrCreateStore( ctx context.Context, workerGroup *errgroup.Group, storeAddr string, -) *storeRequestWorkerPool { - if value, ok := s.workerPools.Load(storeAddr); ok { - return value.(*storeRequestWorkerPool) +) *regionRequestStore { + if value, ok := s.stores.Load(storeAddr); ok { + return value.(*regionRequestStore) } - workerPool := newStoreRequestWorkerPool( + store := newRegionRequestStore( s.upstream, s.eventSink, s.failureHandler, @@ -149,27 +149,27 @@ func (s *regionRequestScheduler) getOrCreateWorkerPool( s.workerWindow, s.maxWindowMultiplier, ) - // The scheduler run loop is the only writer. Publish the pool after its + // The scheduler run loop is the only writer. Publish the store after its // immutable worker list is complete, then start its workers. - s.workerPools.Store(storeAddr, workerPool) - workerPool.startWorkers(ctx, workerGroup) - return workerPool + s.stores.Store(storeAddr, store) + store.startWorkers(ctx, workerGroup) + return store } func (s *regionRequestScheduler) broadcastDeregister( subID SubscriptionID, filterLoop bool, ) { - s.workerPools.Range(func(_, value any) bool { - value.(*storeRequestWorkerPool).broadcastDeregister(subID, filterLoop) + s.stores.Range(func(_, value any) bool { + value.(*regionRequestStore).broadcastDeregister(subID, filterLoop) return true }) } func (s *regionRequestScheduler) inflightCount() int { count := 0 - s.workerPools.Range(func(_, value any) bool { - count += value.(*storeRequestWorkerPool).inflightCount() + s.stores.Range(func(_, value any) bool { + count += value.(*regionRequestStore).inflightCount() return true }) return count @@ -179,9 +179,9 @@ func (s *regionRequestScheduler) close() { s.taskQueue.Close() } -func (s *regionRequestScheduler) closeWorkerPools() { - s.workerPools.Range(func(_, value any) bool { - value.(*storeRequestWorkerPool).close() +func (s *regionRequestScheduler) closeStores() { + s.stores.Range(func(_, value any) bool { + value.(*regionRequestStore).close() return true }) } diff --git a/logservice/logpuller/store_request_worker_pool.go b/logservice/logpuller/region_request_store.go similarity index 67% rename from logservice/logpuller/store_request_worker_pool.go rename to logservice/logpuller/region_request_store.go index 6f84e0019a..92c44d91eb 100644 --- a/logservice/logpuller/store_request_worker_pool.go +++ b/logservice/logpuller/region_request_store.go @@ -21,16 +21,16 @@ import ( "golang.org/x/sync/errgroup" ) -// storeRequestWorkerPool owns the request workers connected to one TiKV store. -// The worker slice is complete before the pool is published and is immutable -// afterwards, so task submission only needs an atomic round-robin counter. -type storeRequestWorkerPool struct { +// regionRequestStore owns the Region request workers connected to one TiKV +// store. The worker slice is complete before the store is published and is +// immutable afterwards, so task submission only needs an atomic round-robin counter. +type regionRequestStore struct { storeAddr string workers []*regionRequestWorker nextWorker atomic.Uint64 } -func newStoreRequestWorkerPool( +func newRegionRequestStore( upstream *upstreamHandle, eventSink *regionEventSink, failureHandler *regionFailureHandler, @@ -38,25 +38,25 @@ func newStoreRequestWorkerPool( workerCount int, workerWindow int, maxWindowMultiplier int, -) *storeRequestWorkerPool { - workerPool := &storeRequestWorkerPool{ +) *regionRequestStore { + store := ®ionRequestStore{ storeAddr: storeAddr, workers: make([]*regionRequestWorker, 0, workerCount), } for i := 0; i < workerCount; i++ { - workerPool.workers = append(workerPool.workers, newRegionRequestWorker( + store.workers = append(store.workers, newRegionRequestWorker( upstream, eventSink, failureHandler, storeAddr, workerWindow, maxWindowMultiplier)) } - return workerPool + return store } -func (s *storeRequestWorkerPool) startWorkers(ctx context.Context, workerGroup *errgroup.Group) { +func (s *regionRequestStore) startWorkers(ctx context.Context, workerGroup *errgroup.Group) { for _, worker := range s.workers { workerGroup.Go(func() error { return worker.Run(ctx) }) } } -func (s *storeRequestWorkerPool) submit(task *regionPriorityTask) bool { +func (s *regionRequestStore) submit(task *regionPriorityTask) bool { if len(s.workers) == 0 { return false } @@ -64,19 +64,19 @@ func (s *storeRequestWorkerPool) submit(task *regionPriorityTask) bool { return s.workers[index].admission.submit(task) } -func (s *storeRequestWorkerPool) broadcastDeregister(subID SubscriptionID, filterLoop bool) { +func (s *regionRequestStore) broadcastDeregister(subID SubscriptionID, filterLoop bool) { for _, worker := range s.workers { worker.controlQueue.push(deregisterRequest{subID: subID, filterLoop: filterLoop}) } } -func (s *storeRequestWorkerPool) close() { +func (s *regionRequestStore) close() { for _, worker := range s.workers { worker.admission.close() } } -func (s *storeRequestWorkerPool) inflightCount() int { +func (s *regionRequestStore) inflightCount() int { count := 0 for _, worker := range s.workers { count += worker.admission.stats().inflight diff --git a/logservice/logpuller/subscription_client_test.go b/logservice/logpuller/subscription_client_test.go index 1a939d50d8..23d6f81fe6 100644 --- a/logservice/logpuller/subscription_client_test.go +++ b/logservice/logpuller/subscription_client_test.go @@ -328,9 +328,9 @@ func TestStopTaskUsesSubscribedSpanFilterLoop(t *testing.T) { res := span.rangeLock.LockRange(context.Background(), rawSpan.StartKey, rawSpan.EndKey, 1, 1) require.Equal(t, regionlock.LockRangeStatusSuccess, res.Status) worker := ®ionRequestWorker{controlQueue: newControlQueue()} - workerPool := &storeRequestWorkerPool{storeAddr: "store-1", workers: []*regionRequestWorker{worker}} + store := ®ionRequestStore{storeAddr: "store-1", workers: []*regionRequestWorker{worker}} client.regionScheduler = ®ionRequestScheduler{} - client.regionScheduler.workerPools.Store(workerPool.storeAddr, workerPool) + client.regionScheduler.stores.Store(store.storeAddr, store) client.setTableStopped(span) @@ -458,8 +458,8 @@ func TestBroadcastDeregisterUsesWorkerControlQueue(t *testing.T) { admission: admission, controlQueue: newControlQueue(), } - workerPool := &storeRequestWorkerPool{storeAddr: "store-1", workers: []*regionRequestWorker{worker}} - scheduler.workerPools.Store(workerPool.storeAddr, workerPool) + store := ®ionRequestStore{storeAddr: "store-1", workers: []*regionRequestWorker{worker}} + scheduler.stores.Store(store.storeAddr, store) dummyRegion := regionInfo{ subscribedSpan: &subscribedSpan{subID: SubscriptionID(2)}, @@ -476,10 +476,10 @@ func TestBroadcastDeregisterUsesWorkerControlQueue(t *testing.T) { require.Equal(t, 1, admission.stats().pending) } -func TestStoreRequestWorkerPoolDistributesRegionsAcrossWorkers(t *testing.T) { +func TestRegionRequestStoreDistributesRegionsAcrossWorkers(t *testing.T) { worker1 := ®ionRequestWorker{admission: newRegionAdmissionController(1, 1)} worker2 := ®ionRequestWorker{admission: newRegionAdmissionController(1, 1)} - workerPool := &storeRequestWorkerPool{ + store := ®ionRequestStore{ storeAddr: "store-1", workers: []*regionRequestWorker{worker1, worker2}, } @@ -490,7 +490,7 @@ func TestStoreRequestWorkerPoolDistributesRegionsAcrossWorkers(t *testing.T) { subscribedSpan: &subscribedSpan{subID: 1}, lockedRangeState: ®ionlock.LockedRangeState{}, } - require.True(t, workerPool.submit(NewRegionPriorityTask(region, 1, i))) + require.True(t, store.submit(NewRegionPriorityTask(region, 1, i))) } require.Equal(t, 2, worker1.admission.stats().pending) From 777dd57a020093c02f02439cc7eb2562c2cefb67 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Fri, 17 Jul 2026 16:24:56 +0800 Subject: [PATCH 32/71] refactor --- .../logpuller/region_request_scheduler.go | 24 ++++++++++++------- logservice/logpuller/region_request_store.go | 5 +--- logservice/logpuller/subscription_client.go | 23 +++++++----------- .../logpuller/subscription_client_test.go | 2 +- pkg/metrics/log_puller.go | 2 +- 5 files changed, 26 insertions(+), 30 deletions(-) diff --git a/logservice/logpuller/region_request_scheduler.go b/logservice/logpuller/region_request_scheduler.go index 2de2322128..b7e3eba394 100644 --- a/logservice/logpuller/region_request_scheduler.go +++ b/logservice/logpuller/region_request_scheduler.go @@ -22,6 +22,7 @@ import ( "github.com/pingcap/log" "github.com/pingcap/ticdc/pkg/config" "github.com/pingcap/ticdc/pkg/errors" + "github.com/pingcap/ticdc/pkg/metrics" "github.com/pingcap/ticdc/utils/priorityqueue" kvclientv2 "github.com/tikv/client-go/v2/kv" "github.com/tikv/client-go/v2/tikv" @@ -31,8 +32,8 @@ import ( const regionRequestWorkerPerStore = 8 -// regionRequestScheduler routes locked Region requests through the global -// priority queue to a worker connected to the Region's TiKV store. Range +// regionRequestScheduler routes locked region requests through the global +// priority queue to a worker connected to the region's TiKV store. Range // resolution and retry policy remain owned by subscriptionClient and // regionFailureHandler respectively. type regionRequestScheduler struct { @@ -40,12 +41,12 @@ type regionRequestScheduler struct { eventSink *regionEventSink failureHandler *regionFailureHandler - // taskQueue orders all Regions before they are assigned to a TiKV store. + // taskQueue orders all regions before they are assigned to a TiKV store. taskQueue *priorityqueue.PriorityQueue[*regionPriorityTask] - // sequence is the FIFO tie-breaker for Regions in the same priority class. + // sequence is the FIFO tie-breaker for regions in the same priority class. sequence atomic.Uint64 // stores maps TiKV addresses to regionRequestStore. Stores are created only - // by run, but are also read by metrics and deregistration goroutines. + // by Run, but are also read by metrics and deregistration goroutines. stores sync.Map // workerCount is the configured number of request workers per store. @@ -75,12 +76,12 @@ func newRegionRequestScheduler( } } -func (s *regionRequestScheduler) submit(region regionInfo) { +func (s *regionRequestScheduler) Submit(region regionInfo) { s.taskQueue.Push(NewRegionPriorityTask( region, s.upstream.pdClock.CurrentTS(), s.sequence.Add(1))) } -func (s *regionRequestScheduler) run(ctx context.Context, workerGroup *errgroup.Group) error { +func (s *regionRequestScheduler) Run(ctx context.Context, workerGroup *errgroup.Group) error { defer s.closeStores() for { select { @@ -156,7 +157,7 @@ func (s *regionRequestScheduler) getOrCreateStore( return store } -func (s *regionRequestScheduler) broadcastDeregister( +func (s *regionRequestScheduler) BroadcastDeregister( subID SubscriptionID, filterLoop bool, ) { @@ -175,7 +176,12 @@ func (s *regionRequestScheduler) inflightCount() int { return count } -func (s *regionRequestScheduler) close() { +func (s *regionRequestScheduler) UpdateMetrics() { + metrics.SubscriptionClientRequestedRegionCount.WithLabelValues("inflight"). + Set(float64(s.inflightCount())) +} + +func (s *regionRequestScheduler) Close() { s.taskQueue.Close() } diff --git a/logservice/logpuller/region_request_store.go b/logservice/logpuller/region_request_store.go index 92c44d91eb..1a8d243e0d 100644 --- a/logservice/logpuller/region_request_store.go +++ b/logservice/logpuller/region_request_store.go @@ -21,7 +21,7 @@ import ( "golang.org/x/sync/errgroup" ) -// regionRequestStore owns the Region request workers connected to one TiKV +// regionRequestStore owns the region request workers connected to one TiKV // store. The worker slice is complete before the store is published and is // immutable afterwards, so task submission only needs an atomic round-robin counter. type regionRequestStore struct { @@ -57,9 +57,6 @@ func (s *regionRequestStore) startWorkers(ctx context.Context, workerGroup *errg } func (s *regionRequestStore) submit(task *regionPriorityTask) bool { - if len(s.workers) == 0 { - return false - } index := (s.nextWorker.Add(1) - 1) % uint64(len(s.workers)) return s.workers[index].admission.submit(task) } diff --git a/logservice/logpuller/subscription_client.go b/logservice/logpuller/subscription_client.go index 0b629d2fa7..924025a386 100644 --- a/logservice/logpuller/subscription_client.go +++ b/logservice/logpuller/subscription_client.go @@ -94,7 +94,7 @@ type upstreamHandle struct { clusterID uint64 } -// initialize loads the cluster metadata needed by Region request workers. It +// initialize loads the cluster metadata needed by region request workers. It // must run before the scheduler starts any workers. func (u *upstreamHandle) initialize(ctx context.Context) { u.clusterID = u.pd.GetClusterID(ctx) @@ -133,7 +133,7 @@ type subscriptionClient struct { eventSink *regionEventSink // spanRegistry tracks subscribed spans and owns span-level background tasks. spanRegistry *spanRegistry - // regionScheduler assigns locked Region requests to per-store workers. + // regionScheduler assigns locked region requests to per-store workers. regionScheduler *regionRequestScheduler // rangeTaskCh is used to receive range tasks. @@ -198,9 +198,7 @@ func (s *subscriptionClient) updateMetrics(ctx context.Context) error { case <-ctx.Done(): return ctx.Err() case <-ticker.C: - inflightRegionRequestCount := s.regionScheduler.inflightCount() - - metrics.SubscriptionClientRequestedRegionCount.WithLabelValues("inflight").Set(float64(inflightRegionRequestCount)) + s.regionScheduler.UpdateMetrics() s.eventSink.UpdateMetrics() s.spanRegistry.UpdateMetrics() } @@ -268,11 +266,6 @@ func (s *subscriptionClient) Unsubscribe(subID SubscriptionID) { } func (s *subscriptionClient) Run(ctx context.Context) error { - // s.consume = consume - if s.upstream == nil || s.upstream.pd == nil { - log.Warn("subscription client should be in test mode, skip run") - return nil - } s.upstream.initialize(ctx) g, ctx := errgroup.WithContext(ctx) @@ -280,7 +273,7 @@ func (s *subscriptionClient) Run(ctx context.Context) error { g.Go(func() error { return s.updateMetrics(ctx) }) g.Go(func() error { return s.eventSink.Run(ctx) }) g.Go(func() error { return s.handleRangeTasks(ctx) }) - g.Go(func() error { return s.regionScheduler.run(ctx, g) }) + g.Go(func() error { return s.regionScheduler.Run(ctx, g) }) g.Go(func() error { return s.failureHandler.Run(ctx) }) g.Go(func() error { return s.handleResolveLockTasks(ctx) }) g.Go(func() error { return s.spanRegistry.Run(ctx) }) @@ -294,7 +287,7 @@ func (s *subscriptionClient) Run(ctx context.Context) error { func (s *subscriptionClient) Close(ctx context.Context) error { s.cancel() s.eventSink.Close() - s.regionScheduler.close() + s.regionScheduler.Close() return nil } @@ -305,7 +298,7 @@ func (s *subscriptionClient) setTableStopped(rt *subscribedSpan) { // Set stopped to true so we can stop handling region events from the table, // then notify every existing worker to deregister the subscription. if rt.stopped.CompareAndSwap(false, true) { - s.regionScheduler.broadcastDeregister(rt.subID, rt.filterLoop) + s.regionScheduler.BroadcastDeregister(rt.subID, rt.filterLoop) if rt.rangeLock.Stop() { s.onTableDrained(rt) } @@ -429,7 +422,7 @@ func (s *subscriptionClient) divideSpanAndScheduleRegionRequests( } } -// scheduleRegionRequest locks the Region's range before submitting it to the +// scheduleRegionRequest locks the region's range before submitting it to the // request scheduler. func (s *subscriptionClient) scheduleRegionRequest(ctx context.Context, region regionInfo) { if region.lockedRangeState != nil && region.lockedRangeState.Initialized.Load() { @@ -445,7 +438,7 @@ func (s *subscriptionClient) scheduleRegionRequest(ctx context.Context, region r switch lockRangeResult.Status { case regionlock.LockRangeStatusSuccess: region.lockedRangeState = lockRangeResult.LockedRangeState - s.regionScheduler.submit(region) + s.regionScheduler.Submit(region) case regionlock.LockRangeStatusStale: for _, r := range lockRangeResult.RetryRanges { s.scheduleRangeRequest(ctx, r, region.subscribedSpan, region.filterLoop, region.wasInitialized) diff --git a/logservice/logpuller/subscription_client_test.go b/logservice/logpuller/subscription_client_test.go index 23d6f81fe6..ef1a387045 100644 --- a/logservice/logpuller/subscription_client_test.go +++ b/logservice/logpuller/subscription_client_test.go @@ -467,7 +467,7 @@ func TestBroadcastDeregisterUsesWorkerControlQueue(t *testing.T) { } require.True(t, admission.submit(NewRegionPriorityTask(dummyRegion, 1, 1))) - scheduler.broadcastDeregister(SubscriptionID(1), true) + scheduler.BroadcastDeregister(SubscriptionID(1), true) require.Equal(t, 1, worker.controlQueue.len()) req, ok := worker.controlQueue.tryPop() require.True(t, ok) diff --git a/pkg/metrics/log_puller.go b/pkg/metrics/log_puller.go index af94bfe89a..a74f1b03c4 100644 --- a/pkg/metrics/log_puller.go +++ b/pkg/metrics/log_puller.go @@ -78,7 +78,7 @@ var ( Namespace: "ticdc", Subsystem: "subscription_client", Name: "requested_region_count", - Help: "The number of Region requests by state.", + Help: "The number of region requests by state.", }, []string{"state"}) RegionRequestFinishScanDuration = prometheus.NewHistogram( prometheus.HistogramOpts{ From 5453d421d408d710077da2a556ad7c0fca43660d Mon Sep 17 00:00:00 2001 From: lidezhu Date: Fri, 17 Jul 2026 16:41:06 +0800 Subject: [PATCH 33/71] refactor --- ...mock_upstream.go => mock_upstream_test.go} | 0 logservice/logpuller/priority_task.go | 67 +++++++---- logservice/logpuller/priority_task_test.go | 106 +++++++++--------- .../region_admission_controller_test.go | 27 ++--- .../logpuller/region_event_handler_test.go | 1 - .../logpuller/region_failure_handler.go | 21 ++-- .../logpuller/region_request_scheduler.go | 4 +- logservice/logpuller/region_request_store.go | 4 +- logservice/logpuller/region_request_worker.go | 6 +- .../logpuller/region_request_worker_test.go | 2 +- logservice/logpuller/region_state.go | 14 +-- logservice/logpuller/region_tracker.go | 10 +- logservice/logpuller/subscription_client.go | 42 +++---- .../logpuller/subscription_client_test.go | 20 ++-- 14 files changed, 176 insertions(+), 148 deletions(-) rename logservice/logpuller/{mock_upstream.go => mock_upstream_test.go} (100%) diff --git a/logservice/logpuller/mock_upstream.go b/logservice/logpuller/mock_upstream_test.go similarity index 100% rename from logservice/logpuller/mock_upstream.go rename to logservice/logpuller/mock_upstream_test.go diff --git a/logservice/logpuller/priority_task.go b/logservice/logpuller/priority_task.go index 38cd813c8b..eb241dec0d 100644 --- a/logservice/logpuller/priority_task.go +++ b/logservice/logpuller/priority_task.go @@ -13,41 +13,55 @@ package logpuller -import "github.com/pingcap/kvproto/pkg/cdcpb" +import ( + "time" -func normalizeScanPriority(priority cdcpb.ScanPriority) cdcpb.ScanPriority { - if priority == cdcpb.ScanPriority_SCAN_PRIORITY_HIGH { - return cdcpb.ScanPriority_SCAN_PRIORITY_HIGH - } - return cdcpb.ScanPriority_SCAN_PRIORITY_LOW -} + "github.com/tikv/client-go/v2/oracle" +) -func isHighScanPriority(priority cdcpb.ScanPriority) bool { - return normalizeScanPriority(priority) == cdcpb.ScanPriority_SCAN_PRIORITY_HIGH -} +const ( + lowLagRegionThreshold = 30 * time.Minute +) + +type regionTaskPriority int + +const ( + initializedRegionPriority regionTaskPriority = iota + lowLagRegionPriority + normalRegionPriority +) type regionPriorityTask struct { regionInfo regionInfo sequence uint64 heapIndex int // for heap.Item interface + priority regionTaskPriority } -// newRegionPriorityTask creates a new priority task for region. -func newRegionPriorityTask(regionInfo regionInfo, sequence uint64) *regionPriorityTask { - regionInfo.scanPriority = normalizeScanPriority(regionInfo.scanPriority) - return ®ionPriorityTask{ - regionInfo: regionInfo, - sequence: sequence, - heapIndex: 0, // 0 means not in heap +func newRegionPriorityTask(regionInfo regionInfo, currentTs, sequence uint64) *regionPriorityTask { + task := ®ionPriorityTask{ + sequence: sequence, + heapIndex: 0, // 0 means not in heap } + task.updateRegion(regionInfo, currentTs) + return task } -func (pt *regionPriorityTask) priority() cdcpb.ScanPriority { - return normalizeScanPriority(pt.regionInfo.scanPriority) +// updateRegion refreshes both the request data and its priority before the task +// enters another scheduling stage. +func (pt *regionPriorityTask) updateRegion(regionInfo regionInfo, currentTs uint64) { + priority := normalRegionPriority + if regionInfo.wasInitialized { + priority = initializedRegionPriority + } else if regionScanLag(currentTs, regionInfo.resolvedTs()) < lowLagRegionThreshold { + priority = lowLagRegionPriority + } + pt.regionInfo = regionInfo + pt.priority = priority } func (pt *regionPriorityTask) canUseMaxWindow() bool { - return isHighScanPriority(pt.regionInfo.scanPriority) + return pt.priority != normalRegionPriority } // SetHeapIndex sets the heap index for heap.Item interface @@ -63,8 +77,17 @@ func (pt *regionPriorityTask) GetHeapIndex() int { // LessThan implements heap.Item interface. Tasks in the same priority class are // processed in submission order. func (pt *regionPriorityTask) LessThan(other *regionPriorityTask) bool { - if isHighScanPriority(pt.regionInfo.scanPriority) != isHighScanPriority(other.regionInfo.scanPriority) { - return isHighScanPriority(pt.regionInfo.scanPriority) + if pt.priority != other.priority { + return pt.priority < other.priority } return pt.sequence < other.sequence } + +func regionScanLag(currentTs, checkpointTs uint64) time.Duration { + currentTime := oracle.GetTimeFromTS(currentTs) + checkpointTime := oracle.GetTimeFromTS(checkpointTs) + if !currentTime.After(checkpointTime) { + return 0 + } + return currentTime.Sub(checkpointTime) +} diff --git a/logservice/logpuller/priority_task_test.go b/logservice/logpuller/priority_task_test.go index 41501181fb..6966779f11 100644 --- a/logservice/logpuller/priority_task_test.go +++ b/logservice/logpuller/priority_task_test.go @@ -18,7 +18,6 @@ import ( "testing" "time" - "github.com/pingcap/kvproto/pkg/cdcpb" "github.com/pingcap/ticdc/heartbeatpb" "github.com/pingcap/ticdc/logservice/logpuller/regionlock" "github.com/pingcap/ticdc/utils/priorityqueue" @@ -30,6 +29,7 @@ import ( func newPriorityTestRegion( regionID uint64, checkpointTs uint64, + wasInitialized bool, ) regionInfo { span := heartbeatpb.TableSpan{TableID: 1, StartKey: []byte("a"), EndKey: []byte("z")} state := ®ionlock.LockedRangeState{} @@ -39,52 +39,31 @@ func newPriorityTestRegion( span: span, subscribedSpan: &subscribedSpan{subID: 1, startTs: checkpointTs, span: span}, lockedRangeState: state, + wasInitialized: wasInitialized, } } -func withScanPriority(region regionInfo, priority cdcpb.ScanPriority) regionInfo { - region.scanPriority = priority - return region -} - -func TestNormalizeScanPriority(t *testing.T) { - require.Equal(t, cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, normalizeScanPriority(cdcpb.ScanPriority_SCAN_PRIORITY_HIGH)) - require.Equal(t, cdcpb.ScanPriority_SCAN_PRIORITY_LOW, normalizeScanPriority(cdcpb.ScanPriority_SCAN_PRIORITY_LOW)) - require.Equal(t, cdcpb.ScanPriority_SCAN_PRIORITY_LOW, normalizeScanPriority(cdcpb.ScanPriority_SCAN_PRIORITY_UNKNOWN)) - require.True(t, isHighScanPriority(cdcpb.ScanPriority_SCAN_PRIORITY_HIGH)) - require.False(t, isHighScanPriority(cdcpb.ScanPriority_SCAN_PRIORITY_LOW)) - require.False(t, isHighScanPriority(cdcpb.ScanPriority_SCAN_PRIORITY_UNKNOWN)) -} - func TestRegionPriorityTaskQueueOrder(t *testing.T) { queue := priorityqueue.New[*regionPriorityTask]() currentTime := time.Now() + currentTs := oracle.GoTimeToTS(currentTime) - lowTask := newRegionPriorityTask( - withScanPriority( - newPriorityTestRegion(1, oracle.GoTimeToTS(currentTime.Add(-time.Hour))), - cdcpb.ScanPriority_SCAN_PRIORITY_LOW, - ), - 3, + normalTask := newRegionPriorityTask( + newPriorityTestRegion(1, oracle.GoTimeToTS(currentTime.Add(-time.Hour)), false), + currentTs, 3, ) - highTask1 := newRegionPriorityTask( - withScanPriority( - newPriorityTestRegion(2, oracle.GoTimeToTS(currentTime.Add(-10*time.Minute))), - cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, - ), - 2, + lowLagTask := newRegionPriorityTask( + newPriorityTestRegion(2, oracle.GoTimeToTS(currentTime.Add(-10*time.Minute)), false), + currentTs, 2, ) - highTask2 := newRegionPriorityTask( - withScanPriority( - newPriorityTestRegion(3, oracle.GoTimeToTS(currentTime.Add(-time.Hour))), - cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, - ), - 1, + initializedTask := newRegionPriorityTask( + newPriorityTestRegion(3, oracle.GoTimeToTS(currentTime.Add(-time.Hour)), true), + currentTs, 1, ) - require.True(t, queue.Push(lowTask)) - require.True(t, queue.Push(highTask1)) - require.True(t, queue.Push(highTask2)) + require.True(t, queue.Push(normalTask)) + require.True(t, queue.Push(lowLagTask)) + require.True(t, queue.Push(initializedTask)) for _, expectedRegionID := range []uint64{3, 2, 1} { task, err := queue.Pop(t.Context()) @@ -96,12 +75,11 @@ func TestRegionPriorityTaskQueueOrder(t *testing.T) { func TestRegionPriorityTaskFIFOWithinPriority(t *testing.T) { queue := priorityqueue.New[*regionPriorityTask]() currentTime := time.Now() + currentTs := oracle.GoTimeToTS(currentTime) checkpointTs := oracle.GoTimeToTS(currentTime.Add(-time.Hour)) - first := newRegionPriorityTask( - withScanPriority(newPriorityTestRegion(1, checkpointTs), cdcpb.ScanPriority_SCAN_PRIORITY_HIGH), 1) - second := newRegionPriorityTask( - withScanPriority(newPriorityTestRegion(2, checkpointTs), cdcpb.ScanPriority_SCAN_PRIORITY_HIGH), 2) + first := newRegionPriorityTask(newPriorityTestRegion(1, checkpointTs, false), currentTs, 1) + second := newRegionPriorityTask(newPriorityTestRegion(2, checkpointTs, false), currentTs, 2) require.True(t, queue.Push(second)) require.True(t, queue.Push(first)) @@ -114,22 +92,42 @@ func TestRegionPriorityTaskFIFOWithinPriority(t *testing.T) { require.Equal(t, uint64(2), task.regionInfo.verID.GetID()) } -func TestRegionPriorityTaskUsesHighPriorityWindow(t *testing.T) { - highTask := newRegionPriorityTask( - withScanPriority(newPriorityTestRegion(1, 1), cdcpb.ScanPriority_SCAN_PRIORITY_HIGH), 1) - lowTask := newRegionPriorityTask( - withScanPriority(newPriorityTestRegion(2, 1), cdcpb.ScanPriority_SCAN_PRIORITY_LOW), 2) +func TestRegionPriorityTaskLowLagBoundary(t *testing.T) { + currentTime := time.Now() + currentTs := oracle.GoTimeToTS(currentTime) + + belowThreshold := newRegionPriorityTask(newPriorityTestRegion( + 1, + oracle.GoTimeToTS(currentTime.Add(-lowLagRegionThreshold+time.Millisecond)), + false, + ), currentTs, 1) + atThreshold := newRegionPriorityTask(newPriorityTestRegion( + 2, + oracle.GoTimeToTS(currentTime.Add(-lowLagRegionThreshold)), + false, + ), currentTs, 2) + futureCheckpoint := newRegionPriorityTask(newPriorityTestRegion( + 3, + oracle.GoTimeToTS(currentTime.Add(time.Second)), + false, + ), currentTs, 3) - require.True(t, highTask.canUseMaxWindow()) - require.False(t, lowTask.canUseMaxWindow()) + require.Equal(t, lowLagRegionPriority, belowThreshold.priority) + require.Equal(t, normalRegionPriority, atThreshold.priority) + require.Equal(t, lowLagRegionPriority, futureCheckpoint.priority) } -func TestRegionPriorityTaskRefreshesRegionInfoBetweenStages(t *testing.T) { - region := withScanPriority(newPriorityTestRegion(1, 1), cdcpb.ScanPriority_SCAN_PRIORITY_LOW) - task := newRegionPriorityTask(region, 1) - require.Equal(t, cdcpb.ScanPriority_SCAN_PRIORITY_LOW, task.priority()) +func TestRegionPriorityTaskRefreshesPriorityBetweenStages(t *testing.T) { + checkpointTime := time.Now() + checkpointTs := oracle.GoTimeToTS(checkpointTime) + region := newPriorityTestRegion(1, checkpointTs, false) + task := newRegionPriorityTask(region, oracle.GoTimeToTS(checkpointTime.Add(time.Minute)), 1) + require.Equal(t, lowLagRegionPriority, task.priority) + + task.updateRegion(region, oracle.GoTimeToTS(checkpointTime.Add(time.Hour))) + require.Equal(t, normalRegionPriority, task.priority) - region.scanPriority = cdcpb.ScanPriority_SCAN_PRIORITY_HIGH - task.regionInfo = region - require.Equal(t, cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, task.priority()) + region.wasInitialized = true + task.updateRegion(region, oracle.GoTimeToTS(checkpointTime.Add(time.Hour))) + require.Equal(t, initializedRegionPriority, task.priority) } diff --git a/logservice/logpuller/region_admission_controller_test.go b/logservice/logpuller/region_admission_controller_test.go index 13fc2ed2bc..abc2c1a6b8 100644 --- a/logservice/logpuller/region_admission_controller_test.go +++ b/logservice/logpuller/region_admission_controller_test.go @@ -20,7 +20,6 @@ import ( "testing" "time" - "github.com/pingcap/kvproto/pkg/cdcpb" "github.com/pingcap/ticdc/heartbeatpb" "github.com/pingcap/ticdc/logservice/logpuller/regionlock" "github.com/stretchr/testify/require" @@ -39,7 +38,6 @@ func createTestRegionInfo(subID SubscriptionID, regionID uint64) regionInfo { span, nil, &subscribedSpan{subID: subID, startTs: 100, span: span}, - false, ) } @@ -56,7 +54,7 @@ func submitRegionForAdmission( currentTs uint64, ) { t.Helper() - task := newRegionPriorityTask(region, region.verID.GetID()) + task := newRegionPriorityTask(region, currentTs, region.verID.GetID()) require.True(t, controller.submit(task)) } @@ -85,10 +83,11 @@ func TestRegionAdmissionControllerNormalWindow(t *testing.T) { require.True(t, req2.abort()) } -func TestRegionAdmissionControllerHighPriorityUsesMaxWindow(t *testing.T) { +func TestRegionAdmissionControllerLowLagUsesMaxWindow(t *testing.T) { controller := newRegionAdmissionController(1, 2) currentTs := oracle.GoTimeToTS(time.Now()) slowCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Hour)) + lowLagCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Minute)) submitRegionForAdmission(t, controller, prepareRegionForAdmission(createTestRegionInfo(1, 1), slowCheckpointTs), @@ -99,9 +98,9 @@ func TestRegionAdmissionControllerHighPriorityUsesMaxWindow(t *testing.T) { submitRegionForAdmission(t, controller, prepareRegionForAdmission(createTestRegionInfo(1, 2), slowCheckpointTs), currentTs) - highPriorityRegion := prepareRegionForAdmission(createTestRegionInfo(1, 3), slowCheckpointTs) - highPriorityRegion.scanPriority = cdcpb.ScanPriority_SCAN_PRIORITY_HIGH - submitRegionForAdmission(t, controller, highPriorityRegion, currentTs) + submitRegionForAdmission(t, controller, + prepareRegionForAdmission(createTestRegionInfo(1, 3), lowLagCheckpointTs), + currentTs) req2, err := controller.pop(t.Context(), nil) require.NoError(t, err) @@ -121,10 +120,11 @@ func TestRegionAdmissionControllerHighPriorityUsesMaxWindow(t *testing.T) { require.True(t, req3.abort()) } -func TestRegionAdmissionControllerPrioritizesHighPriorityRegion(t *testing.T) { +func TestRegionAdmissionControllerPrioritizesInitializedRegion(t *testing.T) { controller := newRegionAdmissionController(1, 2) currentTs := oracle.GoTimeToTS(time.Now()) slowCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Hour)) + lowLagCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Minute)) submitRegionForAdmission(t, controller, prepareRegionForAdmission(createTestRegionInfo(1, 1), slowCheckpointTs), @@ -133,11 +133,12 @@ func TestRegionAdmissionControllerPrioritizesHighPriorityRegion(t *testing.T) { require.NoError(t, err) submitRegionForAdmission(t, controller, - prepareRegionForAdmission(createTestRegionInfo(1, 2), slowCheckpointTs), + prepareRegionForAdmission(createTestRegionInfo(1, 2), lowLagCheckpointTs), currentTs) - highPriorityRegion := prepareRegionForAdmission(createTestRegionInfo(1, 3), slowCheckpointTs) - highPriorityRegion.scanPriority = cdcpb.ScanPriority_SCAN_PRIORITY_HIGH - submitRegionForAdmission(t, controller, highPriorityRegion, currentTs) + initializedRegion := prepareRegionForAdmission(createTestRegionInfo(1, 3), slowCheckpointTs) + initializedRegion.wasInitialized = true + submitRegionForAdmission(t, controller, + initializedRegion, currentTs) req2, err := controller.pop(t.Context(), nil) require.NoError(t, err) @@ -191,7 +192,7 @@ func TestRegionAdmissionControllerClose(t *testing.T) { controller := newRegionAdmissionController(1, 1) controller.close() region := prepareRegionForAdmission(createTestRegionInfo(1, 1), 1) - require.False(t, controller.submit(newRegionPriorityTask(region, 1))) + require.False(t, controller.submit(newRegionPriorityTask(region, 1, 1))) _, err := controller.pop(context.Background(), nil) require.ErrorIs(t, err, context.Canceled) diff --git a/logservice/logpuller/region_event_handler_test.go b/logservice/logpuller/region_event_handler_test.go index fa6e76eaad..f2e1885197 100644 --- a/logservice/logpuller/region_event_handler_test.go +++ b/logservice/logpuller/region_event_handler_test.go @@ -87,7 +87,6 @@ func TestHandleEventEntryEventOutOfOrder(t *testing.T) { span, &tikv.RPCContext{}, subSpan, - false, ) lockResult := subSpan.rangeLock.LockRange( context.Background(), span.StartKey, span.EndKey, 1, 1) diff --git a/logservice/logpuller/region_failure_handler.go b/logservice/logpuller/region_failure_handler.go index f90483b740..7464afb940 100644 --- a/logservice/logpuller/region_failure_handler.go +++ b/logservice/logpuller/region_failure_handler.go @@ -19,7 +19,6 @@ import ( "time" "github.com/pingcap/log" - "github.com/pingcap/ticdc/heartbeatpb" "github.com/pingcap/ticdc/pkg/errors" "github.com/pingcap/ticdc/pkg/metrics" "github.com/tikv/client-go/v2/tikv" @@ -47,14 +46,14 @@ type regionFailureHandler struct { onTableDrained func(*subscribedSpan) scheduleRegionRequest func(context.Context, regionInfo) - scheduleRangeRequest func(context.Context, heartbeatpb.TableSpan, *subscribedSpan, bool, bool) + scheduleRangeRequest func(context.Context, rangeTask) } func newRegionFailureHandler( regionCache *tikv.RegionCache, onTableDrained func(*subscribedSpan), scheduleRegionRequest func(context.Context, regionInfo), - scheduleRangeRequest func(context.Context, heartbeatpb.TableSpan, *subscribedSpan, bool, bool), + scheduleRangeRequest func(context.Context, rangeTask), ) *regionFailureHandler { return ®ionFailureHandler{ cache: newErrCache(), @@ -65,6 +64,14 @@ func newRegionFailureHandler( } } +func (r *regionFailureHandler) retryRange(ctx context.Context, errInfo regionErrorInfo) { + r.scheduleRangeRequest(ctx, rangeTask{ + span: errInfo.span, + subscribedSpan: errInfo.subscribedSpan, + wasInitialized: errInfo.wasInitialized, + }) +} + // Report admits a region failure into the recovery pipeline. It releases the // corresponding range lock before enqueueing the failure so new range tasks are // not blocked by stale region ownership. @@ -118,12 +125,12 @@ func (r *regionFailureHandler) handleError(ctx context.Context, errInfo regionEr } if innerErr.GetEpochNotMatch() != nil { metricFeedEpochNotMatchCounter.Inc() - r.scheduleRangeRequest(ctx, errInfo.span, errInfo.subscribedSpan, errInfo.filterLoop, errInfo.wasInitialized) + r.retryRange(ctx, errInfo) return nil } if innerErr.GetRegionNotFound() != nil { metricFeedRegionNotFoundCounter.Inc() - r.scheduleRangeRequest(ctx, errInfo.span, errInfo.subscribedSpan, errInfo.filterLoop, errInfo.wasInitialized) + r.retryRange(ctx, errInfo) return nil } if innerErr.GetCongested() != nil { @@ -156,14 +163,14 @@ func (r *regionFailureHandler) handleError(ctx context.Context, errInfo regionEr return nil case *rpcCtxUnavailableErr: metricFeedRPCCtxUnavailable.Inc() - r.scheduleRangeRequest(ctx, errInfo.span, errInfo.subscribedSpan, errInfo.filterLoop, errInfo.wasInitialized) + r.retryRange(ctx, errInfo) return nil case *getStoreErr: metricGetStoreErr.Inc() bo := tikv.NewBackoffer(ctx, tikvRequestMaxBackoff) // cannot get the store the region belongs to, so we need to reload the region. r.regionCache.OnSendFail(bo, errInfo.rpcCtx, true, err) - r.scheduleRangeRequest(ctx, errInfo.span, errInfo.subscribedSpan, errInfo.filterLoop, errInfo.wasInitialized) + r.retryRange(ctx, errInfo) return nil case *storeStreamErr: metricStoreSendRequestErr.Inc() diff --git a/logservice/logpuller/region_request_scheduler.go b/logservice/logpuller/region_request_scheduler.go index b7e3eba394..f75fbf7868 100644 --- a/logservice/logpuller/region_request_scheduler.go +++ b/logservice/logpuller/region_request_scheduler.go @@ -77,7 +77,7 @@ func newRegionRequestScheduler( } func (s *regionRequestScheduler) Submit(region regionInfo) { - s.taskQueue.Push(NewRegionPriorityTask( + s.taskQueue.Push(newRegionPriorityTask( region, s.upstream.pdClock.CurrentTS(), s.sequence.Add(1))) } @@ -98,7 +98,7 @@ func (s *regionRequestScheduler) Run(ctx context.Context, workerGroup *errgroup. return err } - region, err := s.attachRPCContext(ctx, task.GetRegionInfo()) + region, err := s.attachRPCContext(ctx, task.regionInfo) if err != nil { s.failureHandler.Report(newRegionErrorInfo(region, err)) continue diff --git a/logservice/logpuller/region_request_store.go b/logservice/logpuller/region_request_store.go index 1a8d243e0d..9b9a5af025 100644 --- a/logservice/logpuller/region_request_store.go +++ b/logservice/logpuller/region_request_store.go @@ -25,7 +25,6 @@ import ( // store. The worker slice is complete before the store is published and is // immutable afterwards, so task submission only needs an atomic round-robin counter. type regionRequestStore struct { - storeAddr string workers []*regionRequestWorker nextWorker atomic.Uint64 } @@ -40,8 +39,7 @@ func newRegionRequestStore( maxWindowMultiplier int, ) *regionRequestStore { store := ®ionRequestStore{ - storeAddr: storeAddr, - workers: make([]*regionRequestWorker, 0, workerCount), + workers: make([]*regionRequestWorker, 0, workerCount), } for i := 0; i < workerCount; i++ { store.workers = append(store.workers, newRegionRequestWorker( diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index b22541bb49..3ef4422d5a 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -159,10 +159,8 @@ func (s *regionRequestWorker) Run(ctx context.Context) error { // failStreamRegions transfers every request sent by a failed stream to the // recovery pipeline. func (s *regionRequestWorker) failStreamRegions(err error) { - for _, states := range s.tracker.Drain() { - for _, state := range states { - s.notifyRegionError(state, err) - } + for _, state := range s.tracker.Drain() { + s.notifyRegionError(state, err) } // The failed stream no longer owns remote registrations. s.controlQueue.drain() diff --git a/logservice/logpuller/region_request_worker_test.go b/logservice/logpuller/region_request_worker_test.go index 9b75667040..c6e3f8d845 100644 --- a/logservice/logpuller/region_request_worker_test.go +++ b/logservice/logpuller/region_request_worker_test.go @@ -393,7 +393,7 @@ func TestFailPendingRegionsReschedulesWorkerBuffer(t *testing.T) { }, } for i, region := range regions { - require.True(t, admission.submit(NewRegionPriorityTask(region, 1, uint64(i+1)))) + require.True(t, admission.submit(newRegionPriorityTask(region, 1, uint64(i+1)))) } worker.failPendingRegions(&storeStreamErr{}) diff --git a/logservice/logpuller/region_state.go b/logservice/logpuller/region_state.go index 0f2f84ce3e..06e3bdd5d4 100644 --- a/logservice/logpuller/region_state.go +++ b/logservice/logpuller/region_state.go @@ -17,7 +17,6 @@ import ( "sync" "sync/atomic" - "github.com/pingcap/kvproto/pkg/cdcpb" "github.com/pingcap/ticdc/heartbeatpb" "github.com/pingcap/ticdc/logservice/logpuller/regionlock" "github.com/tikv/client-go/v2/tikv" @@ -45,12 +44,12 @@ type regionInfo struct { subscribedSpan *subscribedSpan // The state of the locked range of the region. lockedRangeState *regionlock.LockedRangeState + // wasInitialized preserves scheduling priority while a failed region is + // unlocked, resolved again, and subscribed with a new locked range state. + wasInitialized bool // Whether to filter out the value write by cdc itself. // It should be `true` in BDR mode filterLoop bool - // scanPriority is sent to TiKV/CSE so remote incremental scan admission can - // preserve TiCDC's business priority across retries. - scanPriority cdcpb.ScanPriority } func newRegionInfo( @@ -58,15 +57,13 @@ func newRegionInfo( span heartbeatpb.TableSpan, rpcCtx *tikv.RPCContext, subscribedSpan *subscribedSpan, - filterLoop bool, ) regionInfo { return regionInfo{ verID: verID, span: span, rpcCtx: rpcCtx, subscribedSpan: subscribedSpan, - filterLoop: filterLoop, - scanPriority: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, + filterLoop: subscribedSpan.filterLoop, } } @@ -80,6 +77,9 @@ type regionErrorInfo struct { } func newRegionErrorInfo(info regionInfo, err error) regionErrorInfo { + if info.lockedRangeState != nil && info.lockedRangeState.Initialized.Load() { + info.wasInitialized = true + } return regionErrorInfo{ regionInfo: info, err: err, diff --git a/logservice/logpuller/region_tracker.go b/logservice/logpuller/region_tracker.go index 81a27dc86c..7066779eaf 100644 --- a/logservice/logpuller/region_tracker.go +++ b/logservice/logpuller/region_tracker.go @@ -112,13 +112,15 @@ func (t *regionTracker) Drain() []*regionFeedState { t.statesBySubscription = make(map[SubscriptionID]regionStatesByID) t.mu.Unlock() - totalStates := 0 + stateCount := 0 for _, states := range statesBySubscription { - totalStates += len(states) + stateCount += len(states) } - drainedStates := make([]*regionFeedState, 0, totalStates) + drainedStates := make([]*regionFeedState, 0, stateCount) for _, states := range statesBySubscription { - drainedStates = append(drainedStates, slices.Collect(maps.Values(states))...) + for _, state := range states { + drainedStates = append(drainedStates, state) + } } return drainedStates } diff --git a/logservice/logpuller/subscription_client.go b/logservice/logpuller/subscription_client.go index 924025a386..ea4cd33c31 100644 --- a/logservice/logpuller/subscription_client.go +++ b/logservice/logpuller/subscription_client.go @@ -80,7 +80,6 @@ type resolveLockTask struct { type rangeTask struct { span heartbeatpb.TableSpan subscribedSpan *subscribedSpan - filterLoop bool wasInitialized bool } @@ -242,7 +241,7 @@ func (s *subscriptionClient) Subscribe( select { case <-s.ctx.Done(): log.Warn("subscribes span failed, the subscription client has closed") - case s.rangeTaskCh <- rangeTask{span: span, subscribedSpan: rt, filterLoop: rt.filterLoop}: + case s.rangeTaskCh <- rangeTask{span: span, subscribedSpan: rt}: log.Info("subscribes span done", zap.Uint64("subscriptionID", uint64(subID)), zap.Int64("tableID", span.TableID), zap.Uint64("startTs", startTs), zap.String("startKey", spanz.HexKey(span.StartKey)), zap.String("endKey", spanz.HexKey(span.EndKey))) @@ -270,13 +269,15 @@ func (s *subscriptionClient) Run(ctx context.Context) error { g, ctx := errgroup.WithContext(ctx) - g.Go(func() error { return s.updateMetrics(ctx) }) - g.Go(func() error { return s.eventSink.Run(ctx) }) + // The goroutines are listed by data flow; errgroup does not guarantee their + // actual startup order. g.Go(func() error { return s.handleRangeTasks(ctx) }) g.Go(func() error { return s.regionScheduler.Run(ctx, g) }) + g.Go(func() error { return s.eventSink.Run(ctx) }) g.Go(func() error { return s.failureHandler.Run(ctx) }) - g.Go(func() error { return s.handleResolveLockTasks(ctx) }) g.Go(func() error { return s.spanRegistry.Run(ctx) }) + g.Go(func() error { return s.handleResolveLockTasks(ctx) }) + g.Go(func() error { return s.updateMetrics(ctx) }) log.Info("subscription client starts") defer log.Info("subscription client exits") @@ -328,7 +329,7 @@ func (s *subscriptionClient) handleRangeTasks(ctx context.Context) error { return ctx.Err() case task := <-s.rangeTaskCh: g.Go(func() error { - return s.divideSpanAndScheduleRegionRequests(ctx, task.span, task.subscribedSpan, task.filterLoop, task.wasInitialized) + return s.divideSpanAndScheduleRegionRequests(ctx, task) }) } } @@ -341,11 +342,11 @@ func (s *subscriptionClient) handleRangeTasks(ctx context.Context) error { // 3. Schedule a region request to subscribe the region. func (s *subscriptionClient) divideSpanAndScheduleRegionRequests( ctx context.Context, - span heartbeatpb.TableSpan, - subscribedSpan *subscribedSpan, - filterLoop bool, - wasInitialized bool, + task rangeTask, ) error { + span := task.span + subscribedSpan := task.subscribedSpan + // Limit the number of regions loaded at a time to make the load more stable. limit := 1024 nextSpan := span @@ -406,8 +407,8 @@ func (s *subscriptionClient) divideSpanAndScheduleRegionRequests( } verID := tikv.NewRegionVerID(regionMeta.Id, regionMeta.RegionEpoch.ConfVer, regionMeta.RegionEpoch.Version) - regionInfo := newRegionInfo(verID, intersectSpan, nil, subscribedSpan, filterLoop) - regionInfo.wasInitialized = wasInitialized + regionInfo := newRegionInfo(verID, intersectSpan, nil, subscribedSpan) + regionInfo.wasInitialized = task.wasInitialized // Schedule a region request to subscribe the region. s.scheduleRegionRequest(ctx, regionInfo) @@ -441,7 +442,11 @@ func (s *subscriptionClient) scheduleRegionRequest(ctx context.Context, region r s.regionScheduler.Submit(region) case regionlock.LockRangeStatusStale: for _, r := range lockRangeResult.RetryRanges { - s.scheduleRangeRequest(ctx, r, region.subscribedSpan, region.filterLoop, region.wasInitialized) + s.scheduleRangeRequest(ctx, rangeTask{ + span: r, + subscribedSpan: region.subscribedSpan, + wasInitialized: region.wasInitialized, + }) } default: return @@ -449,17 +454,12 @@ func (s *subscriptionClient) scheduleRegionRequest(ctx context.Context, region r } func (s *subscriptionClient) scheduleRangeRequest( - ctx context.Context, span heartbeatpb.TableSpan, - subscribedSpan *subscribedSpan, - filterLoop bool, - wasInitialized bool, + ctx context.Context, + task rangeTask, ) { select { case <-ctx.Done(): - case s.rangeTaskCh <- rangeTask{ - span: span, subscribedSpan: subscribedSpan, - filterLoop: filterLoop, wasInitialized: wasInitialized, - }: + case s.rangeTaskCh <- task: } } diff --git a/logservice/logpuller/subscription_client_test.go b/logservice/logpuller/subscription_client_test.go index ef1a387045..474d3c6af5 100644 --- a/logservice/logpuller/subscription_client_test.go +++ b/logservice/logpuller/subscription_client_test.go @@ -327,10 +327,11 @@ func TestStopTaskUsesSubscribedSpanFilterLoop(t *testing.T) { res := span.rangeLock.LockRange(context.Background(), rawSpan.StartKey, rawSpan.EndKey, 1, 1) require.Equal(t, regionlock.LockRangeStatusSuccess, res.Status) - worker := ®ionRequestWorker{controlQueue: newControlQueue()} - store := ®ionRequestStore{storeAddr: "store-1", workers: []*regionRequestWorker{worker}} + const storeAddr = "store-1" + worker := ®ionRequestWorker{storeAddr: storeAddr, controlQueue: newControlQueue()} + store := ®ionRequestStore{workers: []*regionRequestWorker{worker}} client.regionScheduler = ®ionRequestScheduler{} - client.regionScheduler.stores.Store(store.storeAddr, store) + client.regionScheduler.stores.Store(storeAddr, store) client.setTableStopped(span) @@ -454,18 +455,20 @@ func TestBroadcastDeregisterUsesWorkerControlQueue(t *testing.T) { scheduler := ®ionRequestScheduler{} admission := newRegionAdmissionController(1, 1) + const storeAddr = "store-1" worker := ®ionRequestWorker{ + storeAddr: storeAddr, admission: admission, controlQueue: newControlQueue(), } - store := ®ionRequestStore{storeAddr: "store-1", workers: []*regionRequestWorker{worker}} - scheduler.stores.Store(store.storeAddr, store) + store := ®ionRequestStore{workers: []*regionRequestWorker{worker}} + scheduler.stores.Store(storeAddr, store) dummyRegion := regionInfo{ subscribedSpan: &subscribedSpan{subID: SubscriptionID(2)}, lockedRangeState: ®ionlock.LockedRangeState{}, } - require.True(t, admission.submit(NewRegionPriorityTask(dummyRegion, 1, 1))) + require.True(t, admission.submit(newRegionPriorityTask(dummyRegion, 1, 1))) scheduler.BroadcastDeregister(SubscriptionID(1), true) require.Equal(t, 1, worker.controlQueue.len()) @@ -480,8 +483,7 @@ func TestRegionRequestStoreDistributesRegionsAcrossWorkers(t *testing.T) { worker1 := ®ionRequestWorker{admission: newRegionAdmissionController(1, 1)} worker2 := ®ionRequestWorker{admission: newRegionAdmissionController(1, 1)} store := ®ionRequestStore{ - storeAddr: "store-1", - workers: []*regionRequestWorker{worker1, worker2}, + workers: []*regionRequestWorker{worker1, worker2}, } for i := uint64(1); i <= 4; i++ { @@ -490,7 +492,7 @@ func TestRegionRequestStoreDistributesRegionsAcrossWorkers(t *testing.T) { subscribedSpan: &subscribedSpan{subID: 1}, lockedRangeState: ®ionlock.LockedRangeState{}, } - require.True(t, store.submit(NewRegionPriorityTask(region, 1, i))) + require.True(t, store.submit(newRegionPriorityTask(region, 1, i))) } require.Equal(t, 2, worker1.admission.stats().pending) From ed0c1ae2e991259cead4120739f39ed70b549bd9 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Mon, 3 Aug 2026 22:59:46 +0800 Subject: [PATCH 34/71] fix conflict --- logservice/logpuller/priority_task.go | 67 ++++------- logservice/logpuller/priority_task_test.go | 106 +++++++++--------- .../region_admission_controller_test.go | 27 +++-- .../logpuller/region_event_handler_test.go | 1 + .../logpuller/region_failure_handler.go | 12 +- .../logpuller/region_failure_handler_test.go | 4 +- .../logpuller/region_request_scheduler.go | 5 +- .../logpuller/region_request_worker_test.go | 2 +- logservice/logpuller/region_state.go | 14 +-- logservice/logpuller/scan_priority_test.go | 14 ++- logservice/logpuller/subscription_client.go | 31 +++-- .../logpuller/subscription_client_test.go | 20 ++-- 12 files changed, 156 insertions(+), 147 deletions(-) diff --git a/logservice/logpuller/priority_task.go b/logservice/logpuller/priority_task.go index eb241dec0d..38cd813c8b 100644 --- a/logservice/logpuller/priority_task.go +++ b/logservice/logpuller/priority_task.go @@ -13,55 +13,41 @@ package logpuller -import ( - "time" +import "github.com/pingcap/kvproto/pkg/cdcpb" - "github.com/tikv/client-go/v2/oracle" -) - -const ( - lowLagRegionThreshold = 30 * time.Minute -) - -type regionTaskPriority int +func normalizeScanPriority(priority cdcpb.ScanPriority) cdcpb.ScanPriority { + if priority == cdcpb.ScanPriority_SCAN_PRIORITY_HIGH { + return cdcpb.ScanPriority_SCAN_PRIORITY_HIGH + } + return cdcpb.ScanPriority_SCAN_PRIORITY_LOW +} -const ( - initializedRegionPriority regionTaskPriority = iota - lowLagRegionPriority - normalRegionPriority -) +func isHighScanPriority(priority cdcpb.ScanPriority) bool { + return normalizeScanPriority(priority) == cdcpb.ScanPriority_SCAN_PRIORITY_HIGH +} type regionPriorityTask struct { regionInfo regionInfo sequence uint64 heapIndex int // for heap.Item interface - priority regionTaskPriority } -func newRegionPriorityTask(regionInfo regionInfo, currentTs, sequence uint64) *regionPriorityTask { - task := ®ionPriorityTask{ - sequence: sequence, - heapIndex: 0, // 0 means not in heap +// newRegionPriorityTask creates a new priority task for region. +func newRegionPriorityTask(regionInfo regionInfo, sequence uint64) *regionPriorityTask { + regionInfo.scanPriority = normalizeScanPriority(regionInfo.scanPriority) + return ®ionPriorityTask{ + regionInfo: regionInfo, + sequence: sequence, + heapIndex: 0, // 0 means not in heap } - task.updateRegion(regionInfo, currentTs) - return task } -// updateRegion refreshes both the request data and its priority before the task -// enters another scheduling stage. -func (pt *regionPriorityTask) updateRegion(regionInfo regionInfo, currentTs uint64) { - priority := normalRegionPriority - if regionInfo.wasInitialized { - priority = initializedRegionPriority - } else if regionScanLag(currentTs, regionInfo.resolvedTs()) < lowLagRegionThreshold { - priority = lowLagRegionPriority - } - pt.regionInfo = regionInfo - pt.priority = priority +func (pt *regionPriorityTask) priority() cdcpb.ScanPriority { + return normalizeScanPriority(pt.regionInfo.scanPriority) } func (pt *regionPriorityTask) canUseMaxWindow() bool { - return pt.priority != normalRegionPriority + return isHighScanPriority(pt.regionInfo.scanPriority) } // SetHeapIndex sets the heap index for heap.Item interface @@ -77,17 +63,8 @@ func (pt *regionPriorityTask) GetHeapIndex() int { // LessThan implements heap.Item interface. Tasks in the same priority class are // processed in submission order. func (pt *regionPriorityTask) LessThan(other *regionPriorityTask) bool { - if pt.priority != other.priority { - return pt.priority < other.priority + if isHighScanPriority(pt.regionInfo.scanPriority) != isHighScanPriority(other.regionInfo.scanPriority) { + return isHighScanPriority(pt.regionInfo.scanPriority) } return pt.sequence < other.sequence } - -func regionScanLag(currentTs, checkpointTs uint64) time.Duration { - currentTime := oracle.GetTimeFromTS(currentTs) - checkpointTime := oracle.GetTimeFromTS(checkpointTs) - if !currentTime.After(checkpointTime) { - return 0 - } - return currentTime.Sub(checkpointTime) -} diff --git a/logservice/logpuller/priority_task_test.go b/logservice/logpuller/priority_task_test.go index 6966779f11..41501181fb 100644 --- a/logservice/logpuller/priority_task_test.go +++ b/logservice/logpuller/priority_task_test.go @@ -18,6 +18,7 @@ import ( "testing" "time" + "github.com/pingcap/kvproto/pkg/cdcpb" "github.com/pingcap/ticdc/heartbeatpb" "github.com/pingcap/ticdc/logservice/logpuller/regionlock" "github.com/pingcap/ticdc/utils/priorityqueue" @@ -29,7 +30,6 @@ import ( func newPriorityTestRegion( regionID uint64, checkpointTs uint64, - wasInitialized bool, ) regionInfo { span := heartbeatpb.TableSpan{TableID: 1, StartKey: []byte("a"), EndKey: []byte("z")} state := ®ionlock.LockedRangeState{} @@ -39,31 +39,52 @@ func newPriorityTestRegion( span: span, subscribedSpan: &subscribedSpan{subID: 1, startTs: checkpointTs, span: span}, lockedRangeState: state, - wasInitialized: wasInitialized, } } +func withScanPriority(region regionInfo, priority cdcpb.ScanPriority) regionInfo { + region.scanPriority = priority + return region +} + +func TestNormalizeScanPriority(t *testing.T) { + require.Equal(t, cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, normalizeScanPriority(cdcpb.ScanPriority_SCAN_PRIORITY_HIGH)) + require.Equal(t, cdcpb.ScanPriority_SCAN_PRIORITY_LOW, normalizeScanPriority(cdcpb.ScanPriority_SCAN_PRIORITY_LOW)) + require.Equal(t, cdcpb.ScanPriority_SCAN_PRIORITY_LOW, normalizeScanPriority(cdcpb.ScanPriority_SCAN_PRIORITY_UNKNOWN)) + require.True(t, isHighScanPriority(cdcpb.ScanPriority_SCAN_PRIORITY_HIGH)) + require.False(t, isHighScanPriority(cdcpb.ScanPriority_SCAN_PRIORITY_LOW)) + require.False(t, isHighScanPriority(cdcpb.ScanPriority_SCAN_PRIORITY_UNKNOWN)) +} + func TestRegionPriorityTaskQueueOrder(t *testing.T) { queue := priorityqueue.New[*regionPriorityTask]() currentTime := time.Now() - currentTs := oracle.GoTimeToTS(currentTime) - normalTask := newRegionPriorityTask( - newPriorityTestRegion(1, oracle.GoTimeToTS(currentTime.Add(-time.Hour)), false), - currentTs, 3, + lowTask := newRegionPriorityTask( + withScanPriority( + newPriorityTestRegion(1, oracle.GoTimeToTS(currentTime.Add(-time.Hour))), + cdcpb.ScanPriority_SCAN_PRIORITY_LOW, + ), + 3, ) - lowLagTask := newRegionPriorityTask( - newPriorityTestRegion(2, oracle.GoTimeToTS(currentTime.Add(-10*time.Minute)), false), - currentTs, 2, + highTask1 := newRegionPriorityTask( + withScanPriority( + newPriorityTestRegion(2, oracle.GoTimeToTS(currentTime.Add(-10*time.Minute))), + cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, + ), + 2, ) - initializedTask := newRegionPriorityTask( - newPriorityTestRegion(3, oracle.GoTimeToTS(currentTime.Add(-time.Hour)), true), - currentTs, 1, + highTask2 := newRegionPriorityTask( + withScanPriority( + newPriorityTestRegion(3, oracle.GoTimeToTS(currentTime.Add(-time.Hour))), + cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, + ), + 1, ) - require.True(t, queue.Push(normalTask)) - require.True(t, queue.Push(lowLagTask)) - require.True(t, queue.Push(initializedTask)) + require.True(t, queue.Push(lowTask)) + require.True(t, queue.Push(highTask1)) + require.True(t, queue.Push(highTask2)) for _, expectedRegionID := range []uint64{3, 2, 1} { task, err := queue.Pop(t.Context()) @@ -75,11 +96,12 @@ func TestRegionPriorityTaskQueueOrder(t *testing.T) { func TestRegionPriorityTaskFIFOWithinPriority(t *testing.T) { queue := priorityqueue.New[*regionPriorityTask]() currentTime := time.Now() - currentTs := oracle.GoTimeToTS(currentTime) checkpointTs := oracle.GoTimeToTS(currentTime.Add(-time.Hour)) - first := newRegionPriorityTask(newPriorityTestRegion(1, checkpointTs, false), currentTs, 1) - second := newRegionPriorityTask(newPriorityTestRegion(2, checkpointTs, false), currentTs, 2) + first := newRegionPriorityTask( + withScanPriority(newPriorityTestRegion(1, checkpointTs), cdcpb.ScanPriority_SCAN_PRIORITY_HIGH), 1) + second := newRegionPriorityTask( + withScanPriority(newPriorityTestRegion(2, checkpointTs), cdcpb.ScanPriority_SCAN_PRIORITY_HIGH), 2) require.True(t, queue.Push(second)) require.True(t, queue.Push(first)) @@ -92,42 +114,22 @@ func TestRegionPriorityTaskFIFOWithinPriority(t *testing.T) { require.Equal(t, uint64(2), task.regionInfo.verID.GetID()) } -func TestRegionPriorityTaskLowLagBoundary(t *testing.T) { - currentTime := time.Now() - currentTs := oracle.GoTimeToTS(currentTime) - - belowThreshold := newRegionPriorityTask(newPriorityTestRegion( - 1, - oracle.GoTimeToTS(currentTime.Add(-lowLagRegionThreshold+time.Millisecond)), - false, - ), currentTs, 1) - atThreshold := newRegionPriorityTask(newPriorityTestRegion( - 2, - oracle.GoTimeToTS(currentTime.Add(-lowLagRegionThreshold)), - false, - ), currentTs, 2) - futureCheckpoint := newRegionPriorityTask(newPriorityTestRegion( - 3, - oracle.GoTimeToTS(currentTime.Add(time.Second)), - false, - ), currentTs, 3) +func TestRegionPriorityTaskUsesHighPriorityWindow(t *testing.T) { + highTask := newRegionPriorityTask( + withScanPriority(newPriorityTestRegion(1, 1), cdcpb.ScanPriority_SCAN_PRIORITY_HIGH), 1) + lowTask := newRegionPriorityTask( + withScanPriority(newPriorityTestRegion(2, 1), cdcpb.ScanPriority_SCAN_PRIORITY_LOW), 2) - require.Equal(t, lowLagRegionPriority, belowThreshold.priority) - require.Equal(t, normalRegionPriority, atThreshold.priority) - require.Equal(t, lowLagRegionPriority, futureCheckpoint.priority) + require.True(t, highTask.canUseMaxWindow()) + require.False(t, lowTask.canUseMaxWindow()) } -func TestRegionPriorityTaskRefreshesPriorityBetweenStages(t *testing.T) { - checkpointTime := time.Now() - checkpointTs := oracle.GoTimeToTS(checkpointTime) - region := newPriorityTestRegion(1, checkpointTs, false) - task := newRegionPriorityTask(region, oracle.GoTimeToTS(checkpointTime.Add(time.Minute)), 1) - require.Equal(t, lowLagRegionPriority, task.priority) - - task.updateRegion(region, oracle.GoTimeToTS(checkpointTime.Add(time.Hour))) - require.Equal(t, normalRegionPriority, task.priority) +func TestRegionPriorityTaskRefreshesRegionInfoBetweenStages(t *testing.T) { + region := withScanPriority(newPriorityTestRegion(1, 1), cdcpb.ScanPriority_SCAN_PRIORITY_LOW) + task := newRegionPriorityTask(region, 1) + require.Equal(t, cdcpb.ScanPriority_SCAN_PRIORITY_LOW, task.priority()) - region.wasInitialized = true - task.updateRegion(region, oracle.GoTimeToTS(checkpointTime.Add(time.Hour))) - require.Equal(t, initializedRegionPriority, task.priority) + region.scanPriority = cdcpb.ScanPriority_SCAN_PRIORITY_HIGH + task.regionInfo = region + require.Equal(t, cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, task.priority()) } diff --git a/logservice/logpuller/region_admission_controller_test.go b/logservice/logpuller/region_admission_controller_test.go index abc2c1a6b8..13fc2ed2bc 100644 --- a/logservice/logpuller/region_admission_controller_test.go +++ b/logservice/logpuller/region_admission_controller_test.go @@ -20,6 +20,7 @@ import ( "testing" "time" + "github.com/pingcap/kvproto/pkg/cdcpb" "github.com/pingcap/ticdc/heartbeatpb" "github.com/pingcap/ticdc/logservice/logpuller/regionlock" "github.com/stretchr/testify/require" @@ -38,6 +39,7 @@ func createTestRegionInfo(subID SubscriptionID, regionID uint64) regionInfo { span, nil, &subscribedSpan{subID: subID, startTs: 100, span: span}, + false, ) } @@ -54,7 +56,7 @@ func submitRegionForAdmission( currentTs uint64, ) { t.Helper() - task := newRegionPriorityTask(region, currentTs, region.verID.GetID()) + task := newRegionPriorityTask(region, region.verID.GetID()) require.True(t, controller.submit(task)) } @@ -83,11 +85,10 @@ func TestRegionAdmissionControllerNormalWindow(t *testing.T) { require.True(t, req2.abort()) } -func TestRegionAdmissionControllerLowLagUsesMaxWindow(t *testing.T) { +func TestRegionAdmissionControllerHighPriorityUsesMaxWindow(t *testing.T) { controller := newRegionAdmissionController(1, 2) currentTs := oracle.GoTimeToTS(time.Now()) slowCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Hour)) - lowLagCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Minute)) submitRegionForAdmission(t, controller, prepareRegionForAdmission(createTestRegionInfo(1, 1), slowCheckpointTs), @@ -98,9 +99,9 @@ func TestRegionAdmissionControllerLowLagUsesMaxWindow(t *testing.T) { submitRegionForAdmission(t, controller, prepareRegionForAdmission(createTestRegionInfo(1, 2), slowCheckpointTs), currentTs) - submitRegionForAdmission(t, controller, - prepareRegionForAdmission(createTestRegionInfo(1, 3), lowLagCheckpointTs), - currentTs) + highPriorityRegion := prepareRegionForAdmission(createTestRegionInfo(1, 3), slowCheckpointTs) + highPriorityRegion.scanPriority = cdcpb.ScanPriority_SCAN_PRIORITY_HIGH + submitRegionForAdmission(t, controller, highPriorityRegion, currentTs) req2, err := controller.pop(t.Context(), nil) require.NoError(t, err) @@ -120,11 +121,10 @@ func TestRegionAdmissionControllerLowLagUsesMaxWindow(t *testing.T) { require.True(t, req3.abort()) } -func TestRegionAdmissionControllerPrioritizesInitializedRegion(t *testing.T) { +func TestRegionAdmissionControllerPrioritizesHighPriorityRegion(t *testing.T) { controller := newRegionAdmissionController(1, 2) currentTs := oracle.GoTimeToTS(time.Now()) slowCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Hour)) - lowLagCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Minute)) submitRegionForAdmission(t, controller, prepareRegionForAdmission(createTestRegionInfo(1, 1), slowCheckpointTs), @@ -133,12 +133,11 @@ func TestRegionAdmissionControllerPrioritizesInitializedRegion(t *testing.T) { require.NoError(t, err) submitRegionForAdmission(t, controller, - prepareRegionForAdmission(createTestRegionInfo(1, 2), lowLagCheckpointTs), + prepareRegionForAdmission(createTestRegionInfo(1, 2), slowCheckpointTs), currentTs) - initializedRegion := prepareRegionForAdmission(createTestRegionInfo(1, 3), slowCheckpointTs) - initializedRegion.wasInitialized = true - submitRegionForAdmission(t, controller, - initializedRegion, currentTs) + highPriorityRegion := prepareRegionForAdmission(createTestRegionInfo(1, 3), slowCheckpointTs) + highPriorityRegion.scanPriority = cdcpb.ScanPriority_SCAN_PRIORITY_HIGH + submitRegionForAdmission(t, controller, highPriorityRegion, currentTs) req2, err := controller.pop(t.Context(), nil) require.NoError(t, err) @@ -192,7 +191,7 @@ func TestRegionAdmissionControllerClose(t *testing.T) { controller := newRegionAdmissionController(1, 1) controller.close() region := prepareRegionForAdmission(createTestRegionInfo(1, 1), 1) - require.False(t, controller.submit(newRegionPriorityTask(region, 1, 1))) + require.False(t, controller.submit(newRegionPriorityTask(region, 1))) _, err := controller.pop(context.Background(), nil) require.ErrorIs(t, err, context.Canceled) diff --git a/logservice/logpuller/region_event_handler_test.go b/logservice/logpuller/region_event_handler_test.go index f2e1885197..fa6e76eaad 100644 --- a/logservice/logpuller/region_event_handler_test.go +++ b/logservice/logpuller/region_event_handler_test.go @@ -87,6 +87,7 @@ func TestHandleEventEntryEventOutOfOrder(t *testing.T) { span, &tikv.RPCContext{}, subSpan, + false, ) lockResult := subSpan.rangeLock.LockRange( context.Background(), span.StartKey, span.EndKey, 1, 1) diff --git a/logservice/logpuller/region_failure_handler.go b/logservice/logpuller/region_failure_handler.go index 7464afb940..abf172793d 100644 --- a/logservice/logpuller/region_failure_handler.go +++ b/logservice/logpuller/region_failure_handler.go @@ -18,6 +18,7 @@ import ( "sync" "time" + "github.com/pingcap/kvproto/pkg/cdcpb" "github.com/pingcap/log" "github.com/pingcap/ticdc/pkg/errors" "github.com/pingcap/ticdc/pkg/metrics" @@ -65,10 +66,19 @@ func newRegionFailureHandler( } func (r *regionFailureHandler) retryRange(ctx context.Context, errInfo regionErrorInfo) { + priority := normalizeScanPriority(errInfo.scanPriority) + if priority == cdcpb.ScanPriority_SCAN_PRIORITY_LOW { + priority = errInfo.subscribedSpan.priorityPolicy.resolve( + priority, + errInfo.resolvedTs(), + errInfo.subscribedSpan.priorityPolicy.pdClock.CurrentTime(), + ) + } r.scheduleRangeRequest(ctx, rangeTask{ span: errInfo.span, subscribedSpan: errInfo.subscribedSpan, - wasInitialized: errInfo.wasInitialized, + filterLoop: errInfo.filterLoop, + priority: priority, }) } diff --git a/logservice/logpuller/region_failure_handler_test.go b/logservice/logpuller/region_failure_handler_test.go index 7f419920a7..a7f8b2f883 100644 --- a/logservice/logpuller/region_failure_handler_test.go +++ b/logservice/logpuller/region_failure_handler_test.go @@ -115,8 +115,8 @@ func TestErrCachePopBatch(t *testing.T) { } func TestRegionFailureHandlerRunDrainsErrCacheWithoutDispatcher(t *testing.T) { - handler := newRegionFailureHandler(&subscriptionClient{}) - for i := 0; i < errCacheBatchSize+5; i++ { + handler := newRegionFailureHandler(nil, func(*subscribedSpan) {}, func(context.Context, regionInfo) {}, func(context.Context, rangeTask) {}) + for i := 0; i < errCacheDispatchBatchSize+5; i++ { handler.cache.add(newTestRegionErrorInfo(&requestCancelledErr{})) } diff --git a/logservice/logpuller/region_request_scheduler.go b/logservice/logpuller/region_request_scheduler.go index f75fbf7868..df1a68756c 100644 --- a/logservice/logpuller/region_request_scheduler.go +++ b/logservice/logpuller/region_request_scheduler.go @@ -77,8 +77,7 @@ func newRegionRequestScheduler( } func (s *regionRequestScheduler) Submit(region regionInfo) { - s.taskQueue.Push(newRegionPriorityTask( - region, s.upstream.pdClock.CurrentTS(), s.sequence.Add(1))) + s.taskQueue.Push(newRegionPriorityTask(region, s.sequence.Add(1))) } func (s *regionRequestScheduler) Run(ctx context.Context, workerGroup *errgroup.Group) error { @@ -105,7 +104,7 @@ func (s *regionRequestScheduler) Run(ctx context.Context, workerGroup *errgroup. } store := s.getOrCreateStore(ctx, workerGroup, region.rpcCtx.Addr) - task.updateRegion(region, s.upstream.pdClock.CurrentTS()) + task.regionInfo = region if !store.submit(task) { return context.Canceled } diff --git a/logservice/logpuller/region_request_worker_test.go b/logservice/logpuller/region_request_worker_test.go index c6e3f8d845..7189766f04 100644 --- a/logservice/logpuller/region_request_worker_test.go +++ b/logservice/logpuller/region_request_worker_test.go @@ -393,7 +393,7 @@ func TestFailPendingRegionsReschedulesWorkerBuffer(t *testing.T) { }, } for i, region := range regions { - require.True(t, admission.submit(newRegionPriorityTask(region, 1, uint64(i+1)))) + require.True(t, admission.submit(newRegionPriorityTask(region, uint64(i+1)))) } worker.failPendingRegions(&storeStreamErr{}) diff --git a/logservice/logpuller/region_state.go b/logservice/logpuller/region_state.go index 06e3bdd5d4..0f2f84ce3e 100644 --- a/logservice/logpuller/region_state.go +++ b/logservice/logpuller/region_state.go @@ -17,6 +17,7 @@ import ( "sync" "sync/atomic" + "github.com/pingcap/kvproto/pkg/cdcpb" "github.com/pingcap/ticdc/heartbeatpb" "github.com/pingcap/ticdc/logservice/logpuller/regionlock" "github.com/tikv/client-go/v2/tikv" @@ -44,12 +45,12 @@ type regionInfo struct { subscribedSpan *subscribedSpan // The state of the locked range of the region. lockedRangeState *regionlock.LockedRangeState - // wasInitialized preserves scheduling priority while a failed region is - // unlocked, resolved again, and subscribed with a new locked range state. - wasInitialized bool // Whether to filter out the value write by cdc itself. // It should be `true` in BDR mode filterLoop bool + // scanPriority is sent to TiKV/CSE so remote incremental scan admission can + // preserve TiCDC's business priority across retries. + scanPriority cdcpb.ScanPriority } func newRegionInfo( @@ -57,13 +58,15 @@ func newRegionInfo( span heartbeatpb.TableSpan, rpcCtx *tikv.RPCContext, subscribedSpan *subscribedSpan, + filterLoop bool, ) regionInfo { return regionInfo{ verID: verID, span: span, rpcCtx: rpcCtx, subscribedSpan: subscribedSpan, - filterLoop: subscribedSpan.filterLoop, + filterLoop: filterLoop, + scanPriority: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, } } @@ -77,9 +80,6 @@ type regionErrorInfo struct { } func newRegionErrorInfo(info regionInfo, err error) regionErrorInfo { - if info.lockedRangeState != nil && info.lockedRangeState.Initialized.Load() { - info.wasInitialized = true - } return regionErrorInfo{ regionInfo: info, err: err, diff --git a/logservice/logpuller/scan_priority_test.go b/logservice/logpuller/scan_priority_test.go index 42806ca52a..629069cbf0 100644 --- a/logservice/logpuller/scan_priority_test.go +++ b/logservice/logpuller/scan_priority_test.go @@ -106,8 +106,10 @@ func TestScanPriorityUsesRestoredRegionProgress(t *testing.T) { pdClock := pdutil.NewClock4Test() pdClock.(*pdutil.Clock4Test).SetTS(currentTs) client := &subscriptionClient{ - pdClock: pdClock, - regionTaskQueue: priorityqueue.New[*regionPriorityTask](), + upstream: &upstreamHandle{pdClock: pdClock}, + regionScheduler: ®ionRequestScheduler{ + taskQueue: priorityqueue.New[*regionPriorityTask](), + }, } startTs := oracle.GoTimeToTS(currentTime.Add(-time.Hour)) @@ -125,8 +127,8 @@ func TestScanPriorityUsesRestoredRegionProgress(t *testing.T) { } region := newRegionInfo(tikv.NewRegionVerID(1, 1, 1), rawSpan, nil, span, false) - client.scheduleRegionRequest(context.Background(), region, cdcpb.ScanPriority_SCAN_PRIORITY_LOW) - firstTask := popRegionPriorityTask(t, client.regionTaskQueue) + client.scheduleRegionRequest(context.Background(), region) + firstTask := popRegionPriorityTask(t, client.regionScheduler.taskQueue) require.Equal(t, cdcpb.ScanPriority_SCAN_PRIORITY_LOW, firstTask.priority()) firstRegion := firstTask.regionInfo @@ -139,8 +141,8 @@ func TestScanPriorityUsesRestoredRegionProgress(t *testing.T) { ) retryRegion := newRegionInfo(tikv.NewRegionVerID(1, 1, 2), rawSpan, nil, span, false) - client.scheduleRegionRequest(context.Background(), retryRegion, cdcpb.ScanPriority_SCAN_PRIORITY_LOW) - retryTask := popRegionPriorityTask(t, client.regionTaskQueue) + client.scheduleRegionRequest(context.Background(), retryRegion) + retryTask := popRegionPriorityTask(t, client.regionScheduler.taskQueue) require.Equal(t, cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, retryTask.priority()) require.Equal(t, cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, retryTask.regionInfo.scanPriority) require.False(t, span.priorityPolicy.everCaughtUp.Load()) diff --git a/logservice/logpuller/subscription_client.go b/logservice/logpuller/subscription_client.go index ea4cd33c31..8fa9d17238 100644 --- a/logservice/logpuller/subscription_client.go +++ b/logservice/logpuller/subscription_client.go @@ -18,6 +18,7 @@ import ( "sync/atomic" "time" + "github.com/pingcap/kvproto/pkg/cdcpb" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/log" "github.com/pingcap/ticdc/heartbeatpb" @@ -25,6 +26,7 @@ import ( "github.com/pingcap/ticdc/logservice/txnutil" "github.com/pingcap/ticdc/pkg/common" appcontext "github.com/pingcap/ticdc/pkg/common/context" + "github.com/pingcap/ticdc/pkg/config" "github.com/pingcap/ticdc/pkg/metrics" "github.com/pingcap/ticdc/pkg/pdutil" "github.com/pingcap/ticdc/pkg/security" @@ -80,7 +82,8 @@ type resolveLockTask struct { type rangeTask struct { span heartbeatpb.TableSpan subscribedSpan *subscribedSpan - wasInitialized bool + filterLoop bool + priority cdcpb.ScanPriority } // upstreamHandle contains the stable TiKV and PD dependencies shared by the @@ -170,7 +173,7 @@ func NewSubscriptionClient( subClient.scheduleRegionRequest, subClient.scheduleRangeRequest, ) - subClient.eventSink = newRegionEventSink(subClient.ctx, subClient.failureHandler) + subClient.eventSink = newRegionEventSink(subClient.failureHandler) subClient.spanRegistry = newSpanRegistry(subClient.upstream.pd, subClient.upstream.pdClock) subClient.regionScheduler = newRegionRequestScheduler( subClient.upstream, @@ -234,6 +237,8 @@ func (s *subscriptionClient) Subscribe( advanceResolvedTs, advanceInterval, bdrMode, + s.upstream.pdClock, + time.Duration(config.GetGlobalServerConfig().Debug.Puller.OldStartTsScanLowPriorityThreshold), ) s.spanRegistry.Add(rt) s.eventSink.AddPath(rt) @@ -241,7 +246,12 @@ func (s *subscriptionClient) Subscribe( select { case <-s.ctx.Done(): log.Warn("subscribes span failed, the subscription client has closed") - case s.rangeTaskCh <- rangeTask{span: span, subscribedSpan: rt}: + case s.rangeTaskCh <- rangeTask{ + span: span, + subscribedSpan: rt, + filterLoop: rt.filterLoop, + priority: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, + }: log.Info("subscribes span done", zap.Uint64("subscriptionID", uint64(subID)), zap.Int64("tableID", span.TableID), zap.Uint64("startTs", startTs), zap.String("startKey", spanz.HexKey(span.StartKey)), zap.String("endKey", spanz.HexKey(span.EndKey))) @@ -407,8 +417,8 @@ func (s *subscriptionClient) divideSpanAndScheduleRegionRequests( } verID := tikv.NewRegionVerID(regionMeta.Id, regionMeta.RegionEpoch.ConfVer, regionMeta.RegionEpoch.Version) - regionInfo := newRegionInfo(verID, intersectSpan, nil, subscribedSpan) - regionInfo.wasInitialized = task.wasInitialized + regionInfo := newRegionInfo(verID, intersectSpan, nil, subscribedSpan, task.filterLoop) + regionInfo.scanPriority = normalizeScanPriority(task.priority) // Schedule a region request to subscribe the region. s.scheduleRegionRequest(ctx, regionInfo) @@ -426,9 +436,6 @@ func (s *subscriptionClient) divideSpanAndScheduleRegionRequests( // scheduleRegionRequest locks the region's range before submitting it to the // request scheduler. func (s *subscriptionClient) scheduleRegionRequest(ctx context.Context, region regionInfo) { - if region.lockedRangeState != nil && region.lockedRangeState.Initialized.Load() { - region.wasInitialized = true - } lockRangeResult := region.subscribedSpan.rangeLock.LockRange( ctx, region.span.StartKey, region.span.EndKey, region.verID.GetID(), region.verID.GetVer()) @@ -439,13 +446,19 @@ func (s *subscriptionClient) scheduleRegionRequest(ctx context.Context, region r switch lockRangeResult.Status { case regionlock.LockRangeStatusSuccess: region.lockedRangeState = lockRangeResult.LockedRangeState + region.scanPriority = region.subscribedSpan.priorityPolicy.resolve( + region.scanPriority, + region.resolvedTs(), + s.upstream.pdClock.CurrentTime(), + ) s.regionScheduler.Submit(region) case regionlock.LockRangeStatusStale: for _, r := range lockRangeResult.RetryRanges { s.scheduleRangeRequest(ctx, rangeTask{ span: r, subscribedSpan: region.subscribedSpan, - wasInitialized: region.wasInitialized, + filterLoop: region.filterLoop, + priority: region.scanPriority, }) } default: diff --git a/logservice/logpuller/subscription_client_test.go b/logservice/logpuller/subscription_client_test.go index 474d3c6af5..41230e77b4 100644 --- a/logservice/logpuller/subscription_client_test.go +++ b/logservice/logpuller/subscription_client_test.go @@ -79,6 +79,8 @@ func TestGenerateResolveLockTask(t *testing.T) { advanceResolvedTs, 0, false, + client.upstream.pdClock, + 30*time.Minute, ) client.spanRegistry.Add(span) @@ -147,16 +149,17 @@ func TestResolveLockTaskDeduplicatedAcrossSubscribedSpans(t *testing.T) { consumeKVEvents := func(_ []common.RawKVEntry, _ func()) bool { return false } advanceResolvedTs := func(ts uint64) {} + pdClock := pdutil.NewClock4Test() span1 := newSubscribedSpan(client.ctx, client.resolveLockRateLimiter, client.resolveLockTaskCh, SubscriptionID(1), heartbeatpb.TableSpan{ TableID: 1, StartKey: []byte{'a'}, EndKey: []byte{'z'}, - }, 100, consumeKVEvents, advanceResolvedTs, 0, false) + }, 100, consumeKVEvents, advanceResolvedTs, 0, false, pdClock, 30*time.Minute) span2 := newSubscribedSpan(client.ctx, client.resolveLockRateLimiter, client.resolveLockTaskCh, SubscriptionID(2), heartbeatpb.TableSpan{ TableID: 2, StartKey: []byte{'a'}, EndKey: []byte{'z'}, - }, 100, consumeKVEvents, advanceResolvedTs, 0, false) + }, 100, consumeKVEvents, advanceResolvedTs, 0, false, pdClock, 30*time.Minute) res := span1.rangeLock.LockRange(context.Background(), []byte{'b'}, []byte{'c'}, 1, 100) require.Equal(t, regionlock.LockRangeStatusSuccess, res.Status) @@ -266,6 +269,8 @@ func TestResolveLockTaskDroppedWhenChannelFull(t *testing.T) { advanceResolvedTs, 0, false, + pdutil.NewClock4Test(), + 30*time.Minute, ) res := span.rangeLock.LockRange(context.Background(), []byte{'b'}, []byte{'c'}, 1, 100) @@ -323,6 +328,8 @@ func TestStopTaskUsesSubscribedSpanFilterLoop(t *testing.T) { advanceResolvedTs, 0, true, + pdutil.NewClock4Test(), + 30*time.Minute, ) res := span.rangeLock.LockRange(context.Background(), rawSpan.StartKey, rawSpan.EndKey, 1, 1) @@ -418,8 +425,7 @@ func (s *mockDynamicStream) GetMetrics() dynstream.Metrics[int, SubscriptionID] func TestRegionEventSinkPushUnblocksOnClientClose(t *testing.T) { sink := ®ionEventSink{ - ctx: context.Background(), - ds: &mockDynamicStream{}, + ds: &mockDynamicStream{}, } sink.cond = sync.NewCond(&sink.mu) client := &subscriptionClient{eventSink: sink} @@ -468,7 +474,7 @@ func TestBroadcastDeregisterUsesWorkerControlQueue(t *testing.T) { subscribedSpan: &subscribedSpan{subID: SubscriptionID(2)}, lockedRangeState: ®ionlock.LockedRangeState{}, } - require.True(t, admission.submit(newRegionPriorityTask(dummyRegion, 1, 1))) + require.True(t, admission.submit(newRegionPriorityTask(dummyRegion, 1))) scheduler.BroadcastDeregister(SubscriptionID(1), true) require.Equal(t, 1, worker.controlQueue.len()) @@ -492,7 +498,7 @@ func TestRegionRequestStoreDistributesRegionsAcrossWorkers(t *testing.T) { subscribedSpan: &subscribedSpan{subID: 1}, lockedRangeState: ®ionlock.LockedRangeState{}, } - require.True(t, store.submit(newRegionPriorityTask(region, 1, i))) + require.True(t, store.submit(newRegionPriorityTask(region, i))) } require.Equal(t, 2, worker1.admission.stats().pending) @@ -755,7 +761,7 @@ func TestGetResolvedTargetTs(t *testing.T) { TableID: 1, StartKey: []byte{'a'}, EndKey: []byte{'z'}, - }, 100, consumeKVEvents, advanceResolvedTs, 0, false) + }, 100, consumeKVEvents, advanceResolvedTs, 0, false, pdutil.NewClock4Test(), 30*time.Minute) span.initialized.Store(true) // Replicate the getResolvedTargetTs closure from runResolveLockChecker From bb1ef55c716006d61e8ed6fa2d19b3df71a3d779 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Mon, 3 Aug 2026 23:28:56 +0800 Subject: [PATCH 35/71] more refactor --- .../logpuller/region_failure_handler.go | 89 +++++------ .../logpuller/region_failure_handler_test.go | 2 +- .../logpuller/subscription_client_test.go | 144 ------------------ 3 files changed, 36 insertions(+), 199 deletions(-) diff --git a/logservice/logpuller/region_failure_handler.go b/logservice/logpuller/region_failure_handler.go index abf172793d..4160485c25 100644 --- a/logservice/logpuller/region_failure_handler.go +++ b/logservice/logpuller/region_failure_handler.go @@ -24,7 +24,6 @@ import ( "github.com/pingcap/ticdc/pkg/metrics" "github.com/tikv/client-go/v2/tikv" "go.uber.org/zap" - "golang.org/x/sync/errgroup" ) var ( @@ -96,21 +95,43 @@ func (r *regionFailureHandler) Report(errInfo regionErrorInfo) { } func (r *regionFailureHandler) Run(ctx context.Context) error { - g, ctx := errgroup.WithContext(ctx) - g.Go(func() error { return r.cache.dispatch(ctx) }) - g.Go(func() error { + handleCachedErrors := func() error { for { - select { - case <-ctx.Done(): - return ctx.Err() - case errInfo := <-r.cache.errCh: + batch := r.cache.popBatch(errCacheBatchSize) + for _, errInfo := range batch { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } if err := r.handleError(ctx, errInfo); err != nil { return err } } + if len(batch) < errCacheBatchSize { + return nil + } } - }) - return g.Wait() + } + + // e.notify should surface new failures promptly in normal flow. The ticker is + // only a fallback scan and is not expected to be needed in practice. + ticker := time.NewTicker(200 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + if err := handleCachedErrors(); err != nil { + return err + } + case <-r.cache.ready(): + if err := handleCachedErrors(); err != nil { + return err + } + } + } } func (r *regionFailureHandler) handleError(ctx context.Context, errInfo regionErrorInfo) error { @@ -203,17 +224,15 @@ func (r *regionFailureHandler) handleError(ctx context.Context, errInfo regionEr type errCache struct { sync.Mutex cache []regionErrorInfo - errCh chan regionErrorInfo notify chan struct{} } -const errCacheDispatchBatchSize = 1024 +const errCacheBatchSize = 1024 func newErrCache() *errCache { return &errCache{ cache: make([]regionErrorInfo, 0, 1024), - errCh: make(chan regionErrorInfo, 4096), - notify: make(chan struct{}, 1024), + notify: make(chan struct{}, 1), } } @@ -247,44 +266,6 @@ func (e *errCache) popBatch(limit int) []regionErrorInfo { return batch } -func (e *errCache) dispatchBatch(ctx context.Context, limit int) (int, error) { - batch := e.popBatch(limit) - for _, errInfo := range batch { - select { - case <-ctx.Done(): - return 0, ctx.Err() - case e.errCh <- errInfo: - } - } - return len(batch), nil -} - -func (e *errCache) dispatch(ctx context.Context) error { - ticker := time.NewTicker(10 * time.Millisecond) - defer ticker.Stop() - sendToErrCh := func() error { - for { - n, err := e.dispatchBatch(ctx, errCacheDispatchBatchSize) - if err != nil { - return err - } - if n < errCacheDispatchBatchSize { - return nil - } - } - } - for { - select { - case <-ctx.Done(): - return ctx.Err() - case <-ticker.C: - if err := sendToErrCh(); err != nil { - return err - } - case <-e.notify: - if err := sendToErrCh(); err != nil { - return err - } - } - } +func (e *errCache) ready() <-chan struct{} { + return e.notify } diff --git a/logservice/logpuller/region_failure_handler_test.go b/logservice/logpuller/region_failure_handler_test.go index a7f8b2f883..f47fca3509 100644 --- a/logservice/logpuller/region_failure_handler_test.go +++ b/logservice/logpuller/region_failure_handler_test.go @@ -116,7 +116,7 @@ func TestErrCachePopBatch(t *testing.T) { func TestRegionFailureHandlerRunDrainsErrCacheWithoutDispatcher(t *testing.T) { handler := newRegionFailureHandler(nil, func(*subscribedSpan) {}, func(context.Context, regionInfo) {}, func(context.Context, rangeTask) {}) - for i := 0; i < errCacheDispatchBatchSize+5; i++ { + for i := 0; i < errCacheBatchSize+5; i++ { handler.cache.add(newTestRegionErrorInfo(&requestCancelledErr{})) } diff --git a/logservice/logpuller/subscription_client_test.go b/logservice/logpuller/subscription_client_test.go index 41230e77b4..315aa686ae 100644 --- a/logservice/logpuller/subscription_client_test.go +++ b/logservice/logpuller/subscription_client_test.go @@ -604,150 +604,6 @@ func TestSubscriptionWithFailedTiKV(t *testing.T) { } } -// TestErrCacheDispatchWithFullChannelAndCanceledContext tests that when errCh is full -// and context is canceled, the dispatch method doesn't get stuck. -func TestErrCacheDispatchWithFullChannelAndCanceledContext(t *testing.T) { - // Create errCache with a small errCh to easily fill it up - errCache := &errCache{ - cache: make([]regionErrorInfo, 0, 10), - errCh: make(chan regionErrorInfo, 2), // Small buffer to easily fill - notify: make(chan struct{}, 10), - } - - // Create a mock regionErrorInfo - mockErrInfo := regionErrorInfo{ - regionInfo: regionInfo{ - verID: tikv.NewRegionVerID(1, 1, 1), - span: heartbeatpb.TableSpan{TableID: 1, StartKey: []byte("a"), EndKey: []byte("b")}, - }, - err: errors.New("test error"), - } - - // Fill up the errCh channel to make it full - errCache.errCh <- mockErrInfo - errCache.errCh <- mockErrInfo - - // Add some errors to the cache - for i := 0; i < 5; i++ { - errCache.add(mockErrInfo) - } - - // Create a context that will be canceled - ctx, cancel := context.WithCancel(context.Background()) - - // Channel to signal when dispatch returns - dispatchDone := make(chan error, 1) - - // Start dispatch in a goroutine - go func() { - err := errCache.dispatch(ctx) - dispatchDone <- err - }() - - // Give dispatch some time to start and potentially get stuck - time.Sleep(50 * time.Millisecond) - - // Cancel the context - cancel() - - // Wait for dispatch to return with a timeout - select { - case err := <-dispatchDone: - // Verify that dispatch returned with context.Canceled error - require.Equal(t, context.Canceled, err) - case <-time.After(5 * time.Second): - // If we timeout here, it means dispatch is stuck - t.Fatal("dispatch method is stuck and didn't return after context cancellation") - } -} - -func TestErrCacheDispatchBatch(t *testing.T) { - mockErrInfo := regionErrorInfo{ - regionInfo: regionInfo{ - verID: tikv.NewRegionVerID(1, 1, 1), - span: heartbeatpb.TableSpan{TableID: 1, StartKey: []byte("a"), EndKey: []byte("b")}, - }, - err: errors.New("test error"), - } - - tests := []struct { - name string - cacheLen int - limit int - expectedN int - expectedCache int - expectedErrCh int - }{ - { - name: "dispatch all when limit equals cache length", - cacheLen: 5, - limit: 5, - expectedN: 5, - expectedCache: 0, - expectedErrCh: 5, - }, - { - name: "keep remaining cache when limit is smaller", - cacheLen: 5, - limit: 2, - expectedN: 2, - expectedCache: 3, - expectedErrCh: 2, - }, - { - name: "dispatch all when limit is larger", - cacheLen: 5, - limit: 10, - expectedN: 5, - expectedCache: 0, - expectedErrCh: 5, - }, - { - name: "dispatch all when limit is zero", - cacheLen: 5, - limit: 0, - expectedN: 5, - expectedCache: 0, - expectedErrCh: 5, - }, - { - name: "dispatch all when limit is negative", - cacheLen: 5, - limit: -1, - expectedN: 5, - expectedCache: 0, - expectedErrCh: 5, - }, - { - name: "empty cache", - cacheLen: 0, - limit: 5, - expectedN: 0, - expectedCache: 0, - expectedErrCh: 0, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - errCache := &errCache{ - cache: make([]regionErrorInfo, 0, 10), - errCh: make(chan regionErrorInfo, 10), - notify: make(chan struct{}, 1), - } - for i := 0; i < tc.cacheLen; i++ { - errCache.add(mockErrInfo) - } - - n, err := errCache.dispatchBatch(context.Background(), tc.limit) - require.NoError(t, err) - require.Equal(t, tc.expectedN, n) - require.Len(t, errCache.cache, tc.expectedCache) - require.Len(t, errCache.errCh, tc.expectedErrCh) - }) - } -} - func TestGetResolvedTargetTs(t *testing.T) { client := &subscriptionClient{ resolveLockTaskCh: make(chan resolveLockTask, 10), From 024206cd0a0bff8b46e7ffd6964e56bf3b09d5ce Mon Sep 17 00:00:00 2001 From: lidezhu Date: Mon, 3 Aug 2026 23:34:00 +0800 Subject: [PATCH 36/71] more refactor --- .../logpuller/region_failure_handler.go | 49 ++++++++++--------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/logservice/logpuller/region_failure_handler.go b/logservice/logpuller/region_failure_handler.go index 4160485c25..3622d927d9 100644 --- a/logservice/logpuller/region_failure_handler.go +++ b/logservice/logpuller/region_failure_handler.go @@ -64,23 +64,6 @@ func newRegionFailureHandler( } } -func (r *regionFailureHandler) retryRange(ctx context.Context, errInfo regionErrorInfo) { - priority := normalizeScanPriority(errInfo.scanPriority) - if priority == cdcpb.ScanPriority_SCAN_PRIORITY_LOW { - priority = errInfo.subscribedSpan.priorityPolicy.resolve( - priority, - errInfo.resolvedTs(), - errInfo.subscribedSpan.priorityPolicy.pdClock.CurrentTime(), - ) - } - r.scheduleRangeRequest(ctx, rangeTask{ - span: errInfo.span, - subscribedSpan: errInfo.subscribedSpan, - filterLoop: errInfo.filterLoop, - priority: priority, - }) -} - // Report admits a region failure into the recovery pipeline. It releases the // corresponding range lock before enqueueing the failure so new range tasks are // not blocked by stale region ownership. @@ -95,6 +78,9 @@ func (r *regionFailureHandler) Report(errInfo regionErrorInfo) { } func (r *regionFailureHandler) Run(ctx context.Context) error { + log.Info("region failure handler starts") + defer log.Info("region failure handler exits") + handleCachedErrors := func() error { for { batch := r.cache.popBatch(errCacheBatchSize) @@ -114,8 +100,8 @@ func (r *regionFailureHandler) Run(ctx context.Context) error { } } - // e.notify should surface new failures promptly in normal flow. The ticker is - // only a fallback scan and is not expected to be needed in practice. + // r.cache.ready() should handle failures promptly in normal flow. The ticker is only a + // fallback scan and is not expected to be needed in practice. ticker := time.NewTicker(200 * time.Millisecond) defer ticker.Stop() for { @@ -136,6 +122,23 @@ func (r *regionFailureHandler) Run(ctx context.Context) error { func (r *regionFailureHandler) handleError(ctx context.Context, errInfo regionErrorInfo) error { err := errors.Cause(errInfo.err) + rescheduleRange := func() { + priority := normalizeScanPriority(errInfo.scanPriority) + if priority == cdcpb.ScanPriority_SCAN_PRIORITY_LOW { + priority = errInfo.subscribedSpan.priorityPolicy.resolve( + priority, + errInfo.resolvedTs(), + errInfo.subscribedSpan.priorityPolicy.pdClock.CurrentTime(), + ) + } + r.scheduleRangeRequest(ctx, rangeTask{ + span: errInfo.span, + subscribedSpan: errInfo.subscribedSpan, + filterLoop: errInfo.filterLoop, + priority: priority, + }) + } + //nolint:errorlint // converting large type switch to errors.As is a significant refactor if _, requestCancelled := err.(*requestCancelledErr); !requestCancelled { log.Debug("cdc region error", @@ -156,12 +159,12 @@ func (r *regionFailureHandler) handleError(ctx context.Context, errInfo regionEr } if innerErr.GetEpochNotMatch() != nil { metricFeedEpochNotMatchCounter.Inc() - r.retryRange(ctx, errInfo) + rescheduleRange() return nil } if innerErr.GetRegionNotFound() != nil { metricFeedRegionNotFoundCounter.Inc() - r.retryRange(ctx, errInfo) + rescheduleRange() return nil } if innerErr.GetCongested() != nil { @@ -194,14 +197,14 @@ func (r *regionFailureHandler) handleError(ctx context.Context, errInfo regionEr return nil case *rpcCtxUnavailableErr: metricFeedRPCCtxUnavailable.Inc() - r.retryRange(ctx, errInfo) + rescheduleRange() return nil case *getStoreErr: metricGetStoreErr.Inc() bo := tikv.NewBackoffer(ctx, tikvRequestMaxBackoff) // cannot get the store the region belongs to, so we need to reload the region. r.regionCache.OnSendFail(bo, errInfo.rpcCtx, true, err) - r.retryRange(ctx, errInfo) + rescheduleRange() return nil case *storeStreamErr: metricStoreSendRequestErr.Inc() From 0cdb4d734b45b08816e15a5982590ac1e9bb729d Mon Sep 17 00:00:00 2001 From: lidezhu Date: Mon, 3 Aug 2026 23:45:29 +0800 Subject: [PATCH 37/71] remove unnecessary change --- logservice/logpuller/region_request_worker.go | 114 +++++++++--------- .../logpuller/region_request_worker_test.go | 34 ++++-- logservice/logpuller/region_tracker.go | 10 +- 3 files changed, 82 insertions(+), 76 deletions(-) diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index 3ef4422d5a..1211e699b6 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -131,55 +131,47 @@ func (s *regionRequestWorker) Run(ctx context.Context) error { return err } - regionErr := error(&storeStreamErr{}) - if err := s.checkStoreVersion(ctx); err != nil { - regionErr = err - } else if err := s.runStream(ctx, firstReq); err != nil { - regionErr = err - } + regionErr := s.runStream(ctx, firstReq) if ctx.Err() != nil { firstReq.abort() return ctx.Err() } - - // Stop sent requests first so their states release the admission leases. - // firstReq still owns its lease only if the stream failed before Send. - s.failStreamRegions(regionErr) - if firstReq.abort() { - s.failureHandler.Report(newRegionErrorInfo(firstReq.regionInfo, regionErr)) + // Treat an unexpected clean stream exit as a recoverable store-stream failure. + if regionErr == nil { + regionErr = &storeStreamErr{} + } + if err := s.handleStreamFailure(firstReq, regionErr); err != nil { + return err } - s.failPendingRegions(regionErr) - if err := util.Hang(ctx, storeReconnectBackoff); err != nil { return err } } } -// failStreamRegions transfers every request sent by a failed stream to the -// recovery pipeline. -func (s *regionRequestWorker) failStreamRegions(err error) { +func (s *regionRequestWorker) handleStreamFailure(firstReq *regionReq, regionErr error) error { + // Stream failure recovery: + // - tracker: requests already sent to this stream. + // - firstReq: popped from admission for this stream, but not necessarily + // added to tracker yet if the stream fails before sendRegionRequest calls + // tracker.Add. + // - admission: requests owned by this worker but not sent yet. for _, state := range s.tracker.Drain() { - s.notifyRegionError(state, err) + state.markStopped(regionErr) + s.eventSink.Push( + SubscriptionID(state.requestID), + regionEvent{states: []*regionFeedState{state}}, + ) } // The failed stream no longer owns remote registrations. s.controlQueue.drain() -} - -// failPendingRegions transfers requests owned by this worker but not yet sent -// to the recovery pipeline, so they can be resolved and routed again. -func (s *regionRequestWorker) failPendingRegions(err error) { + if firstReq != nil && firstReq.abort() { + s.failureHandler.Report(newRegionErrorInfo(firstReq.regionInfo, regionErr)) + } for _, task := range s.admission.drain() { - s.failureHandler.Report(newRegionErrorInfo(task.regionInfo, err)) + s.failureHandler.Report(newRegionErrorInfo(task.regionInfo, regionErr)) } -} - -func (s *regionRequestWorker) notifyRegionError(state *regionFeedState, err error) { - state.markStopped(err) - s.eventSink.Push( - SubscriptionID(state.requestID), - regionEvent{states: []*regionFeedState{state}}, - ) + return nil } func (s *regionRequestWorker) waitForRegionRequest(ctx context.Context) (*regionReq, error) { @@ -214,6 +206,10 @@ func (s *regionRequestWorker) checkStoreVersion(ctx context.Context) error { } func (s *regionRequestWorker) runStream(ctx context.Context, firstReq *regionReq) (err error) { + if err := s.checkStoreVersion(ctx); err != nil { + return err + } + log.Info("region request worker going to create grpc stream", zap.Uint64("workerID", s.workerID), zap.String("addr", s.storeAddr)) @@ -297,7 +293,7 @@ func (s *regionRequestWorker) dispatchRegionChangeEvents(events []*cdcpb.Event) subscriptionID := SubscriptionID(event.RequestId) state := s.tracker.Get(subscriptionID, regionID) if state != nil { - regionEvent := regionEvent{states: []*regionFeedState{state}} + eventToPush := regionEvent{states: []*regionFeedState{state}} switch eventData := event.Event.(type) { case *cdcpb.Event_Entries_: if eventData == nil { @@ -307,7 +303,7 @@ func (s *regionRequestWorker) dispatchRegionChangeEvents(events []*cdcpb.Event) zap.Uint64("regionID", regionID)) continue } - regionEvent.entries = eventData + eventToPush.entries = eventData case *cdcpb.Event_Admin_: continue case *cdcpb.Event_Error: @@ -316,16 +312,17 @@ func (s *regionRequestWorker) dispatchRegionChangeEvents(events []*cdcpb.Event) zap.Uint64("subscriptionID", uint64(subscriptionID)), zap.Uint64("regionID", event.RegionId), zap.Any("error", eventData.Error)) - s.notifyRegionError(state, &eventError{err: eventData.Error}) + state.markStopped(&eventError{err: eventData.Error}) + s.eventSink.Push(subscriptionID, regionEvent{states: []*regionFeedState{state}}) continue case *cdcpb.Event_ResolvedTs: - regionEvent.resolvedTs = eventData.ResolvedTs + eventToPush.resolvedTs = eventData.ResolvedTs case *cdcpb.Event_LongTxn_: continue default: log.Panic("unknown event type", zap.Any("event", event)) } - s.eventSink.Push(subscriptionID, regionEvent) + s.eventSink.Push(subscriptionID, eventToPush) continue } @@ -422,30 +419,19 @@ func (s *regionRequestWorker) sendDeregisterRequest( return err } for _, state := range s.tracker.TakeSubscription(req.subID) { - s.notifyRegionError(state, &requestCancelledErr{}) + state.markStopped(&requestCancelledErr{}) + s.eventSink.Push(req.subID, regionEvent{states: []*regionFeedState{state}}) } return nil } -func (s *regionRequestWorker) drainControlQueue(conn *ConnAndClient) error { - for { - req, ok := s.controlQueue.tryPop() - if !ok { - return nil - } - if err := s.sendDeregisterRequest(conn, req); err != nil { - return err - } - } -} - func (s *regionRequestWorker) sendRegionRequest(conn *ConnAndClient, req *regionReq) error { if !req.isActive() { - return &storeStreamErr{} + return nil } region := req.regionInfo subID := region.subscribedSpan.subID - log.Debug("region request worker gets a singleRegionInfo", + log.Debug("region request worker sends region request", zap.Uint64("workerID", s.workerID), zap.Uint64("subscriptionID", uint64(subID)), zap.Uint64("regionID", region.verID.GetID()), @@ -473,7 +459,7 @@ func (s *regionRequestWorker) sendRegionRequest(conn *ConnAndClient, req *region zap.Uint64("regionID", region.verID.GetID())) return nil } - if err := s.sendChangeDataRequest(conn, s.createRegionRequest(region)); err != nil { + if err := s.sendChangeDataRequest(conn, createRegionRequest(s.upstream.clusterID, region)); err != nil { // Transport failures are always recoverable at the region level. Preserve // the stream error as the function result, but classify the region for // rescheduling instead of exposing an arbitrary gRPC error downstream. @@ -490,17 +476,28 @@ func (s *regionRequestWorker) processRegionSendTask( ) error { regionReq := firstReq for { + // Send the current region request before handling anything newly queued. if regionReq != nil { if err := s.sendRegionRequest(conn, regionReq); err != nil { return err } regionReq = nil - continue } - if err := s.drainControlQueue(conn); err != nil { - return err + // Flush pending deregisters before admitting the next region request. + // Sending a region request after a deregister is safe because + // sendRegionRequest re-checks subscription liveness before tracker.Add and Send. + for { + req, ok := s.controlQueue.tryPop() + if !ok { + break + } + if err := s.sendDeregisterRequest(conn, req); err != nil { + return err + } } + + // Block for the next request, but wake early when deregisters arrive. var err error regionReq, err = s.admission.pop(ctx, s.controlQueue.ready()) if err != nil { @@ -509,9 +506,9 @@ func (s *regionRequestWorker) processRegionSendTask( } } -func (s *regionRequestWorker) createRegionRequest(region regionInfo) *cdcpb.ChangeDataRequest { +func createRegionRequest(clusterID uint64, region regionInfo) *cdcpb.ChangeDataRequest { return &cdcpb.ChangeDataRequest{ - Header: &cdcpb.Header{ClusterId: s.upstream.clusterID, TicdcVersion: version.ReleaseSemver()}, + Header: &cdcpb.Header{ClusterId: clusterID, TicdcVersion: version.ReleaseSemver()}, RegionId: region.verID.GetID(), RequestId: uint64(region.subscribedSpan.subID), RegionEpoch: region.rpcCtx.Meta.RegionEpoch, @@ -520,5 +517,6 @@ func (s *regionRequestWorker) createRegionRequest(region regionInfo) *cdcpb.Chan EndKey: region.span.EndKey, ExtraOp: kvrpcpb.ExtraOp_ReadOldValue, FilterLoop: region.filterLoop, + ScanPriority: normalizeScanPriority(region.scanPriority), } } diff --git a/logservice/logpuller/region_request_worker_test.go b/logservice/logpuller/region_request_worker_test.go index 7189766f04..7b08a114d3 100644 --- a/logservice/logpuller/region_request_worker_test.go +++ b/logservice/logpuller/region_request_worker_test.go @@ -336,7 +336,7 @@ func TestStoppedStateRemovesSentRequest(t *testing.T) { require.Equal(t, 0, admission.stats().inflight) } -func TestFailStreamRegionsReleasesSentAdmission(t *testing.T) { +func TestHandleStreamFailureReleasesSentAdmission(t *testing.T) { admission := newRegionAdmissionController(1, 1) ds := &mockRegionEventDynamicStream{} worker := ®ionRequestWorker{ @@ -350,14 +350,14 @@ func TestFailStreamRegionsReleasesSentAdmission(t *testing.T) { state := newRegionFeedState(region, uint64(region.subscribedSpan.subID), worker, req) require.True(t, worker.tracker.Add(region.subscribedSpan.subID, region.verID.GetID(), state)) - worker.failStreamRegions(&storeStreamErr{}) + require.NoError(t, worker.handleStreamFailure(nil, &storeStreamErr{})) require.Zero(t, admission.stats().inflight) require.False(t, req.abort()) require.Equal(t, 1, ds.pushCount) } -func TestFailPendingRegionsReschedulesWorkerBuffer(t *testing.T) { +func TestHandleStreamFailureReschedulesWorkerBuffer(t *testing.T) { rawSpan := heartbeatpb.TableSpan{ TableID: 1, StartKey: []byte("a"), @@ -375,7 +375,12 @@ func TestFailPendingRegionsReschedulesWorkerBuffer(t *testing.T) { admission := newRegionAdmissionController(1, 1) failureHandler := ®ionFailureHandler{cache: newErrCache()} - worker := ®ionRequestWorker{failureHandler: failureHandler, admission: admission} + worker := ®ionRequestWorker{ + failureHandler: failureHandler, + admission: admission, + controlQueue: newControlQueue(), + tracker: newRegionTracker(), + } regions := []regionInfo{ { verID: tikv.NewRegionVerID(1, 1, 1), @@ -396,7 +401,7 @@ func TestFailPendingRegionsReschedulesWorkerBuffer(t *testing.T) { require.True(t, admission.submit(newRegionPriorityTask(region, uint64(i+1)))) } - worker.failPendingRegions(&storeStreamErr{}) + require.NoError(t, worker.handleStreamFailure(nil, &storeStreamErr{})) require.Zero(t, admission.stats().pending) require.Len(t, failureHandler.cache.cache, 2) @@ -446,18 +451,23 @@ func TestProcessRegionSendTaskDoesNotSendRemovedRequest(t *testing.T) { req := admitRegionRequest(t, admission, region) require.True(t, req.abort()) + ctx, cancel := context.WithCancel(context.Background()) sendCh := make(chan *cdcpb.ChangeDataRequest, 1) - err := worker.processRegionSendTask(t.Context(), &ConnAndClient{ - Client: &mockEventFeedV2Client{sendCh: sendCh}, - Conn: &grpc.ClientConn{}, - }, req) - var streamErr *storeStreamErr - require.ErrorAs(t, err, &streamErr) + done := make(chan error, 1) + go func() { + done <- worker.processRegionSendTask(ctx, &ConnAndClient{ + Client: &mockEventFeedV2Client{sendCh: sendCh}, + Conn: &grpc.ClientConn{}, + }, req) + }() + select { case sentReq := <-sendCh: t.Fatalf("removed request was sent: %+v", sentReq) - default: + case <-time.After(50 * time.Millisecond): } + cancel() + require.ErrorIs(t, <-done, context.Canceled) } func TestProcessRegionSendTaskSendEOFIsRetriable(t *testing.T) { diff --git a/logservice/logpuller/region_tracker.go b/logservice/logpuller/region_tracker.go index 7066779eaf..81a27dc86c 100644 --- a/logservice/logpuller/region_tracker.go +++ b/logservice/logpuller/region_tracker.go @@ -112,15 +112,13 @@ func (t *regionTracker) Drain() []*regionFeedState { t.statesBySubscription = make(map[SubscriptionID]regionStatesByID) t.mu.Unlock() - stateCount := 0 + totalStates := 0 for _, states := range statesBySubscription { - stateCount += len(states) + totalStates += len(states) } - drainedStates := make([]*regionFeedState, 0, stateCount) + drainedStates := make([]*regionFeedState, 0, totalStates) for _, states := range statesBySubscription { - for _, state := range states { - drainedStates = append(drainedStates, state) - } + drainedStates = append(drainedStates, slices.Collect(maps.Values(states))...) } return drainedStates } From a03750b6d6977ace8a9d29e5752463226c5ecad1 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Mon, 3 Aug 2026 23:54:18 +0800 Subject: [PATCH 38/71] add more test --- .../region_request_scheduler_test.go | 78 ++++++++++++++++++ .../logpuller/region_request_store_test.go | 70 ++++++++++++++++ logservice/logpuller/region_request_worker.go | 81 +++++++++---------- .../logpuller/region_request_worker_test.go | 72 ----------------- .../logpuller/subscription_client_test.go | 48 ----------- 5 files changed, 187 insertions(+), 162 deletions(-) create mode 100644 logservice/logpuller/region_request_scheduler_test.go create mode 100644 logservice/logpuller/region_request_store_test.go diff --git a/logservice/logpuller/region_request_scheduler_test.go b/logservice/logpuller/region_request_scheduler_test.go new file mode 100644 index 0000000000..33f7da4f2a --- /dev/null +++ b/logservice/logpuller/region_request_scheduler_test.go @@ -0,0 +1,78 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package logpuller + +import ( + "testing" + + "github.com/pingcap/ticdc/logservice/logpuller/regionlock" + "github.com/stretchr/testify/require" +) + +func TestRegionRequestSchedulerBroadcastDeregisterUsesWorkerControlQueue(t *testing.T) { + scheduler := ®ionRequestScheduler{} + + worker1 := ®ionRequestWorker{ + storeAddr: "store-1", + admission: newRegionAdmissionController(1, 1), + controlQueue: newControlQueue(), + } + worker2 := ®ionRequestWorker{ + storeAddr: "store-2", + admission: newRegionAdmissionController(1, 1), + controlQueue: newControlQueue(), + } + store1 := ®ionRequestStore{workers: []*regionRequestWorker{worker1}} + store2 := ®ionRequestStore{workers: []*regionRequestWorker{worker2}} + scheduler.stores.Store("store-1", store1) + scheduler.stores.Store("store-2", store2) + + dummyRegion := regionInfo{ + subscribedSpan: &subscribedSpan{subID: SubscriptionID(2)}, + lockedRangeState: ®ionlock.LockedRangeState{}, + } + require.True(t, worker1.admission.submit(newRegionPriorityTask(dummyRegion, 1))) + + scheduler.BroadcastDeregister(SubscriptionID(1), true) + + req1, ok := worker1.controlQueue.tryPop() + require.True(t, ok) + require.Equal(t, SubscriptionID(1), req1.subID) + require.True(t, req1.filterLoop) + + req2, ok := worker2.controlQueue.tryPop() + require.True(t, ok) + require.Equal(t, SubscriptionID(1), req2.subID) + require.True(t, req2.filterLoop) + + require.Equal(t, 1, worker1.admission.stats().pending) +} + +func TestRegionRequestSchedulerInflightCountAggregatesStores(t *testing.T) { + scheduler := ®ionRequestScheduler{} + + worker1 := ®ionRequestWorker{admission: newRegionAdmissionController(2, 1)} + worker2 := ®ionRequestWorker{admission: newRegionAdmissionController(2, 1)} + scheduler.stores.Store("store-1", ®ionRequestStore{workers: []*regionRequestWorker{worker1}}) + scheduler.stores.Store("store-2", ®ionRequestStore{workers: []*regionRequestWorker{worker2}}) + + req1 := admitRegionRequest(t, worker1.admission, prepareRegionForAdmission(createTestRegionInfo(1, 1), 100)) + req2 := admitRegionRequest(t, worker2.admission, prepareRegionForAdmission(createTestRegionInfo(1, 2), 100)) + + require.Equal(t, 2, scheduler.inflightCount()) + + require.True(t, req1.abort()) + require.True(t, req2.abort()) +} diff --git a/logservice/logpuller/region_request_store_test.go b/logservice/logpuller/region_request_store_test.go new file mode 100644 index 0000000000..c555a9441e --- /dev/null +++ b/logservice/logpuller/region_request_store_test.go @@ -0,0 +1,70 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package logpuller + +import ( + "testing" + + "github.com/stretchr/testify/require" + "github.com/tikv/client-go/v2/tikv" +) + +func TestRegionRequestStoreDistributesRegionsAcrossWorkers(t *testing.T) { + worker1 := ®ionRequestWorker{admission: newRegionAdmissionController(1, 1)} + worker2 := ®ionRequestWorker{admission: newRegionAdmissionController(1, 1)} + store := ®ionRequestStore{ + workers: []*regionRequestWorker{worker1, worker2}, + } + + for i := uint64(1); i <= 4; i++ { + region := prepareRegionForAdmission(createTestRegionInfo(1, i), 100) + region.verID = tikv.NewRegionVerID(i, 1, 1) + require.True(t, store.submit(newRegionPriorityTask(region, i))) + } + + require.Equal(t, 2, worker1.admission.stats().pending) + require.Equal(t, 2, worker2.admission.stats().pending) +} + +func TestRegionRequestStoreInflightCountAggregatesWorkers(t *testing.T) { + worker1 := ®ionRequestWorker{admission: newRegionAdmissionController(2, 1)} + worker2 := ®ionRequestWorker{admission: newRegionAdmissionController(2, 1)} + store := ®ionRequestStore{ + workers: []*regionRequestWorker{worker1, worker2}, + } + + req1 := admitRegionRequest(t, worker1.admission, prepareRegionForAdmission(createTestRegionInfo(1, 1), 100)) + req2 := admitRegionRequest(t, worker2.admission, prepareRegionForAdmission(createTestRegionInfo(1, 2), 100)) + + require.Equal(t, 2, store.inflightCount()) + + require.True(t, req1.abort()) + require.True(t, req2.abort()) +} + +func TestRegionRequestStoreCloseClosesWorkerAdmissions(t *testing.T) { + worker1 := ®ionRequestWorker{admission: newRegionAdmissionController(1, 1)} + worker2 := ®ionRequestWorker{admission: newRegionAdmissionController(1, 1)} + store := ®ionRequestStore{ + workers: []*regionRequestWorker{worker1, worker2}, + } + + store.close() + + region1 := prepareRegionForAdmission(createTestRegionInfo(1, 1), 100) + region2 := prepareRegionForAdmission(createTestRegionInfo(1, 2), 100) + require.False(t, worker1.admission.submit(newRegionPriorityTask(region1, 1))) + require.False(t, worker2.admission.submit(newRegionPriorityTask(region2, 2))) +} diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index 1211e699b6..1e38d08a1e 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -124,6 +124,30 @@ func newRegionRequestWorker( } func (s *regionRequestWorker) Run(ctx context.Context) error { + handleStreamFailure := func(firstReq *regionReq, regionErr error) { + // Stream failure recovery: + // - tracker: requests already sent to this stream. + // - firstReq: popped from admission for this stream, but not necessarily + // added to tracker yet if the stream fails before sendRegionRequest calls + // tracker.Add. + // - admission: requests owned by this worker but not sent yet. + for _, state := range s.tracker.Drain() { + state.markStopped(regionErr) + s.eventSink.Push( + SubscriptionID(state.requestID), + regionEvent{states: []*regionFeedState{state}}, + ) + } + // The failed stream no longer owns remote registrations. + s.controlQueue.drain() + if firstReq != nil && firstReq.abort() { + s.failureHandler.Report(newRegionErrorInfo(firstReq.regionInfo, regionErr)) + } + for _, task := range s.admission.drain() { + s.failureHandler.Report(newRegionErrorInfo(task.regionInfo, regionErr)) + } + } + for { // Do not connect an idle worker to an unavailable store indefinitely. firstReq, err := s.waitForRegionRequest(ctx) @@ -140,40 +164,13 @@ func (s *regionRequestWorker) Run(ctx context.Context) error { if regionErr == nil { regionErr = &storeStreamErr{} } - if err := s.handleStreamFailure(firstReq, regionErr); err != nil { - return err - } + handleStreamFailure(firstReq, regionErr) if err := util.Hang(ctx, storeReconnectBackoff); err != nil { return err } } } -func (s *regionRequestWorker) handleStreamFailure(firstReq *regionReq, regionErr error) error { - // Stream failure recovery: - // - tracker: requests already sent to this stream. - // - firstReq: popped from admission for this stream, but not necessarily - // added to tracker yet if the stream fails before sendRegionRequest calls - // tracker.Add. - // - admission: requests owned by this worker but not sent yet. - for _, state := range s.tracker.Drain() { - state.markStopped(regionErr) - s.eventSink.Push( - SubscriptionID(state.requestID), - regionEvent{states: []*regionFeedState{state}}, - ) - } - // The failed stream no longer owns remote registrations. - s.controlQueue.drain() - if firstReq != nil && firstReq.abort() { - s.failureHandler.Report(newRegionErrorInfo(firstReq.regionInfo, regionErr)) - } - for _, task := range s.admission.drain() { - s.failureHandler.Report(newRegionErrorInfo(task.regionInfo, regionErr)) - } - return nil -} - func (s *regionRequestWorker) waitForRegionRequest(ctx context.Context) (*regionReq, error) { // Without a stream there are no remote registrations to deregister. s.controlQueue.drain() @@ -288,13 +285,13 @@ func (s *regionRequestWorker) receiveAndDispatchChangeEvents(conn *ConnAndClient } func (s *regionRequestWorker) dispatchRegionChangeEvents(events []*cdcpb.Event) { - for _, event := range events { - regionID := event.RegionId - subscriptionID := SubscriptionID(event.RequestId) + for _, cdcEvent := range events { + regionID := cdcEvent.RegionId + subscriptionID := SubscriptionID(cdcEvent.RequestId) state := s.tracker.Get(subscriptionID, regionID) if state != nil { - eventToPush := regionEvent{states: []*regionFeedState{state}} - switch eventData := event.Event.(type) { + regionEvent := regionEvent{states: []*regionFeedState{state}} + switch eventData := cdcEvent.Event.(type) { case *cdcpb.Event_Entries_: if eventData == nil { log.Warn("region request worker receives a region event with nil entries, ignore it", @@ -303,40 +300,40 @@ func (s *regionRequestWorker) dispatchRegionChangeEvents(events []*cdcpb.Event) zap.Uint64("regionID", regionID)) continue } - eventToPush.entries = eventData + regionEvent.entries = eventData case *cdcpb.Event_Admin_: continue case *cdcpb.Event_Error: log.Debug("region request worker receives a region error", zap.Uint64("workerID", s.workerID), zap.Uint64("subscriptionID", uint64(subscriptionID)), - zap.Uint64("regionID", event.RegionId), + zap.Uint64("regionID", cdcEvent.RegionId), zap.Any("error", eventData.Error)) state.markStopped(&eventError{err: eventData.Error}) - s.eventSink.Push(subscriptionID, regionEvent{states: []*regionFeedState{state}}) + s.eventSink.Push(subscriptionID, regionEvent) continue case *cdcpb.Event_ResolvedTs: - eventToPush.resolvedTs = eventData.ResolvedTs + regionEvent.resolvedTs = eventData.ResolvedTs case *cdcpb.Event_LongTxn_: continue default: - log.Panic("unknown event type", zap.Any("event", event)) + log.Panic("unknown event type", zap.Any("event", cdcEvent)) } - s.eventSink.Push(subscriptionID, eventToPush) + s.eventSink.Push(subscriptionID, regionEvent) continue } - switch event.Event.(type) { + switch cdcEvent.Event.(type) { case *cdcpb.Event_Error: log.Debug("region request worker receives an error for a stale region, ignore it", zap.Uint64("workerID", s.workerID), zap.Uint64("subscriptionID", uint64(subscriptionID)), - zap.Uint64("regionID", event.RegionId)) + zap.Uint64("regionID", cdcEvent.RegionId)) default: log.Warn("region request worker receives a region event for an untracked region", zap.Uint64("workerID", s.workerID), zap.Uint64("subscriptionID", uint64(subscriptionID)), - zap.Uint64("regionID", event.RegionId)) + zap.Uint64("regionID", cdcEvent.RegionId)) } } } diff --git a/logservice/logpuller/region_request_worker_test.go b/logservice/logpuller/region_request_worker_test.go index 7b08a114d3..2c27325319 100644 --- a/logservice/logpuller/region_request_worker_test.go +++ b/logservice/logpuller/region_request_worker_test.go @@ -22,7 +22,6 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/cdcpb" "github.com/pingcap/kvproto/pkg/metapb" - "github.com/pingcap/ticdc/heartbeatpb" "github.com/pingcap/ticdc/logservice/logpuller/regionlock" "github.com/pingcap/ticdc/utils/dynstream" "github.com/stretchr/testify/require" @@ -336,77 +335,6 @@ func TestStoppedStateRemovesSentRequest(t *testing.T) { require.Equal(t, 0, admission.stats().inflight) } -func TestHandleStreamFailureReleasesSentAdmission(t *testing.T) { - admission := newRegionAdmissionController(1, 1) - ds := &mockRegionEventDynamicStream{} - worker := ®ionRequestWorker{ - admission: admission, - controlQueue: newControlQueue(), - eventSink: ®ionEventSink{ds: ds}, - tracker: newRegionTracker(), - } - region := prepareRegionForSendTest(createTestRegionInfo(1, 1)) - req := admitRegionRequest(t, admission, region) - state := newRegionFeedState(region, uint64(region.subscribedSpan.subID), worker, req) - require.True(t, worker.tracker.Add(region.subscribedSpan.subID, region.verID.GetID(), state)) - - require.NoError(t, worker.handleStreamFailure(nil, &storeStreamErr{})) - - require.Zero(t, admission.stats().inflight) - require.False(t, req.abort()) - require.Equal(t, 1, ds.pushCount) -} - -func TestHandleStreamFailureReschedulesWorkerBuffer(t *testing.T) { - rawSpan := heartbeatpb.TableSpan{ - TableID: 1, - StartKey: []byte("a"), - EndKey: []byte("z"), - } - span := &subscribedSpan{ - subID: 1, - span: rawSpan, - rangeLock: regionlock.NewRangeLock(1, rawSpan.StartKey, rawSpan.EndKey, 100), - } - lock1 := span.rangeLock.LockRange(t.Context(), []byte("a"), []byte("m"), 1, 1) - lock2 := span.rangeLock.LockRange(t.Context(), []byte("m"), []byte("z"), 2, 1) - require.Equal(t, regionlock.LockRangeStatusSuccess, lock1.Status) - require.Equal(t, regionlock.LockRangeStatusSuccess, lock2.Status) - - admission := newRegionAdmissionController(1, 1) - failureHandler := ®ionFailureHandler{cache: newErrCache()} - worker := ®ionRequestWorker{ - failureHandler: failureHandler, - admission: admission, - controlQueue: newControlQueue(), - tracker: newRegionTracker(), - } - regions := []regionInfo{ - { - verID: tikv.NewRegionVerID(1, 1, 1), - span: heartbeatpb.TableSpan{ - TableID: 1, StartKey: []byte("a"), EndKey: []byte("m"), - }, - subscribedSpan: span, lockedRangeState: lock1.LockedRangeState, - }, - { - verID: tikv.NewRegionVerID(2, 1, 1), - span: heartbeatpb.TableSpan{ - TableID: 1, StartKey: []byte("m"), EndKey: []byte("z"), - }, - subscribedSpan: span, lockedRangeState: lock2.LockedRangeState, - }, - } - for i, region := range regions { - require.True(t, admission.submit(newRegionPriorityTask(region, uint64(i+1)))) - } - - require.NoError(t, worker.handleStreamFailure(nil, &storeStreamErr{})) - - require.Zero(t, admission.stats().pending) - require.Len(t, failureHandler.cache.cache, 2) -} - func TestProcessRegionSendTaskSendFailureCleansSentRequest(t *testing.T) { admission := newRegionAdmissionController(10, 1) worker := ®ionRequestWorker{ diff --git a/logservice/logpuller/subscription_client_test.go b/logservice/logpuller/subscription_client_test.go index 315aa686ae..99600666ea 100644 --- a/logservice/logpuller/subscription_client_test.go +++ b/logservice/logpuller/subscription_client_test.go @@ -457,54 +457,6 @@ func TestRegionEventSinkPushUnblocksOnClientClose(t *testing.T) { } } -func TestBroadcastDeregisterUsesWorkerControlQueue(t *testing.T) { - scheduler := ®ionRequestScheduler{} - admission := newRegionAdmissionController(1, 1) - - const storeAddr = "store-1" - worker := ®ionRequestWorker{ - storeAddr: storeAddr, - admission: admission, - controlQueue: newControlQueue(), - } - store := ®ionRequestStore{workers: []*regionRequestWorker{worker}} - scheduler.stores.Store(storeAddr, store) - - dummyRegion := regionInfo{ - subscribedSpan: &subscribedSpan{subID: SubscriptionID(2)}, - lockedRangeState: ®ionlock.LockedRangeState{}, - } - require.True(t, admission.submit(newRegionPriorityTask(dummyRegion, 1))) - - scheduler.BroadcastDeregister(SubscriptionID(1), true) - require.Equal(t, 1, worker.controlQueue.len()) - req, ok := worker.controlQueue.tryPop() - require.True(t, ok) - require.Equal(t, SubscriptionID(1), req.subID) - require.True(t, req.filterLoop) - require.Equal(t, 1, admission.stats().pending) -} - -func TestRegionRequestStoreDistributesRegionsAcrossWorkers(t *testing.T) { - worker1 := ®ionRequestWorker{admission: newRegionAdmissionController(1, 1)} - worker2 := ®ionRequestWorker{admission: newRegionAdmissionController(1, 1)} - store := ®ionRequestStore{ - workers: []*regionRequestWorker{worker1, worker2}, - } - - for i := uint64(1); i <= 4; i++ { - region := regionInfo{ - verID: tikv.NewRegionVerID(i, 1, 1), - subscribedSpan: &subscribedSpan{subID: 1}, - lockedRangeState: ®ionlock.LockedRangeState{}, - } - require.True(t, store.submit(newRegionPriorityTask(region, i))) - } - - require.Equal(t, 2, worker1.admission.stats().pending) - require.Equal(t, 2, worker2.admission.stats().pending) -} - func TestSubscriptionWithFailedTiKV(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) mockPDClock := pdutil.NewClock4Test() From 841c0da95cefb879832db588e3da6546a715a784 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Tue, 4 Aug 2026 00:17:49 +0800 Subject: [PATCH 39/71] more refactor --- logservice/logpuller/region_request_worker.go | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index 1e38d08a1e..e58211838b 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -125,7 +125,7 @@ func newRegionRequestWorker( func (s *regionRequestWorker) Run(ctx context.Context) error { handleStreamFailure := func(firstReq *regionReq, regionErr error) { - // Stream failure recovery: + // Stream failure handle cases: // - tracker: requests already sent to this stream. // - firstReq: popped from admission for this stream, but not necessarily // added to tracker yet if the stream fails before sendRegionRequest calls @@ -217,8 +217,7 @@ func (s *regionRequestWorker) runStream(ctx context.Context, firstReq *regionReq zap.Error(err)) }() - g, gctx := errgroup.WithContext(ctx) - conn, err := Connect(gctx, s.upstream.credential, s.storeAddr) + conn, err := Connect(ctx, s.upstream.credential, s.storeAddr) if err != nil { log.Warn("region request worker create grpc stream failed", zap.Uint64("workerID", s.workerID), @@ -234,6 +233,7 @@ func (s *regionRequestWorker) runStream(ctx context.Context, firstReq *regionReq } defer func() { _ = conn.Conn.Close() }() + g, gctx := errgroup.WithContext(ctx) g.Go(func() error { return s.receiveAndDispatchChangeEvents(conn) }) g.Go(func() error { return s.processRegionSendTask(gctx, conn, firstReq) }) @@ -285,13 +285,13 @@ func (s *regionRequestWorker) receiveAndDispatchChangeEvents(conn *ConnAndClient } func (s *regionRequestWorker) dispatchRegionChangeEvents(events []*cdcpb.Event) { - for _, cdcEvent := range events { - regionID := cdcEvent.RegionId - subscriptionID := SubscriptionID(cdcEvent.RequestId) + for _, event := range events { + regionID := event.RegionId + subscriptionID := SubscriptionID(event.RequestId) state := s.tracker.Get(subscriptionID, regionID) if state != nil { regionEvent := regionEvent{states: []*regionFeedState{state}} - switch eventData := cdcEvent.Event.(type) { + switch eventData := event.Event.(type) { case *cdcpb.Event_Entries_: if eventData == nil { log.Warn("region request worker receives a region event with nil entries, ignore it", @@ -307,7 +307,7 @@ func (s *regionRequestWorker) dispatchRegionChangeEvents(events []*cdcpb.Event) log.Debug("region request worker receives a region error", zap.Uint64("workerID", s.workerID), zap.Uint64("subscriptionID", uint64(subscriptionID)), - zap.Uint64("regionID", cdcEvent.RegionId), + zap.Uint64("regionID", event.RegionId), zap.Any("error", eventData.Error)) state.markStopped(&eventError{err: eventData.Error}) s.eventSink.Push(subscriptionID, regionEvent) @@ -317,23 +317,23 @@ func (s *regionRequestWorker) dispatchRegionChangeEvents(events []*cdcpb.Event) case *cdcpb.Event_LongTxn_: continue default: - log.Panic("unknown event type", zap.Any("event", cdcEvent)) + log.Panic("unknown event type", zap.Any("event", event)) } s.eventSink.Push(subscriptionID, regionEvent) continue } - switch cdcEvent.Event.(type) { + switch event.Event.(type) { case *cdcpb.Event_Error: log.Debug("region request worker receives an error for a stale region, ignore it", zap.Uint64("workerID", s.workerID), zap.Uint64("subscriptionID", uint64(subscriptionID)), - zap.Uint64("regionID", cdcEvent.RegionId)) + zap.Uint64("regionID", event.RegionId)) default: log.Warn("region request worker receives a region event for an untracked region", zap.Uint64("workerID", s.workerID), zap.Uint64("subscriptionID", uint64(subscriptionID)), - zap.Uint64("regionID", cdcEvent.RegionId)) + zap.Uint64("regionID", event.RegionId)) } } } From 76571094183248bc14e57bccb4ff6a97463b2270 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Tue, 4 Aug 2026 13:28:53 +0800 Subject: [PATCH 40/71] small fix --- logservice/logpuller/region_event_handler.go | 2 +- logservice/logpuller/region_request_worker.go | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/logservice/logpuller/region_event_handler.go b/logservice/logpuller/region_event_handler.go index e13b91cbd0..b7f214922d 100644 --- a/logservice/logpuller/region_event_handler.go +++ b/logservice/logpuller/region_event_handler.go @@ -29,7 +29,7 @@ import ( ) var ( - metricsEventCount = metrics.PullerEventCounter.WithLabelValues("event") + metricsEventCount = metrics.PullerEventCounter.WithLabelValues("event") metricRegionEventHandleDurationEntries = metrics.SubscriptionClientRegionEventHandleDuration.WithLabelValues("entries") metricRegionEventHandleDurationResolved = metrics.SubscriptionClientRegionEventHandleDuration.WithLabelValues("resolved") diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index e58211838b..3b472c88d1 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -36,11 +36,14 @@ import ( const storeReconnectBackoff = time.Second +var ( + metricsResolvedTsCount = metrics.PullerEventCounter.WithLabelValues("resolved_ts") + metricBatchResolvedSize = metrics.BatchResolvedEventSize.WithLabelValues("event-store") +) + // To generate a workerID in `newRegionRequestWorker`. var workerIDGen atomic.Uint64 -var metricBatchResolvedSize = metrics.BatchResolvedEventSize.WithLabelValues("event-store") - type deregisterRequest struct { subID SubscriptionID filterLoop bool From 4e4d4ed8287805814596cca9636e81abfcd0741e Mon Sep 17 00:00:00 2001 From: lidezhu Date: Tue, 4 Aug 2026 14:54:59 +0800 Subject: [PATCH 41/71] small fix --- logservice/logpuller/region_event_handler.go | 2 +- logservice/logpuller/region_request_worker.go | 45 ++++++++++--------- 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/logservice/logpuller/region_event_handler.go b/logservice/logpuller/region_event_handler.go index b7f214922d..e13b91cbd0 100644 --- a/logservice/logpuller/region_event_handler.go +++ b/logservice/logpuller/region_event_handler.go @@ -29,7 +29,7 @@ import ( ) var ( - metricsEventCount = metrics.PullerEventCounter.WithLabelValues("event") + metricsEventCount = metrics.PullerEventCounter.WithLabelValues("event") metricRegionEventHandleDurationEntries = metrics.SubscriptionClientRegionEventHandleDuration.WithLabelValues("entries") metricRegionEventHandleDurationResolved = metrics.SubscriptionClientRegionEventHandleDuration.WithLabelValues("resolved") diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index 3b472c88d1..db55144362 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -36,14 +36,14 @@ import ( const storeReconnectBackoff = time.Second +// To generate a workerID in `newRegionRequestWorker`. +var workerIDGen atomic.Uint64 + var ( metricsResolvedTsCount = metrics.PullerEventCounter.WithLabelValues("resolved_ts") metricBatchResolvedSize = metrics.BatchResolvedEventSize.WithLabelValues("event-store") ) -// To generate a workerID in `newRegionRequestWorker`. -var workerIDGen atomic.Uint64 - type deregisterRequest struct { subID SubscriptionID filterLoop bool @@ -313,8 +313,6 @@ func (s *regionRequestWorker) dispatchRegionChangeEvents(events []*cdcpb.Event) zap.Uint64("regionID", event.RegionId), zap.Any("error", eventData.Error)) state.markStopped(&eventError{err: eventData.Error}) - s.eventSink.Push(subscriptionID, regionEvent) - continue case *cdcpb.Event_ResolvedTs: regionEvent.resolvedTs = eventData.ResolvedTs case *cdcpb.Event_LongTxn_: @@ -323,20 +321,19 @@ func (s *regionRequestWorker) dispatchRegionChangeEvents(events []*cdcpb.Event) log.Panic("unknown event type", zap.Any("event", event)) } s.eventSink.Push(subscriptionID, regionEvent) - continue - } - - switch event.Event.(type) { - case *cdcpb.Event_Error: - log.Debug("region request worker receives an error for a stale region, ignore it", - zap.Uint64("workerID", s.workerID), - zap.Uint64("subscriptionID", uint64(subscriptionID)), - zap.Uint64("regionID", event.RegionId)) - default: - log.Warn("region request worker receives a region event for an untracked region", - zap.Uint64("workerID", s.workerID), - zap.Uint64("subscriptionID", uint64(subscriptionID)), - zap.Uint64("regionID", event.RegionId)) + } else { + switch event.Event.(type) { + case *cdcpb.Event_Error: + log.Debug("region request worker receives an error for a stale region, ignore it", + zap.Uint64("workerID", s.workerID), + zap.Uint64("subscriptionID", uint64(subscriptionID)), + zap.Uint64("regionID", event.RegionId)) + default: + log.Warn("region request worker receives a region event for an untracked region", + zap.Uint64("workerID", s.workerID), + zap.Uint64("subscriptionID", uint64(subscriptionID)), + zap.Uint64("regionID", event.RegionId)) + } } } } @@ -345,6 +342,7 @@ func (s *regionRequestWorker) dispatchResolvedTsEvent(resolvedTsEvent *cdcpb.Res subscriptionID := SubscriptionID(resolvedTsEvent.RequestId) metricsResolvedTsCount.Add(float64(len(resolvedTsEvent.Regions))) metricBatchResolvedSize.Observe(float64(len(resolvedTsEvent.Regions))) + // TODO: resolvedTsEvent.Ts be 0 is impossible, we need find the root cause. if resolvedTsEvent.Ts == 0 { log.Warn("region request worker receives a resolved ts event with zero value, ignore it", zap.Uint64("workerID", s.workerID), @@ -354,6 +352,8 @@ func (s *regionRequestWorker) dispatchResolvedTsEvent(resolvedTsEvent *cdcpb.Res } const resolvedTsStateBatchSize = 1024 + // Avoid allocating a huge states slice when resolvedTsEvent.Regions is large. + // Push resolved-ts events in batches to reduce peak memory usage and improve GC behavior. capHint := min(len(resolvedTsEvent.Regions), resolvedTsStateBatchSize) resolvedStates := make([]*regionFeedState, 0, capHint) flush := func() { @@ -481,12 +481,12 @@ func (s *regionRequestWorker) processRegionSendTask( if err := s.sendRegionRequest(conn, regionReq); err != nil { return err } - regionReq = nil } // Flush pending deregisters before admitting the next region request. - // Sending a region request after a deregister is safe because - // sendRegionRequest re-checks subscription liveness before tracker.Add and Send. + // Admission may still contain stale tasks from a stopped subscription, but + // sendRegionRequest re-checks subscription liveness before tracker.Add/Send, + // so those tasks are dropped locally instead of recreating remote registrations. for { req, ok := s.controlQueue.tryPop() if !ok { @@ -498,6 +498,7 @@ func (s *regionRequestWorker) processRegionSendTask( } // Block for the next request, but wake early when deregisters arrive. + // regionReq above is already consumed and will be replaced by the next pop. var err error regionReq, err = s.admission.pop(ctx, s.controlQueue.ready()) if err != nil { From a8600f47c792efeba1ba13199aa124388f21ffb4 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Tue, 4 Aug 2026 14:55:54 +0800 Subject: [PATCH 42/71] small fix --- logservice/logpuller/region_request_worker.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index db55144362..5f69af6fe3 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -482,7 +482,6 @@ func (s *regionRequestWorker) processRegionSendTask( return err } } - // Flush pending deregisters before admitting the next region request. // Admission may still contain stale tasks from a stopped subscription, but // sendRegionRequest re-checks subscription liveness before tracker.Add/Send, @@ -496,7 +495,6 @@ func (s *regionRequestWorker) processRegionSendTask( return err } } - // Block for the next request, but wake early when deregisters arrive. // regionReq above is already consumed and will be replaced by the next pop. var err error From d0ae6d8f1437ef497426406485b10dac21a75d50 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Tue, 4 Aug 2026 15:30:22 +0800 Subject: [PATCH 43/71] small fix --- .../logpuller/region_request_scheduler.go | 20 +- .../region_request_scheduler_test.go | 89 ++++++++ logservice/logpuller/region_request_worker.go | 2 + .../logpuller/region_request_worker_test.go | 215 ++++++++++++++++++ 4 files changed, 325 insertions(+), 1 deletion(-) diff --git a/logservice/logpuller/region_request_scheduler.go b/logservice/logpuller/region_request_scheduler.go index df1a68756c..964f39e88a 100644 --- a/logservice/logpuller/region_request_scheduler.go +++ b/logservice/logpuller/region_request_scheduler.go @@ -20,6 +20,7 @@ import ( "sync/atomic" "github.com/pingcap/log" + "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/config" "github.com/pingcap/ticdc/pkg/errors" "github.com/pingcap/ticdc/pkg/metrics" @@ -27,6 +28,7 @@ import ( kvclientv2 "github.com/tikv/client-go/v2/kv" "github.com/tikv/client-go/v2/tikv" "go.uber.org/zap" + "go.uber.org/zap/zapcore" "golang.org/x/sync/errgroup" ) @@ -77,6 +79,18 @@ func newRegionRequestScheduler( } func (s *regionRequestScheduler) Submit(region regionInfo) { + if log.GetLevel() <= zapcore.DebugLevel { + log.Debug("cdc region scan task enqueued", + zap.Uint64("subscriptionID", uint64(region.subscribedSpan.subID)), + zap.Int64("tableID", region.subscribedSpan.span.TableID), + zap.Uint64("startTs", region.subscribedSpan.startTs), + zap.Uint64("regionID", region.verID.GetID()), + zap.Uint64("regionEpochVersion", region.verID.GetVer()), + zap.Uint64("regionEpochConfVer", region.verID.GetConfVer()), + zap.String("priority", normalizeScanPriority(region.scanPriority).String()), + zap.String("scanPriority", region.scanPriority.String()), + zap.String("span", common.FormatTableSpan(®ion.span))) + } s.taskQueue.Push(newRegionPriorityTask(region, s.sequence.Add(1))) } @@ -106,7 +120,11 @@ func (s *regionRequestScheduler) Run(ctx context.Context, workerGroup *errgroup. store := s.getOrCreateStore(ctx, workerGroup, region.rpcCtx.Addr) task.regionInfo = region if !store.submit(task) { - return context.Canceled + if ctx.Err() != nil { + return ctx.Err() + } + s.failureHandler.Report(newRegionErrorInfo(region, &storeStreamErr{})) + continue } } } diff --git a/logservice/logpuller/region_request_scheduler_test.go b/logservice/logpuller/region_request_scheduler_test.go index 33f7da4f2a..98d69be0cd 100644 --- a/logservice/logpuller/region_request_scheduler_test.go +++ b/logservice/logpuller/region_request_scheduler_test.go @@ -15,10 +15,19 @@ package logpuller import ( + "context" "testing" + "time" + "github.com/pingcap/ticdc/heartbeatpb" "github.com/pingcap/ticdc/logservice/logpuller/regionlock" + "github.com/pingcap/ticdc/pkg/pdutil" + "github.com/pingcap/ticdc/utils/priorityqueue" + "github.com/pingcap/tidb/pkg/store/mockstore/mockcopr" "github.com/stretchr/testify/require" + "github.com/tikv/client-go/v2/testutils" + "github.com/tikv/client-go/v2/tikv" + "golang.org/x/sync/errgroup" ) func TestRegionRequestSchedulerBroadcastDeregisterUsesWorkerControlQueue(t *testing.T) { @@ -76,3 +85,83 @@ func TestRegionRequestSchedulerInflightCountAggregatesStores(t *testing.T) { require.True(t, req1.abort()) require.True(t, req2.abort()) } + +func TestRegionRequestSchedulerReschedulesRegionWhenStoreSubmitFails(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + _, cluster, pdClient, _ := testutils.NewMockTiKV("", mockcopr.NewCoprRPCHandler()) + pdClient = &mockPDClient{Client: pdClient, versionGen: defaultVersionGen} + defer pdClient.Close() + + const storeAddr = "store-1" + cluster.AddStore(1, storeAddr) + cluster.Bootstrap(11, []uint64{1}, []uint64{2}, 2) + + regionCache := tikv.NewRegionCache(pdClient) + defer regionCache.Close() + + bo := tikv.NewBackoffer(ctx, tikvRequestMaxBackoff) + location, err := regionCache.LocateKey(bo, []byte("a")) + require.NoError(t, err) + + rawSpan := heartbeatpb.TableSpan{ + TableID: 1, + StartKey: []byte("a"), + EndKey: []byte("b"), + } + span := &subscribedSpan{ + subID: SubscriptionID(1), + startTs: 100, + span: rawSpan, + rangeLock: regionlock.NewRangeLock(1, rawSpan.StartKey, rawSpan.EndKey, 100), + priorityPolicy: newScanPriorityPolicy(pdutil.NewClock4Test(), 30*time.Minute), + } + lockRes := span.rangeLock.LockRange( + context.Background(), rawSpan.StartKey, rawSpan.EndKey, location.Region.GetID(), location.Region.GetVer()) + require.Equal(t, regionlock.LockRangeStatusSuccess, lockRes.Status) + + admission := newRegionAdmissionController(1, 1) + admission.close() + store := ®ionRequestStore{workers: []*regionRequestWorker{{admission: admission}}} + + handler := newRegionFailureHandler(nil, func(*subscribedSpan) {}, nil, nil) + scheduler := ®ionRequestScheduler{ + upstream: &upstreamHandle{ + pd: pdClient, + regionCache: regionCache, + }, + failureHandler: handler, + taskQueue: priorityqueue.New[*regionPriorityTask](), + } + scheduler.stores.Store(storeAddr, store) + + region := newRegionInfo(location.Region, rawSpan, nil, span, false) + region.lockedRangeState = lockRes.LockedRangeState + scheduler.taskQueue.Push(newRegionPriorityTask(region, 1)) + + var workerGroup errgroup.Group + errCh := make(chan error, 1) + go func() { + errCh <- scheduler.Run(ctx, &workerGroup) + }() + + require.Eventually(t, func() bool { + return errCacheLen(handler) == 1 + }, time.Second, 20*time.Millisecond) + + select { + case err := <-errCh: + t.Fatalf("scheduler exited unexpectedly: %v", err) + default: + } + + batch := handler.cache.popBatch(1) + require.Len(t, batch, 1) + require.Equal(t, region.verID, batch[0].verID) + var streamErr *storeStreamErr + require.ErrorAs(t, batch[0].err, &streamErr) + + cancel() + require.ErrorIs(t, <-errCh, context.Canceled) +} diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index 5f69af6fe3..13f9c8d005 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -305,6 +305,7 @@ func (s *regionRequestWorker) dispatchRegionChangeEvents(events []*cdcpb.Event) } regionEvent.entries = eventData case *cdcpb.Event_Admin_: + // ignore continue case *cdcpb.Event_Error: log.Debug("region request worker receives a region error", @@ -324,6 +325,7 @@ func (s *regionRequestWorker) dispatchRegionChangeEvents(events []*cdcpb.Event) } else { switch event.Event.(type) { case *cdcpb.Event_Error: + // it is normal to receive region error after deregister a subscription log.Debug("region request worker receives an error for a stale region, ignore it", zap.Uint64("workerID", s.workerID), zap.Uint64("subscriptionID", uint64(subscriptionID)), diff --git a/logservice/logpuller/region_request_worker_test.go b/logservice/logpuller/region_request_worker_test.go index 2c27325319..8735f3021c 100644 --- a/logservice/logpuller/region_request_worker_test.go +++ b/logservice/logpuller/region_request_worker_test.go @@ -22,10 +22,14 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/cdcpb" "github.com/pingcap/kvproto/pkg/metapb" + "github.com/pingcap/ticdc/heartbeatpb" "github.com/pingcap/ticdc/logservice/logpuller/regionlock" + "github.com/pingcap/ticdc/pkg/security" "github.com/pingcap/ticdc/utils/dynstream" + "github.com/pingcap/tidb/pkg/store/mockstore/mockcopr" "github.com/stretchr/testify/require" "github.com/tikv/client-go/v2/oracle" + "github.com/tikv/client-go/v2/testutils" "github.com/tikv/client-go/v2/tikv" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -64,6 +68,38 @@ func prepareRegionForSendTest(region regionInfo) regionInfo { return region } +func TestCreateRegionRequestScanPriority(t *testing.T) { + for _, tc := range []struct { + name string + priority cdcpb.ScanPriority + expected cdcpb.ScanPriority + }{ + { + name: "high", + priority: cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, + expected: cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, + }, + { + name: "low", + priority: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, + expected: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, + }, + { + name: "unknown defaults to low", + priority: cdcpb.ScanPriority_SCAN_PRIORITY_UNKNOWN, + expected: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, + }, + } { + t.Run(tc.name, func(t *testing.T) { + region := prepareRegionForSendTest(createTestRegionInfo(1, 1)) + region.scanPriority = tc.priority + + req := createRegionRequest(1, region) + require.Equal(t, tc.expected, req.GetScanPriority()) + }) + } +} + func admitRegionRequest( t *testing.T, controller *regionAdmissionController, @@ -77,6 +113,99 @@ func admitRegionRequest( return req } +type pushedRegionEvent struct { + subscriptionID SubscriptionID + event regionEvent +} + +type recordingRegionEventDynamicStream struct { + events chan pushedRegionEvent +} + +func (m *recordingRegionEventDynamicStream) Start() {} + +func (m *recordingRegionEventDynamicStream) Close() {} + +func (m *recordingRegionEventDynamicStream) Push(path SubscriptionID, event regionEvent) { + m.events <- pushedRegionEvent{subscriptionID: path, event: event} +} + +func (m *recordingRegionEventDynamicStream) Wake(SubscriptionID) {} + +func (m *recordingRegionEventDynamicStream) Feedback() <-chan dynstream.Feedback[int, SubscriptionID, *subscribedSpan] { + return nil +} + +func (m *recordingRegionEventDynamicStream) AddPath(SubscriptionID, *subscribedSpan, ...dynstream.AreaSettings) error { + return nil +} + +func (m *recordingRegionEventDynamicStream) RemovePath(SubscriptionID) error { + return nil +} + +func (m *recordingRegionEventDynamicStream) Release(SubscriptionID) {} + +func (m *recordingRegionEventDynamicStream) SetAreaSettings(int, dynstream.AreaSettings) {} + +func (m *recordingRegionEventDynamicStream) GetMetrics() dynstream.Metrics[int, SubscriptionID] { + return dynstream.Metrics[int, SubscriptionID]{} +} + +func createFailureRecoveryTestRegion(t *testing.T, subID SubscriptionID, regionID uint64) regionInfo { + t.Helper() + + fullSpan := heartbeatpb.TableSpan{ + TableID: 1, + StartKey: []byte("a"), + EndKey: []byte("z"), + } + subSpan := &subscribedSpan{ + subID: subID, + startTs: 100, + span: fullSpan, + rangeLock: regionlock.NewRangeLock(1, fullSpan.StartKey, fullSpan.EndKey, 100), + } + regionSpan := heartbeatpb.TableSpan{ + TableID: 1, + StartKey: []byte("a"), + EndKey: []byte("m"), + } + locked := subSpan.rangeLock.LockRange(context.Background(), regionSpan.StartKey, regionSpan.EndKey, regionID, 1) + require.Equal(t, regionlock.LockRangeStatusSuccess, locked.Status) + other := subSpan.rangeLock.LockRange(context.Background(), []byte("m"), []byte("z"), regionID+1000, 1) + require.Equal(t, regionlock.LockRangeStatusSuccess, other.Status) + + region := newRegionInfo(tikv.NewRegionVerID(regionID, 1, 1), regionSpan, nil, subSpan, false) + region.lockedRangeState = locked.LockedRangeState + region.lockedRangeState.ResolvedTs.Store(100) + return region +} + +func newFailureRecoveryTestPDClient(t *testing.T) *mockPDClient { + t.Helper() + + _, _, pdClient, _ := testutils.NewMockTiKV("", mockcopr.NewCoprRPCHandler()) + return &mockPDClient{Client: pdClient, versionGen: defaultVersionGen} +} + +func snapshotErrCacheRegionIDs(handler *regionFailureHandler) []uint64 { + handler.cache.Lock() + defer handler.cache.Unlock() + + regionIDs := make([]uint64, 0, len(handler.cache.cache)) + for _, errInfo := range handler.cache.cache { + regionIDs = append(regionIDs, errInfo.verID.GetID()) + } + return regionIDs +} + +func errCacheLen(handler *regionFailureHandler) int { + handler.cache.Lock() + defer handler.cache.Unlock() + return len(handler.cache.cache) +} + func TestRegionRequestWorkerIgnoresDuplicateActiveRegion(t *testing.T) { admission := newRegionAdmissionController(10, 1) worker := ®ionRequestWorker{ @@ -335,6 +464,92 @@ func TestStoppedStateRemovesSentRequest(t *testing.T) { require.Equal(t, 0, admission.stats().inflight) } +func TestRunStreamFailurePushesTrackedRegionToEventSink(t *testing.T) { + pdClient := newFailureRecoveryTestPDClient(t) + defer pdClient.Close() + + ds := &recordingRegionEventDynamicStream{events: make(chan pushedRegionEvent, 4)} + handler := newRegionFailureHandler(nil, func(*subscribedSpan) {}, nil, nil) + worker := ®ionRequestWorker{ + upstream: &upstreamHandle{pd: pdClient, credential: &security.Credential{}}, + eventSink: ®ionEventSink{ds: ds}, + failureHandler: handler, + admission: newRegionAdmissionController(10, 1), + controlQueue: newControlQueue(), + tracker: newRegionTracker(), + storeAddr: "127.0.0.1:1", + } + + sentRegion := createFailureRecoveryTestRegion(t, 1, 1) + sentReq := admitRegionRequest(t, worker.admission, sentRegion) + sentState := newRegionFeedState(sentRegion, uint64(sentRegion.subscribedSpan.subID), worker, sentReq) + require.True(t, worker.tracker.Add(sentRegion.subscribedSpan.subID, sentRegion.verID.GetID(), sentState)) + + firstRegion := createFailureRecoveryTestRegion(t, 2, 2) + submitRegionForAdmission(t, worker.admission, firstRegion, 100) + + ctx, cancel := context.WithCancel(context.Background()) + runErrCh := make(chan error, 1) + go func() { + runErrCh <- worker.Run(ctx) + }() + + var pushed pushedRegionEvent + select { + case pushed = <-ds.events: + case <-time.After(5 * time.Second): + t.Fatal("worker did not push tracked region after stream failure") + } + + require.Equal(t, SubscriptionID(1), pushed.subscriptionID) + require.Len(t, pushed.event.states, 1) + require.Same(t, sentState, pushed.event.states[0]) + require.Equal(t, 0, worker.admission.stats().inflight) + + var streamErr *storeStreamErr + require.ErrorAs(t, sentState.takeError(), &streamErr) + + cancel() + require.ErrorIs(t, <-runErrCh, context.Canceled) +} + +func TestRunStreamFailureReportsPendingRegionsToFailureHandler(t *testing.T) { + pdClient := newFailureRecoveryTestPDClient(t) + defer pdClient.Close() + + handler := newRegionFailureHandler(nil, func(*subscribedSpan) {}, nil, nil) + worker := ®ionRequestWorker{ + upstream: &upstreamHandle{pd: pdClient, credential: &security.Credential{}}, + eventSink: ®ionEventSink{ds: &mockDynamicStream{}}, + failureHandler: handler, + admission: newRegionAdmissionController(10, 1), + controlQueue: newControlQueue(), + tracker: newRegionTracker(), + storeAddr: "127.0.0.1:1", + } + + firstRegion := createFailureRecoveryTestRegion(t, 1, 1) + pendingRegion := createFailureRecoveryTestRegion(t, 2, 2) + submitRegionForAdmission(t, worker.admission, firstRegion, 100) + submitRegionForAdmission(t, worker.admission, pendingRegion, 100) + + ctx, cancel := context.WithCancel(context.Background()) + runErrCh := make(chan error, 1) + go func() { + runErrCh <- worker.Run(ctx) + }() + + require.Eventually(t, func() bool { + return errCacheLen(handler) == 2 + }, 5*time.Second, 10*time.Millisecond) + require.ElementsMatch(t, []uint64{1, 2}, snapshotErrCacheRegionIDs(handler)) + require.Equal(t, 0, worker.admission.stats().pending) + require.Equal(t, 0, worker.admission.stats().inflight) + + cancel() + require.ErrorIs(t, <-runErrCh, context.Canceled) +} + func TestProcessRegionSendTaskSendFailureCleansSentRequest(t *testing.T) { admission := newRegionAdmissionController(10, 1) worker := ®ionRequestWorker{ From c27b62fcbdbd4ee89118b767fd89629f0bf0cef7 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Tue, 4 Aug 2026 15:48:12 +0800 Subject: [PATCH 44/71] fix --- .../logpuller/region_request_scheduler.go | 23 ++++-- .../region_request_scheduler_test.go | 80 +++++++++++++++++++ 2 files changed, 95 insertions(+), 8 deletions(-) diff --git a/logservice/logpuller/region_request_scheduler.go b/logservice/logpuller/region_request_scheduler.go index 964f39e88a..633e0e6374 100644 --- a/logservice/logpuller/region_request_scheduler.go +++ b/logservice/logpuller/region_request_scheduler.go @@ -95,7 +95,13 @@ func (s *regionRequestScheduler) Submit(region regionInfo) { } func (s *regionRequestScheduler) Run(ctx context.Context, workerGroup *errgroup.Group) error { - defer s.closeStores() + defer func() { + s.stores.Range(func(_, value any) bool { + value.(*regionRequestStore).close() + return true + }) + }() + for { select { case <-ctx.Done(): @@ -116,6 +122,10 @@ func (s *regionRequestScheduler) Run(ctx context.Context, workerGroup *errgroup. s.failureHandler.Report(newRegionErrorInfo(region, err)) continue } + if region.subscribedSpan.stopped.Load() { + s.failureHandler.Report(newRegionErrorInfo(region, &requestCancelledErr{})) + continue + } store := s.getOrCreateStore(ctx, workerGroup, region.rpcCtx.Addr) task.regionInfo = region @@ -126,6 +136,10 @@ func (s *regionRequestScheduler) Run(ctx context.Context, workerGroup *errgroup. s.failureHandler.Report(newRegionErrorInfo(region, &storeStreamErr{})) continue } + log.Debug("subscription client will request a region", + zap.Uint64("subscriptionID", uint64(region.subscribedSpan.subID)), + zap.Uint64("regionID", region.verID.GetID()), + zap.String("addr", region.rpcCtx.Addr)) } } @@ -201,10 +215,3 @@ func (s *regionRequestScheduler) UpdateMetrics() { func (s *regionRequestScheduler) Close() { s.taskQueue.Close() } - -func (s *regionRequestScheduler) closeStores() { - s.stores.Range(func(_, value any) bool { - value.(*regionRequestStore).close() - return true - }) -} diff --git a/logservice/logpuller/region_request_scheduler_test.go b/logservice/logpuller/region_request_scheduler_test.go index 98d69be0cd..59cc940651 100644 --- a/logservice/logpuller/region_request_scheduler_test.go +++ b/logservice/logpuller/region_request_scheduler_test.go @@ -165,3 +165,83 @@ func TestRegionRequestSchedulerReschedulesRegionWhenStoreSubmitFails(t *testing. cancel() require.ErrorIs(t, <-errCh, context.Canceled) } + +func TestRegionRequestSchedulerSkipsStoppedSubscriptionBeforeCreatingStore(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + _, cluster, pdClient, _ := testutils.NewMockTiKV("", mockcopr.NewCoprRPCHandler()) + pdClient = &mockPDClient{Client: pdClient, versionGen: defaultVersionGen} + defer pdClient.Close() + + const storeAddr = "store-1" + cluster.AddStore(1, storeAddr) + cluster.Bootstrap(11, []uint64{1}, []uint64{2}, 2) + + regionCache := tikv.NewRegionCache(pdClient) + defer regionCache.Close() + + bo := tikv.NewBackoffer(ctx, tikvRequestMaxBackoff) + location, err := regionCache.LocateKey(bo, []byte("a")) + require.NoError(t, err) + + rawSpan := heartbeatpb.TableSpan{ + TableID: 1, + StartKey: []byte("a"), + EndKey: []byte("b"), + } + span := &subscribedSpan{ + subID: SubscriptionID(1), + startTs: 100, + span: rawSpan, + rangeLock: regionlock.NewRangeLock(1, rawSpan.StartKey, rawSpan.EndKey, 100), + priorityPolicy: newScanPriorityPolicy(pdutil.NewClock4Test(), 30*time.Minute), + } + lockRes := span.rangeLock.LockRange( + context.Background(), rawSpan.StartKey, rawSpan.EndKey, location.Region.GetID(), location.Region.GetVer()) + require.Equal(t, regionlock.LockRangeStatusSuccess, lockRes.Status) + span.stopped.Store(true) + require.False(t, span.rangeLock.Stop()) + + drainedCh := make(chan *subscribedSpan, 1) + handler := newRegionFailureHandler(nil, func(rt *subscribedSpan) { + drainedCh <- rt + }, nil, nil) + scheduler := ®ionRequestScheduler{ + upstream: &upstreamHandle{ + pd: pdClient, + regionCache: regionCache, + }, + failureHandler: handler, + taskQueue: priorityqueue.New[*regionPriorityTask](), + } + + region := newRegionInfo(location.Region, rawSpan, nil, span, false) + region.lockedRangeState = lockRes.LockedRangeState + scheduler.taskQueue.Push(newRegionPriorityTask(region, 1)) + + var workerGroup errgroup.Group + errCh := make(chan error, 1) + go func() { + errCh <- scheduler.Run(ctx, &workerGroup) + }() + + select { + case drained := <-drainedCh: + require.Same(t, span, drained) + case <-time.After(time.Second): + t.Fatal("stopped subscription was not drained") + } + + _, ok := scheduler.stores.Load(storeAddr) + require.False(t, ok) + + select { + case err := <-errCh: + t.Fatalf("scheduler exited unexpectedly: %v", err) + default: + } + + cancel() + require.ErrorIs(t, <-errCh, context.Canceled) +} From 7e3df1d9c885fad6990c6e8bd746104e105ad6a4 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Tue, 4 Aug 2026 15:49:37 +0800 Subject: [PATCH 45/71] small fix --- logservice/logpuller/region_request_scheduler.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/logservice/logpuller/region_request_scheduler.go b/logservice/logpuller/region_request_scheduler.go index 633e0e6374..463cce32b8 100644 --- a/logservice/logpuller/region_request_scheduler.go +++ b/logservice/logpuller/region_request_scheduler.go @@ -136,10 +136,12 @@ func (s *regionRequestScheduler) Run(ctx context.Context, workerGroup *errgroup. s.failureHandler.Report(newRegionErrorInfo(region, &storeStreamErr{})) continue } - log.Debug("subscription client will request a region", - zap.Uint64("subscriptionID", uint64(region.subscribedSpan.subID)), - zap.Uint64("regionID", region.verID.GetID()), - zap.String("addr", region.rpcCtx.Addr)) + if log.GetLevel() <= zapcore.DebugLevel { + log.Debug("subscription client will request a region", + zap.Uint64("subscriptionID", uint64(region.subscribedSpan.subID)), + zap.Uint64("regionID", region.verID.GetID()), + zap.String("addr", region.rpcCtx.Addr)) + } } } From cfb0623b5a2f960c790b24b8f54a64527ed6b981 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Fri, 17 Jul 2026 09:54:45 +0800 Subject: [PATCH 46/71] logpuller: extract region request scheduler from subscription client --- .../logpuller/region_request_scheduler.go | 123 ++++++++++++++++++ logservice/logpuller/requested_store.go | 83 ++++++++++++ logservice/logpuller/subscription_client.go | 68 ++++++++++ .../logpuller/subscription_client_test.go | 62 +++++++++ 4 files changed, 336 insertions(+) create mode 100644 logservice/logpuller/requested_store.go diff --git a/logservice/logpuller/region_request_scheduler.go b/logservice/logpuller/region_request_scheduler.go index 463cce32b8..33a01f726b 100644 --- a/logservice/logpuller/region_request_scheduler.go +++ b/logservice/logpuller/region_request_scheduler.go @@ -20,14 +20,20 @@ import ( "sync/atomic" "github.com/pingcap/log" +<<<<<<< HEAD "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/config" "github.com/pingcap/ticdc/pkg/errors" "github.com/pingcap/ticdc/pkg/metrics" +======= + "github.com/pingcap/ticdc/pkg/config" + "github.com/pingcap/ticdc/pkg/errors" +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) "github.com/pingcap/ticdc/utils/priorityqueue" kvclientv2 "github.com/tikv/client-go/v2/kv" "github.com/tikv/client-go/v2/tikv" "go.uber.org/zap" +<<<<<<< HEAD "go.uber.org/zap/zapcore" "golang.org/x/sync/errgroup" ) @@ -49,6 +55,24 @@ type regionRequestScheduler struct { sequence atomic.Uint64 // stores maps TiKV addresses to regionRequestStore. Stores are created only // by Run, but are also read by metrics and deregistration goroutines. +======= + "golang.org/x/sync/errgroup" +) + +// regionRequestScheduler routes locked Region requests through the global +// priority queue to a worker connected to the Region's TiKV store. Range +// resolution and retry policy remain owned by subscriptionClient and +// regionFailureHandler respectively. +type regionRequestScheduler struct { + client *subscriptionClient + + // taskQueue orders all Regions before they are assigned to a TiKV store. + taskQueue *priorityqueue.PriorityQueue[*regionPriorityTask] + // sequence is the FIFO tie-breaker for Regions in the same priority class. + sequence atomic.Uint64 + // stores maps TiKV addresses to requestedStore. Stores are created only by + // run, but are also read by metrics and deregistration goroutines. +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) stores sync.Map // workerCount is the configured number of request workers per store. @@ -59,6 +83,7 @@ type regionRequestScheduler struct { maxWindowMultiplier int } +<<<<<<< HEAD func newRegionRequestScheduler( upstream *upstreamHandle, eventSink *regionEventSink, @@ -71,6 +96,17 @@ func newRegionRequestScheduler( upstream: upstream, eventSink: eventSink, failureHandler: failureHandler, +======= +func newRegionRequestScheduler(client *subscriptionClient) *regionRequestScheduler { + pullerConfig := config.GetGlobalServerConfig().Debug.Puller + workerCount := int(client.config.RegionRequestWorkerPerStore) + if workerCount <= 0 { + workerCount = 1 + } + workerWindow := (pullerConfig.PendingRegionRequestQueueSize + workerCount - 1) / workerCount + return ®ionRequestScheduler{ + client: client, +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) taskQueue: priorityqueue.New[*regionPriorityTask](), workerCount: workerCount, workerWindow: workerWindow, @@ -78,6 +114,7 @@ func newRegionRequestScheduler( } } +<<<<<<< HEAD func (s *regionRequestScheduler) Submit(region regionInfo) { if log.GetLevel() <= zapcore.DebugLevel { log.Debug("cdc region scan task enqueued", @@ -102,6 +139,15 @@ func (s *regionRequestScheduler) Run(ctx context.Context, workerGroup *errgroup. }) }() +======= +func (s *regionRequestScheduler) submit(region regionInfo) { + s.taskQueue.Push(NewRegionPriorityTask( + region, s.client.pdClock.CurrentTS(), s.sequence.Add(1))) +} + +func (s *regionRequestScheduler) run(ctx context.Context, group *errgroup.Group) error { + defer s.closeStores() +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) for { select { case <-ctx.Done(): @@ -117,6 +163,7 @@ func (s *regionRequestScheduler) Run(ctx context.Context, workerGroup *errgroup. return err } +<<<<<<< HEAD region, err := s.attachRPCContext(ctx, task.regionInfo) if err != nil { s.failureHandler.Report(newRegionErrorInfo(region, err)) @@ -142,12 +189,30 @@ func (s *regionRequestScheduler) Run(ctx context.Context, workerGroup *errgroup. zap.Uint64("regionID", region.verID.GetID()), zap.String("addr", region.rpcCtx.Addr)) } +======= + region, ok := s.attachRPCContext(ctx, task.GetRegionInfo()) + if !ok { + continue + } + + store := s.getOrCreateStore(ctx, group, region.rpcCtx.Addr) + task.updateRegion(region, s.client.pdClock.CurrentTS()) + if !store.submit(task) { + return context.Canceled + } + + log.Debug("subscription client will request a region", + zap.Uint64("subscriptionID", uint64(region.subscribedSpan.subID)), + zap.Uint64("regionID", region.verID.GetID()), + zap.String("addr", store.storeAddr)) +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) } } func (s *regionRequestScheduler) attachRPCContext( ctx context.Context, region regionInfo, +<<<<<<< HEAD ) (regionInfo, error) { bo := tikv.NewBackoffer(ctx, tikvRequestMaxBackoff) rpcCtx, err := s.upstream.regionCache.GetTiKVRPCContext( @@ -158,15 +223,33 @@ func (s *regionRequestScheduler) attachRPCContext( } if err != nil { log.Debug("region request scheduler failed to get RPC context", +======= +) (regionInfo, bool) { + bo := tikv.NewBackoffer(ctx, tikvRequestMaxBackoff) + rpcCtx, err := s.client.regionCache.GetTiKVRPCContext( + bo, region.verID, kvclientv2.ReplicaReadLeader, 0) + if rpcCtx != nil { + region.rpcCtx = rpcCtx + return region, true + } + if err != nil { + log.Debug("subscription client get rpc context fail", +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) zap.Uint64("subscriptionID", uint64(region.subscribedSpan.subID)), zap.Uint64("regionID", region.verID.GetID()), zap.Error(err)) } +<<<<<<< HEAD return region, &rpcCtxUnavailableErr{verID: region.verID} +======= + s.client.onRegionFail(newRegionErrorInfo(region, &rpcCtxUnavailableErr{verID: region.verID})) + return region, false +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) } func (s *regionRequestScheduler) getOrCreateStore( ctx context.Context, +<<<<<<< HEAD workerGroup *errgroup.Group, storeAddr string, ) *regionRequestStore { @@ -191,11 +274,34 @@ func (s *regionRequestScheduler) getOrCreateStore( } func (s *regionRequestScheduler) BroadcastDeregister( +======= + group *errgroup.Group, + storeAddr string, +) *requestedStore { + if value, ok := s.stores.Load(storeAddr); ok { + return value.(*requestedStore) + } + + store := newRequestedStore( + s.client, storeAddr, s.workerCount, s.workerWindow, s.maxWindowMultiplier) + // run is the only writer. Publish the store after its immutable worker list + // is complete, then start its workers. + s.stores.Store(storeAddr, store) + store.run(ctx, group) + return store +} + +func (s *regionRequestScheduler) broadcastDeregister( +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) subID SubscriptionID, filterLoop bool, ) { s.stores.Range(func(_, value any) bool { +<<<<<<< HEAD value.(*regionRequestStore).broadcastDeregister(subID, filterLoop) +======= + value.(*requestedStore).broadcastDeregister(subID, filterLoop) +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) return true }) } @@ -203,12 +309,17 @@ func (s *regionRequestScheduler) BroadcastDeregister( func (s *regionRequestScheduler) inflightCount() int { count := 0 s.stores.Range(func(_, value any) bool { +<<<<<<< HEAD count += value.(*regionRequestStore).inflightCount() +======= + count += value.(*requestedStore).inflightCount() +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) return true }) return count } +<<<<<<< HEAD func (s *regionRequestScheduler) UpdateMetrics() { metrics.SubscriptionClientRequestedRegionCount.WithLabelValues("inflight"). Set(float64(s.inflightCount())) @@ -217,3 +328,15 @@ func (s *regionRequestScheduler) UpdateMetrics() { func (s *regionRequestScheduler) Close() { s.taskQueue.Close() } +======= +func (s *regionRequestScheduler) close() { + s.taskQueue.Close() +} + +func (s *regionRequestScheduler) closeStores() { + s.stores.Range(func(_, value any) bool { + value.(*requestedStore).close() + return true + }) +} +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) diff --git a/logservice/logpuller/requested_store.go b/logservice/logpuller/requested_store.go new file mode 100644 index 0000000000..f98b59b679 --- /dev/null +++ b/logservice/logpuller/requested_store.go @@ -0,0 +1,83 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package logpuller + +import ( + "context" + "sync/atomic" + + "golang.org/x/sync/errgroup" +) + +// requestedStore owns the request workers connected to one TiKV store. The +// worker slice is complete before the store is published and is immutable +// afterwards, so task submission only needs an atomic round-robin counter. +type requestedStore struct { + storeAddr string + workers []*regionRequestWorker + nextWorker atomic.Uint64 +} + +func newRequestedStore( + client *subscriptionClient, + storeAddr string, + workerCount int, + workerWindow int, + maxWindowMultiplier int, +) *requestedStore { + store := &requestedStore{ + storeAddr: storeAddr, + workers: make([]*regionRequestWorker, 0, workerCount), + } + for i := 0; i < workerCount; i++ { + store.workers = append(store.workers, newRegionRequestWorker( + client, store, workerWindow, maxWindowMultiplier)) + } + return store +} + +func (s *requestedStore) run(ctx context.Context, group *errgroup.Group) { + for _, worker := range s.workers { + group.Go(func() error { return worker.Run(ctx) }) + } +} + +func (s *requestedStore) submit(task *regionPriorityTask) bool { + if len(s.workers) == 0 { + return false + } + index := (s.nextWorker.Add(1) - 1) % uint64(len(s.workers)) + return s.workers[index].admission.submit(task) +} + +func (s *requestedStore) broadcastDeregister(subID SubscriptionID, filterLoop bool) { + for _, worker := range s.workers { + worker.controlQueue.push(deregisterRequest{subID: subID, filterLoop: filterLoop}) + } +} + +func (s *requestedStore) close() { + for _, worker := range s.workers { + worker.admission.close() + } +} + +func (s *requestedStore) inflightCount() int { + count := 0 + for _, worker := range s.workers { + count += worker.admission.stats().inflight + } + return count +} diff --git a/logservice/logpuller/subscription_client.go b/logservice/logpuller/subscription_client.go index 8fa9d17238..7614ab01dc 100644 --- a/logservice/logpuller/subscription_client.go +++ b/logservice/logpuller/subscription_client.go @@ -26,12 +26,19 @@ import ( "github.com/pingcap/ticdc/logservice/txnutil" "github.com/pingcap/ticdc/pkg/common" appcontext "github.com/pingcap/ticdc/pkg/common/context" +<<<<<<< HEAD "github.com/pingcap/ticdc/pkg/config" +======= +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) "github.com/pingcap/ticdc/pkg/metrics" "github.com/pingcap/ticdc/pkg/pdutil" "github.com/pingcap/ticdc/pkg/security" "github.com/pingcap/ticdc/pkg/spanz" "github.com/pingcap/ticdc/pkg/util" +<<<<<<< HEAD +======= + "github.com/prometheus/client_golang/prometheus" +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) "github.com/tikv/client-go/v2/tikv" pd "github.com/tikv/pd/client" "go.uber.org/zap" @@ -129,13 +136,23 @@ type subscriptionClient struct { lockResolver txnutil.LockResolver +<<<<<<< HEAD +======= + // the credential to connect tikv + credential *security.Credential + +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) // failureHandler handles failed regions and owns reschedule/retry decisions. failureHandler *regionFailureHandler // eventSink delivers region events and owns dynstream interaction. eventSink *regionEventSink // spanRegistry tracks subscribed spans and owns span-level background tasks. spanRegistry *spanRegistry +<<<<<<< HEAD // regionScheduler assigns locked region requests to per-store workers. +======= + // regionScheduler assigns locked Region requests to per-store workers. +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) regionScheduler *regionRequestScheduler // rangeTaskCh is used to receive range tasks. @@ -154,12 +171,20 @@ func NewSubscriptionClient( credential *security.Credential, ) SubscriptionClient { subClient := &subscriptionClient{ +<<<<<<< HEAD upstream: &upstreamHandle{ pd: pd, regionCache: appcontext.GetService[*tikv.RegionCache](appcontext.RegionCache), pdClock: appcontext.GetService[pdutil.Clock](appcontext.DefaultPDClock), credential: credential, }, +======= + config: config, + + pd: pd, + regionCache: appcontext.GetService[*tikv.RegionCache](appcontext.RegionCache), + pdClock: appcontext.GetService[pdutil.Clock](appcontext.DefaultPDClock), +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) lockResolver: lockResolver, rangeTaskCh: make(chan rangeTask, 1024), @@ -167,6 +192,7 @@ func NewSubscriptionClient( resolveLockRateLimiter: newResolveLockRateLimiter(), } subClient.ctx, subClient.cancel = context.WithCancel(context.Background()) +<<<<<<< HEAD subClient.failureHandler = newRegionFailureHandler( subClient.upstream.regionCache, subClient.onTableDrained, @@ -180,6 +206,14 @@ func NewSubscriptionClient( subClient.eventSink, subClient.failureHandler, ) +======= + subClient.failureHandler = newRegionFailureHandler(subClient) + subClient.eventSink = newRegionEventSink(subClient.ctx, subClient.failureHandler) + subClient.spanRegistry = newSpanRegistry(subClient.pd, subClient.pdClock) + subClient.regionScheduler = newRegionRequestScheduler(subClient) + + subClient.initMetrics() +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) return subClient } @@ -200,7 +234,13 @@ func (s *subscriptionClient) updateMetrics(ctx context.Context) error { case <-ctx.Done(): return ctx.Err() case <-ticker.C: +<<<<<<< HEAD s.regionScheduler.UpdateMetrics() +======= + pendingRegionReqCount := s.regionScheduler.inflightCount() + + metrics.SubscriptionClientRequestedRegionCount.WithLabelValues("pending").Set(float64(pendingRegionReqCount)) +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) s.eventSink.UpdateMetrics() s.spanRegistry.UpdateMetrics() } @@ -282,8 +322,12 @@ func (s *subscriptionClient) Run(ctx context.Context) error { // The goroutines are listed by data flow; errgroup does not guarantee their // actual startup order. g.Go(func() error { return s.handleRangeTasks(ctx) }) +<<<<<<< HEAD g.Go(func() error { return s.regionScheduler.Run(ctx, g) }) g.Go(func() error { return s.eventSink.Run(ctx) }) +======= + g.Go(func() error { return s.regionScheduler.run(ctx, g) }) +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) g.Go(func() error { return s.failureHandler.Run(ctx) }) g.Go(func() error { return s.spanRegistry.Run(ctx) }) g.Go(func() error { return s.handleResolveLockTasks(ctx) }) @@ -298,7 +342,11 @@ func (s *subscriptionClient) Run(ctx context.Context) error { func (s *subscriptionClient) Close(ctx context.Context) error { s.cancel() s.eventSink.Close() +<<<<<<< HEAD s.regionScheduler.Close() +======= + s.regionScheduler.close() +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) return nil } @@ -309,7 +357,11 @@ func (s *subscriptionClient) setTableStopped(rt *subscribedSpan) { // Set stopped to true so we can stop handling region events from the table, // then notify every existing worker to deregister the subscription. if rt.stopped.CompareAndSwap(false, true) { +<<<<<<< HEAD s.regionScheduler.BroadcastDeregister(rt.subID, rt.filterLoop) +======= + s.regionScheduler.broadcastDeregister(rt.subID, rt.filterLoop) +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) if rt.rangeLock.Stop() { s.onTableDrained(rt) } @@ -329,6 +381,14 @@ func (s *subscriptionClient) onTableDrained(rt *subscribedSpan) { s.spanRegistry.Remove(rt.subID) } +<<<<<<< HEAD +======= +// Note: don't block the caller, otherwise there may be deadlock +func (s *subscriptionClient) onRegionFail(errInfo regionErrorInfo) { + s.failureHandler.Report(errInfo) +} + +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) func (s *subscriptionClient) handleRangeTasks(ctx context.Context) error { g, ctx := errgroup.WithContext(ctx) // Limit the concurrent number of goroutines to convert range tasks to region tasks. @@ -433,7 +493,11 @@ func (s *subscriptionClient) divideSpanAndScheduleRegionRequests( } } +<<<<<<< HEAD // scheduleRegionRequest locks the region's range before submitting it to the +======= +// scheduleRegionRequest locks the Region's range before submitting it to the +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) // request scheduler. func (s *subscriptionClient) scheduleRegionRequest(ctx context.Context, region regionInfo) { lockRangeResult := region.subscribedSpan.rangeLock.LockRange( @@ -446,12 +510,16 @@ func (s *subscriptionClient) scheduleRegionRequest(ctx context.Context, region r switch lockRangeResult.Status { case regionlock.LockRangeStatusSuccess: region.lockedRangeState = lockRangeResult.LockedRangeState +<<<<<<< HEAD region.scanPriority = region.subscribedSpan.priorityPolicy.resolve( region.scanPriority, region.resolvedTs(), s.upstream.pdClock.CurrentTime(), ) s.regionScheduler.Submit(region) +======= + s.regionScheduler.submit(region) +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) case regionlock.LockRangeStatusStale: for _, r := range lockRangeResult.RetryRanges { s.scheduleRangeRequest(ctx, rangeTask{ diff --git a/logservice/logpuller/subscription_client_test.go b/logservice/logpuller/subscription_client_test.go index 99600666ea..16121f7b5d 100644 --- a/logservice/logpuller/subscription_client_test.go +++ b/logservice/logpuller/subscription_client_test.go @@ -334,11 +334,18 @@ func TestStopTaskUsesSubscribedSpanFilterLoop(t *testing.T) { res := span.rangeLock.LockRange(context.Background(), rawSpan.StartKey, rawSpan.EndKey, 1, 1) require.Equal(t, regionlock.LockRangeStatusSuccess, res.Status) +<<<<<<< HEAD const storeAddr = "store-1" worker := ®ionRequestWorker{storeAddr: storeAddr, controlQueue: newControlQueue()} store := ®ionRequestStore{workers: []*regionRequestWorker{worker}} client.regionScheduler = ®ionRequestScheduler{} client.regionScheduler.stores.Store(storeAddr, store) +======= + worker := ®ionRequestWorker{controlQueue: newControlQueue()} + store := &requestedStore{storeAddr: "store-1", workers: []*regionRequestWorker{worker}} + client.regionScheduler = ®ionRequestScheduler{client: client} + client.regionScheduler.stores.Store(store.storeAddr, store) +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) client.setTableStopped(span) @@ -430,6 +437,10 @@ func TestRegionEventSinkPushUnblocksOnClientClose(t *testing.T) { sink.cond = sync.NewCond(&sink.mu) client := &subscriptionClient{eventSink: sink} client.regionScheduler = ®ionRequestScheduler{ +<<<<<<< HEAD +======= + client: client, +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) taskQueue: priorityqueue.New[*regionPriorityTask](), } client.ctx, client.cancel = context.WithCancel(context.Background()) @@ -457,6 +468,57 @@ func TestRegionEventSinkPushUnblocksOnClientClose(t *testing.T) { } } +<<<<<<< HEAD +======= +func TestBroadcastDeregisterUsesWorkerControlQueue(t *testing.T) { + client := &subscriptionClient{} + scheduler := ®ionRequestScheduler{client: client} + admission := newRegionAdmissionController(1, 1) + + worker := ®ionRequestWorker{ + admission: admission, + controlQueue: newControlQueue(), + } + store := &requestedStore{storeAddr: "store-1", workers: []*regionRequestWorker{worker}} + scheduler.stores.Store(store.storeAddr, store) + + dummyRegion := regionInfo{ + subscribedSpan: &subscribedSpan{subID: SubscriptionID(2)}, + lockedRangeState: ®ionlock.LockedRangeState{}, + } + require.True(t, admission.submit(NewRegionPriorityTask(dummyRegion, 1, 1))) + + scheduler.broadcastDeregister(SubscriptionID(1), true) + require.Equal(t, 1, worker.controlQueue.len()) + req, ok := worker.controlQueue.tryPop() + require.True(t, ok) + require.Equal(t, SubscriptionID(1), req.subID) + require.True(t, req.filterLoop) + require.Equal(t, 1, admission.stats().pending) +} + +func TestRequestedStoreDistributesRegionsAcrossWorkerBuffers(t *testing.T) { + worker1 := ®ionRequestWorker{admission: newRegionAdmissionController(1, 1)} + worker2 := ®ionRequestWorker{admission: newRegionAdmissionController(1, 1)} + store := &requestedStore{ + storeAddr: "store-1", + workers: []*regionRequestWorker{worker1, worker2}, + } + + for i := uint64(1); i <= 4; i++ { + region := regionInfo{ + verID: tikv.NewRegionVerID(i, 1, 1), + subscribedSpan: &subscribedSpan{subID: 1}, + lockedRangeState: ®ionlock.LockedRangeState{}, + } + require.True(t, store.submit(NewRegionPriorityTask(region, 1, i))) + } + + require.Equal(t, 2, worker1.admission.stats().pending) + require.Equal(t, 2, worker2.admission.stats().pending) +} + +>>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) func TestSubscriptionWithFailedTiKV(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) mockPDClock := pdutil.NewClock4Test() From 2b9f207637b7d9e8d3214d0e119926768a9863d2 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Fri, 17 Jul 2026 11:07:32 +0800 Subject: [PATCH 47/71] refactor --- logservice/logpuller/requested_store.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/logservice/logpuller/requested_store.go b/logservice/logpuller/requested_store.go index f98b59b679..97294a8e33 100644 --- a/logservice/logpuller/requested_store.go +++ b/logservice/logpuller/requested_store.go @@ -31,7 +31,9 @@ type requestedStore struct { } func newRequestedStore( - client *subscriptionClient, + upstream *upstreamHandle, + eventSink *regionEventSink, + failureHandler *regionFailureHandler, storeAddr string, workerCount int, workerWindow int, @@ -43,7 +45,7 @@ func newRequestedStore( } for i := 0; i < workerCount; i++ { store.workers = append(store.workers, newRegionRequestWorker( - client, store, workerWindow, maxWindowMultiplier)) + upstream, eventSink, failureHandler, store, workerWindow, maxWindowMultiplier)) } return store } From a097472804a26251f4c3989cdcb76387e3221a9b Mon Sep 17 00:00:00 2001 From: lidezhu Date: Fri, 17 Jul 2026 16:41:06 +0800 Subject: [PATCH 48/71] refactor --- logservice/logpuller/priority_task.go | 21 ++++++++ logservice/logpuller/priority_task_test.go | 49 +++++++++++++++++++ .../region_admission_controller_test.go | 9 +++- .../logpuller/region_event_handler_test.go | 1 - .../logpuller/region_failure_handler.go | 24 +++++++++ .../logpuller/region_request_scheduler.go | 8 +++ logservice/logpuller/region_request_worker.go | 29 +++++++++++ .../logpuller/region_request_worker_test.go | 24 +++++++++ logservice/logpuller/region_state.go | 5 +- logservice/logpuller/region_tracker.go | 12 +++++ logservice/logpuller/subscription_client.go | 20 ++++++++ .../logpuller/subscription_client_test.go | 22 ++++++++- 12 files changed, 219 insertions(+), 5 deletions(-) diff --git a/logservice/logpuller/priority_task.go b/logservice/logpuller/priority_task.go index 38cd813c8b..89d43dc03a 100644 --- a/logservice/logpuller/priority_task.go +++ b/logservice/logpuller/priority_task.go @@ -32,6 +32,7 @@ type regionPriorityTask struct { heapIndex int // for heap.Item interface } +<<<<<<< HEAD // newRegionPriorityTask creates a new priority task for region. func newRegionPriorityTask(regionInfo regionInfo, sequence uint64) *regionPriorityTask { regionInfo.scanPriority = normalizeScanPriority(regionInfo.scanPriority) @@ -39,11 +40,31 @@ func newRegionPriorityTask(regionInfo regionInfo, sequence uint64) *regionPriori regionInfo: regionInfo, sequence: sequence, heapIndex: 0, // 0 means not in heap +======= +func newRegionPriorityTask(regionInfo regionInfo, currentTs, sequence uint64) *regionPriorityTask { + task := ®ionPriorityTask{ + sequence: sequence, + heapIndex: 0, // 0 means not in heap +>>>>>>> 9903a1be7 (refactor) } } +<<<<<<< HEAD func (pt *regionPriorityTask) priority() cdcpb.ScanPriority { return normalizeScanPriority(pt.regionInfo.scanPriority) +======= +// updateRegion refreshes both the request data and its priority before the task +// enters another scheduling stage. +func (pt *regionPriorityTask) updateRegion(regionInfo regionInfo, currentTs uint64) { + priority := normalRegionPriority + if regionInfo.wasInitialized { + priority = initializedRegionPriority + } else if regionScanLag(currentTs, regionInfo.resolvedTs()) < lowLagRegionThreshold { + priority = lowLagRegionPriority + } + pt.regionInfo = regionInfo + pt.priority = priority +>>>>>>> 9903a1be7 (refactor) } func (pt *regionPriorityTask) canUseMaxWindow() bool { diff --git a/logservice/logpuller/priority_task_test.go b/logservice/logpuller/priority_task_test.go index 41501181fb..35b8209c2b 100644 --- a/logservice/logpuller/priority_task_test.go +++ b/logservice/logpuller/priority_task_test.go @@ -60,6 +60,7 @@ func TestRegionPriorityTaskQueueOrder(t *testing.T) { queue := priorityqueue.New[*regionPriorityTask]() currentTime := time.Now() +<<<<<<< HEAD lowTask := newRegionPriorityTask( withScanPriority( newPriorityTestRegion(1, oracle.GoTimeToTS(currentTime.Add(-time.Hour))), @@ -80,6 +81,19 @@ func TestRegionPriorityTaskQueueOrder(t *testing.T) { cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, ), 1, +======= + normalTask := newRegionPriorityTask( + newPriorityTestRegion(1, oracle.GoTimeToTS(currentTime.Add(-time.Hour)), false), + currentTs, 3, + ) + lowLagTask := newRegionPriorityTask( + newPriorityTestRegion(2, oracle.GoTimeToTS(currentTime.Add(-10*time.Minute)), false), + currentTs, 2, + ) + initializedTask := newRegionPriorityTask( + newPriorityTestRegion(3, oracle.GoTimeToTS(currentTime.Add(-time.Hour)), true), + currentTs, 1, +>>>>>>> 9903a1be7 (refactor) ) require.True(t, queue.Push(lowTask)) @@ -98,10 +112,15 @@ func TestRegionPriorityTaskFIFOWithinPriority(t *testing.T) { currentTime := time.Now() checkpointTs := oracle.GoTimeToTS(currentTime.Add(-time.Hour)) +<<<<<<< HEAD first := newRegionPriorityTask( withScanPriority(newPriorityTestRegion(1, checkpointTs), cdcpb.ScanPriority_SCAN_PRIORITY_HIGH), 1) second := newRegionPriorityTask( withScanPriority(newPriorityTestRegion(2, checkpointTs), cdcpb.ScanPriority_SCAN_PRIORITY_HIGH), 2) +======= + first := newRegionPriorityTask(newPriorityTestRegion(1, checkpointTs, false), currentTs, 1) + second := newRegionPriorityTask(newPriorityTestRegion(2, checkpointTs, false), currentTs, 2) +>>>>>>> 9903a1be7 (refactor) require.True(t, queue.Push(second)) require.True(t, queue.Push(first)) @@ -120,6 +139,7 @@ func TestRegionPriorityTaskUsesHighPriorityWindow(t *testing.T) { lowTask := newRegionPriorityTask( withScanPriority(newPriorityTestRegion(2, 1), cdcpb.ScanPriority_SCAN_PRIORITY_LOW), 2) +<<<<<<< HEAD require.True(t, highTask.canUseMaxWindow()) require.False(t, lowTask.canUseMaxWindow()) } @@ -128,6 +148,35 @@ func TestRegionPriorityTaskRefreshesRegionInfoBetweenStages(t *testing.T) { region := withScanPriority(newPriorityTestRegion(1, 1), cdcpb.ScanPriority_SCAN_PRIORITY_LOW) task := newRegionPriorityTask(region, 1) require.Equal(t, cdcpb.ScanPriority_SCAN_PRIORITY_LOW, task.priority()) +======= + belowThreshold := newRegionPriorityTask(newPriorityTestRegion( + 1, + oracle.GoTimeToTS(currentTime.Add(-lowLagRegionThreshold+time.Millisecond)), + false, + ), currentTs, 1) + atThreshold := newRegionPriorityTask(newPriorityTestRegion( + 2, + oracle.GoTimeToTS(currentTime.Add(-lowLagRegionThreshold)), + false, + ), currentTs, 2) + futureCheckpoint := newRegionPriorityTask(newPriorityTestRegion( + 3, + oracle.GoTimeToTS(currentTime.Add(time.Second)), + false, + ), currentTs, 3) + + require.Equal(t, lowLagRegionPriority, belowThreshold.priority) + require.Equal(t, normalRegionPriority, atThreshold.priority) + require.Equal(t, lowLagRegionPriority, futureCheckpoint.priority) +} + +func TestRegionPriorityTaskRefreshesPriorityBetweenStages(t *testing.T) { + checkpointTime := time.Now() + checkpointTs := oracle.GoTimeToTS(checkpointTime) + region := newPriorityTestRegion(1, checkpointTs, false) + task := newRegionPriorityTask(region, oracle.GoTimeToTS(checkpointTime.Add(time.Minute)), 1) + require.Equal(t, lowLagRegionPriority, task.priority) +>>>>>>> 9903a1be7 (refactor) region.scanPriority = cdcpb.ScanPriority_SCAN_PRIORITY_HIGH task.regionInfo = region diff --git a/logservice/logpuller/region_admission_controller_test.go b/logservice/logpuller/region_admission_controller_test.go index 13fc2ed2bc..706e140501 100644 --- a/logservice/logpuller/region_admission_controller_test.go +++ b/logservice/logpuller/region_admission_controller_test.go @@ -39,7 +39,6 @@ func createTestRegionInfo(subID SubscriptionID, regionID uint64) regionInfo { span, nil, &subscribedSpan{subID: subID, startTs: 100, span: span}, - false, ) } @@ -56,7 +55,11 @@ func submitRegionForAdmission( currentTs uint64, ) { t.Helper() +<<<<<<< HEAD task := newRegionPriorityTask(region, region.verID.GetID()) +======= + task := newRegionPriorityTask(region, currentTs, region.verID.GetID()) +>>>>>>> 9903a1be7 (refactor) require.True(t, controller.submit(task)) } @@ -191,7 +194,11 @@ func TestRegionAdmissionControllerClose(t *testing.T) { controller := newRegionAdmissionController(1, 1) controller.close() region := prepareRegionForAdmission(createTestRegionInfo(1, 1), 1) +<<<<<<< HEAD require.False(t, controller.submit(newRegionPriorityTask(region, 1))) +======= + require.False(t, controller.submit(newRegionPriorityTask(region, 1, 1))) +>>>>>>> 9903a1be7 (refactor) _, err := controller.pop(context.Background(), nil) require.ErrorIs(t, err, context.Canceled) diff --git a/logservice/logpuller/region_event_handler_test.go b/logservice/logpuller/region_event_handler_test.go index fa6e76eaad..f2e1885197 100644 --- a/logservice/logpuller/region_event_handler_test.go +++ b/logservice/logpuller/region_event_handler_test.go @@ -87,7 +87,6 @@ func TestHandleEventEntryEventOutOfOrder(t *testing.T) { span, &tikv.RPCContext{}, subSpan, - false, ) lockResult := subSpan.rangeLock.LockRange( context.Background(), span.StartKey, span.EndKey, 1, 1) diff --git a/logservice/logpuller/region_failure_handler.go b/logservice/logpuller/region_failure_handler.go index 3622d927d9..5ca0f03c96 100644 --- a/logservice/logpuller/region_failure_handler.go +++ b/logservice/logpuller/region_failure_handler.go @@ -64,6 +64,14 @@ func newRegionFailureHandler( } } +func (r *regionFailureHandler) retryRange(ctx context.Context, errInfo regionErrorInfo) { + r.scheduleRangeRequest(ctx, rangeTask{ + span: errInfo.span, + subscribedSpan: errInfo.subscribedSpan, + wasInitialized: errInfo.wasInitialized, + }) +} + // Report admits a region failure into the recovery pipeline. It releases the // corresponding range lock before enqueueing the failure so new range tasks are // not blocked by stale region ownership. @@ -159,12 +167,20 @@ func (r *regionFailureHandler) handleError(ctx context.Context, errInfo regionEr } if innerErr.GetEpochNotMatch() != nil { metricFeedEpochNotMatchCounter.Inc() +<<<<<<< HEAD rescheduleRange() +======= + r.retryRange(ctx, errInfo) +>>>>>>> 9903a1be7 (refactor) return nil } if innerErr.GetRegionNotFound() != nil { metricFeedRegionNotFoundCounter.Inc() +<<<<<<< HEAD rescheduleRange() +======= + r.retryRange(ctx, errInfo) +>>>>>>> 9903a1be7 (refactor) return nil } if innerErr.GetCongested() != nil { @@ -197,14 +213,22 @@ func (r *regionFailureHandler) handleError(ctx context.Context, errInfo regionEr return nil case *rpcCtxUnavailableErr: metricFeedRPCCtxUnavailable.Inc() +<<<<<<< HEAD rescheduleRange() +======= + r.retryRange(ctx, errInfo) +>>>>>>> 9903a1be7 (refactor) return nil case *getStoreErr: metricGetStoreErr.Inc() bo := tikv.NewBackoffer(ctx, tikvRequestMaxBackoff) // cannot get the store the region belongs to, so we need to reload the region. r.regionCache.OnSendFail(bo, errInfo.rpcCtx, true, err) +<<<<<<< HEAD rescheduleRange() +======= + r.retryRange(ctx, errInfo) +>>>>>>> 9903a1be7 (refactor) return nil case *storeStreamErr: metricStoreSendRequestErr.Inc() diff --git a/logservice/logpuller/region_request_scheduler.go b/logservice/logpuller/region_request_scheduler.go index 33a01f726b..196030c695 100644 --- a/logservice/logpuller/region_request_scheduler.go +++ b/logservice/logpuller/region_request_scheduler.go @@ -116,6 +116,7 @@ func newRegionRequestScheduler(client *subscriptionClient) *regionRequestSchedul <<<<<<< HEAD func (s *regionRequestScheduler) Submit(region regionInfo) { +<<<<<<< HEAD if log.GetLevel() <= zapcore.DebugLevel { log.Debug("cdc region scan task enqueued", zap.Uint64("subscriptionID", uint64(region.subscribedSpan.subID)), @@ -129,6 +130,10 @@ func (s *regionRequestScheduler) Submit(region regionInfo) { zap.String("span", common.FormatTableSpan(®ion.span))) } s.taskQueue.Push(newRegionPriorityTask(region, s.sequence.Add(1))) +======= + s.taskQueue.Push(newRegionPriorityTask( + region, s.upstream.pdClock.CurrentTS(), s.sequence.Add(1))) +>>>>>>> 9903a1be7 (refactor) } func (s *regionRequestScheduler) Run(ctx context.Context, workerGroup *errgroup.Group) error { @@ -164,6 +169,9 @@ func (s *regionRequestScheduler) run(ctx context.Context, group *errgroup.Group) } <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 9903a1be7 (refactor) region, err := s.attachRPCContext(ctx, task.regionInfo) if err != nil { s.failureHandler.Report(newRegionErrorInfo(region, err)) diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index 13f9c8d005..f717321db2 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -174,6 +174,35 @@ func (s *regionRequestWorker) Run(ctx context.Context) error { } } +<<<<<<< HEAD +======= +// failStreamRegions transfers every request sent by a failed stream to the +// recovery pipeline. +func (s *regionRequestWorker) failStreamRegions(err error) { + for _, state := range s.tracker.Drain() { + s.notifyRegionError(state, err) + } + // The failed stream no longer owns remote registrations. + s.controlQueue.drain() +} + +// failPendingRegions transfers requests owned by this worker but not yet sent +// to the recovery pipeline, so they can be resolved and routed again. +func (s *regionRequestWorker) failPendingRegions(err error) { + for _, task := range s.admission.drain() { + s.failureHandler.Report(newRegionErrorInfo(task.regionInfo, err)) + } +} + +func (s *regionRequestWorker) notifyRegionError(state *regionFeedState, err error) { + state.markStopped(err) + s.eventSink.Push( + SubscriptionID(state.requestID), + regionEvent{states: []*regionFeedState{state}}, + ) +} + +>>>>>>> 9903a1be7 (refactor) func (s *regionRequestWorker) waitForRegionRequest(ctx context.Context) (*regionReq, error) { // Without a stream there are no remote registrations to deregister. s.controlQueue.drain() diff --git a/logservice/logpuller/region_request_worker_test.go b/logservice/logpuller/region_request_worker_test.go index 8735f3021c..aa54d2ea65 100644 --- a/logservice/logpuller/region_request_worker_test.go +++ b/logservice/logpuller/region_request_worker_test.go @@ -517,6 +517,7 @@ func TestRunStreamFailureReportsPendingRegionsToFailureHandler(t *testing.T) { pdClient := newFailureRecoveryTestPDClient(t) defer pdClient.Close() +<<<<<<< HEAD handler := newRegionFailureHandler(nil, func(*subscribedSpan) {}, nil, nil) worker := ®ionRequestWorker{ upstream: &upstreamHandle{pd: pdClient, credential: &security.Credential{}}, @@ -526,6 +527,29 @@ func TestRunStreamFailureReportsPendingRegionsToFailureHandler(t *testing.T) { controlQueue: newControlQueue(), tracker: newRegionTracker(), storeAddr: "127.0.0.1:1", +======= + admission := newRegionAdmissionController(1, 1) + failureHandler := ®ionFailureHandler{cache: newErrCache()} + worker := ®ionRequestWorker{failureHandler: failureHandler, admission: admission} + regions := []regionInfo{ + { + verID: tikv.NewRegionVerID(1, 1, 1), + span: heartbeatpb.TableSpan{ + TableID: 1, StartKey: []byte("a"), EndKey: []byte("m"), + }, + subscribedSpan: span, lockedRangeState: lock1.LockedRangeState, + }, + { + verID: tikv.NewRegionVerID(2, 1, 1), + span: heartbeatpb.TableSpan{ + TableID: 1, StartKey: []byte("m"), EndKey: []byte("z"), + }, + subscribedSpan: span, lockedRangeState: lock2.LockedRangeState, + }, + } + for i, region := range regions { + require.True(t, admission.submit(newRegionPriorityTask(region, 1, uint64(i+1)))) +>>>>>>> 9903a1be7 (refactor) } firstRegion := createFailureRecoveryTestRegion(t, 1, 1) diff --git a/logservice/logpuller/region_state.go b/logservice/logpuller/region_state.go index 0f2f84ce3e..6a2837564f 100644 --- a/logservice/logpuller/region_state.go +++ b/logservice/logpuller/region_state.go @@ -58,15 +58,18 @@ func newRegionInfo( span heartbeatpb.TableSpan, rpcCtx *tikv.RPCContext, subscribedSpan *subscribedSpan, - filterLoop bool, ) regionInfo { return regionInfo{ verID: verID, span: span, rpcCtx: rpcCtx, subscribedSpan: subscribedSpan, +<<<<<<< HEAD filterLoop: filterLoop, scanPriority: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, +======= + filterLoop: subscribedSpan.filterLoop, +>>>>>>> 9903a1be7 (refactor) } } diff --git a/logservice/logpuller/region_tracker.go b/logservice/logpuller/region_tracker.go index 81a27dc86c..4d69bf8bb3 100644 --- a/logservice/logpuller/region_tracker.go +++ b/logservice/logpuller/region_tracker.go @@ -112,6 +112,7 @@ func (t *regionTracker) Drain() []*regionFeedState { t.statesBySubscription = make(map[SubscriptionID]regionStatesByID) t.mu.Unlock() +<<<<<<< HEAD totalStates := 0 for _, states := range statesBySubscription { totalStates += len(states) @@ -119,6 +120,17 @@ func (t *regionTracker) Drain() []*regionFeedState { drainedStates := make([]*regionFeedState, 0, totalStates) for _, states := range statesBySubscription { drainedStates = append(drainedStates, slices.Collect(maps.Values(states))...) +======= + stateCount := 0 + for _, states := range statesBySubscription { + stateCount += len(states) + } + drainedStates := make([]*regionFeedState, 0, stateCount) + for _, states := range statesBySubscription { + for _, state := range states { + drainedStates = append(drainedStates, state) + } +>>>>>>> 9903a1be7 (refactor) } return drainedStates } diff --git a/logservice/logpuller/subscription_client.go b/logservice/logpuller/subscription_client.go index 7614ab01dc..d77de7dc5b 100644 --- a/logservice/logpuller/subscription_client.go +++ b/logservice/logpuller/subscription_client.go @@ -89,8 +89,12 @@ type resolveLockTask struct { type rangeTask struct { span heartbeatpb.TableSpan subscribedSpan *subscribedSpan +<<<<<<< HEAD filterLoop bool priority cdcpb.ScanPriority +======= + wasInitialized bool +>>>>>>> 9903a1be7 (refactor) } // upstreamHandle contains the stable TiKV and PD dependencies shared by the @@ -286,12 +290,16 @@ func (s *subscriptionClient) Subscribe( select { case <-s.ctx.Done(): log.Warn("subscribes span failed, the subscription client has closed") +<<<<<<< HEAD case s.rangeTaskCh <- rangeTask{ span: span, subscribedSpan: rt, filterLoop: rt.filterLoop, priority: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, }: +======= + case s.rangeTaskCh <- rangeTask{span: span, subscribedSpan: rt}: +>>>>>>> 9903a1be7 (refactor) log.Info("subscribes span done", zap.Uint64("subscriptionID", uint64(subID)), zap.Int64("tableID", span.TableID), zap.Uint64("startTs", startTs), zap.String("startKey", spanz.HexKey(span.StartKey)), zap.String("endKey", spanz.HexKey(span.EndKey))) @@ -325,9 +333,12 @@ func (s *subscriptionClient) Run(ctx context.Context) error { <<<<<<< HEAD g.Go(func() error { return s.regionScheduler.Run(ctx, g) }) g.Go(func() error { return s.eventSink.Run(ctx) }) +<<<<<<< HEAD ======= g.Go(func() error { return s.regionScheduler.run(ctx, g) }) >>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) +======= +>>>>>>> 9903a1be7 (refactor) g.Go(func() error { return s.failureHandler.Run(ctx) }) g.Go(func() error { return s.spanRegistry.Run(ctx) }) g.Go(func() error { return s.handleResolveLockTasks(ctx) }) @@ -477,8 +488,13 @@ func (s *subscriptionClient) divideSpanAndScheduleRegionRequests( } verID := tikv.NewRegionVerID(regionMeta.Id, regionMeta.RegionEpoch.ConfVer, regionMeta.RegionEpoch.Version) +<<<<<<< HEAD regionInfo := newRegionInfo(verID, intersectSpan, nil, subscribedSpan, task.filterLoop) regionInfo.scanPriority = normalizeScanPriority(task.priority) +======= + regionInfo := newRegionInfo(verID, intersectSpan, nil, subscribedSpan) + regionInfo.wasInitialized = task.wasInitialized +>>>>>>> 9903a1be7 (refactor) // Schedule a region request to subscribe the region. s.scheduleRegionRequest(ctx, regionInfo) @@ -525,8 +541,12 @@ func (s *subscriptionClient) scheduleRegionRequest(ctx context.Context, region r s.scheduleRangeRequest(ctx, rangeTask{ span: r, subscribedSpan: region.subscribedSpan, +<<<<<<< HEAD filterLoop: region.filterLoop, priority: region.scanPriority, +======= + wasInitialized: region.wasInitialized, +>>>>>>> 9903a1be7 (refactor) }) } default: diff --git a/logservice/logpuller/subscription_client_test.go b/logservice/logpuller/subscription_client_test.go index 16121f7b5d..4e6354479f 100644 --- a/logservice/logpuller/subscription_client_test.go +++ b/logservice/logpuller/subscription_client_test.go @@ -335,17 +335,23 @@ func TestStopTaskUsesSubscribedSpanFilterLoop(t *testing.T) { res := span.rangeLock.LockRange(context.Background(), rawSpan.StartKey, rawSpan.EndKey, 1, 1) require.Equal(t, regionlock.LockRangeStatusSuccess, res.Status) <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 9903a1be7 (refactor) const storeAddr = "store-1" worker := ®ionRequestWorker{storeAddr: storeAddr, controlQueue: newControlQueue()} store := ®ionRequestStore{workers: []*regionRequestWorker{worker}} client.regionScheduler = ®ionRequestScheduler{} client.regionScheduler.stores.Store(storeAddr, store) +<<<<<<< HEAD ======= worker := ®ionRequestWorker{controlQueue: newControlQueue()} store := &requestedStore{storeAddr: "store-1", workers: []*regionRequestWorker{worker}} client.regionScheduler = ®ionRequestScheduler{client: client} client.regionScheduler.stores.Store(store.storeAddr, store) >>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) +======= +>>>>>>> 9903a1be7 (refactor) client.setTableStopped(span) @@ -475,18 +481,25 @@ func TestBroadcastDeregisterUsesWorkerControlQueue(t *testing.T) { scheduler := ®ionRequestScheduler{client: client} admission := newRegionAdmissionController(1, 1) + const storeAddr = "store-1" worker := ®ionRequestWorker{ + storeAddr: storeAddr, admission: admission, controlQueue: newControlQueue(), } +<<<<<<< HEAD store := &requestedStore{storeAddr: "store-1", workers: []*regionRequestWorker{worker}} scheduler.stores.Store(store.storeAddr, store) +======= + store := ®ionRequestStore{workers: []*regionRequestWorker{worker}} + scheduler.stores.Store(storeAddr, store) +>>>>>>> 9903a1be7 (refactor) dummyRegion := regionInfo{ subscribedSpan: &subscribedSpan{subID: SubscriptionID(2)}, lockedRangeState: ®ionlock.LockedRangeState{}, } - require.True(t, admission.submit(NewRegionPriorityTask(dummyRegion, 1, 1))) + require.True(t, admission.submit(newRegionPriorityTask(dummyRegion, 1, 1))) scheduler.broadcastDeregister(SubscriptionID(1), true) require.Equal(t, 1, worker.controlQueue.len()) @@ -500,9 +513,14 @@ func TestBroadcastDeregisterUsesWorkerControlQueue(t *testing.T) { func TestRequestedStoreDistributesRegionsAcrossWorkerBuffers(t *testing.T) { worker1 := ®ionRequestWorker{admission: newRegionAdmissionController(1, 1)} worker2 := ®ionRequestWorker{admission: newRegionAdmissionController(1, 1)} +<<<<<<< HEAD store := &requestedStore{ storeAddr: "store-1", workers: []*regionRequestWorker{worker1, worker2}, +======= + store := ®ionRequestStore{ + workers: []*regionRequestWorker{worker1, worker2}, +>>>>>>> 9903a1be7 (refactor) } for i := uint64(1); i <= 4; i++ { @@ -511,7 +529,7 @@ func TestRequestedStoreDistributesRegionsAcrossWorkerBuffers(t *testing.T) { subscribedSpan: &subscribedSpan{subID: 1}, lockedRangeState: ®ionlock.LockedRangeState{}, } - require.True(t, store.submit(NewRegionPriorityTask(region, 1, i))) + require.True(t, store.submit(newRegionPriorityTask(region, 1, i))) } require.Equal(t, 2, worker1.admission.stats().pending) From 7b39abef18ac2ec4a080a1e9ba9d3195e8ceebb4 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Fri, 17 Jul 2026 19:31:15 +0800 Subject: [PATCH 49/71] refactor --- logservice/eventstore/pebble.go | 2 +- logservice/logpuller/memory_quota.go | 415 ++++++++++++++++++ logservice/logpuller/memory_quota_test.go | 296 +++++++++++++ logservice/logpuller/priority_task.go | 21 - logservice/logpuller/priority_task_test.go | 49 --- .../logpuller/region_admission_controller.go | 48 +- .../region_admission_controller_test.go | 42 +- logservice/logpuller/region_event_handler.go | 52 ++- .../logpuller/region_event_handler_test.go | 183 ++++---- logservice/logpuller/region_event_sink.go | 115 +++-- .../logpuller/region_event_sink_test.go | 120 +++-- .../logpuller/region_failure_handler.go | 24 - .../logpuller/region_request_scheduler.go | 140 +----- logservice/logpuller/region_request_store.go | 16 +- logservice/logpuller/region_request_worker.go | 41 +- .../logpuller/region_request_worker_test.go | 24 - logservice/logpuller/region_state.go | 5 +- logservice/logpuller/region_tracker.go | 12 - logservice/logpuller/span_registry.go | 12 + logservice/logpuller/subscription_client.go | 103 +---- .../logpuller/subscription_client_test.go | 80 ---- metrics/grafana/ticdc_new_arch.json | 90 ++++ .../ticdc_new_arch_next_gen.json | 90 ++++ pkg/config/debug.go | 18 + pkg/config/debug_test.go | 15 +- pkg/metrics/log_puller.go | 8 + 26 files changed, 1293 insertions(+), 728 deletions(-) create mode 100644 logservice/logpuller/memory_quota.go create mode 100644 logservice/logpuller/memory_quota_test.go diff --git a/logservice/eventstore/pebble.go b/logservice/eventstore/pebble.go index f5f48bf71b..0d950c99d0 100644 --- a/logservice/eventstore/pebble.go +++ b/logservice/eventstore/pebble.go @@ -42,7 +42,7 @@ func newPebbleOptions(dbNum int) *pebble.Options { MaxOpenFiles: maxOpenFilesPerDB, - MaxConcurrentCompactions: func() int { return 6 }, + MaxConcurrentCompactions: func() int { return 2 }, // Decrease compaction frequency L0CompactionThreshold: 20, diff --git a/logservice/logpuller/memory_quota.go b/logservice/logpuller/memory_quota.go new file mode 100644 index 0000000000..6596fc0f29 --- /dev/null +++ b/logservice/logpuller/memory_quota.go @@ -0,0 +1,415 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package logpuller + +import ( + "context" + "math" + "sync" + "sync/atomic" + "time" +) + +const ( + defaultLogPullerMemoryQuota uint64 = 1024 * 1024 * 1024 + + defaultPauseWarmingRatio = 0.15 + defaultResumeWarmingRatio = 0.05 + defaultFreezeAllRatio = 0.8 + defaultResumeAllRatio = 0.6 + defaultHardLimitRatio = 5.0 + + defaultScanBaseSize uint64 = 8 * 1024 * 1024 + defaultScanLagUnit = 10 * time.Minute + defaultScanLagWeight = 0.22 + defaultMaxScanLagFactor = 16 +) + +type admissionLevel uint8 + +const ( + admissionNormal admissionLevel = iota + admissionPauseWarming + admissionFreezeAllNewScans +) + +type memoryQuotaLease struct { + once sync.Once + release func() +} + +func (l *memoryQuotaLease) Release() { + if l == nil { + return + } + l.once.Do(func() { + if l.release != nil { + l.release() + } + }) +} + +type subscriptionQuotaState struct { + eventLeases map[*memoryQuotaLease]struct{} + scanLeases map[*memoryQuotaLease]struct{} +} + +func newSubscriptionQuotaState() *subscriptionQuotaState { + return &subscriptionQuotaState{ + eventLeases: make(map[*memoryQuotaLease]struct{}), + scanLeases: make(map[*memoryQuotaLease]struct{}), + } +} + +// memoryQuotaController tracks event memory retained by downstream callbacks +// and estimated memory for admitted initial scans. Event memory is allowed to +// exceed the soft capacity, but the receive path waits at the hard limit. Scan +// admission first pauses uninitialized high-lag spans and freezes all new scans +// only under heavier pressure; both transitions use hysteresis when resuming. +type memoryQuotaController struct { + mu sync.Mutex + cond *sync.Cond + + capacity uint64 + // used tracks event bytes retained until downstream finishes consuming them. + used uint64 + + // scanUsed tracks the estimated memory of all admitted initial scans. + // warmingScanUsed is the subset used by uninitialized, high-lag spans. + scanUsed uint64 + warmingScanUsed uint64 + level admissionLevel + + pauseWarmingRatio float64 + resumeWarmingRatio float64 + freezeAllRatio float64 + resumeAllRatio float64 + hardLimitRatio float64 + + scanEstimate uint64 + + subscriptions map[SubscriptionID]*subscriptionQuotaState + onAvailable atomic.Value // func() +} + +func newMemoryQuotaController(capacity, scanBaseSize uint64) *memoryQuotaController { + if capacity == 0 { + capacity = defaultLogPullerMemoryQuota + } + if scanBaseSize == 0 { + scanBaseSize = defaultScanBaseSize + } + c := &memoryQuotaController{ + capacity: capacity, + level: admissionNormal, + pauseWarmingRatio: defaultPauseWarmingRatio, + resumeWarmingRatio: defaultResumeWarmingRatio, + freezeAllRatio: defaultFreezeAllRatio, + resumeAllRatio: defaultResumeAllRatio, + hardLimitRatio: defaultHardLimitRatio, + scanEstimate: scanBaseSize, + subscriptions: make(map[SubscriptionID]*subscriptionQuotaState), + } + c.cond = sync.NewCond(&c.mu) + return c +} + +func (c *memoryQuotaController) setOnAvailable(fn func()) { + c.onAvailable.Store(fn) +} + +func (c *memoryQuotaController) notifyAvailable() { + if fn, ok := c.onAvailable.Load().(func()); ok && fn != nil { + fn() + } +} + +func (c *memoryQuotaController) wakeAll() { + c.mu.Lock() + c.cond.Broadcast() + c.mu.Unlock() +} + +func (c *memoryQuotaController) snapshot() (used, capacity uint64, level admissionLevel) { + c.mu.Lock() + defer c.mu.Unlock() + return c.used, c.capacity, c.level +} + +func (c *memoryQuotaController) scanSnapshot() ( + scanUsed uint64, + warmingScanUsed uint64, + warmingScanBudget uint64, + scanEstimate uint64, + hardLimit uint64, +) { + c.mu.Lock() + defer c.mu.Unlock() + return c.scanUsed, c.warmingScanUsed, c.warmingScanBudgetLocked(), + c.scanEstimate, c.hardLimitLocked() +} + +func (c *memoryQuotaController) addSubscription(span *subscribedSpan) { + if span == nil { + return + } + c.mu.Lock() + c.subscriptions[span.subID] = newSubscriptionQuotaState() + c.mu.Unlock() +} + +func (c *memoryQuotaController) removeSubscription(span *subscribedSpan) { + if span == nil { + return + } + + c.mu.Lock() + state, ok := c.subscriptions[span.subID] + if !ok { + c.mu.Unlock() + return + } + delete(c.subscriptions, span.subID) + leases := make([]*memoryQuotaLease, 0, len(state.eventLeases)+len(state.scanLeases)) + for lease := range state.eventLeases { + leases = append(leases, lease) + } + for lease := range state.scanLeases { + leases = append(leases, lease) + } + // Wake event receivers so they can observe that the subscription was removed. + c.cond.Broadcast() + c.mu.Unlock() + + for _, lease := range leases { + lease.Release() + } + // Removing a subscription can make a pending request eligible even when the + // subscription itself did not own any lease. + c.notifyAvailable() +} + +func (c *memoryQuotaController) markSubscriptionInitialized() { + c.notifyAvailable() +} + +func (c *memoryQuotaController) acquireScan( + region regionInfo, + currentTs uint64, +) (*memoryQuotaLease, bool) { + span := region.subscribedSpan + if span == nil { + return nil, true + } + + c.mu.Lock() + state, ok := c.subscriptions[span.subID] + if !ok { + // The subscription has already been removed. Let the request continue to + // the worker, where the normal stopped-subscription path will discard it. + c.mu.Unlock() + return nil, true + } + c.refreshLevelLocked() + if c.level == admissionFreezeAllNewScans { + c.mu.Unlock() + return nil, false + } + + bytes := c.estimateScanSizeLocked(region, currentTs) + warming := isWarmingScan(region, currentTs) + if c.isWarmingScanBlockedLocked(warming, bytes) { + c.mu.Unlock() + return nil, false + } + + lease := &memoryQuotaLease{} + lease.release = func() { + c.mu.Lock() + previousLevel := c.level + c.scanUsed = subtractFloor(c.scanUsed, bytes) + if warming { + c.warmingScanUsed = subtractFloor(c.warmingScanUsed, bytes) + } + delete(state.scanLeases, lease) + c.refreshLevelLocked() + shouldNotifyAdmission := c.level < previousLevel || + (warming && c.level == admissionNormal) + c.mu.Unlock() + + if shouldNotifyAdmission { + c.notifyAvailable() + } + } + c.scanUsed += bytes + if warming { + c.warmingScanUsed += bytes + } + state.scanLeases[lease] = struct{}{} + c.refreshLevelLocked() + c.mu.Unlock() + return lease, true +} + +func (c *memoryQuotaController) trackEvent( + ctx context.Context, + span *subscribedSpan, + bytes uint64, +) *memoryQuotaLease { + if span == nil || bytes == 0 { + return nil + } + + c.mu.Lock() + if ctx.Err() != nil { + c.mu.Unlock() + return nil + } + state := c.subscriptions[span.subID] + if state == nil { + c.mu.Unlock() + return nil + } + + for c.used > 0 && wouldExceed(c.used, bytes, c.hardLimitLocked()) { + c.cond.Wait() + if ctx.Err() != nil { + c.mu.Unlock() + return nil + } + if c.subscriptions[span.subID] != state { + c.mu.Unlock() + return nil + } + } + + c.used += bytes + c.refreshLevelLocked() + lease := &memoryQuotaLease{} + lease.release = func() { + c.mu.Lock() + previousLevel := c.level + c.used = subtractFloor(c.used, bytes) + delete(state.eventLeases, lease) + c.refreshLevelLocked() + shouldNotifyAdmission := c.level < previousLevel + c.cond.Broadcast() + c.mu.Unlock() + + if shouldNotifyAdmission { + c.notifyAvailable() + } + } + state.eventLeases[lease] = struct{}{} + c.mu.Unlock() + return lease +} + +func (c *memoryQuotaController) estimateScanSizeLocked(region regionInfo, currentTs uint64) uint64 { + raw := float64(c.scanEstimate) * scanLagFactor(region.resolvedTs(), currentTs) + estimate := uint64(raw) + if estimate < c.scanEstimate { + estimate = c.scanEstimate + } + maxEstimate := uint64(math.MaxUint64) + if c.scanEstimate <= math.MaxUint64/defaultMaxScanLagFactor { + maxEstimate = c.scanEstimate * defaultMaxScanLagFactor + } + if estimate > maxEstimate { + estimate = maxEstimate + } + if estimate == 0 { + estimate = c.scanEstimate + } + return estimate +} + +func scanLagFactor(startTs, currentTs uint64) float64 { + lag := regionScanLag(currentTs, startTs) + if lag <= 0 { + return 1 + } + return min(defaultMaxScanLagFactor, + 1+defaultScanLagWeight*math.Log2(1+float64(lag)/float64(defaultScanLagUnit))) +} + +func isWarmingScan(region regionInfo, currentTs uint64) bool { + span := region.subscribedSpan + if span == nil || span.initialized.Load() { + return false + } + return regionScanLag(currentTs, region.resolvedTs()) >= lowLagRegionThreshold +} + +func (c *memoryQuotaController) isWarmingScanBlockedLocked(warming bool, bytes uint64) bool { + if !warming { + return false + } + if c.level == admissionPauseWarming { + return true + } + return wouldExceed(c.warmingScanUsed, bytes, c.warmingScanBudgetLocked()) +} + +func (c *memoryQuotaController) warmingScanBudgetLocked() uint64 { + budget := uint64(float64(c.capacity) * c.pauseWarmingRatio) + return max(budget, c.scanEstimate) +} + +func (c *memoryQuotaController) hardLimitLocked() uint64 { + return uint64(float64(c.capacity) * c.hardLimitRatio) +} + +func (c *memoryQuotaController) refreshLevelLocked() { + // scanUsed predicts the event memory an initial scan may produce, so adding + // it to actual event bytes would count the same pressure twice. + pressure := max(c.used, c.scanUsed) + usage := float64(pressure) / float64(c.capacity) + switch c.level { + case admissionFreezeAllNewScans: + if usage <= c.resumeAllRatio { + if usage >= c.pauseWarmingRatio { + c.level = admissionPauseWarming + } else { + c.level = admissionNormal + } + } + case admissionPauseWarming: + switch { + case usage >= c.freezeAllRatio: + c.level = admissionFreezeAllNewScans + case usage <= c.resumeWarmingRatio: + c.level = admissionNormal + } + default: + switch { + case usage >= c.freezeAllRatio: + c.level = admissionFreezeAllNewScans + case usage >= c.pauseWarmingRatio: + c.level = admissionPauseWarming + } + } +} + +func wouldExceed(used, bytes, limit uint64) bool { + return bytes > limit || used > limit-bytes +} + +func subtractFloor(value, delta uint64) uint64 { + if value < delta { + return 0 + } + return value - delta +} diff --git a/logservice/logpuller/memory_quota_test.go b/logservice/logpuller/memory_quota_test.go new file mode 100644 index 0000000000..be4d88d89d --- /dev/null +++ b/logservice/logpuller/memory_quota_test.go @@ -0,0 +1,296 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package logpuller + +import ( + "context" + "testing" + "time" + + "github.com/pingcap/ticdc/logservice/logpuller/regionlock" + "github.com/stretchr/testify/require" + "github.com/tikv/client-go/v2/oracle" +) + +func newTestQuotaSpan(subID SubscriptionID) *subscribedSpan { + span := &subscribedSpan{subID: subID} + span.resolvedTs.Store(oracle.GoTimeToTS(time.Now())) + return span +} + +func newTestQuotaRegion(span *subscribedSpan) regionInfo { + state := ®ionlock.LockedRangeState{} + state.ResolvedTs.Store(span.resolvedTs.Load()) + return regionInfo{ + subscribedSpan: span, + lockedRangeState: state, + } +} + +func setTestQuotaSpanLag(span *subscribedSpan, lag time.Duration) uint64 { + now := time.Now() + span.resolvedTs.Store(oracle.GoTimeToTS(now.Add(-lag))) + return oracle.GoTimeToTS(now) +} + +func TestMemoryQuotaAdmissionLevels(t *testing.T) { + quota := newMemoryQuotaController(100, 10) + warmingSpan := newTestQuotaSpan(1) + initializedSpan := newTestQuotaSpan(2) + initializedSpan.initialized.Store(true) + warmingTs := setTestQuotaSpanLag(warmingSpan, lowLagRegionThreshold+time.Minute) + initializedTs := setTestQuotaSpanLag(initializedSpan, lowLagRegionThreshold+time.Minute) + quota.addSubscription(warmingSpan) + quota.addSubscription(initializedSpan) + + lowLease := quota.trackEvent(context.Background(), initializedSpan, 5) + pauseLease := quota.trackEvent(context.Background(), initializedSpan, 10) + require.NotNil(t, lowLease) + require.NotNil(t, pauseLease) + scanLease, admitted := quota.acquireScan(newTestQuotaRegion(warmingSpan), warmingTs) + require.False(t, admitted) + require.Nil(t, scanLease) + + scanLease, admitted = quota.acquireScan( + newTestQuotaRegion(initializedSpan), initializedTs) + require.True(t, admitted) + scanLease.Release() + + middleLease := quota.trackEvent(context.Background(), initializedSpan, 45) + freezeLease := quota.trackEvent(context.Background(), initializedSpan, 20) + require.NotNil(t, middleLease) + require.NotNil(t, freezeLease) + scanLease, admitted = quota.acquireScan( + newTestQuotaRegion(initializedSpan), initializedTs) + require.False(t, admitted) + require.Nil(t, scanLease) + + freezeLease.Release() + _, _, level := quota.snapshot() + require.Equal(t, admissionPauseWarming, level) + middleLease.Release() + _, _, level = quota.snapshot() + require.Equal(t, admissionPauseWarming, level) + pauseLease.Release() + _, _, level = quota.snapshot() + require.Equal(t, admissionNormal, level) + lowLease.Release() +} + +func TestMemoryQuotaRemoveSubscriptionReleasesOwnedMemory(t *testing.T) { + quota := newMemoryQuotaController(100, 10) + span1 := newTestQuotaSpan(1) + span2 := newTestQuotaSpan(2) + quota.addSubscription(span1) + quota.addSubscription(span2) + + lease1 := quota.trackEvent(context.Background(), span1, 30) + lease2 := quota.trackEvent(context.Background(), span2, 40) + require.NotNil(t, lease1) + require.NotNil(t, lease2) + scanLease, admitted := quota.acquireScan(newTestQuotaRegion(span1), span1.resolvedTs.Load()) + require.True(t, admitted) + require.NotNil(t, scanLease) + + quota.removeSubscription(span1) + used, _, _ := quota.snapshot() + require.Equal(t, uint64(40), used) + scanUsed, _, _, _, _ := quota.scanSnapshot() + require.Zero(t, scanUsed) + require.NotContains(t, quota.subscriptions, span1.subID) + quota.removeSubscription(span1) + + lease1.Release() + scanLease.Release() + used, _, _ = quota.snapshot() + require.Equal(t, uint64(40), used) + + // Late region tasks are allowed to reach the stopped-subscription cleanup + // path without recreating quota state. + scanLease, admitted = quota.acquireScan(newTestQuotaRegion(span1), span1.resolvedTs.Load()) + require.True(t, admitted) + require.Nil(t, scanLease) + require.NotContains(t, quota.subscriptions, span1.subID) + + lease2.Release() + used, _, _ = quota.snapshot() + require.Zero(t, used) +} + +func TestMemoryQuotaBlockedEventStopsWhenSubscriptionIsRemoved(t *testing.T) { + quota := newMemoryQuotaController(100, 10) + quota.hardLimitRatio = 1 + span := newTestQuotaSpan(1) + quota.addSubscription(span) + + lease := quota.trackEvent(context.Background(), span, 100) + require.NotNil(t, lease) + acquired := make(chan *memoryQuotaLease, 1) + go func() { + acquired <- quota.trackEvent(context.Background(), span, 1) + }() + + select { + case <-acquired: + t.Fatal("event memory should wait at the hard limit") + case <-time.After(100 * time.Millisecond): + } + + quota.removeSubscription(span) + select { + case blockedLease := <-acquired: + require.Nil(t, blockedLease) + case <-time.After(time.Second): + t.Fatal("removing the subscription did not wake the blocked event") + } + require.NotContains(t, quota.subscriptions, span.subID) + lease.Release() +} + +func TestMemoryQuotaBlockedEventResumesAfterRelease(t *testing.T) { + quota := newMemoryQuotaController(100, 10) + quota.hardLimitRatio = 1 + span := newTestQuotaSpan(1) + quota.addSubscription(span) + + lease := quota.trackEvent(context.Background(), span, 100) + require.NotNil(t, lease) + acquired := make(chan *memoryQuotaLease, 1) + go func() { + acquired <- quota.trackEvent(context.Background(), span, 1) + }() + + select { + case <-acquired: + t.Fatal("event memory should wait at the hard limit") + case <-time.After(100 * time.Millisecond): + } + + lease.Release() + select { + case nextLease := <-acquired: + require.NotNil(t, nextLease) + nextLease.Release() + case <-time.After(time.Second): + t.Fatal("event memory did not resume after memory was released") + } +} + +func TestMemoryQuotaWarmingScanBudget(t *testing.T) { + quota := newMemoryQuotaController(200, 10) + span := newTestQuotaSpan(1) + currentTs := setTestQuotaSpanLag(span, lowLagRegionThreshold+time.Minute) + quota.addSubscription(span) + region := newTestQuotaRegion(span) + + lease1, admitted := quota.acquireScan(region, currentTs) + require.True(t, admitted) + lease2, admitted := quota.acquireScan(region, currentTs) + require.True(t, admitted) + + lease3, admitted := quota.acquireScan(region, currentTs) + require.False(t, admitted) + require.Nil(t, lease3) + + lease1.Release() + lease3, admitted = quota.acquireScan(region, currentTs) + require.True(t, admitted) + lease2.Release() + lease3.Release() +} + +func TestMemoryQuotaLowLagScanBypassesWarmingGate(t *testing.T) { + quota := newMemoryQuotaController(100, 10) + span := newTestQuotaSpan(1) + currentTs := setTestQuotaSpanLag(span, lowLagRegionThreshold-time.Second) + quota.addSubscription(span) + + pressureLease := quota.trackEvent(context.Background(), span, 20) + require.NotNil(t, pressureLease) + scanLease, admitted := quota.acquireScan(newTestQuotaRegion(span), currentTs) + require.True(t, admitted) + require.NotNil(t, scanLease) + _, warmingScanUsed, _, _, _ := quota.scanSnapshot() + require.Zero(t, warmingScanUsed) + + scanLease.Release() + pressureLease.Release() +} + +func TestMemoryQuotaRemovalNotifiesAdmissionWithoutLeases(t *testing.T) { + quota := newMemoryQuotaController(100, 10) + span := newTestQuotaSpan(1) + quota.addSubscription(span) + + notified := make(chan struct{}, 1) + quota.setOnAvailable(func() { + select { + case notified <- struct{}{}: + default: + } + }) + quota.removeSubscription(span) + select { + case <-notified: + case <-time.After(time.Second): + t.Fatal("subscription removal did not notify admission") + } +} + +func TestAdmissionWaitsForMemoryAndReleasesScanLease(t *testing.T) { + quota := newMemoryQuotaController(100, 10) + span := newTestQuotaSpan(1) + currentTs := setTestQuotaSpanLag(span, lowLagRegionThreshold+time.Minute) + quota.addSubscription(span) + controller := newRegionAdmissionController(1, 1, quota, func() uint64 { + return currentTs + }) + quota.setOnAvailable(controller.notifyAvailable) + + pressureLease := quota.trackEvent(context.Background(), span, 20) + require.NotNil(t, pressureLease) + region := newTestQuotaRegion(span) + require.True(t, controller.submit(newRegionPriorityTask(region, currentTs, 1))) + + type popResult struct { + req *regionReq + err error + } + result := make(chan popResult, 1) + go func() { + req, err := controller.pop(context.Background(), nil) + result <- popResult{req: req, err: err} + }() + select { + case <-result: + t.Fatal("warming scan should wait while memory is under pressure") + case <-time.After(100 * time.Millisecond): + } + + pressureLease.Release() + var resultValue popResult + select { + case resultValue = <-result: + case <-time.After(time.Second): + t.Fatal("scan admission was not notified after memory became available") + } + require.NoError(t, resultValue.err) + req := resultValue.req + scanUsed, _, _, _, _ := quota.scanSnapshot() + require.NotZero(t, scanUsed) + require.True(t, req.abort()) + scanUsed, _, _, _, _ = quota.scanSnapshot() + require.Zero(t, scanUsed) +} diff --git a/logservice/logpuller/priority_task.go b/logservice/logpuller/priority_task.go index 89d43dc03a..38cd813c8b 100644 --- a/logservice/logpuller/priority_task.go +++ b/logservice/logpuller/priority_task.go @@ -32,7 +32,6 @@ type regionPriorityTask struct { heapIndex int // for heap.Item interface } -<<<<<<< HEAD // newRegionPriorityTask creates a new priority task for region. func newRegionPriorityTask(regionInfo regionInfo, sequence uint64) *regionPriorityTask { regionInfo.scanPriority = normalizeScanPriority(regionInfo.scanPriority) @@ -40,31 +39,11 @@ func newRegionPriorityTask(regionInfo regionInfo, sequence uint64) *regionPriori regionInfo: regionInfo, sequence: sequence, heapIndex: 0, // 0 means not in heap -======= -func newRegionPriorityTask(regionInfo regionInfo, currentTs, sequence uint64) *regionPriorityTask { - task := ®ionPriorityTask{ - sequence: sequence, - heapIndex: 0, // 0 means not in heap ->>>>>>> 9903a1be7 (refactor) } } -<<<<<<< HEAD func (pt *regionPriorityTask) priority() cdcpb.ScanPriority { return normalizeScanPriority(pt.regionInfo.scanPriority) -======= -// updateRegion refreshes both the request data and its priority before the task -// enters another scheduling stage. -func (pt *regionPriorityTask) updateRegion(regionInfo regionInfo, currentTs uint64) { - priority := normalRegionPriority - if regionInfo.wasInitialized { - priority = initializedRegionPriority - } else if regionScanLag(currentTs, regionInfo.resolvedTs()) < lowLagRegionThreshold { - priority = lowLagRegionPriority - } - pt.regionInfo = regionInfo - pt.priority = priority ->>>>>>> 9903a1be7 (refactor) } func (pt *regionPriorityTask) canUseMaxWindow() bool { diff --git a/logservice/logpuller/priority_task_test.go b/logservice/logpuller/priority_task_test.go index 35b8209c2b..41501181fb 100644 --- a/logservice/logpuller/priority_task_test.go +++ b/logservice/logpuller/priority_task_test.go @@ -60,7 +60,6 @@ func TestRegionPriorityTaskQueueOrder(t *testing.T) { queue := priorityqueue.New[*regionPriorityTask]() currentTime := time.Now() -<<<<<<< HEAD lowTask := newRegionPriorityTask( withScanPriority( newPriorityTestRegion(1, oracle.GoTimeToTS(currentTime.Add(-time.Hour))), @@ -81,19 +80,6 @@ func TestRegionPriorityTaskQueueOrder(t *testing.T) { cdcpb.ScanPriority_SCAN_PRIORITY_HIGH, ), 1, -======= - normalTask := newRegionPriorityTask( - newPriorityTestRegion(1, oracle.GoTimeToTS(currentTime.Add(-time.Hour)), false), - currentTs, 3, - ) - lowLagTask := newRegionPriorityTask( - newPriorityTestRegion(2, oracle.GoTimeToTS(currentTime.Add(-10*time.Minute)), false), - currentTs, 2, - ) - initializedTask := newRegionPriorityTask( - newPriorityTestRegion(3, oracle.GoTimeToTS(currentTime.Add(-time.Hour)), true), - currentTs, 1, ->>>>>>> 9903a1be7 (refactor) ) require.True(t, queue.Push(lowTask)) @@ -112,15 +98,10 @@ func TestRegionPriorityTaskFIFOWithinPriority(t *testing.T) { currentTime := time.Now() checkpointTs := oracle.GoTimeToTS(currentTime.Add(-time.Hour)) -<<<<<<< HEAD first := newRegionPriorityTask( withScanPriority(newPriorityTestRegion(1, checkpointTs), cdcpb.ScanPriority_SCAN_PRIORITY_HIGH), 1) second := newRegionPriorityTask( withScanPriority(newPriorityTestRegion(2, checkpointTs), cdcpb.ScanPriority_SCAN_PRIORITY_HIGH), 2) -======= - first := newRegionPriorityTask(newPriorityTestRegion(1, checkpointTs, false), currentTs, 1) - second := newRegionPriorityTask(newPriorityTestRegion(2, checkpointTs, false), currentTs, 2) ->>>>>>> 9903a1be7 (refactor) require.True(t, queue.Push(second)) require.True(t, queue.Push(first)) @@ -139,7 +120,6 @@ func TestRegionPriorityTaskUsesHighPriorityWindow(t *testing.T) { lowTask := newRegionPriorityTask( withScanPriority(newPriorityTestRegion(2, 1), cdcpb.ScanPriority_SCAN_PRIORITY_LOW), 2) -<<<<<<< HEAD require.True(t, highTask.canUseMaxWindow()) require.False(t, lowTask.canUseMaxWindow()) } @@ -148,35 +128,6 @@ func TestRegionPriorityTaskRefreshesRegionInfoBetweenStages(t *testing.T) { region := withScanPriority(newPriorityTestRegion(1, 1), cdcpb.ScanPriority_SCAN_PRIORITY_LOW) task := newRegionPriorityTask(region, 1) require.Equal(t, cdcpb.ScanPriority_SCAN_PRIORITY_LOW, task.priority()) -======= - belowThreshold := newRegionPriorityTask(newPriorityTestRegion( - 1, - oracle.GoTimeToTS(currentTime.Add(-lowLagRegionThreshold+time.Millisecond)), - false, - ), currentTs, 1) - atThreshold := newRegionPriorityTask(newPriorityTestRegion( - 2, - oracle.GoTimeToTS(currentTime.Add(-lowLagRegionThreshold)), - false, - ), currentTs, 2) - futureCheckpoint := newRegionPriorityTask(newPriorityTestRegion( - 3, - oracle.GoTimeToTS(currentTime.Add(time.Second)), - false, - ), currentTs, 3) - - require.Equal(t, lowLagRegionPriority, belowThreshold.priority) - require.Equal(t, normalRegionPriority, atThreshold.priority) - require.Equal(t, lowLagRegionPriority, futureCheckpoint.priority) -} - -func TestRegionPriorityTaskRefreshesPriorityBetweenStages(t *testing.T) { - checkpointTime := time.Now() - checkpointTs := oracle.GoTimeToTS(checkpointTime) - region := newPriorityTestRegion(1, checkpointTs, false) - task := newRegionPriorityTask(region, oracle.GoTimeToTS(checkpointTime.Add(time.Minute)), 1) - require.Equal(t, lowLagRegionPriority, task.priority) ->>>>>>> 9903a1be7 (refactor) region.scanPriority = cdcpb.ScanPriority_SCAN_PRIORITY_HIGH task.regionInfo = region diff --git a/logservice/logpuller/region_admission_controller.go b/logservice/logpuller/region_admission_controller.go index 6994e6e52c..83a5c7c5bd 100644 --- a/logservice/logpuller/region_admission_controller.go +++ b/logservice/logpuller/region_admission_controller.go @@ -36,6 +36,7 @@ type regionReq struct { regionInfo regionInfo createTime time.Time controller *regionAdmissionController + scanQuota *memoryQuotaLease released atomic.Bool } @@ -72,6 +73,7 @@ func (r *regionReq) release() bool { if !r.released.CompareAndSwap(false, true) { return false } + r.scanQuota.Release() r.controller.release() return true } @@ -92,6 +94,10 @@ type regionAdmissionController struct { // pending keeps requests that have not entered the initial-scan window. // It is guarded by mu. pending *heap.Heap[*regionPriorityTask] + // memoryQuota gates initial scans using the log puller's global memory + // pressure. currentTs is sampled when a request is admitted. + memoryQuota *memoryQuotaController + currentTs func() uint64 // notify wakes workers when a request is submitted or an admission slot is // released. The one-element buffer prevents a wakeup from being lost between // checking the admission condition and waiting on this channel. Notifications @@ -107,7 +113,12 @@ type regionAdmissionStats struct { inflight int } -func newRegionAdmissionController(currentWindow, maxWindowMultiplier int) *regionAdmissionController { +func newRegionAdmissionController( + currentWindow int, + maxWindowMultiplier int, + memoryQuota *memoryQuotaController, + currentTs func() uint64, +) *regionAdmissionController { if currentWindow <= 0 { currentWindow = 1 } @@ -122,6 +133,8 @@ func newRegionAdmissionController(currentWindow, maxWindowMultiplier int) *regio currentWindow: currentWindow, maxWindow: maxWindow, pending: heap.NewHeap[*regionPriorityTask](), + memoryQuota: memoryQuota, + currentTs: currentTs, notify: make(chan struct{}, 1), } } @@ -150,7 +163,7 @@ func (c *regionAdmissionController) pop( c.mu.Unlock() return nil, context.Canceled } - request := c.popEligibleLocked() + request, scanQuota := c.popEligibleLocked() if request != nil { c.inflight++ c.mu.Unlock() @@ -158,6 +171,7 @@ func (c *regionAdmissionController) pop( regionInfo: request.regionInfo, createTime: time.Now(), controller: c, + scanQuota: scanQuota, }, nil } c.mu.Unlock() @@ -172,16 +186,32 @@ func (c *regionAdmissionController) pop( } } -func (c *regionAdmissionController) popEligibleLocked() *regionPriorityTask { +func (c *regionAdmissionController) popEligibleLocked() ( + *regionPriorityTask, + *memoryQuotaLease, +) { request, ok := c.pending.PeekTop() if !ok { - return nil + return nil, nil } if c.inflight >= c.windowFor(request) { - return nil + return nil, nil + } + + var scanQuota *memoryQuotaLease + if c.memoryQuota != nil { + currentTs := uint64(0) + if c.currentTs != nil { + currentTs = c.currentTs() + } + var admitted bool + scanQuota, admitted = c.memoryQuota.acquireScan(request.regionInfo, currentTs) + if !admitted { + return nil, nil + } } request, _ = c.pending.PopTop() - return request + return request, scanQuota } func (c *regionAdmissionController) windowFor(request *regionPriorityTask) int { @@ -200,6 +230,12 @@ func (c *regionAdmissionController) release() { c.mu.Unlock() } +func (c *regionAdmissionController) notifyAvailable() { + c.mu.Lock() + c.notifyOneLocked() + c.mu.Unlock() +} + func (c *regionAdmissionController) close() { c.mu.Lock() if !c.closed { diff --git a/logservice/logpuller/region_admission_controller_test.go b/logservice/logpuller/region_admission_controller_test.go index 706e140501..ed68f14a2f 100644 --- a/logservice/logpuller/region_admission_controller_test.go +++ b/logservice/logpuller/region_admission_controller_test.go @@ -20,7 +20,6 @@ import ( "testing" "time" - "github.com/pingcap/kvproto/pkg/cdcpb" "github.com/pingcap/ticdc/heartbeatpb" "github.com/pingcap/ticdc/logservice/logpuller/regionlock" "github.com/stretchr/testify/require" @@ -55,16 +54,12 @@ func submitRegionForAdmission( currentTs uint64, ) { t.Helper() -<<<<<<< HEAD - task := newRegionPriorityTask(region, region.verID.GetID()) -======= task := newRegionPriorityTask(region, currentTs, region.verID.GetID()) ->>>>>>> 9903a1be7 (refactor) require.True(t, controller.submit(task)) } func TestRegionAdmissionControllerNormalWindow(t *testing.T) { - controller := newRegionAdmissionController(1, 2) + controller := newRegionAdmissionController(1, 2, nil, nil) currentTs := oracle.GoTimeToTS(time.Now()) checkpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Hour)) region1 := prepareRegionForAdmission(createTestRegionInfo(1, 1), checkpointTs) @@ -88,10 +83,11 @@ func TestRegionAdmissionControllerNormalWindow(t *testing.T) { require.True(t, req2.abort()) } -func TestRegionAdmissionControllerHighPriorityUsesMaxWindow(t *testing.T) { - controller := newRegionAdmissionController(1, 2) +func TestRegionAdmissionControllerLowLagUsesMaxWindow(t *testing.T) { + controller := newRegionAdmissionController(1, 2, nil, nil) currentTs := oracle.GoTimeToTS(time.Now()) slowCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Hour)) + lowLagCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Minute)) submitRegionForAdmission(t, controller, prepareRegionForAdmission(createTestRegionInfo(1, 1), slowCheckpointTs), @@ -102,9 +98,9 @@ func TestRegionAdmissionControllerHighPriorityUsesMaxWindow(t *testing.T) { submitRegionForAdmission(t, controller, prepareRegionForAdmission(createTestRegionInfo(1, 2), slowCheckpointTs), currentTs) - highPriorityRegion := prepareRegionForAdmission(createTestRegionInfo(1, 3), slowCheckpointTs) - highPriorityRegion.scanPriority = cdcpb.ScanPriority_SCAN_PRIORITY_HIGH - submitRegionForAdmission(t, controller, highPriorityRegion, currentTs) + submitRegionForAdmission(t, controller, + prepareRegionForAdmission(createTestRegionInfo(1, 3), lowLagCheckpointTs), + currentTs) req2, err := controller.pop(t.Context(), nil) require.NoError(t, err) @@ -124,10 +120,11 @@ func TestRegionAdmissionControllerHighPriorityUsesMaxWindow(t *testing.T) { require.True(t, req3.abort()) } -func TestRegionAdmissionControllerPrioritizesHighPriorityRegion(t *testing.T) { - controller := newRegionAdmissionController(1, 2) +func TestRegionAdmissionControllerPrioritizesInitializedRegion(t *testing.T) { + controller := newRegionAdmissionController(1, 2, nil, nil) currentTs := oracle.GoTimeToTS(time.Now()) slowCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Hour)) + lowLagCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Minute)) submitRegionForAdmission(t, controller, prepareRegionForAdmission(createTestRegionInfo(1, 1), slowCheckpointTs), @@ -136,11 +133,12 @@ func TestRegionAdmissionControllerPrioritizesHighPriorityRegion(t *testing.T) { require.NoError(t, err) submitRegionForAdmission(t, controller, - prepareRegionForAdmission(createTestRegionInfo(1, 2), slowCheckpointTs), + prepareRegionForAdmission(createTestRegionInfo(1, 2), lowLagCheckpointTs), currentTs) - highPriorityRegion := prepareRegionForAdmission(createTestRegionInfo(1, 3), slowCheckpointTs) - highPriorityRegion.scanPriority = cdcpb.ScanPriority_SCAN_PRIORITY_HIGH - submitRegionForAdmission(t, controller, highPriorityRegion, currentTs) + initializedRegion := prepareRegionForAdmission(createTestRegionInfo(1, 3), slowCheckpointTs) + initializedRegion.wasInitialized = true + submitRegionForAdmission(t, controller, + initializedRegion, currentTs) req2, err := controller.pop(t.Context(), nil) require.NoError(t, err) @@ -155,7 +153,7 @@ func TestRegionAdmissionControllerPrioritizesHighPriorityRegion(t *testing.T) { } func TestRegionAdmissionLeaseReleasedOnce(t *testing.T) { - controller := newRegionAdmissionController(1, 1) + controller := newRegionAdmissionController(1, 1, nil, nil) currentTs := oracle.GoTimeToTS(time.Now()) region := prepareRegionForAdmission(createTestRegionInfo(1, 1), currentTs) submitRegionForAdmission(t, controller, region, currentTs) @@ -191,21 +189,17 @@ func TestRegionAdmissionLeaseReleasedOnce(t *testing.T) { } func TestRegionAdmissionControllerClose(t *testing.T) { - controller := newRegionAdmissionController(1, 1) + controller := newRegionAdmissionController(1, 1, nil, nil) controller.close() region := prepareRegionForAdmission(createTestRegionInfo(1, 1), 1) -<<<<<<< HEAD - require.False(t, controller.submit(newRegionPriorityTask(region, 1))) -======= require.False(t, controller.submit(newRegionPriorityTask(region, 1, 1))) ->>>>>>> 9903a1be7 (refactor) _, err := controller.pop(context.Background(), nil) require.ErrorIs(t, err, context.Canceled) } func TestRegionAdmissionControllerDrainPending(t *testing.T) { - controller := newRegionAdmissionController(1, 1) + controller := newRegionAdmissionController(1, 1, nil, nil) region1 := prepareRegionForAdmission(createTestRegionInfo(1, 1), 1) region2 := prepareRegionForAdmission(createTestRegionInfo(1, 2), 1) submitRegionForAdmission(t, controller, region1, 1) diff --git a/logservice/logpuller/region_event_handler.go b/logservice/logpuller/region_event_handler.go index e13b91cbd0..c23184ddab 100644 --- a/logservice/logpuller/region_event_handler.go +++ b/logservice/logpuller/region_event_handler.go @@ -29,7 +29,8 @@ import ( ) var ( - metricsEventCount = metrics.PullerEventCounter.WithLabelValues("event") + metricsResolvedTsCount = metrics.PullerEventCounter.WithLabelValues("resolved_ts") + metricsEventCount = metrics.PullerEventCounter.WithLabelValues("event") metricRegionEventHandleDurationEntries = metrics.SubscriptionClientRegionEventHandleDuration.WithLabelValues("entries") metricRegionEventHandleDurationResolved = metrics.SubscriptionClientRegionEventHandleDuration.WithLabelValues("resolved") @@ -53,8 +54,18 @@ type regionEvent struct { // Resolved-ts events: `resolvedTs` is set and `states` contains all related regions. states []*regionFeedState - entries *cdcpb.Event_Entries_ - resolvedTs uint64 + entries *cdcpb.Event_Entries_ + resolvedTs uint64 + memoryQuota *memoryQuotaLease +} + +func (event *regionEvent) needMemoryQuota() bool { + return event.entries != nil +} + +func (event *regionEvent) releaseMemoryQuota() { + event.memoryQuota.Release() + event.memoryQuota = nil } func (event *regionEvent) getSize() int { @@ -121,7 +132,12 @@ func (h *regionEventHandler) Handle(span *subscribedSpan, events ...regionEvent) } newResolvedTs := uint64(0) + wasInitialized := span.initialized.Load() + quotaLeases := make([]*memoryQuotaLease, 0, len(events)) for _, event := range events { + if event.memoryQuota != nil { + quotaLeases = append(quotaLeases, event.memoryQuota) + } if len(event.states) == 1 && event.states[0].isStale() { hasError = true h.handleRegionError(event.states[0]) @@ -142,14 +158,24 @@ func (h *regionEventHandler) Handle(span *subscribedSpan, events ...regionEvent) log.Panic("should not reach", zap.Any("event", event), zap.Any("events", events)) } } + if !wasInitialized && span.initialized.Load() && + h.eventSink != nil && h.eventSink.memoryQuota != nil { + h.eventSink.memoryQuota.markSubscriptionInitialized() + } tryAdvanceResolvedTs := func() { if newResolvedTs != 0 { span.advanceResolvedTs(newResolvedTs) } } + releaseMemoryQuota := func() { + for _, lease := range quotaLeases { + lease.Release() + } + } if len(span.kvEventsCache) > 0 { metricsEventCount.Add(float64(len(span.kvEventsCache))) await := span.consumeKVEvents(span.kvEventsCache, func() { + defer releaseMemoryQuota() start := time.Now() span.clearKVEventsCache() metricConsumeKVEventsCallbackDurationClearCache.Observe(time.Since(start).Seconds()) @@ -166,10 +192,12 @@ func (h *regionEventHandler) Handle(span *subscribedSpan, events ...regionEvent) if !await { span.clearKVEventsCache() tryAdvanceResolvedTs() + releaseMemoryQuota() } return await } else { tryAdvanceResolvedTs() + releaseMemoryQuota() } return false } @@ -226,6 +254,7 @@ func (h *regionEventHandler) GetType(event regionEvent) dynstream.EventType { } func (h *regionEventHandler) OnDrop(event regionEvent) interface{} { + event.releaseMemoryQuota() // TODO: Distinguish between drop events caused by "path not found" errors and memory control. state := event.mustFirstState() fields := []zap.Field{ @@ -285,7 +314,7 @@ func handleEventEntries(span *subscribedSpan, state *regionFeedState, entries *c for _, entry := range entries.Entries.GetEntries() { switch entry.Type { case cdcpb.Event_INITIALIZED: - span.markRegionInitialized(state) + state.setInitialized() log.Debug("region is initialized", zap.Int64("tableID", span.span.TableID), zap.Uint64("regionID", regionID), @@ -392,6 +421,7 @@ func handleResolvedTs(span *subscribedSpan, state *regionFeedState, resolvedTs u } if shouldAdvance { + span.tryMarkInitialized(regionID, ts) lastResolvedTs := span.resolvedTs.Load() nextResolvedPhyTs := oracle.ExtractPhysical(ts) // Generally, we don't want to send duplicate resolved ts, @@ -399,16 +429,7 @@ func handleResolvedTs(span *subscribedSpan, state *regionFeedState, resolvedTs u // but when `ts` == `lastResolvedTs` == `span.startTs`, // the span may just be initialized and have not receive any resolved ts before, // so we also send ts in this case for quick notification to downstream. - if ts > lastResolvedTs || - (span.initialized.Load() && ts == lastResolvedTs && lastResolvedTs == span.startTs) { - if lastResolvedTs == span.startTs && ts > span.startTs && !span.initialized.Load() { - log.Warn("should not happen: resolved ts advances before span is initialized", - zap.Uint64("subscriptionID", uint64(span.subID)), - zap.Int64("tableID", span.span.TableID), - zap.Uint64("regionID", regionID), - zap.Uint64("startTs", span.startTs), - zap.Uint64("resolvedTs", ts)) - } + if ts > lastResolvedTs || (ts == lastResolvedTs && lastResolvedTs == span.startTs) { resolvedPhyTs := oracle.ExtractPhysical(lastResolvedTs) decreaseLag := float64(nextResolvedPhyTs-resolvedPhyTs) / 1e3 const largeResolvedTsAdvanceStepInSecs = 30 @@ -421,7 +442,8 @@ func handleResolvedTs(span *subscribedSpan, state *regionFeedState, resolvedTs u zap.Uint64("lastResolvedTs", lastResolvedTs), zap.Float64("decreaseLag(s)", decreaseLag)) } - span.recordResolvedTs(ts) + span.resolvedTs.Store(ts) + span.resolvedTsUpdated.Store(time.Now().Unix()) return ts } } diff --git a/logservice/logpuller/region_event_handler_test.go b/logservice/logpuller/region_event_handler_test.go index f2e1885197..92fc3a8e03 100644 --- a/logservice/logpuller/region_event_handler_test.go +++ b/logservice/logpuller/region_event_handler_test.go @@ -23,7 +23,6 @@ import ( "github.com/pingcap/ticdc/heartbeatpb" "github.com/pingcap/ticdc/logservice/logpuller/regionlock" "github.com/pingcap/ticdc/pkg/common" - "github.com/pingcap/ticdc/pkg/pdutil" "github.com/pingcap/ticdc/utils/dynstream" "github.com/stretchr/testify/require" "github.com/tikv/client-go/v2/tikv" @@ -72,7 +71,6 @@ func TestHandleEventEntryEventOutOfOrder(t *testing.T) { subID: subID, span: span, startTs: 1000, // not used - rangeLock: regionlock.NewRangeLock(uint64(subID), span.StartKey, span.EndKey, 1000), consumeKVEvents: consumeKVEvents, advanceResolvedTs: advanceResolvedTs, advanceInterval: 0, @@ -83,15 +81,12 @@ func TestHandleEventEntryEventOutOfOrder(t *testing.T) { tracker: newRegionTracker(), } region := newRegionInfo( - tikv.NewRegionVerID(1, 1, 1), + tikv.RegionVerID{}, span, &tikv.RPCContext{}, subSpan, ) - lockResult := subSpan.rangeLock.LockRange( - context.Background(), span.StartKey, span.EndKey, 1, 1) - require.Equal(t, regionlock.LockRangeStatusSuccess, lockResult.Status) - region.lockedRangeState = lockResult.LockedRangeState + region.lockedRangeState = ®ionlock.LockedRangeState{} state := newRegionFeedState(region, 1, worker, nil) // Receive prewrite2 with empty value. @@ -209,8 +204,6 @@ func TestHandleEventEntryEventOutOfOrder(t *testing.T) { func TestHandleResolvedTs(t *testing.T) { // initialize option := dynstream.NewOption() - pdClock := pdutil.NewClock4Test() - pdClock.(*pdutil.Clock4Test).SetTS(10) ds := dynstream.NewParallelDynamicStream("test", ®ionEventHandler{}, option) ds.Start() @@ -225,28 +218,23 @@ func TestHandleResolvedTs(t *testing.T) { tracker: newRegionTracker(), } state1 := newRegionFeedState(regionInfo{verID: tikv.NewRegionVerID(1, 1, 1)}, uint64(subID1), worker, nil) - var subSpan1 *subscribedSpan { span := heartbeatpb.TableSpan{ TableID: 100, StartKey: common.ToComparableKey([]byte{}), // TODO: remove spanz dependency EndKey: common.ToComparableKey(common.UpperBoundKey), } - subSpan1 = &subscribedSpan{ + subSpan := &subscribedSpan{ subID: subID1, span: heartbeatpb.TableSpan{}, rangeLock: regionlock.NewRangeLock(uint64(subID1), span.StartKey, span.EndKey, 1), consumeKVEvents: consumeKVEvents, advanceResolvedTs: advanceResolvedTs, advanceInterval: 0, - priorityPolicy: newScanPriorityPolicy(pdClock, 30*time.Minute), } - ds.AddPath(subID1, subSpan1, dynstream.AreaSettings{}) - state1.region.subscribedSpan = subSpan1 - lockResult := subSpan1.rangeLock.LockRange( - context.Background(), span.StartKey, span.EndKey, 1, 1) - require.Equal(t, regionlock.LockRangeStatusSuccess, lockResult.Status) - state1.region.lockedRangeState = lockResult.LockedRangeState + ds.AddPath(subID1, subSpan, dynstream.AreaSettings{}) + state1.region.subscribedSpan = subSpan + state1.region.lockedRangeState = ®ionlock.LockedRangeState{} state1.setInitialized() state1.updateResolvedTs(9) } @@ -266,14 +254,10 @@ func TestHandleResolvedTs(t *testing.T) { consumeKVEvents: consumeKVEvents, advanceResolvedTs: advanceResolvedTs, advanceInterval: 0, - priorityPolicy: newScanPriorityPolicy(pdClock, 30*time.Minute), } ds.AddPath(subID2, subSpan, dynstream.AreaSettings{}) state2.region.subscribedSpan = subSpan - lockResult := subSpan.rangeLock.LockRange( - context.Background(), span.StartKey, span.EndKey, 2, 2) - require.Equal(t, regionlock.LockRangeStatusSuccess, lockResult.Status) - state2.region.lockedRangeState = lockResult.LockedRangeState + state2.region.lockedRangeState = ®ionlock.LockedRangeState{} state2.setInitialized() state2.updateResolvedTs(11) } @@ -293,14 +277,10 @@ func TestHandleResolvedTs(t *testing.T) { consumeKVEvents: consumeKVEvents, advanceResolvedTs: advanceResolvedTs, advanceInterval: 0, - priorityPolicy: newScanPriorityPolicy(pdClock, 30*time.Minute), } ds.AddPath(subID3, subSpan, dynstream.AreaSettings{}) state3.region.subscribedSpan = subSpan - lockResult := subSpan.rangeLock.LockRange( - context.Background(), span.StartKey, span.EndKey, 3, 3) - require.Equal(t, regionlock.LockRangeStatusSuccess, lockResult.Status) - state3.region.lockedRangeState = lockResult.LockedRangeState + state3.region.lockedRangeState = ®ionlock.LockedRangeState{} state3.updateResolvedTs(8) } @@ -345,7 +325,6 @@ func TestHandleResolvedTs(t *testing.T) { require.Equal(t, uint64(10), state1.getLastResolvedTs()) require.Equal(t, uint64(11), state2.getLastResolvedTs()) require.Equal(t, uint64(8), state3.getLastResolvedTs()) - require.True(t, subSpan1.priorityPolicy.everCaughtUp.Load()) } func TestHandleResolvedTsThrottled(t *testing.T) { @@ -375,7 +354,6 @@ func TestHandleResolvedTsThrottled(t *testing.T) { subID: SubscriptionID(1), rangeLock: l, advanceInterval: 100, - priorityPolicy: newScanPriorityPolicy(pdutil.NewClock4Test(), 30*time.Minute), } span.lastAdvanceTime.Store(0) worker := ®ionRequestWorker{tracker: newRegionTracker()} @@ -393,63 +371,104 @@ func TestHandleResolvedTsThrottled(t *testing.T) { require.Equal(t, uint64(200), handleResolvedTs(span, state, 300)) } -func TestSpanInitializedAfterAllRangesInitialized(t *testing.T) { - ctx := context.Background() - rangeLock := regionlock.NewRangeLock(1, []byte("a"), []byte("z"), 100) - firstLock := rangeLock.LockRange(ctx, []byte("a"), []byte("m"), 1, 1) - require.Equal(t, regionlock.LockRangeStatusSuccess, firstLock.Status) - secondLock := rangeLock.LockRange(ctx, []byte("m"), []byte("z"), 2, 1) - require.Equal(t, regionlock.LockRangeStatusSuccess, secondLock.Status) - - span := &subscribedSpan{ - subID: SubscriptionID(1), - startTs: 100, - span: heartbeatpb.TableSpan{StartKey: []byte("a"), EndKey: []byte("z")}, - rangeLock: rangeLock, - priorityPolicy: newScanPriorityPolicy(pdutil.NewClock4Test(), 30*time.Minute), +func TestHandleEntriesReleasesMemoryAfterDownstreamCallback(t *testing.T) { + quota := newMemoryQuotaController(1024, 8) + span := newTestQuotaSpan(1) + quota.addSubscription(span) + callbackCh := make(chan func(), 1) + span.consumeKVEvents = func(_ []common.RawKVEntry, callback func()) bool { + callbackCh <- callback + return true } - span.resolvedTs.Store(span.startTs) - worker := ®ionRequestWorker{tracker: newRegionTracker()} - newState := func( - regionID uint64, regionSpan heartbeatpb.TableSpan, - lockedRangeState *regionlock.LockedRangeState, - ) *regionFeedState { - state := newRegionFeedState( - regionInfo{ - verID: tikv.NewRegionVerID(regionID, 1, 1), - span: regionSpan, - rpcCtx: &tikv.RPCContext{}, - subscribedSpan: span, - lockedRangeState: lockedRangeState, - }, - uint64(span.subID), - worker, - nil, - ) - return state + span.advanceResolvedTs = func(uint64) {} + + lockedState := ®ionlock.LockedRangeState{} + lockedState.ResolvedTs.Store(100) + state := ®ionFeedState{ + region: regionInfo{ + verID: tikv.NewRegionVerID(1, 1, 1), + rpcCtx: &tikv.RPCContext{}, + subscribedSpan: span, + lockedRangeState: lockedState, + }, } - firstState := newState(1, - heartbeatpb.TableSpan{StartKey: []byte("a"), EndKey: []byte("m")}, - firstLock.LockedRangeState) - secondState := newState(2, - heartbeatpb.TableSpan{StartKey: []byte("m"), EndKey: []byte("z")}, - secondLock.LockedRangeState) - - handler := ®ionEventHandler{} - initializedEvent := func(state *regionFeedState) regionEvent { - return regionEvent{ - states: []*regionFeedState{state}, - entries: &cdcpb.Event_Entries_{Entries: &cdcpb.Event_Entries{ - Entries: []*cdcpb.Event_Row{{Type: cdcpb.Event_INITIALIZED}}, + lease := quota.trackEvent(context.Background(), span, 10) + require.NotNil(t, lease) + handler := ®ionEventHandler{eventSink: ®ionEventSink{ + ds: newMockRegionEventSinkStream(), + memoryQuota: quota, + }} + + await := handler.Handle(span, regionEvent{ + states: []*regionFeedState{state}, + memoryQuota: lease, + entries: &cdcpb.Event_Entries_{Entries: &cdcpb.Event_Entries{ + Entries: []*cdcpb.Event_Row{{ + Type: cdcpb.Event_COMMITTED, + OpType: cdcpb.Event_Row_PUT, + CommitTs: 101, }}, - } - } + }}, + }) + require.True(t, await) + used, _, _ := quota.snapshot() + require.Equal(t, uint64(10), used) + + callback := <-callbackCh + callback() + used, _, _ = quota.snapshot() + require.Zero(t, used) +} - require.False(t, handler.Handle(span, initializedEvent(firstState))) +func TestTryMarkSpanInitializedByResolvedTs(t *testing.T) { + span := &subscribedSpan{subID: 1, startTs: 100} + require.False(t, span.tryMarkInitialized(1, 100)) require.False(t, span.initialized.Load()) - require.Equal(t, uint64(0), handleResolvedTs(span, firstState, span.startTs)) + require.True(t, span.tryMarkInitialized(1, 101)) + require.True(t, span.initialized.Load()) + require.False(t, span.tryMarkInitialized(1, 102)) +} + +func TestSpanInitializationNotifiesMemoryAdmission(t *testing.T) { + quota := newMemoryQuotaController(1024, 8) + notified := make(chan struct{}, 1) + quota.setOnAvailable(func() { + select { + case notified <- struct{}{}: + default: + } + }) + + const startTs = 100 + rangeLock := regionlock.NewRangeLock(1, []byte("a"), []byte("z"), startTs) + lockResult := rangeLock.LockRange(t.Context(), []byte("a"), []byte("z"), 1, 1) + require.Equal(t, regionlock.LockRangeStatusSuccess, lockResult.Status) + lockResult.LockedRangeState.Initialized.Store(true) - require.False(t, handler.Handle(span, initializedEvent(secondState))) + span := &subscribedSpan{ + subID: 1, + startTs: startTs, + rangeLock: rangeLock, + consumeKVEvents: func([]common.RawKVEntry, func()) bool { return false }, + advanceResolvedTs: func(uint64) {}, + } + span.resolvedTs.Store(startTs) + quota.addSubscription(span) + state := newRegionFeedState(regionInfo{ + verID: tikv.NewRegionVerID(1, 1, 1), + subscribedSpan: span, + lockedRangeState: lockResult.LockedRangeState, + }, uint64(span.subID), ®ionRequestWorker{}, nil) + handler := ®ionEventHandler{eventSink: ®ionEventSink{memoryQuota: quota}} + + require.False(t, handler.Handle(span, regionEvent{ + states: []*regionFeedState{state}, + resolvedTs: startTs + 1, + })) require.True(t, span.initialized.Load()) - require.Equal(t, span.startTs, handleResolvedTs(span, secondState, span.startTs)) + select { + case <-notified: + case <-time.After(time.Second): + t.Fatal("span initialization did not notify memory admission") + } } diff --git a/logservice/logpuller/region_event_sink.go b/logservice/logpuller/region_event_sink.go index a89a0555bc..3c161fda1a 100644 --- a/logservice/logpuller/region_event_sink.go +++ b/logservice/logpuller/region_event_sink.go @@ -26,21 +26,22 @@ import ( // regionEventSink delivers region events to dynstream and owns push-side flow control. type regionEventSink struct { - // mu/cond coordinate the paused push path with pause/resume and shutdown signals. - mu sync.Mutex - cond *sync.Cond - // paused tracks whether region event pushing is temporarily held back by feedback. - paused atomic.Bool - // stopped marks the sink as shutting down so blocked pushers can exit instead of waiting for resume. - stopped atomic.Bool + ctx context.Context + ds dynstream.DynamicStream[int, SubscriptionID, regionEvent, *subscribedSpan, *regionEventHandler] - // ds owns the dynstream used to deliver region events and receive flow-control feedback. - ds dynstream.DynamicStream[int, SubscriptionID, regionEvent, *subscribedSpan, *regionEventHandler] + memoryQuota *memoryQuotaController + // the following three fields are used to manage feedback from ds and notify other goroutines + mu sync.Mutex + cond *sync.Cond + paused atomic.Bool } -func newRegionEventSink(failureHandler *regionFailureHandler) *regionEventSink { - sink := ®ionEventSink{} - sink.cond = sync.NewCond(&sink.mu) +func newRegionEventSink( + ctx context.Context, + failureHandler *regionFailureHandler, + memoryQuota *memoryQuotaController, +) *regionEventSink { + sink := ®ionEventSink{ctx: ctx, memoryQuota: memoryQuota} option := dynstream.NewOption() // Note: it is max batch size of the kv sent from tikv(not committed rows) @@ -48,7 +49,7 @@ func newRegionEventSink(failureHandler *regionFailureHandler) *regionEventSink { // TODO: Set `UseBuffer` to true until we refactor the `regionEventHandler.Handle` method so that it doesn't call any method of the dynamic stream. Currently, if `UseBuffer` is set to false, there will be a deadlock: // ds.handleLoop fetch events from `ch` -> regionEventHandler.Handle -> ds.RemovePath -> send event to `ch` option.UseBuffer = true - option.EnableMemoryControl = true + option.EnableMemoryControl = false ds := dynstream.NewParallelDynamicStream( "log-puller", ®ionEventHandler{eventSink: sink, failureHandler: failureHandler}, @@ -56,12 +57,12 @@ func newRegionEventSink(failureHandler *regionFailureHandler) *regionEventSink { ) ds.Start() sink.ds = ds + sink.cond = sync.NewCond(&sink.mu) return sink } func (s *regionEventSink) AddPath(rt *subscribedSpan) { - areaSetting := dynstream.NewAreaSettingsWithMaxPendingSize(1*1024*1024*1024, dynstream.MemoryControlForPuller, "logPuller") // 1GB - if err := s.ds.AddPath(rt.subID, rt, areaSetting); err != nil { + if err := s.ds.AddPath(rt.subID, rt); err != nil { log.Warn("subscription client add path failed", zap.Uint64("subscriptionID", uint64(rt.subID)), zap.Error(err)) @@ -77,8 +78,12 @@ func (s *regionEventSink) Wake(subID SubscriptionID) { } func (s *regionEventSink) Push(subID SubscriptionID, event regionEvent) { - if s.stopped.Load() { - return + if event.needMemoryQuota() && s.memoryQuota != nil { + span := event.mustFirstState().region.subscribedSpan + event.memoryQuota = s.memoryQuota.trackEvent(s.ctx, span, uint64(event.getSize())) + if event.memoryQuota == nil { + return + } } // fast path if !s.paused.Load() { @@ -88,15 +93,17 @@ func (s *regionEventSink) Push(subID SubscriptionID, event regionEvent) { // slow path: wait until paused is false s.mu.Lock() - for s.paused.Load() && !s.stopped.Load() { - s.cond.Wait() + for s.paused.Load() { + select { + case <-s.ctx.Done(): + s.mu.Unlock() + event.releaseMemoryQuota() + return + default: + s.cond.Wait() + } } - stopped := s.stopped.Load() s.mu.Unlock() - - if stopped { - return - } s.ds.Push(subID, event) } @@ -104,15 +111,19 @@ func (s *regionEventSink) Run(ctx context.Context) error { for { select { case <-ctx.Done(): - s.stop() return nil case feedback := <-s.ds.Feedback(): switch feedback.FeedbackType { case dynstream.PauseArea: - s.pause() + s.mu.Lock() + s.paused.Store(true) + s.mu.Unlock() log.Info("subscription client pause push region event") case dynstream.ResumeArea: - s.resume() + s.mu.Lock() + s.paused.Store(false) + s.cond.Broadcast() + s.mu.Unlock() log.Info("subscription client resume push region event") case dynstream.ReleasePath, dynstream.ResumePath: // Ignore it, because it is no need to pause and resume a path in puller. @@ -126,59 +137,41 @@ func (s *regionEventSink) UpdateMetrics() { metricSubscriptionClientDSChannelSize.Set(float64(dsMetrics.EventChanSize)) metricSubscriptionClientDSPendingQueueLen.Set(float64(dsMetrics.PendingQueueLen)) - if len(dsMetrics.MemoryControl.AreaMemoryMetrics) == 0 { - return - } - if len(dsMetrics.MemoryControl.AreaMemoryMetrics) != 1 { - log.Warn("subscription client should have exactly one area") + if s.memoryQuota == nil { return } - areaMetric := dsMetrics.MemoryControl.AreaMemoryMetrics[0] + used, capacity, _ := s.memoryQuota.snapshot() + scanUsed, warmingScanUsed, warmingScanBudget, scanEstimate, hardLimit := + s.memoryQuota.scanSnapshot() + metrics.LogPullerMemoryQuota.WithLabelValues("max").Set(float64(capacity)) + metrics.LogPullerMemoryQuota.WithLabelValues("used").Set(float64(used)) + metrics.LogPullerMemoryQuota.WithLabelValues("scan_used").Set(float64(scanUsed)) + metrics.LogPullerMemoryQuota.WithLabelValues("warming_scan_used").Set(float64(warmingScanUsed)) + metrics.LogPullerMemoryQuota.WithLabelValues("warming_scan_budget").Set(float64(warmingScanBudget)) + metrics.LogPullerMemoryQuota.WithLabelValues("scan_estimate").Set(float64(scanEstimate)) + metrics.LogPullerMemoryQuota.WithLabelValues("hard_limit").Set(float64(hardLimit)) metrics.DynamicStreamMemoryUsage.WithLabelValues( "log-puller", "max", "default", "default", - ).Set(float64(areaMetric.MaxMemory())) + ).Set(float64(capacity)) metrics.DynamicStreamMemoryUsage.WithLabelValues( "log-puller", "used", "default", "default", - ).Set(float64(areaMetric.MemoryUsage())) + ).Set(float64(used)) } func (s *regionEventSink) Close() { - s.stop() - s.ds.Close() -} - -func (s *regionEventSink) pause() { - s.mu.Lock() - defer s.mu.Unlock() - if s.stopped.Load() || s.paused.Load() { - return - } - s.paused.Store(true) -} - -func (s *regionEventSink) resume() { - s.mu.Lock() - defer s.mu.Unlock() - if !s.paused.Load() { - return - } - s.paused.Store(false) - s.cond.Broadcast() -} - -func (s *regionEventSink) stop() { - if !s.stopped.CompareAndSwap(false, true) { - return + if s.memoryQuota != nil { + s.memoryQuota.wakeAll() } s.mu.Lock() s.paused.Store(false) s.cond.Broadcast() s.mu.Unlock() + s.ds.Close() } diff --git a/logservice/logpuller/region_event_sink_test.go b/logservice/logpuller/region_event_sink_test.go index 0698d3fb4a..1f08aa51f9 100644 --- a/logservice/logpuller/region_event_sink_test.go +++ b/logservice/logpuller/region_event_sink_test.go @@ -20,6 +20,7 @@ import ( "testing" "time" + "github.com/pingcap/kvproto/pkg/cdcpb" "github.com/pingcap/ticdc/pkg/metrics" "github.com/pingcap/ticdc/utils/dynstream" "github.com/prometheus/client_golang/prometheus/testutil" @@ -30,6 +31,7 @@ type mockRegionEventSinkStream struct { feedbackCh chan dynstream.Feedback[int, SubscriptionID, *subscribedSpan] pushCount atomic.Int32 pushCh chan struct{} + eventCh chan regionEvent metrics dynstream.Metrics[int, SubscriptionID] } @@ -37,6 +39,7 @@ func newMockRegionEventSinkStream() *mockRegionEventSinkStream { return &mockRegionEventSinkStream{ feedbackCh: make(chan dynstream.Feedback[int, SubscriptionID, *subscribedSpan], 2), pushCh: make(chan struct{}, 1), + eventCh: make(chan regionEvent, 1), } } @@ -44,9 +47,10 @@ func (s *mockRegionEventSinkStream) Start() {} func (s *mockRegionEventSinkStream) Close() {} -func (s *mockRegionEventSinkStream) Push(_ SubscriptionID, _ regionEvent) { +func (s *mockRegionEventSinkStream) Push(_ SubscriptionID, event regionEvent) { s.pushCount.Add(1) s.pushCh <- struct{}{} + s.eventCh <- event } func (s *mockRegionEventSinkStream) Wake(_ SubscriptionID) {} @@ -71,20 +75,16 @@ func (s *mockRegionEventSinkStream) GetMetrics() dynstream.Metrics[int, Subscrip return s.metrics } -func newTestRegionEventSink( - ds dynstream.DynamicStream[int, SubscriptionID, regionEvent, *subscribedSpan, *regionEventHandler], -) *regionEventSink { - sink := ®ionEventSink{ds: ds} - sink.cond = sync.NewCond(&sink.mu) - return sink -} - func TestRegionEventSinkRunPausesAndResumesPush(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() ds := newMockRegionEventSinkStream() - sink := newTestRegionEventSink(ds) + sink := ®ionEventSink{ + ctx: ctx, + ds: ds, + } + sink.cond = sync.NewCond(&sink.mu) runErrCh := make(chan error, 1) go func() { @@ -135,7 +135,7 @@ func TestRegionEventSinkRunPausesAndResumesPush(t *testing.T) { } func TestRegionEventSinkUpdateMetrics(t *testing.T) { - t.Run("empty area metrics returns after queue gauges", func(t *testing.T) { + t.Run("without quota only updates queue gauges", func(t *testing.T) { ds := newMockRegionEventSinkStream() ds.metrics = dynstream.Metrics[int, SubscriptionID]{ EventChanSize: 11, @@ -156,8 +156,10 @@ func TestRegionEventSinkUpdateMetrics(t *testing.T) { ).Set(456) sink := ®ionEventSink{ - ds: ds, + ctx: context.Background(), + ds: ds, } + sink.cond = sync.NewCond(&sink.mu) sink.UpdateMetrics() require.Equal(t, float64(11), testutil.ToFloat64(metricSubscriptionClientDSChannelSize)) @@ -176,25 +178,25 @@ func TestRegionEventSinkUpdateMetrics(t *testing.T) { ))) }) - t.Run("single area metrics updates memory gauges", func(t *testing.T) { + t.Run("quota updates memory gauges", func(t *testing.T) { ds := newMockRegionEventSinkStream() ds.metrics = dynstream.Metrics[int, SubscriptionID]{ EventChanSize: 33, PendingQueueLen: 44, - MemoryControl: dynstream.MemoryMetric[int, SubscriptionID]{ - AreaMemoryMetrics: []dynstream.AreaMemoryMetric[int, SubscriptionID]{ - { - UsedMemoryValue: 55, - MaxMemoryValue: 66, - PathMaxMemoryValue: 66, - }, - }, - }, } + quota := newMemoryQuotaController(66, 8) + span := newTestQuotaSpan(1) + quota.addSubscription(span) + lease := quota.trackEvent(context.Background(), span, 55) + require.NotNil(t, lease) + t.Cleanup(lease.Release) sink := ®ionEventSink{ - ds: ds, + ctx: context.Background(), + ds: ds, + memoryQuota: quota, } + sink.cond = sync.NewCond(&sink.mu) sink.UpdateMetrics() require.Equal(t, float64(33), testutil.ToFloat64(metricSubscriptionClientDSChannelSize)) @@ -211,51 +213,41 @@ func TestRegionEventSinkUpdateMetrics(t *testing.T) { "default", "default", ))) + require.Equal(t, float64(66), testutil.ToFloat64( + metrics.LogPullerMemoryQuota.WithLabelValues("max"))) + require.Equal(t, float64(55), testutil.ToFloat64( + metrics.LogPullerMemoryQuota.WithLabelValues("used"))) }) } -func TestRegionEventSinkRunCancelUnblocksPush(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - - ds := newMockRegionEventSinkStream() - sink := newTestRegionEventSink(ds) - - runErrCh := make(chan error, 1) - go func() { - runErrCh <- sink.Run(ctx) - }() - - ds.feedbackCh <- dynstream.Feedback[int, SubscriptionID, *subscribedSpan]{ - FeedbackType: dynstream.PauseArea, - } - require.Eventually(t, sink.paused.Load, time.Second, 10*time.Millisecond) - - pushDone := make(chan struct{}) - go func() { - sink.Push(SubscriptionID(1), regionEvent{resolvedTs: 100}) - close(pushDone) - }() - - select { - case <-pushDone: - t.Fatal("Push should block while the sink is paused") - case <-time.After(100 * time.Millisecond): +func TestRegionEventSinkTracksEntriesUntilDrop(t *testing.T) { + quota := newMemoryQuotaController(1024, 8) + span := newTestQuotaSpan(1) + quota.addSubscription(span) + state := ®ionFeedState{ + region: regionInfo{subscribedSpan: span}, + worker: ®ionRequestWorker{}, } - require.Equal(t, int32(0), ds.pushCount.Load()) - - cancel() - - select { - case <-pushDone: - case <-time.After(time.Second): - t.Fatal("Push should be unblocked by Run context cancellation") + ds := newMockRegionEventSinkStream() + sink := ®ionEventSink{ + ctx: context.Background(), + ds: ds, + memoryQuota: quota, } - require.Equal(t, int32(0), ds.pushCount.Load()) + sink.cond = sync.NewCond(&sink.mu) - select { - case err := <-runErrCh: - require.NoError(t, err) - case <-time.After(time.Second): - t.Fatal("Run should exit after context cancellation") - } + sink.Push(span.subID, regionEvent{ + states: []*regionFeedState{state}, + entries: &cdcpb.Event_Entries_{Entries: &cdcpb.Event_Entries{ + Entries: []*cdcpb.Event_Row{{Key: []byte("key"), Value: []byte("value")}}, + }}, + }) + pushed := <-ds.eventCh + require.NotNil(t, pushed.memoryQuota) + used, _, _ := quota.snapshot() + require.NotZero(t, used) + + (®ionEventHandler{}).OnDrop(pushed) + used, _, _ = quota.snapshot() + require.Zero(t, used) } diff --git a/logservice/logpuller/region_failure_handler.go b/logservice/logpuller/region_failure_handler.go index 5ca0f03c96..3622d927d9 100644 --- a/logservice/logpuller/region_failure_handler.go +++ b/logservice/logpuller/region_failure_handler.go @@ -64,14 +64,6 @@ func newRegionFailureHandler( } } -func (r *regionFailureHandler) retryRange(ctx context.Context, errInfo regionErrorInfo) { - r.scheduleRangeRequest(ctx, rangeTask{ - span: errInfo.span, - subscribedSpan: errInfo.subscribedSpan, - wasInitialized: errInfo.wasInitialized, - }) -} - // Report admits a region failure into the recovery pipeline. It releases the // corresponding range lock before enqueueing the failure so new range tasks are // not blocked by stale region ownership. @@ -167,20 +159,12 @@ func (r *regionFailureHandler) handleError(ctx context.Context, errInfo regionEr } if innerErr.GetEpochNotMatch() != nil { metricFeedEpochNotMatchCounter.Inc() -<<<<<<< HEAD rescheduleRange() -======= - r.retryRange(ctx, errInfo) ->>>>>>> 9903a1be7 (refactor) return nil } if innerErr.GetRegionNotFound() != nil { metricFeedRegionNotFoundCounter.Inc() -<<<<<<< HEAD rescheduleRange() -======= - r.retryRange(ctx, errInfo) ->>>>>>> 9903a1be7 (refactor) return nil } if innerErr.GetCongested() != nil { @@ -213,22 +197,14 @@ func (r *regionFailureHandler) handleError(ctx context.Context, errInfo regionEr return nil case *rpcCtxUnavailableErr: metricFeedRPCCtxUnavailable.Inc() -<<<<<<< HEAD rescheduleRange() -======= - r.retryRange(ctx, errInfo) ->>>>>>> 9903a1be7 (refactor) return nil case *getStoreErr: metricGetStoreErr.Inc() bo := tikv.NewBackoffer(ctx, tikvRequestMaxBackoff) // cannot get the store the region belongs to, so we need to reload the region. r.regionCache.OnSendFail(bo, errInfo.rpcCtx, true, err) -<<<<<<< HEAD rescheduleRange() -======= - r.retryRange(ctx, errInfo) ->>>>>>> 9903a1be7 (refactor) return nil case *storeStreamErr: metricStoreSendRequestErr.Inc() diff --git a/logservice/logpuller/region_request_scheduler.go b/logservice/logpuller/region_request_scheduler.go index 196030c695..4bf6dd3ea0 100644 --- a/logservice/logpuller/region_request_scheduler.go +++ b/logservice/logpuller/region_request_scheduler.go @@ -20,20 +20,14 @@ import ( "sync/atomic" "github.com/pingcap/log" -<<<<<<< HEAD "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/config" "github.com/pingcap/ticdc/pkg/errors" "github.com/pingcap/ticdc/pkg/metrics" -======= - "github.com/pingcap/ticdc/pkg/config" - "github.com/pingcap/ticdc/pkg/errors" ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) "github.com/pingcap/ticdc/utils/priorityqueue" kvclientv2 "github.com/tikv/client-go/v2/kv" "github.com/tikv/client-go/v2/tikv" "go.uber.org/zap" -<<<<<<< HEAD "go.uber.org/zap/zapcore" "golang.org/x/sync/errgroup" ) @@ -48,6 +42,7 @@ type regionRequestScheduler struct { upstream *upstreamHandle eventSink *regionEventSink failureHandler *regionFailureHandler + memoryQuota *memoryQuotaController // taskQueue orders all regions before they are assigned to a TiKV store. taskQueue *priorityqueue.PriorityQueue[*regionPriorityTask] @@ -55,24 +50,6 @@ type regionRequestScheduler struct { sequence atomic.Uint64 // stores maps TiKV addresses to regionRequestStore. Stores are created only // by Run, but are also read by metrics and deregistration goroutines. -======= - "golang.org/x/sync/errgroup" -) - -// regionRequestScheduler routes locked Region requests through the global -// priority queue to a worker connected to the Region's TiKV store. Range -// resolution and retry policy remain owned by subscriptionClient and -// regionFailureHandler respectively. -type regionRequestScheduler struct { - client *subscriptionClient - - // taskQueue orders all Regions before they are assigned to a TiKV store. - taskQueue *priorityqueue.PriorityQueue[*regionPriorityTask] - // sequence is the FIFO tie-breaker for Regions in the same priority class. - sequence atomic.Uint64 - // stores maps TiKV addresses to requestedStore. Stores are created only by - // run, but are also read by metrics and deregistration goroutines. ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) stores sync.Map // workerCount is the configured number of request workers per store. @@ -83,11 +60,11 @@ type regionRequestScheduler struct { maxWindowMultiplier int } -<<<<<<< HEAD func newRegionRequestScheduler( upstream *upstreamHandle, eventSink *regionEventSink, failureHandler *regionFailureHandler, + memoryQuota *memoryQuotaController, ) *regionRequestScheduler { pullerConfig := config.GetGlobalServerConfig().Debug.Puller workerCount := regionRequestWorkerPerStore @@ -96,17 +73,7 @@ func newRegionRequestScheduler( upstream: upstream, eventSink: eventSink, failureHandler: failureHandler, -======= -func newRegionRequestScheduler(client *subscriptionClient) *regionRequestScheduler { - pullerConfig := config.GetGlobalServerConfig().Debug.Puller - workerCount := int(client.config.RegionRequestWorkerPerStore) - if workerCount <= 0 { - workerCount = 1 - } - workerWindow := (pullerConfig.PendingRegionRequestQueueSize + workerCount - 1) / workerCount - return ®ionRequestScheduler{ - client: client, ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) + memoryQuota: memoryQuota, taskQueue: priorityqueue.New[*regionPriorityTask](), workerCount: workerCount, workerWindow: workerWindow, @@ -114,9 +81,7 @@ func newRegionRequestScheduler(client *subscriptionClient) *regionRequestSchedul } } -<<<<<<< HEAD func (s *regionRequestScheduler) Submit(region regionInfo) { -<<<<<<< HEAD if log.GetLevel() <= zapcore.DebugLevel { log.Debug("cdc region scan task enqueued", zap.Uint64("subscriptionID", uint64(region.subscribedSpan.subID)), @@ -130,10 +95,6 @@ func (s *regionRequestScheduler) Submit(region regionInfo) { zap.String("span", common.FormatTableSpan(®ion.span))) } s.taskQueue.Push(newRegionPriorityTask(region, s.sequence.Add(1))) -======= - s.taskQueue.Push(newRegionPriorityTask( - region, s.upstream.pdClock.CurrentTS(), s.sequence.Add(1))) ->>>>>>> 9903a1be7 (refactor) } func (s *regionRequestScheduler) Run(ctx context.Context, workerGroup *errgroup.Group) error { @@ -144,15 +105,6 @@ func (s *regionRequestScheduler) Run(ctx context.Context, workerGroup *errgroup. }) }() -======= -func (s *regionRequestScheduler) submit(region regionInfo) { - s.taskQueue.Push(NewRegionPriorityTask( - region, s.client.pdClock.CurrentTS(), s.sequence.Add(1))) -} - -func (s *regionRequestScheduler) run(ctx context.Context, group *errgroup.Group) error { - defer s.closeStores() ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) for { select { case <-ctx.Done(): @@ -168,10 +120,6 @@ func (s *regionRequestScheduler) run(ctx context.Context, group *errgroup.Group) return err } -<<<<<<< HEAD -<<<<<<< HEAD -======= ->>>>>>> 9903a1be7 (refactor) region, err := s.attachRPCContext(ctx, task.regionInfo) if err != nil { s.failureHandler.Report(newRegionErrorInfo(region, err)) @@ -197,30 +145,12 @@ func (s *regionRequestScheduler) run(ctx context.Context, group *errgroup.Group) zap.Uint64("regionID", region.verID.GetID()), zap.String("addr", region.rpcCtx.Addr)) } -======= - region, ok := s.attachRPCContext(ctx, task.GetRegionInfo()) - if !ok { - continue - } - - store := s.getOrCreateStore(ctx, group, region.rpcCtx.Addr) - task.updateRegion(region, s.client.pdClock.CurrentTS()) - if !store.submit(task) { - return context.Canceled - } - - log.Debug("subscription client will request a region", - zap.Uint64("subscriptionID", uint64(region.subscribedSpan.subID)), - zap.Uint64("regionID", region.verID.GetID()), - zap.String("addr", store.storeAddr)) ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) } } func (s *regionRequestScheduler) attachRPCContext( ctx context.Context, region regionInfo, -<<<<<<< HEAD ) (regionInfo, error) { bo := tikv.NewBackoffer(ctx, tikvRequestMaxBackoff) rpcCtx, err := s.upstream.regionCache.GetTiKVRPCContext( @@ -231,33 +161,15 @@ func (s *regionRequestScheduler) attachRPCContext( } if err != nil { log.Debug("region request scheduler failed to get RPC context", -======= -) (regionInfo, bool) { - bo := tikv.NewBackoffer(ctx, tikvRequestMaxBackoff) - rpcCtx, err := s.client.regionCache.GetTiKVRPCContext( - bo, region.verID, kvclientv2.ReplicaReadLeader, 0) - if rpcCtx != nil { - region.rpcCtx = rpcCtx - return region, true - } - if err != nil { - log.Debug("subscription client get rpc context fail", ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) zap.Uint64("subscriptionID", uint64(region.subscribedSpan.subID)), zap.Uint64("regionID", region.verID.GetID()), zap.Error(err)) } -<<<<<<< HEAD return region, &rpcCtxUnavailableErr{verID: region.verID} -======= - s.client.onRegionFail(newRegionErrorInfo(region, &rpcCtxUnavailableErr{verID: region.verID})) - return region, false ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) } func (s *regionRequestScheduler) getOrCreateStore( ctx context.Context, -<<<<<<< HEAD workerGroup *errgroup.Group, storeAddr string, ) *regionRequestStore { @@ -273,6 +185,7 @@ func (s *regionRequestScheduler) getOrCreateStore( s.workerCount, s.workerWindow, s.maxWindowMultiplier, + s.memoryQuota, ) // The scheduler run loop is the only writer. Publish the store after its // immutable worker list is complete, then start its workers. @@ -282,34 +195,11 @@ func (s *regionRequestScheduler) getOrCreateStore( } func (s *regionRequestScheduler) BroadcastDeregister( -======= - group *errgroup.Group, - storeAddr string, -) *requestedStore { - if value, ok := s.stores.Load(storeAddr); ok { - return value.(*requestedStore) - } - - store := newRequestedStore( - s.client, storeAddr, s.workerCount, s.workerWindow, s.maxWindowMultiplier) - // run is the only writer. Publish the store after its immutable worker list - // is complete, then start its workers. - s.stores.Store(storeAddr, store) - store.run(ctx, group) - return store -} - -func (s *regionRequestScheduler) broadcastDeregister( ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) subID SubscriptionID, filterLoop bool, ) { s.stores.Range(func(_, value any) bool { -<<<<<<< HEAD value.(*regionRequestStore).broadcastDeregister(subID, filterLoop) -======= - value.(*requestedStore).broadcastDeregister(subID, filterLoop) ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) return true }) } @@ -317,34 +207,24 @@ func (s *regionRequestScheduler) broadcastDeregister( func (s *regionRequestScheduler) inflightCount() int { count := 0 s.stores.Range(func(_, value any) bool { -<<<<<<< HEAD count += value.(*regionRequestStore).inflightCount() -======= - count += value.(*requestedStore).inflightCount() ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) return true }) return count } -<<<<<<< HEAD func (s *regionRequestScheduler) UpdateMetrics() { metrics.SubscriptionClientRequestedRegionCount.WithLabelValues("inflight"). Set(float64(s.inflightCount())) } -func (s *regionRequestScheduler) Close() { - s.taskQueue.Close() -} -======= -func (s *regionRequestScheduler) close() { - s.taskQueue.Close() -} - -func (s *regionRequestScheduler) closeStores() { +func (s *regionRequestScheduler) notifyAvailable() { s.stores.Range(func(_, value any) bool { - value.(*requestedStore).close() + value.(*regionRequestStore).notifyAvailable() return true }) } ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) + +func (s *regionRequestScheduler) Close() { + s.taskQueue.Close() +} diff --git a/logservice/logpuller/region_request_store.go b/logservice/logpuller/region_request_store.go index 9b9a5af025..509c7449df 100644 --- a/logservice/logpuller/region_request_store.go +++ b/logservice/logpuller/region_request_store.go @@ -37,13 +37,21 @@ func newRegionRequestStore( workerCount int, workerWindow int, maxWindowMultiplier int, + memoryQuota *memoryQuotaController, ) *regionRequestStore { store := ®ionRequestStore{ workers: make([]*regionRequestWorker, 0, workerCount), } for i := 0; i < workerCount; i++ { store.workers = append(store.workers, newRegionRequestWorker( - upstream, eventSink, failureHandler, storeAddr, workerWindow, maxWindowMultiplier)) + upstream, + eventSink, + failureHandler, + storeAddr, + workerWindow, + maxWindowMultiplier, + memoryQuota, + )) } return store } @@ -78,3 +86,9 @@ func (s *regionRequestStore) inflightCount() int { } return count } + +func (s *regionRequestStore) notifyAvailable() { + for _, worker := range s.workers { + worker.admission.notifyAvailable() + } +} diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index f717321db2..288bb384f7 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -112,6 +112,7 @@ func newRegionRequestWorker( storeAddr string, currentWindow int, maxWindowMultiplier int, + memoryQuota *memoryQuotaController, ) *regionRequestWorker { workerID := workerIDGen.Add(1) return ®ionRequestWorker{ @@ -120,9 +121,14 @@ func newRegionRequestWorker( eventSink: eventSink, failureHandler: failureHandler, storeAddr: storeAddr, - admission: newRegionAdmissionController(currentWindow, maxWindowMultiplier), - controlQueue: newControlQueue(), - tracker: newRegionTracker(), + admission: newRegionAdmissionController( + currentWindow, + maxWindowMultiplier, + memoryQuota, + upstream.pdClock.CurrentTS, + ), + controlQueue: newControlQueue(), + tracker: newRegionTracker(), } } @@ -174,35 +180,6 @@ func (s *regionRequestWorker) Run(ctx context.Context) error { } } -<<<<<<< HEAD -======= -// failStreamRegions transfers every request sent by a failed stream to the -// recovery pipeline. -func (s *regionRequestWorker) failStreamRegions(err error) { - for _, state := range s.tracker.Drain() { - s.notifyRegionError(state, err) - } - // The failed stream no longer owns remote registrations. - s.controlQueue.drain() -} - -// failPendingRegions transfers requests owned by this worker but not yet sent -// to the recovery pipeline, so they can be resolved and routed again. -func (s *regionRequestWorker) failPendingRegions(err error) { - for _, task := range s.admission.drain() { - s.failureHandler.Report(newRegionErrorInfo(task.regionInfo, err)) - } -} - -func (s *regionRequestWorker) notifyRegionError(state *regionFeedState, err error) { - state.markStopped(err) - s.eventSink.Push( - SubscriptionID(state.requestID), - regionEvent{states: []*regionFeedState{state}}, - ) -} - ->>>>>>> 9903a1be7 (refactor) func (s *regionRequestWorker) waitForRegionRequest(ctx context.Context) (*regionReq, error) { // Without a stream there are no remote registrations to deregister. s.controlQueue.drain() diff --git a/logservice/logpuller/region_request_worker_test.go b/logservice/logpuller/region_request_worker_test.go index aa54d2ea65..8735f3021c 100644 --- a/logservice/logpuller/region_request_worker_test.go +++ b/logservice/logpuller/region_request_worker_test.go @@ -517,7 +517,6 @@ func TestRunStreamFailureReportsPendingRegionsToFailureHandler(t *testing.T) { pdClient := newFailureRecoveryTestPDClient(t) defer pdClient.Close() -<<<<<<< HEAD handler := newRegionFailureHandler(nil, func(*subscribedSpan) {}, nil, nil) worker := ®ionRequestWorker{ upstream: &upstreamHandle{pd: pdClient, credential: &security.Credential{}}, @@ -527,29 +526,6 @@ func TestRunStreamFailureReportsPendingRegionsToFailureHandler(t *testing.T) { controlQueue: newControlQueue(), tracker: newRegionTracker(), storeAddr: "127.0.0.1:1", -======= - admission := newRegionAdmissionController(1, 1) - failureHandler := ®ionFailureHandler{cache: newErrCache()} - worker := ®ionRequestWorker{failureHandler: failureHandler, admission: admission} - regions := []regionInfo{ - { - verID: tikv.NewRegionVerID(1, 1, 1), - span: heartbeatpb.TableSpan{ - TableID: 1, StartKey: []byte("a"), EndKey: []byte("m"), - }, - subscribedSpan: span, lockedRangeState: lock1.LockedRangeState, - }, - { - verID: tikv.NewRegionVerID(2, 1, 1), - span: heartbeatpb.TableSpan{ - TableID: 1, StartKey: []byte("m"), EndKey: []byte("z"), - }, - subscribedSpan: span, lockedRangeState: lock2.LockedRangeState, - }, - } - for i, region := range regions { - require.True(t, admission.submit(newRegionPriorityTask(region, 1, uint64(i+1)))) ->>>>>>> 9903a1be7 (refactor) } firstRegion := createFailureRecoveryTestRegion(t, 1, 1) diff --git a/logservice/logpuller/region_state.go b/logservice/logpuller/region_state.go index 6a2837564f..0f2f84ce3e 100644 --- a/logservice/logpuller/region_state.go +++ b/logservice/logpuller/region_state.go @@ -58,18 +58,15 @@ func newRegionInfo( span heartbeatpb.TableSpan, rpcCtx *tikv.RPCContext, subscribedSpan *subscribedSpan, + filterLoop bool, ) regionInfo { return regionInfo{ verID: verID, span: span, rpcCtx: rpcCtx, subscribedSpan: subscribedSpan, -<<<<<<< HEAD filterLoop: filterLoop, scanPriority: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, -======= - filterLoop: subscribedSpan.filterLoop, ->>>>>>> 9903a1be7 (refactor) } } diff --git a/logservice/logpuller/region_tracker.go b/logservice/logpuller/region_tracker.go index 4d69bf8bb3..81a27dc86c 100644 --- a/logservice/logpuller/region_tracker.go +++ b/logservice/logpuller/region_tracker.go @@ -112,7 +112,6 @@ func (t *regionTracker) Drain() []*regionFeedState { t.statesBySubscription = make(map[SubscriptionID]regionStatesByID) t.mu.Unlock() -<<<<<<< HEAD totalStates := 0 for _, states := range statesBySubscription { totalStates += len(states) @@ -120,17 +119,6 @@ func (t *regionTracker) Drain() []*regionFeedState { drainedStates := make([]*regionFeedState, 0, totalStates) for _, states := range statesBySubscription { drainedStates = append(drainedStates, slices.Collect(maps.Values(states))...) -======= - stateCount := 0 - for _, states := range statesBySubscription { - stateCount += len(states) - } - drainedStates := make([]*regionFeedState, 0, stateCount) - for _, states := range statesBySubscription { - for _, state := range states { - drainedStates = append(drainedStates, state) - } ->>>>>>> 9903a1be7 (refactor) } return drainedStates } diff --git a/logservice/logpuller/span_registry.go b/logservice/logpuller/span_registry.go index 797e42d833..5c8f70fba8 100644 --- a/logservice/logpuller/span_registry.go +++ b/logservice/logpuller/span_registry.go @@ -190,6 +190,18 @@ func (span *subscribedSpan) resolveStaleLocks(targetTs uint64) { zap.Any("ranges", res)) } +func (span *subscribedSpan) tryMarkInitialized(regionID, resolvedTs uint64) bool { + if resolvedTs <= span.startTs || !span.initialized.CompareAndSwap(false, true) { + return false + } + log.Info("subscription client is initialized", + zap.Uint64("subscriptionID", uint64(span.subID)), + zap.Uint64("regionID", regionID), + zap.Uint64("resolvedTs", resolvedTs), + zap.Uint64("startTs", span.startTs)) + return true +} + func newSpanRegistry(pd pd.Client, pdClock pdutil.Clock) *spanRegistry { return &spanRegistry{ spans: make(map[SubscriptionID]*subscribedSpan), diff --git a/logservice/logpuller/subscription_client.go b/logservice/logpuller/subscription_client.go index d77de7dc5b..28b26944c2 100644 --- a/logservice/logpuller/subscription_client.go +++ b/logservice/logpuller/subscription_client.go @@ -26,19 +26,12 @@ import ( "github.com/pingcap/ticdc/logservice/txnutil" "github.com/pingcap/ticdc/pkg/common" appcontext "github.com/pingcap/ticdc/pkg/common/context" -<<<<<<< HEAD "github.com/pingcap/ticdc/pkg/config" -======= ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) "github.com/pingcap/ticdc/pkg/metrics" "github.com/pingcap/ticdc/pkg/pdutil" "github.com/pingcap/ticdc/pkg/security" "github.com/pingcap/ticdc/pkg/spanz" "github.com/pingcap/ticdc/pkg/util" -<<<<<<< HEAD -======= - "github.com/prometheus/client_golang/prometheus" ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) "github.com/tikv/client-go/v2/tikv" pd "github.com/tikv/pd/client" "go.uber.org/zap" @@ -89,12 +82,8 @@ type resolveLockTask struct { type rangeTask struct { span heartbeatpb.TableSpan subscribedSpan *subscribedSpan -<<<<<<< HEAD filterLoop bool priority cdcpb.ScanPriority -======= - wasInitialized bool ->>>>>>> 9903a1be7 (refactor) } // upstreamHandle contains the stable TiKV and PD dependencies shared by the @@ -140,24 +129,16 @@ type subscriptionClient struct { lockResolver txnutil.LockResolver -<<<<<<< HEAD -======= - // the credential to connect tikv - credential *security.Credential - ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) // failureHandler handles failed regions and owns reschedule/retry decisions. failureHandler *regionFailureHandler // eventSink delivers region events and owns dynstream interaction. eventSink *regionEventSink // spanRegistry tracks subscribed spans and owns span-level background tasks. spanRegistry *spanRegistry -<<<<<<< HEAD // regionScheduler assigns locked region requests to per-store workers. -======= - // regionScheduler assigns locked Region requests to per-store workers. ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) regionScheduler *regionRequestScheduler + // memoryQuota owns event-memory accounting and initial-scan admission. + memoryQuota *memoryQuotaController // rangeTaskCh is used to receive range tasks. // The tasks will be handled in `handleRangeTask` goroutine. @@ -175,20 +156,12 @@ func NewSubscriptionClient( credential *security.Credential, ) SubscriptionClient { subClient := &subscriptionClient{ -<<<<<<< HEAD upstream: &upstreamHandle{ pd: pd, regionCache: appcontext.GetService[*tikv.RegionCache](appcontext.RegionCache), pdClock: appcontext.GetService[pdutil.Clock](appcontext.DefaultPDClock), credential: credential, }, -======= - config: config, - - pd: pd, - regionCache: appcontext.GetService[*tikv.RegionCache](appcontext.RegionCache), - pdClock: appcontext.GetService[pdutil.Clock](appcontext.DefaultPDClock), ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) lockResolver: lockResolver, rangeTaskCh: make(chan rangeTask, 1024), @@ -196,28 +169,28 @@ func NewSubscriptionClient( resolveLockRateLimiter: newResolveLockRateLimiter(), } subClient.ctx, subClient.cancel = context.WithCancel(context.Background()) -<<<<<<< HEAD + pullerConfig := config.GetGlobalServerConfig().Debug.Puller + subClient.memoryQuota = newMemoryQuotaController( + pullerConfig.MemoryQuota, pullerConfig.ScanBaseSize) subClient.failureHandler = newRegionFailureHandler( subClient.upstream.regionCache, subClient.onTableDrained, subClient.scheduleRegionRequest, subClient.scheduleRangeRequest, ) - subClient.eventSink = newRegionEventSink(subClient.failureHandler) + subClient.eventSink = newRegionEventSink( + subClient.ctx, + subClient.failureHandler, + subClient.memoryQuota, + ) subClient.spanRegistry = newSpanRegistry(subClient.upstream.pd, subClient.upstream.pdClock) subClient.regionScheduler = newRegionRequestScheduler( subClient.upstream, subClient.eventSink, subClient.failureHandler, + subClient.memoryQuota, ) -======= - subClient.failureHandler = newRegionFailureHandler(subClient) - subClient.eventSink = newRegionEventSink(subClient.ctx, subClient.failureHandler) - subClient.spanRegistry = newSpanRegistry(subClient.pd, subClient.pdClock) - subClient.regionScheduler = newRegionRequestScheduler(subClient) - - subClient.initMetrics() ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) + subClient.memoryQuota.setOnAvailable(subClient.regionScheduler.notifyAvailable) return subClient } @@ -238,13 +211,7 @@ func (s *subscriptionClient) updateMetrics(ctx context.Context) error { case <-ctx.Done(): return ctx.Err() case <-ticker.C: -<<<<<<< HEAD s.regionScheduler.UpdateMetrics() -======= - pendingRegionReqCount := s.regionScheduler.inflightCount() - - metrics.SubscriptionClientRequestedRegionCount.WithLabelValues("pending").Set(float64(pendingRegionReqCount)) ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) s.eventSink.UpdateMetrics() s.spanRegistry.UpdateMetrics() } @@ -285,21 +252,18 @@ func (s *subscriptionClient) Subscribe( time.Duration(config.GetGlobalServerConfig().Debug.Puller.OldStartTsScanLowPriorityThreshold), ) s.spanRegistry.Add(rt) + s.memoryQuota.addSubscription(rt) s.eventSink.AddPath(rt) select { case <-s.ctx.Done(): log.Warn("subscribes span failed, the subscription client has closed") -<<<<<<< HEAD case s.rangeTaskCh <- rangeTask{ span: span, subscribedSpan: rt, filterLoop: rt.filterLoop, priority: cdcpb.ScanPriority_SCAN_PRIORITY_LOW, }: -======= - case s.rangeTaskCh <- rangeTask{span: span, subscribedSpan: rt}: ->>>>>>> 9903a1be7 (refactor) log.Info("subscribes span done", zap.Uint64("subscriptionID", uint64(subID)), zap.Int64("tableID", span.TableID), zap.Uint64("startTs", startTs), zap.String("startKey", spanz.HexKey(span.StartKey)), zap.String("endKey", spanz.HexKey(span.EndKey))) @@ -330,15 +294,8 @@ func (s *subscriptionClient) Run(ctx context.Context) error { // The goroutines are listed by data flow; errgroup does not guarantee their // actual startup order. g.Go(func() error { return s.handleRangeTasks(ctx) }) -<<<<<<< HEAD g.Go(func() error { return s.regionScheduler.Run(ctx, g) }) g.Go(func() error { return s.eventSink.Run(ctx) }) -<<<<<<< HEAD -======= - g.Go(func() error { return s.regionScheduler.run(ctx, g) }) ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) -======= ->>>>>>> 9903a1be7 (refactor) g.Go(func() error { return s.failureHandler.Run(ctx) }) g.Go(func() error { return s.spanRegistry.Run(ctx) }) g.Go(func() error { return s.handleResolveLockTasks(ctx) }) @@ -353,11 +310,7 @@ func (s *subscriptionClient) Run(ctx context.Context) error { func (s *subscriptionClient) Close(ctx context.Context) error { s.cancel() s.eventSink.Close() -<<<<<<< HEAD s.regionScheduler.Close() -======= - s.regionScheduler.close() ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) return nil } @@ -368,11 +321,7 @@ func (s *subscriptionClient) setTableStopped(rt *subscribedSpan) { // Set stopped to true so we can stop handling region events from the table, // then notify every existing worker to deregister the subscription. if rt.stopped.CompareAndSwap(false, true) { -<<<<<<< HEAD s.regionScheduler.BroadcastDeregister(rt.subID, rt.filterLoop) -======= - s.regionScheduler.broadcastDeregister(rt.subID, rt.filterLoop) ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) if rt.rangeLock.Stop() { s.onTableDrained(rt) } @@ -389,17 +338,10 @@ func (s *subscriptionClient) onTableDrained(rt *subscribedSpan) { zap.Uint64("subscriptionID", uint64(rt.subID)), zap.Error(err)) } + s.memoryQuota.removeSubscription(rt) s.spanRegistry.Remove(rt.subID) } -<<<<<<< HEAD -======= -// Note: don't block the caller, otherwise there may be deadlock -func (s *subscriptionClient) onRegionFail(errInfo regionErrorInfo) { - s.failureHandler.Report(errInfo) -} - ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) func (s *subscriptionClient) handleRangeTasks(ctx context.Context) error { g, ctx := errgroup.WithContext(ctx) // Limit the concurrent number of goroutines to convert range tasks to region tasks. @@ -488,13 +430,8 @@ func (s *subscriptionClient) divideSpanAndScheduleRegionRequests( } verID := tikv.NewRegionVerID(regionMeta.Id, regionMeta.RegionEpoch.ConfVer, regionMeta.RegionEpoch.Version) -<<<<<<< HEAD regionInfo := newRegionInfo(verID, intersectSpan, nil, subscribedSpan, task.filterLoop) regionInfo.scanPriority = normalizeScanPriority(task.priority) -======= - regionInfo := newRegionInfo(verID, intersectSpan, nil, subscribedSpan) - regionInfo.wasInitialized = task.wasInitialized ->>>>>>> 9903a1be7 (refactor) // Schedule a region request to subscribe the region. s.scheduleRegionRequest(ctx, regionInfo) @@ -509,11 +446,7 @@ func (s *subscriptionClient) divideSpanAndScheduleRegionRequests( } } -<<<<<<< HEAD // scheduleRegionRequest locks the region's range before submitting it to the -======= -// scheduleRegionRequest locks the Region's range before submitting it to the ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) // request scheduler. func (s *subscriptionClient) scheduleRegionRequest(ctx context.Context, region regionInfo) { lockRangeResult := region.subscribedSpan.rangeLock.LockRange( @@ -526,27 +459,19 @@ func (s *subscriptionClient) scheduleRegionRequest(ctx context.Context, region r switch lockRangeResult.Status { case regionlock.LockRangeStatusSuccess: region.lockedRangeState = lockRangeResult.LockedRangeState -<<<<<<< HEAD region.scanPriority = region.subscribedSpan.priorityPolicy.resolve( region.scanPriority, region.resolvedTs(), s.upstream.pdClock.CurrentTime(), ) s.regionScheduler.Submit(region) -======= - s.regionScheduler.submit(region) ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) case regionlock.LockRangeStatusStale: for _, r := range lockRangeResult.RetryRanges { s.scheduleRangeRequest(ctx, rangeTask{ span: r, subscribedSpan: region.subscribedSpan, -<<<<<<< HEAD filterLoop: region.filterLoop, priority: region.scanPriority, -======= - wasInitialized: region.wasInitialized, ->>>>>>> 9903a1be7 (refactor) }) } default: diff --git a/logservice/logpuller/subscription_client_test.go b/logservice/logpuller/subscription_client_test.go index 4e6354479f..99600666ea 100644 --- a/logservice/logpuller/subscription_client_test.go +++ b/logservice/logpuller/subscription_client_test.go @@ -334,24 +334,11 @@ func TestStopTaskUsesSubscribedSpanFilterLoop(t *testing.T) { res := span.rangeLock.LockRange(context.Background(), rawSpan.StartKey, rawSpan.EndKey, 1, 1) require.Equal(t, regionlock.LockRangeStatusSuccess, res.Status) -<<<<<<< HEAD -<<<<<<< HEAD -======= ->>>>>>> 9903a1be7 (refactor) const storeAddr = "store-1" worker := ®ionRequestWorker{storeAddr: storeAddr, controlQueue: newControlQueue()} store := ®ionRequestStore{workers: []*regionRequestWorker{worker}} client.regionScheduler = ®ionRequestScheduler{} client.regionScheduler.stores.Store(storeAddr, store) -<<<<<<< HEAD -======= - worker := ®ionRequestWorker{controlQueue: newControlQueue()} - store := &requestedStore{storeAddr: "store-1", workers: []*regionRequestWorker{worker}} - client.regionScheduler = ®ionRequestScheduler{client: client} - client.regionScheduler.stores.Store(store.storeAddr, store) ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) -======= ->>>>>>> 9903a1be7 (refactor) client.setTableStopped(span) @@ -443,10 +430,6 @@ func TestRegionEventSinkPushUnblocksOnClientClose(t *testing.T) { sink.cond = sync.NewCond(&sink.mu) client := &subscriptionClient{eventSink: sink} client.regionScheduler = ®ionRequestScheduler{ -<<<<<<< HEAD -======= - client: client, ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) taskQueue: priorityqueue.New[*regionPriorityTask](), } client.ctx, client.cancel = context.WithCancel(context.Background()) @@ -474,69 +457,6 @@ func TestRegionEventSinkPushUnblocksOnClientClose(t *testing.T) { } } -<<<<<<< HEAD -======= -func TestBroadcastDeregisterUsesWorkerControlQueue(t *testing.T) { - client := &subscriptionClient{} - scheduler := ®ionRequestScheduler{client: client} - admission := newRegionAdmissionController(1, 1) - - const storeAddr = "store-1" - worker := ®ionRequestWorker{ - storeAddr: storeAddr, - admission: admission, - controlQueue: newControlQueue(), - } -<<<<<<< HEAD - store := &requestedStore{storeAddr: "store-1", workers: []*regionRequestWorker{worker}} - scheduler.stores.Store(store.storeAddr, store) -======= - store := ®ionRequestStore{workers: []*regionRequestWorker{worker}} - scheduler.stores.Store(storeAddr, store) ->>>>>>> 9903a1be7 (refactor) - - dummyRegion := regionInfo{ - subscribedSpan: &subscribedSpan{subID: SubscriptionID(2)}, - lockedRangeState: ®ionlock.LockedRangeState{}, - } - require.True(t, admission.submit(newRegionPriorityTask(dummyRegion, 1, 1))) - - scheduler.broadcastDeregister(SubscriptionID(1), true) - require.Equal(t, 1, worker.controlQueue.len()) - req, ok := worker.controlQueue.tryPop() - require.True(t, ok) - require.Equal(t, SubscriptionID(1), req.subID) - require.True(t, req.filterLoop) - require.Equal(t, 1, admission.stats().pending) -} - -func TestRequestedStoreDistributesRegionsAcrossWorkerBuffers(t *testing.T) { - worker1 := ®ionRequestWorker{admission: newRegionAdmissionController(1, 1)} - worker2 := ®ionRequestWorker{admission: newRegionAdmissionController(1, 1)} -<<<<<<< HEAD - store := &requestedStore{ - storeAddr: "store-1", - workers: []*regionRequestWorker{worker1, worker2}, -======= - store := ®ionRequestStore{ - workers: []*regionRequestWorker{worker1, worker2}, ->>>>>>> 9903a1be7 (refactor) - } - - for i := uint64(1); i <= 4; i++ { - region := regionInfo{ - verID: tikv.NewRegionVerID(i, 1, 1), - subscribedSpan: &subscribedSpan{subID: 1}, - lockedRangeState: ®ionlock.LockedRangeState{}, - } - require.True(t, store.submit(newRegionPriorityTask(region, 1, i))) - } - - require.Equal(t, 2, worker1.admission.stats().pending) - require.Equal(t, 2, worker2.admission.stats().pending) -} - ->>>>>>> 23171df8f (logpuller: extract region request scheduler from subscription client) func TestSubscriptionWithFailedTiKV(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) mockPDClock := pdutil.NewClock4Test() diff --git a/metrics/grafana/ticdc_new_arch.json b/metrics/grafana/ticdc_new_arch.json index 0a3f4bbd37..29123f9715 100644 --- a/metrics/grafana/ticdc_new_arch.json +++ b/metrics/grafana/ticdc_new_arch.json @@ -8815,6 +8815,96 @@ "align": false, "alignLevel": null } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Log puller memory quota tracked by the local memory quota controller.", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 57 + }, + "hiddenSeries": false, + "id": 26001, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(ticdc_log_puller_memory_quota{k8s_cluster=~\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance, type)", + "interval": "", + "legendFormat": "{{instance}}-{{type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Quota Controller", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } } ], "title": "Log Puller", diff --git a/metrics/nextgengrafana/ticdc_new_arch_next_gen.json b/metrics/nextgengrafana/ticdc_new_arch_next_gen.json index f920ac2ccd..2bfe4b8015 100644 --- a/metrics/nextgengrafana/ticdc_new_arch_next_gen.json +++ b/metrics/nextgengrafana/ticdc_new_arch_next_gen.json @@ -8815,6 +8815,96 @@ "align": false, "alignLevel": null } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Log puller memory quota tracked by the local memory quota controller.", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 57 + }, + "hiddenSeries": false, + "id": 26001, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(ticdc_log_puller_memory_quota{k8s_cluster=~\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance, type)", + "interval": "", + "legendFormat": "{{instance}}-{{type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Quota Controller", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } } ], "title": "Log Puller", diff --git a/pkg/config/debug.go b/pkg/config/debug.go index 6df9c767a7..c96527044e 100644 --- a/pkg/config/debug.go +++ b/pkg/config/debug.go @@ -25,6 +25,8 @@ const ( // DefaultOldStartTsScanLowPriorityThreshold is the default lag threshold for // classifying scan tasks as low priority. DefaultOldStartTsScanLowPriorityThreshold = 10 * time.Minute + defaultLogPullerMemoryQuota uint64 = 1024 * 1024 * 1024 + defaultLogPullerScanBaseSize uint64 = 8 * 1024 * 1024 ) // DebugConfig represents config for ticdc unexposed feature configurations @@ -84,6 +86,10 @@ type PullerConfig struct { // Scans within this threshold are scheduled as high priority. Older scans // remain low priority until their span catches up once. OldStartTsScanLowPriorityThreshold TomlDuration `toml:"old-start-ts-scan-low-priority-threshold" json:"old_start_ts_scan_low_priority_threshold"` + // MemoryQuota is the log puller's local soft memory limit in bytes. + MemoryQuota uint64 `toml:"memory-quota" json:"memory_quota"` + // ScanBaseSize is the base memory estimate for one admitted initial scan. + ScanBaseSize uint64 `toml:"scan-base-size" json:"scan_base_size"` } // NewDefaultPullerConfig return the default puller configuration @@ -96,6 +102,8 @@ func NewDefaultPullerConfig() *PullerConfig { RegionRequestMaxWindowMultiplier: 4, OldStartTsScanLowPriorityThreshold: TomlDuration( DefaultOldStartTsScanLowPriorityThreshold), + MemoryQuota: defaultLogPullerMemoryQuota, + ScanBaseSize: defaultLogPullerScanBaseSize, } } @@ -117,6 +125,16 @@ func (c *PullerConfig) ValidateAndAdjust() { if c.OldStartTsScanLowPriorityThreshold <= 0 { c.OldStartTsScanLowPriorityThreshold = TomlDuration(DefaultOldStartTsScanLowPriorityThreshold) } + if c.MemoryQuota == 0 { + log.Warn("log puller memory quota must be positive, use default value", + zap.Uint64("default", defaultCfg.MemoryQuota)) + c.MemoryQuota = defaultCfg.MemoryQuota + } + if c.ScanBaseSize == 0 { + log.Warn("log puller scan base size must be positive, use default value", + zap.Uint64("default", defaultCfg.ScanBaseSize)) + c.ScanBaseSize = defaultCfg.ScanBaseSize + } } type EventStoreConfig struct { diff --git a/pkg/config/debug_test.go b/pkg/config/debug_test.go index ced2d50bff..82904dcabf 100644 --- a/pkg/config/debug_test.go +++ b/pkg/config/debug_test.go @@ -24,19 +24,16 @@ func TestPullerConfigValidateAndAdjustRegionRequestWindow(t *testing.T) { defaultCfg := NewDefaultPullerConfig() require.Equal(t, 32, defaultCfg.PendingRegionRequestQueueSize) require.Equal(t, 4, defaultCfg.RegionRequestMaxWindowMultiplier) - require.Equal( - t, - TomlDuration(DefaultOldStartTsScanLowPriorityThreshold), - defaultCfg.OldStartTsScanLowPriorityThreshold, - ) + require.Equal(t, uint64(1024*1024*1024), defaultCfg.MemoryQuota) + require.Equal(t, uint64(8*1024*1024), defaultCfg.ScanBaseSize) cfg := &PullerConfig{ - PendingRegionRequestQueueSize: -1, - RegionRequestMaxWindowMultiplier: 0, - OldStartTsScanLowPriorityThreshold: 0, + PendingRegionRequestQueueSize: -1, + RegionRequestMaxWindowMultiplier: 0, } cfg.ValidateAndAdjust() require.Equal(t, defaultCfg.PendingRegionRequestQueueSize, cfg.PendingRegionRequestQueueSize) require.Equal(t, defaultCfg.RegionRequestMaxWindowMultiplier, cfg.RegionRequestMaxWindowMultiplier) - require.Equal(t, defaultCfg.OldStartTsScanLowPriorityThreshold, cfg.OldStartTsScanLowPriorityThreshold) + require.Equal(t, defaultCfg.MemoryQuota, cfg.MemoryQuota) + require.Equal(t, defaultCfg.ScanBaseSize, cfg.ScanBaseSize) } diff --git a/pkg/metrics/log_puller.go b/pkg/metrics/log_puller.go index a74f1b03c4..0608d3e699 100644 --- a/pkg/metrics/log_puller.go +++ b/pkg/metrics/log_puller.go @@ -64,6 +64,13 @@ var ( Name: "resolved_ts_lag", Help: "The lag of resolved ts", }) + LogPullerMemoryQuota = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "ticdc", + Subsystem: "log_puller", + Name: "memory_quota", + Help: "The log puller local memory quota usage.", + }, []string{"type"}) SubscriptionClientResolvedTsLagGauge = prometheus.NewGauge( prometheus.GaugeOpts{ @@ -164,6 +171,7 @@ func initLogPullerMetrics(registry *prometheus.Registry) { registry.MustRegister(LogPullerPrewriteCacheRowNum) registry.MustRegister(LogPullerMatcherCount) registry.MustRegister(LogPullerResolvedTsLag) + registry.MustRegister(LogPullerMemoryQuota) registry.MustRegister(SubscriptionClientRequestedRegionCount) registry.MustRegister(SubscriptionClientAddRegionRequestDuration) registry.MustRegister(RegionRequestFinishScanDuration) From 413d67ecef1fb3c6625f0bdba39afda3c1796823 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Fri, 17 Jul 2026 20:02:36 +0800 Subject: [PATCH 50/71] small fix --- logservice/logpuller/memory_quota.go | 2 +- logservice/logpuller/memory_quota_test.go | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/logservice/logpuller/memory_quota.go b/logservice/logpuller/memory_quota.go index 6596fc0f29..85657987cd 100644 --- a/logservice/logpuller/memory_quota.go +++ b/logservice/logpuller/memory_quota.go @@ -29,7 +29,7 @@ const ( defaultResumeWarmingRatio = 0.05 defaultFreezeAllRatio = 0.8 defaultResumeAllRatio = 0.6 - defaultHardLimitRatio = 5.0 + defaultHardLimitRatio = 2.0 defaultScanBaseSize uint64 = 8 * 1024 * 1024 defaultScanLagUnit = 10 * time.Minute diff --git a/logservice/logpuller/memory_quota_test.go b/logservice/logpuller/memory_quota_test.go index be4d88d89d..4ffe57078f 100644 --- a/logservice/logpuller/memory_quota_test.go +++ b/logservice/logpuller/memory_quota_test.go @@ -161,11 +161,10 @@ func TestMemoryQuotaBlockedEventStopsWhenSubscriptionIsRemoved(t *testing.T) { func TestMemoryQuotaBlockedEventResumesAfterRelease(t *testing.T) { quota := newMemoryQuotaController(100, 10) - quota.hardLimitRatio = 1 span := newTestQuotaSpan(1) quota.addSubscription(span) - lease := quota.trackEvent(context.Background(), span, 100) + lease := quota.trackEvent(context.Background(), span, 200) require.NotNil(t, lease) acquired := make(chan *memoryQuotaLease, 1) go func() { From 23b1936c73250ab96477bd87d42a070f68dd3cbf Mon Sep 17 00:00:00 2001 From: lidezhu Date: Fri, 17 Jul 2026 22:22:43 +0800 Subject: [PATCH 51/71] refactor --- logservice/logpuller/memory_quota.go | 81 ++++++++++--------- logservice/logpuller/memory_quota_test.go | 3 +- .../logpuller/region_admission_controller.go | 15 +--- .../region_admission_controller_test.go | 24 ++++-- logservice/logpuller/region_event_handler.go | 11 ++- .../logpuller/region_event_handler_test.go | 30 ++++--- logservice/logpuller/region_event_sink.go | 10 +-- .../logpuller/region_event_sink_test.go | 48 +---------- 8 files changed, 98 insertions(+), 124 deletions(-) diff --git a/logservice/logpuller/memory_quota.go b/logservice/logpuller/memory_quota.go index 85657987cd..c5d90837b1 100644 --- a/logservice/logpuller/memory_quota.go +++ b/logservice/logpuller/memory_quota.go @@ -23,18 +23,44 @@ import ( ) const ( + // defaultLogPullerMemoryQuota is the soft memory capacity shared by event + // accounting and initial-scan admission. defaultLogPullerMemoryQuota uint64 = 1024 * 1024 * 1024 - defaultPauseWarmingRatio = 0.15 + // Admission ratios compare max(accounted event memory, estimated scan + // memory) with the soft capacity. + + // defaultPauseWarmingRatio pauses new high-lag scans when memory pressure + // reaches 15% of the soft capacity. + defaultPauseWarmingRatio = 0.15 + + // defaultResumeWarmingRatio resumes high-lag scans after memory pressure + // falls to 5% of the soft capacity. defaultResumeWarmingRatio = 0.05 - defaultFreezeAllRatio = 0.8 - defaultResumeAllRatio = 0.6 - defaultHardLimitRatio = 2.0 - - defaultScanBaseSize uint64 = 8 * 1024 * 1024 - defaultScanLagUnit = 10 * time.Minute - defaultScanLagWeight = 0.22 - defaultMaxScanLagFactor = 16 + + // defaultFreezeAllRatio pauses every new scan when memory pressure reaches + // 80% of the soft capacity. + defaultFreezeAllRatio = 0.8 + + // defaultResumeAllRatio allows new scans again after memory pressure falls + // to 60% of the soft capacity. + defaultResumeAllRatio = 0.6 + + // defaultHardLimitRatio blocks receiving more events when accounted event + // memory reaches twice the soft capacity. + defaultHardLimitRatio = 2.0 + + // defaultScanBaseSize is the minimum memory estimate for one admitted scan. + defaultScanBaseSize uint64 = 8 * 1024 * 1024 + + // defaultScanLagUnit is the lag unit used by the logarithmic scan estimate. + defaultScanLagUnit = 10 * time.Minute + + // defaultScanLagWeight controls how quickly the scan estimate grows with lag. + defaultScanLagWeight = 0.22 + + // defaultMaxScanLagFactor caps one scan estimate at this multiple of the base. + defaultMaxScanLagFactor = 16 ) type admissionLevel uint8 @@ -50,15 +76,12 @@ type memoryQuotaLease struct { release func() } +func newMemoryQuotaLease(release func()) *memoryQuotaLease { + return &memoryQuotaLease{release: release} +} + func (l *memoryQuotaLease) Release() { - if l == nil { - return - } - l.once.Do(func() { - if l.release != nil { - l.release() - } - }) + l.once.Do(l.release) } type subscriptionQuotaState struct { @@ -123,6 +146,7 @@ func newMemoryQuotaController(capacity, scanBaseSize uint64) *memoryQuotaControl subscriptions: make(map[SubscriptionID]*subscriptionQuotaState), } c.cond = sync.NewCond(&c.mu) + c.onAvailable.Store(func() {}) return c } @@ -131,9 +155,7 @@ func (c *memoryQuotaController) setOnAvailable(fn func()) { } func (c *memoryQuotaController) notifyAvailable() { - if fn, ok := c.onAvailable.Load().(func()); ok && fn != nil { - fn() - } + c.onAvailable.Load().(func())() } func (c *memoryQuotaController) wakeAll() { @@ -162,19 +184,12 @@ func (c *memoryQuotaController) scanSnapshot() ( } func (c *memoryQuotaController) addSubscription(span *subscribedSpan) { - if span == nil { - return - } c.mu.Lock() c.subscriptions[span.subID] = newSubscriptionQuotaState() c.mu.Unlock() } func (c *memoryQuotaController) removeSubscription(span *subscribedSpan) { - if span == nil { - return - } - c.mu.Lock() state, ok := c.subscriptions[span.subID] if !ok { @@ -210,17 +225,13 @@ func (c *memoryQuotaController) acquireScan( currentTs uint64, ) (*memoryQuotaLease, bool) { span := region.subscribedSpan - if span == nil { - return nil, true - } - c.mu.Lock() state, ok := c.subscriptions[span.subID] if !ok { // The subscription has already been removed. Let the request continue to // the worker, where the normal stopped-subscription path will discard it. c.mu.Unlock() - return nil, true + return newMemoryQuotaLease(func() {}), true } c.refreshLevelLocked() if c.level == admissionFreezeAllNewScans { @@ -268,10 +279,6 @@ func (c *memoryQuotaController) trackEvent( span *subscribedSpan, bytes uint64, ) *memoryQuotaLease { - if span == nil || bytes == 0 { - return nil - } - c.mu.Lock() if ctx.Err() != nil { c.mu.Unlock() @@ -347,7 +354,7 @@ func scanLagFactor(startTs, currentTs uint64) float64 { func isWarmingScan(region regionInfo, currentTs uint64) bool { span := region.subscribedSpan - if span == nil || span.initialized.Load() { + if span.initialized.Load() { return false } return regionScanLag(currentTs, region.resolvedTs()) >= lowLagRegionThreshold diff --git a/logservice/logpuller/memory_quota_test.go b/logservice/logpuller/memory_quota_test.go index 4ffe57078f..5f45602f43 100644 --- a/logservice/logpuller/memory_quota_test.go +++ b/logservice/logpuller/memory_quota_test.go @@ -121,7 +121,8 @@ func TestMemoryQuotaRemoveSubscriptionReleasesOwnedMemory(t *testing.T) { // path without recreating quota state. scanLease, admitted = quota.acquireScan(newTestQuotaRegion(span1), span1.resolvedTs.Load()) require.True(t, admitted) - require.Nil(t, scanLease) + require.NotNil(t, scanLease) + scanLease.Release() require.NotContains(t, quota.subscriptions, span1.subID) lease2.Release() diff --git a/logservice/logpuller/region_admission_controller.go b/logservice/logpuller/region_admission_controller.go index 83a5c7c5bd..87096bac49 100644 --- a/logservice/logpuller/region_admission_controller.go +++ b/logservice/logpuller/region_admission_controller.go @@ -198,17 +198,10 @@ func (c *regionAdmissionController) popEligibleLocked() ( return nil, nil } - var scanQuota *memoryQuotaLease - if c.memoryQuota != nil { - currentTs := uint64(0) - if c.currentTs != nil { - currentTs = c.currentTs() - } - var admitted bool - scanQuota, admitted = c.memoryQuota.acquireScan(request.regionInfo, currentTs) - if !admitted { - return nil, nil - } + scanQuota, admitted := c.memoryQuota.acquireScan( + request.regionInfo, c.currentTs()) + if !admitted { + return nil, nil } request, _ = c.pending.PopTop() return request, scanQuota diff --git a/logservice/logpuller/region_admission_controller_test.go b/logservice/logpuller/region_admission_controller_test.go index ed68f14a2f..a6277f80de 100644 --- a/logservice/logpuller/region_admission_controller_test.go +++ b/logservice/logpuller/region_admission_controller_test.go @@ -58,8 +58,20 @@ func submitRegionForAdmission( require.True(t, controller.submit(task)) } +func newTestRegionAdmissionController( + currentWindow int, + maxWindowMultiplier int, +) *regionAdmissionController { + return newRegionAdmissionController( + currentWindow, + maxWindowMultiplier, + newMemoryQuotaController(0, 0), + func() uint64 { return 0 }, + ) +} + func TestRegionAdmissionControllerNormalWindow(t *testing.T) { - controller := newRegionAdmissionController(1, 2, nil, nil) + controller := newTestRegionAdmissionController(1, 2) currentTs := oracle.GoTimeToTS(time.Now()) checkpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Hour)) region1 := prepareRegionForAdmission(createTestRegionInfo(1, 1), checkpointTs) @@ -84,7 +96,7 @@ func TestRegionAdmissionControllerNormalWindow(t *testing.T) { } func TestRegionAdmissionControllerLowLagUsesMaxWindow(t *testing.T) { - controller := newRegionAdmissionController(1, 2, nil, nil) + controller := newTestRegionAdmissionController(1, 2) currentTs := oracle.GoTimeToTS(time.Now()) slowCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Hour)) lowLagCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Minute)) @@ -121,7 +133,7 @@ func TestRegionAdmissionControllerLowLagUsesMaxWindow(t *testing.T) { } func TestRegionAdmissionControllerPrioritizesInitializedRegion(t *testing.T) { - controller := newRegionAdmissionController(1, 2, nil, nil) + controller := newTestRegionAdmissionController(1, 2) currentTs := oracle.GoTimeToTS(time.Now()) slowCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Hour)) lowLagCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Minute)) @@ -153,7 +165,7 @@ func TestRegionAdmissionControllerPrioritizesInitializedRegion(t *testing.T) { } func TestRegionAdmissionLeaseReleasedOnce(t *testing.T) { - controller := newRegionAdmissionController(1, 1, nil, nil) + controller := newTestRegionAdmissionController(1, 1) currentTs := oracle.GoTimeToTS(time.Now()) region := prepareRegionForAdmission(createTestRegionInfo(1, 1), currentTs) submitRegionForAdmission(t, controller, region, currentTs) @@ -189,7 +201,7 @@ func TestRegionAdmissionLeaseReleasedOnce(t *testing.T) { } func TestRegionAdmissionControllerClose(t *testing.T) { - controller := newRegionAdmissionController(1, 1, nil, nil) + controller := newTestRegionAdmissionController(1, 1) controller.close() region := prepareRegionForAdmission(createTestRegionInfo(1, 1), 1) require.False(t, controller.submit(newRegionPriorityTask(region, 1, 1))) @@ -199,7 +211,7 @@ func TestRegionAdmissionControllerClose(t *testing.T) { } func TestRegionAdmissionControllerDrainPending(t *testing.T) { - controller := newRegionAdmissionController(1, 1, nil, nil) + controller := newTestRegionAdmissionController(1, 1) region1 := prepareRegionForAdmission(createTestRegionInfo(1, 1), 1) region2 := prepareRegionForAdmission(createTestRegionInfo(1, 2), 1) submitRegionForAdmission(t, controller, region1, 1) diff --git a/logservice/logpuller/region_event_handler.go b/logservice/logpuller/region_event_handler.go index c23184ddab..92a649349a 100644 --- a/logservice/logpuller/region_event_handler.go +++ b/logservice/logpuller/region_event_handler.go @@ -64,14 +64,14 @@ func (event *regionEvent) needMemoryQuota() bool { } func (event *regionEvent) releaseMemoryQuota() { + if event.memoryQuota == nil { + return + } event.memoryQuota.Release() event.memoryQuota = nil } func (event *regionEvent) getSize() int { - if event == nil { - return 0 - } size := int(unsafe.Sizeof(*event)) if event.entries != nil { size += int(unsafe.Sizeof(*event.entries)) @@ -135,7 +135,7 @@ func (h *regionEventHandler) Handle(span *subscribedSpan, events ...regionEvent) wasInitialized := span.initialized.Load() quotaLeases := make([]*memoryQuotaLease, 0, len(events)) for _, event := range events { - if event.memoryQuota != nil { + if event.needMemoryQuota() { quotaLeases = append(quotaLeases, event.memoryQuota) } if len(event.states) == 1 && event.states[0].isStale() { @@ -158,8 +158,7 @@ func (h *regionEventHandler) Handle(span *subscribedSpan, events ...regionEvent) log.Panic("should not reach", zap.Any("event", event), zap.Any("events", events)) } } - if !wasInitialized && span.initialized.Load() && - h.eventSink != nil && h.eventSink.memoryQuota != nil { + if !wasInitialized && span.initialized.Load() { h.eventSink.memoryQuota.markSubscriptionInitialized() } tryAdvanceResolvedTs := func() { diff --git a/logservice/logpuller/region_event_handler_test.go b/logservice/logpuller/region_event_handler_test.go index 92fc3a8e03..dd4fc1d77b 100644 --- a/logservice/logpuller/region_event_handler_test.go +++ b/logservice/logpuller/region_event_handler_test.go @@ -48,7 +48,10 @@ import ( func TestHandleEventEntryEventOutOfOrder(t *testing.T) { // initialize option := dynstream.NewOption() - ds := dynstream.NewParallelDynamicStream("test", ®ionEventHandler{}, option) + handler := ®ionEventHandler{eventSink: ®ionEventSink{ + memoryQuota: newMemoryQuotaController(0, 0), + }} + ds := dynstream.NewParallelDynamicStream("test", handler, option) ds.Start() span := heartbeatpb.TableSpan{ @@ -104,8 +107,9 @@ func TestHandleEventEntryEventOutOfOrder(t *testing.T) { }, } regionEvent := regionEvent{ - states: []*regionFeedState{state}, - entries: events, + states: []*regionFeedState{state}, + entries: events, + memoryQuota: newMemoryQuotaLease(func() {}), } ds.Push(subID, regionEvent) } @@ -124,8 +128,9 @@ func TestHandleEventEntryEventOutOfOrder(t *testing.T) { }, } regionEvent := regionEvent{ - states: []*regionFeedState{state}, - entries: events, + states: []*regionFeedState{state}, + entries: events, + memoryQuota: newMemoryQuotaLease(func() {}), } ds.Push(subID, regionEvent) } @@ -154,8 +159,9 @@ func TestHandleEventEntryEventOutOfOrder(t *testing.T) { }, } regionEvent := regionEvent{ - states: []*regionFeedState{state}, - entries: events, + states: []*regionFeedState{state}, + entries: events, + memoryQuota: newMemoryQuotaLease(func() {}), } ds.Push(subID, regionEvent) } @@ -181,8 +187,9 @@ func TestHandleEventEntryEventOutOfOrder(t *testing.T) { }, } regionEvent := regionEvent{ - states: []*regionFeedState{state}, - entries: events, + states: []*regionFeedState{state}, + entries: events, + memoryQuota: newMemoryQuotaLease(func() {}), } ds.Push(subID, regionEvent) } @@ -204,7 +211,10 @@ func TestHandleEventEntryEventOutOfOrder(t *testing.T) { func TestHandleResolvedTs(t *testing.T) { // initialize option := dynstream.NewOption() - ds := dynstream.NewParallelDynamicStream("test", ®ionEventHandler{}, option) + handler := ®ionEventHandler{eventSink: ®ionEventSink{ + memoryQuota: newMemoryQuotaController(0, 0), + }} + ds := dynstream.NewParallelDynamicStream("test", handler, option) ds.Start() consumeKVEvents := func(events []common.RawKVEntry, _ func()) bool { return false } // not used diff --git a/logservice/logpuller/region_event_sink.go b/logservice/logpuller/region_event_sink.go index 3c161fda1a..25987ff997 100644 --- a/logservice/logpuller/region_event_sink.go +++ b/logservice/logpuller/region_event_sink.go @@ -78,7 +78,7 @@ func (s *regionEventSink) Wake(subID SubscriptionID) { } func (s *regionEventSink) Push(subID SubscriptionID, event regionEvent) { - if event.needMemoryQuota() && s.memoryQuota != nil { + if event.needMemoryQuota() { span := event.mustFirstState().region.subscribedSpan event.memoryQuota = s.memoryQuota.trackEvent(s.ctx, span, uint64(event.getSize())) if event.memoryQuota == nil { @@ -137,10 +137,6 @@ func (s *regionEventSink) UpdateMetrics() { metricSubscriptionClientDSChannelSize.Set(float64(dsMetrics.EventChanSize)) metricSubscriptionClientDSPendingQueueLen.Set(float64(dsMetrics.PendingQueueLen)) - if s.memoryQuota == nil { - return - } - used, capacity, _ := s.memoryQuota.snapshot() scanUsed, warmingScanUsed, warmingScanBudget, scanEstimate, hardLimit := s.memoryQuota.scanSnapshot() @@ -166,9 +162,7 @@ func (s *regionEventSink) UpdateMetrics() { } func (s *regionEventSink) Close() { - if s.memoryQuota != nil { - s.memoryQuota.wakeAll() - } + s.memoryQuota.wakeAll() s.mu.Lock() s.paused.Store(false) s.cond.Broadcast() diff --git a/logservice/logpuller/region_event_sink_test.go b/logservice/logpuller/region_event_sink_test.go index 1f08aa51f9..43fd6f088d 100644 --- a/logservice/logpuller/region_event_sink_test.go +++ b/logservice/logpuller/region_event_sink_test.go @@ -81,8 +81,9 @@ func TestRegionEventSinkRunPausesAndResumesPush(t *testing.T) { ds := newMockRegionEventSinkStream() sink := ®ionEventSink{ - ctx: ctx, - ds: ds, + ctx: ctx, + ds: ds, + memoryQuota: newMemoryQuotaController(0, 0), } sink.cond = sync.NewCond(&sink.mu) @@ -135,49 +136,6 @@ func TestRegionEventSinkRunPausesAndResumesPush(t *testing.T) { } func TestRegionEventSinkUpdateMetrics(t *testing.T) { - t.Run("without quota only updates queue gauges", func(t *testing.T) { - ds := newMockRegionEventSinkStream() - ds.metrics = dynstream.Metrics[int, SubscriptionID]{ - EventChanSize: 11, - PendingQueueLen: 22, - } - - metrics.DynamicStreamMemoryUsage.WithLabelValues( - "log-puller", - "max", - "default", - "default", - ).Set(123) - metrics.DynamicStreamMemoryUsage.WithLabelValues( - "log-puller", - "used", - "default", - "default", - ).Set(456) - - sink := ®ionEventSink{ - ctx: context.Background(), - ds: ds, - } - sink.cond = sync.NewCond(&sink.mu) - sink.UpdateMetrics() - - require.Equal(t, float64(11), testutil.ToFloat64(metricSubscriptionClientDSChannelSize)) - require.Equal(t, float64(22), testutil.ToFloat64(metricSubscriptionClientDSPendingQueueLen)) - require.Equal(t, float64(123), testutil.ToFloat64(metrics.DynamicStreamMemoryUsage.WithLabelValues( - "log-puller", - "max", - "default", - "default", - ))) - require.Equal(t, float64(456), testutil.ToFloat64(metrics.DynamicStreamMemoryUsage.WithLabelValues( - "log-puller", - "used", - "default", - "default", - ))) - }) - t.Run("quota updates memory gauges", func(t *testing.T) { ds := newMockRegionEventSinkStream() ds.metrics = dynstream.Metrics[int, SubscriptionID]{ From 4e14c70d6d8fef388db1efe8739bcec6ba7400bc Mon Sep 17 00:00:00 2001 From: lidezhu Date: Sun, 19 Jul 2026 18:54:38 +0800 Subject: [PATCH 52/71] remove dependency on ds memory control --- logservice/logpuller/memory_quota.go | 383 +++++++++--------- logservice/logpuller/memory_quota_test.go | 309 ++++++++------ .../logpuller/region_admission_controller.go | 36 +- .../region_admission_controller_test.go | 12 +- logservice/logpuller/region_event_handler.go | 32 +- .../logpuller/region_event_handler_test.go | 37 +- logservice/logpuller/region_event_sink.go | 81 +--- .../logpuller/region_event_sink_test.go | 103 +---- .../logpuller/region_request_scheduler.go | 7 - logservice/logpuller/region_request_store.go | 6 - logservice/logpuller/subscription_client.go | 6 +- 11 files changed, 438 insertions(+), 574 deletions(-) diff --git a/logservice/logpuller/memory_quota.go b/logservice/logpuller/memory_quota.go index c5d90837b1..aa5396171e 100644 --- a/logservice/logpuller/memory_quota.go +++ b/logservice/logpuller/memory_quota.go @@ -71,29 +71,68 @@ const ( admissionFreezeAllNewScans ) -type memoryQuotaLease struct { - once sync.Once - release func() -} - -func newMemoryQuotaLease(release func()) *memoryQuotaLease { - return &memoryQuotaLease{release: release} +// eventMemoryNotifier wakes event receivers that are waiting for memory. Each +// notification closes the current ready channel to wake all current waiters, +// then creates a new channel for future waiters. +// +// To avoid missing a notification, a receiver waits in this order: +// +// 1. Register the waiter. +// 2. Read the current ready channel under mu. +// 3. Recheck memory and the span state before blocking on that channel. +// +// If a notification happens just before registration, the final recheck sees +// the released memory or stopped span, so the receiver does not block. +type eventMemoryNotifier struct { + mu sync.Mutex + ready chan struct{} + waiters atomic.Int64 } -func (l *memoryQuotaLease) Release() { - l.once.Do(l.release) +func newEventMemoryNotifier() *eventMemoryNotifier { + return &eventMemoryNotifier{ready: make(chan struct{})} } -type subscriptionQuotaState struct { - eventLeases map[*memoryQuotaLease]struct{} - scanLeases map[*memoryQuotaLease]struct{} +func (n *eventMemoryNotifier) wait( + ctx context.Context, + span *subscribedSpan, + tryAcquire func() bool, +) bool { + n.waiters.Add(1) + defer n.waiters.Add(-1) + for { + n.mu.Lock() + ready := n.ready + n.mu.Unlock() + + // This check must stay after waiter registration and loading ready. It + // closes both windows in which notify could otherwise be lost. + if tryAcquire() { + return true + } + if span.stopped.Load() { + return false + } + select { + case <-ready: + case <-ctx.Done(): + return false + } + } } -func newSubscriptionQuotaState() *subscriptionQuotaState { - return &subscriptionQuotaState{ - eventLeases: make(map[*memoryQuotaLease]struct{}), - scanLeases: make(map[*memoryQuotaLease]struct{}), +func (n *eventMemoryNotifier) notify() { + // waiters is only a fast-path hint. A waiter that registers after this load + // rechecks memory and the span state before blocking, so observing a stale + // zero cannot lose a wakeup. Observing a stale nonzero only causes a harmless + // extra broadcast. + if n.waiters.Load() == 0 { + return } + n.mu.Lock() + close(n.ready) + n.ready = make(chan struct{}) + n.mu.Unlock() } // memoryQuotaController tracks event memory retained by downstream callbacks @@ -102,29 +141,34 @@ func newSubscriptionQuotaState() *subscriptionQuotaState { // admission first pauses uninitialized high-lag spans and freezes all new scans // only under heavier pressure; both transitions use hysteresis when resuming. type memoryQuotaController struct { - mu sync.Mutex - cond *sync.Cond - capacity uint64 // used tracks event bytes retained until downstream finishes consuming them. - used uint64 + // Event accounting is on the receive hot path, so acquiring and releasing + // memory only use atomic operations while usage is below the hard limit. + used atomic.Uint64 - // scanUsed tracks the estimated memory of all admitted initial scans. - // warmingScanUsed is the subset used by uninitialized, high-lag spans. - scanUsed uint64 - warmingScanUsed uint64 - level admissionLevel + // eventNotifier owns the wait protocol used after the hard limit is reached. + eventNotifier *eventMemoryNotifier - pauseWarmingRatio float64 - resumeWarmingRatio float64 - freezeAllRatio float64 - resumeAllRatio float64 - hardLimitRatio float64 + // scanMu guards scan admission state and scanReady. Scan admission happens + // once per region rather than once per event batch, so it is intentionally + // kept simple instead of adding atomics to every field. + scanMu sync.Mutex + // scanUsed tracks the estimated memory of all admitted initial scans. + scanUsed uint64 + level admissionLevel + // scanReady is replaced and closed when a memory transition can make a + // rejected scan eligible. Workers wait on this channel directly, avoiding a + // synchronous broadcast to every store and request worker. + scanReady chan struct{} + + pauseWarmingLimit uint64 + resumeWarmingLimit uint64 + freezeAllLimit uint64 + resumeAllLimit uint64 + hardLimit uint64 scanEstimate uint64 - - subscriptions map[SubscriptionID]*subscriptionQuotaState - onAvailable atomic.Value // func() } func newMemoryQuotaController(capacity, scanBaseSize uint64) *memoryQuotaController { @@ -137,191 +181,150 @@ func newMemoryQuotaController(capacity, scanBaseSize uint64) *memoryQuotaControl c := &memoryQuotaController{ capacity: capacity, level: admissionNormal, - pauseWarmingRatio: defaultPauseWarmingRatio, - resumeWarmingRatio: defaultResumeWarmingRatio, - freezeAllRatio: defaultFreezeAllRatio, - resumeAllRatio: defaultResumeAllRatio, - hardLimitRatio: defaultHardLimitRatio, + pauseWarmingLimit: uint64(math.Ceil(float64(capacity) * defaultPauseWarmingRatio)), + resumeWarmingLimit: uint64(float64(capacity) * defaultResumeWarmingRatio), + freezeAllLimit: uint64(math.Ceil(float64(capacity) * defaultFreezeAllRatio)), + resumeAllLimit: uint64(float64(capacity) * defaultResumeAllRatio), + hardLimit: uint64(float64(capacity) * defaultHardLimitRatio), scanEstimate: scanBaseSize, - subscriptions: make(map[SubscriptionID]*subscriptionQuotaState), + eventNotifier: newEventMemoryNotifier(), + scanReady: make(chan struct{}), } - c.cond = sync.NewCond(&c.mu) - c.onAvailable.Store(func() {}) return c } -func (c *memoryQuotaController) setOnAvailable(fn func()) { - c.onAvailable.Store(fn) -} - -func (c *memoryQuotaController) notifyAvailable() { - c.onAvailable.Load().(func())() -} - func (c *memoryQuotaController) wakeAll() { - c.mu.Lock() - c.cond.Broadcast() - c.mu.Unlock() + c.eventNotifier.notify() + c.notifyScanAdmission() } func (c *memoryQuotaController) snapshot() (used, capacity uint64, level admissionLevel) { - c.mu.Lock() - defer c.mu.Unlock() - return c.used, c.capacity, c.level + c.scanMu.Lock() + defer c.scanMu.Unlock() + return c.used.Load(), c.capacity, c.level } func (c *memoryQuotaController) scanSnapshot() ( scanUsed uint64, - warmingScanUsed uint64, - warmingScanBudget uint64, scanEstimate uint64, hardLimit uint64, ) { - c.mu.Lock() - defer c.mu.Unlock() - return c.scanUsed, c.warmingScanUsed, c.warmingScanBudgetLocked(), - c.scanEstimate, c.hardLimitLocked() -} - -func (c *memoryQuotaController) addSubscription(span *subscribedSpan) { - c.mu.Lock() - c.subscriptions[span.subID] = newSubscriptionQuotaState() - c.mu.Unlock() -} - -func (c *memoryQuotaController) removeSubscription(span *subscribedSpan) { - c.mu.Lock() - state, ok := c.subscriptions[span.subID] - if !ok { - c.mu.Unlock() - return - } - delete(c.subscriptions, span.subID) - leases := make([]*memoryQuotaLease, 0, len(state.eventLeases)+len(state.scanLeases)) - for lease := range state.eventLeases { - leases = append(leases, lease) - } - for lease := range state.scanLeases { - leases = append(leases, lease) - } - // Wake event receivers so they can observe that the subscription was removed. - c.cond.Broadcast() - c.mu.Unlock() - - for _, lease := range leases { - lease.Release() - } - // Removing a subscription can make a pending request eligible even when the - // subscription itself did not own any lease. - c.notifyAvailable() -} - -func (c *memoryQuotaController) markSubscriptionInitialized() { - c.notifyAvailable() + c.scanMu.Lock() + defer c.scanMu.Unlock() + return c.scanUsed, c.scanEstimate, c.hardLimit } func (c *memoryQuotaController) acquireScan( region regionInfo, currentTs uint64, -) (*memoryQuotaLease, bool) { +) (bytes uint64, retry <-chan struct{}, admitted bool) { span := region.subscribedSpan - c.mu.Lock() - state, ok := c.subscriptions[span.subID] - if !ok { - // The subscription has already been removed. Let the request continue to - // the worker, where the normal stopped-subscription path will discard it. - c.mu.Unlock() - return newMemoryQuotaLease(func() {}), true + if span.stopped.Load() { + // Let stale tasks reach the worker's stopped-subscription cleanup path + // without consuming scan quota. + return 0, nil, true } + + c.scanMu.Lock() + defer c.scanMu.Unlock() c.refreshLevelLocked() if c.level == admissionFreezeAllNewScans { - c.mu.Unlock() - return nil, false + return 0, c.scanReady, false } - bytes := c.estimateScanSizeLocked(region, currentTs) warming := isWarmingScan(region, currentTs) - if c.isWarmingScanBlockedLocked(warming, bytes) { - c.mu.Unlock() - return nil, false - } - - lease := &memoryQuotaLease{} - lease.release = func() { - c.mu.Lock() - previousLevel := c.level - c.scanUsed = subtractFloor(c.scanUsed, bytes) - if warming { - c.warmingScanUsed = subtractFloor(c.warmingScanUsed, bytes) - } - delete(state.scanLeases, lease) - c.refreshLevelLocked() - shouldNotifyAdmission := c.level < previousLevel || - (warming && c.level == admissionNormal) - c.mu.Unlock() - - if shouldNotifyAdmission { - c.notifyAvailable() - } + // Admission is based on the pressure before accounting this scan. This lets + // one scan make progress even when its estimate alone exceeds the threshold. + if warming && c.level == admissionPauseWarming { + return 0, c.scanReady, false } + bytes = c.estimateScanSizeLocked(region, currentTs) c.scanUsed += bytes - if warming { - c.warmingScanUsed += bytes + c.refreshLevelLocked() + return bytes, nil, true +} + +func (c *memoryQuotaController) releaseScan(bytes uint64) { + if bytes == 0 { + return } - state.scanLeases[lease] = struct{}{} + c.scanMu.Lock() + previousLevel := c.level + c.scanUsed = subtractFloor(c.scanUsed, bytes) c.refreshLevelLocked() - c.mu.Unlock() - return lease, true + if c.level < previousLevel { + c.notifyScanAdmissionLocked() + } + c.scanMu.Unlock() } -func (c *memoryQuotaController) trackEvent( +// acquireEvent accounts one event batch. Below the hard limit its hot path is +// a context check and an atomic compare-and-swap; it does not allocate or take +// a mutex. +func (c *memoryQuotaController) acquireEvent( ctx context.Context, span *subscribedSpan, bytes uint64, -) *memoryQuotaLease { - c.mu.Lock() +) bool { if ctx.Err() != nil { - c.mu.Unlock() - return nil + return false } - state := c.subscriptions[span.subID] - if state == nil { - c.mu.Unlock() - return nil + if c.tryAcquireEvent(bytes) { + return true } - for c.used > 0 && wouldExceed(c.used, bytes, c.hardLimitLocked()) { - c.cond.Wait() - if ctx.Err() != nil { - c.mu.Unlock() - return nil + return c.eventNotifier.wait(ctx, span, func() bool { + return c.tryAcquireEvent(bytes) + }) +} + +func (c *memoryQuotaController) tryAcquireEvent(bytes uint64) bool { + for { + used := c.used.Load() + if used > 0 && wouldExceed(used, bytes, c.hardLimit) { + return false + } + if bytes > math.MaxUint64-used { + return false } - if c.subscriptions[span.subID] != state { - c.mu.Unlock() - return nil + if c.used.CompareAndSwap(used, used+bytes) { + return true } } +} + +func (c *memoryQuotaController) releaseEvent(bytes uint64) { + if bytes == 0 { + return + } + used := c.used.Add(^(bytes - 1)) + previousUsed := used + bytes + if crossesDown(previousUsed, used, c.resumeWarmingLimit) || + crossesDown(previousUsed, used, c.resumeAllLimit) { + c.refreshAdmissionAndNotify() + } + c.eventNotifier.notify() +} - c.used += bytes +func (c *memoryQuotaController) notifyScanAdmission() { + c.scanMu.Lock() + c.notifyScanAdmissionLocked() + c.scanMu.Unlock() +} + +func (c *memoryQuotaController) notifyScanAdmissionLocked() { + close(c.scanReady) + c.scanReady = make(chan struct{}) +} + +func (c *memoryQuotaController) refreshAdmissionAndNotify() { + c.scanMu.Lock() + previousLevel := c.level c.refreshLevelLocked() - lease := &memoryQuotaLease{} - lease.release = func() { - c.mu.Lock() - previousLevel := c.level - c.used = subtractFloor(c.used, bytes) - delete(state.eventLeases, lease) - c.refreshLevelLocked() - shouldNotifyAdmission := c.level < previousLevel - c.cond.Broadcast() - c.mu.Unlock() - - if shouldNotifyAdmission { - c.notifyAvailable() - } + if c.level < previousLevel { + c.notifyScanAdmissionLocked() } - state.eventLeases[lease] = struct{}{} - c.mu.Unlock() - return lease + c.scanMu.Unlock() } func (c *memoryQuotaController) estimateScanSizeLocked(region regionInfo, currentTs uint64) uint64 { @@ -360,34 +363,14 @@ func isWarmingScan(region regionInfo, currentTs uint64) bool { return regionScanLag(currentTs, region.resolvedTs()) >= lowLagRegionThreshold } -func (c *memoryQuotaController) isWarmingScanBlockedLocked(warming bool, bytes uint64) bool { - if !warming { - return false - } - if c.level == admissionPauseWarming { - return true - } - return wouldExceed(c.warmingScanUsed, bytes, c.warmingScanBudgetLocked()) -} - -func (c *memoryQuotaController) warmingScanBudgetLocked() uint64 { - budget := uint64(float64(c.capacity) * c.pauseWarmingRatio) - return max(budget, c.scanEstimate) -} - -func (c *memoryQuotaController) hardLimitLocked() uint64 { - return uint64(float64(c.capacity) * c.hardLimitRatio) -} - func (c *memoryQuotaController) refreshLevelLocked() { // scanUsed predicts the event memory an initial scan may produce, so adding // it to actual event bytes would count the same pressure twice. - pressure := max(c.used, c.scanUsed) - usage := float64(pressure) / float64(c.capacity) + pressure := max(c.used.Load(), c.scanUsed) switch c.level { case admissionFreezeAllNewScans: - if usage <= c.resumeAllRatio { - if usage >= c.pauseWarmingRatio { + if pressure <= c.resumeAllLimit { + if pressure >= c.pauseWarmingLimit { c.level = admissionPauseWarming } else { c.level = admissionNormal @@ -395,16 +378,16 @@ func (c *memoryQuotaController) refreshLevelLocked() { } case admissionPauseWarming: switch { - case usage >= c.freezeAllRatio: + case pressure >= c.freezeAllLimit: c.level = admissionFreezeAllNewScans - case usage <= c.resumeWarmingRatio: + case pressure <= c.resumeWarmingLimit: c.level = admissionNormal } default: switch { - case usage >= c.freezeAllRatio: + case pressure >= c.freezeAllLimit: c.level = admissionFreezeAllNewScans - case usage >= c.pauseWarmingRatio: + case pressure >= c.pauseWarmingLimit: c.level = admissionPauseWarming } } @@ -414,6 +397,10 @@ func wouldExceed(used, bytes, limit uint64) bool { return bytes > limit || used > limit-bytes } +func crossesDown(previous, current, threshold uint64) bool { + return previous > threshold && current <= threshold +} + func subtractFloor(value, delta uint64) uint64 { if value < delta { return 0 diff --git a/logservice/logpuller/memory_quota_test.go b/logservice/logpuller/memory_quota_test.go index 5f45602f43..1fac91945e 100644 --- a/logservice/logpuller/memory_quota_test.go +++ b/logservice/logpuller/memory_quota_test.go @@ -52,95 +52,80 @@ func TestMemoryQuotaAdmissionLevels(t *testing.T) { initializedSpan.initialized.Store(true) warmingTs := setTestQuotaSpanLag(warmingSpan, lowLagRegionThreshold+time.Minute) initializedTs := setTestQuotaSpanLag(initializedSpan, lowLagRegionThreshold+time.Minute) - quota.addSubscription(warmingSpan) - quota.addSubscription(initializedSpan) - - lowLease := quota.trackEvent(context.Background(), initializedSpan, 5) - pauseLease := quota.trackEvent(context.Background(), initializedSpan, 10) - require.NotNil(t, lowLease) - require.NotNil(t, pauseLease) - scanLease, admitted := quota.acquireScan(newTestQuotaRegion(warmingSpan), warmingTs) + + require.True(t, quota.acquireEvent(context.Background(), initializedSpan, 5)) + require.True(t, quota.acquireEvent(context.Background(), initializedSpan, 10)) + _, _, admitted := quota.acquireScan(newTestQuotaRegion(warmingSpan), warmingTs) require.False(t, admitted) - require.Nil(t, scanLease) - scanLease, admitted = quota.acquireScan( + scanBytes, _, admitted := quota.acquireScan( newTestQuotaRegion(initializedSpan), initializedTs) require.True(t, admitted) - scanLease.Release() + quota.releaseScan(scanBytes) - middleLease := quota.trackEvent(context.Background(), initializedSpan, 45) - freezeLease := quota.trackEvent(context.Background(), initializedSpan, 20) - require.NotNil(t, middleLease) - require.NotNil(t, freezeLease) - scanLease, admitted = quota.acquireScan( + require.True(t, quota.acquireEvent(context.Background(), initializedSpan, 45)) + require.True(t, quota.acquireEvent(context.Background(), initializedSpan, 20)) + _, _, admitted = quota.acquireScan( newTestQuotaRegion(initializedSpan), initializedTs) require.False(t, admitted) - require.Nil(t, scanLease) - freezeLease.Release() + quota.releaseEvent(20) _, _, level := quota.snapshot() require.Equal(t, admissionPauseWarming, level) - middleLease.Release() + quota.releaseEvent(45) _, _, level = quota.snapshot() require.Equal(t, admissionPauseWarming, level) - pauseLease.Release() + quota.releaseEvent(10) _, _, level = quota.snapshot() require.Equal(t, admissionNormal, level) - lowLease.Release() + quota.releaseEvent(5) } -func TestMemoryQuotaRemoveSubscriptionReleasesOwnedMemory(t *testing.T) { +func TestMemoryQuotaSpanStopKeepsOwnedMemoryUntilRelease(t *testing.T) { quota := newMemoryQuotaController(100, 10) span1 := newTestQuotaSpan(1) span2 := newTestQuotaSpan(2) - quota.addSubscription(span1) - quota.addSubscription(span2) - - lease1 := quota.trackEvent(context.Background(), span1, 30) - lease2 := quota.trackEvent(context.Background(), span2, 40) - require.NotNil(t, lease1) - require.NotNil(t, lease2) - scanLease, admitted := quota.acquireScan(newTestQuotaRegion(span1), span1.resolvedTs.Load()) + + require.True(t, quota.acquireEvent(context.Background(), span1, 30)) + require.True(t, quota.acquireEvent(context.Background(), span2, 40)) + scanBytes, _, admitted := quota.acquireScan( + newTestQuotaRegion(span1), span1.resolvedTs.Load()) require.True(t, admitted) - require.NotNil(t, scanLease) + require.NotZero(t, scanBytes) - quota.removeSubscription(span1) + span1.stopped.Store(true) + quota.wakeAll() used, _, _ := quota.snapshot() - require.Equal(t, uint64(40), used) - scanUsed, _, _, _, _ := quota.scanSnapshot() - require.Zero(t, scanUsed) - require.NotContains(t, quota.subscriptions, span1.subID) - quota.removeSubscription(span1) + require.Equal(t, uint64(70), used) + scanUsed, _, _ := quota.scanSnapshot() + require.Equal(t, scanBytes, scanUsed) - lease1.Release() - scanLease.Release() + quota.releaseEvent(30) + quota.releaseScan(scanBytes) used, _, _ = quota.snapshot() require.Equal(t, uint64(40), used) - // Late region tasks are allowed to reach the stopped-subscription cleanup - // path without recreating quota state. - scanLease, admitted = quota.acquireScan(newTestQuotaRegion(span1), span1.resolvedTs.Load()) + // Late tasks reach the stopped-subscription cleanup path without consuming + // scan quota. + scanBytes, _, admitted = quota.acquireScan( + newTestQuotaRegion(span1), span1.resolvedTs.Load()) require.True(t, admitted) - require.NotNil(t, scanLease) - scanLease.Release() - require.NotContains(t, quota.subscriptions, span1.subID) + require.Zero(t, scanBytes) - lease2.Release() + quota.releaseEvent(40) used, _, _ = quota.snapshot() require.Zero(t, used) } -func TestMemoryQuotaBlockedEventStopsWhenSubscriptionIsRemoved(t *testing.T) { +func TestMemoryQuotaBlockedEventStopsWhenSpanStops(t *testing.T) { quota := newMemoryQuotaController(100, 10) - quota.hardLimitRatio = 1 + quota.hardLimit = 100 span := newTestQuotaSpan(1) - quota.addSubscription(span) - lease := quota.trackEvent(context.Background(), span, 100) - require.NotNil(t, lease) - acquired := make(chan *memoryQuotaLease, 1) + require.True(t, quota.acquireEvent(context.Background(), span, 100)) + acquired := make(chan bool, 1) go func() { - acquired <- quota.trackEvent(context.Background(), span, 1) + acquired <- quota.acquireEvent(context.Background(), span, 1) }() select { @@ -149,27 +134,25 @@ func TestMemoryQuotaBlockedEventStopsWhenSubscriptionIsRemoved(t *testing.T) { case <-time.After(100 * time.Millisecond): } - quota.removeSubscription(span) + span.stopped.Store(true) + quota.wakeAll() select { - case blockedLease := <-acquired: - require.Nil(t, blockedLease) + case ok := <-acquired: + require.False(t, ok) case <-time.After(time.Second): - t.Fatal("removing the subscription did not wake the blocked event") + t.Fatal("stopping the subscription did not wake the blocked event") } - require.NotContains(t, quota.subscriptions, span.subID) - lease.Release() + quota.releaseEvent(100) } func TestMemoryQuotaBlockedEventResumesAfterRelease(t *testing.T) { quota := newMemoryQuotaController(100, 10) span := newTestQuotaSpan(1) - quota.addSubscription(span) - lease := quota.trackEvent(context.Background(), span, 200) - require.NotNil(t, lease) - acquired := make(chan *memoryQuotaLease, 1) + require.True(t, quota.acquireEvent(context.Background(), span, 200)) + acquired := make(chan bool, 1) go func() { - acquired <- quota.trackEvent(context.Background(), span, 1) + acquired <- quota.acquireEvent(context.Background(), span, 1) }() select { @@ -178,89 +161,127 @@ func TestMemoryQuotaBlockedEventResumesAfterRelease(t *testing.T) { case <-time.After(100 * time.Millisecond): } - lease.Release() + quota.releaseEvent(200) select { - case nextLease := <-acquired: - require.NotNil(t, nextLease) - nextLease.Release() + case ok := <-acquired: + require.True(t, ok) + quota.releaseEvent(1) case <-time.After(time.Second): t.Fatal("event memory did not resume after memory was released") } } -func TestMemoryQuotaWarmingScanBudget(t *testing.T) { - quota := newMemoryQuotaController(200, 10) +func TestMemoryQuotaBlockedEventStopsOnContextCancellation(t *testing.T) { + quota := newMemoryQuotaController(100, 10) + quota.hardLimit = 100 + span := newTestQuotaSpan(1) + ctx, cancel := context.WithCancel(context.Background()) + + require.True(t, quota.acquireEvent(context.Background(), span, 100)) + acquired := make(chan bool, 1) + go func() { + acquired <- quota.acquireEvent(ctx, span, 1) + }() + require.Eventually(t, func() bool { + return quota.eventNotifier.waiters.Load() == 1 + }, time.Second, time.Millisecond) + + // Cancellation must stop the waiter without a memory release or an explicit + // quota notification. + cancel() + select { + case ok := <-acquired: + require.False(t, ok) + case <-time.After(time.Second): + t.Fatal("context cancellation did not stop the blocked event") + } + quota.releaseEvent(100) +} + +func TestMemoryQuotaConcurrentWaitersDoNotLoseWakeups(t *testing.T) { + const waiterCount = 32 + quota := newMemoryQuotaController(100, 10) + quota.hardLimit = 1 + span := newTestQuotaSpan(1) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + // Hold the only available byte until every goroutine is waiting. Releasing + // it wakes all waiters; each successful waiter then releases it for the next. + require.True(t, quota.acquireEvent(ctx, span, 1)) + results := make(chan bool, waiterCount) + for range waiterCount { + go func() { + acquired := quota.acquireEvent(ctx, span, 1) + if acquired { + quota.releaseEvent(1) + } + results <- acquired + }() + } + require.Eventually(t, func() bool { + return quota.eventNotifier.waiters.Load() == waiterCount + }, time.Second, time.Millisecond) + + quota.releaseEvent(1) + for range waiterCount { + select { + case acquired := <-results: + require.True(t, acquired) + case <-ctx.Done(): + t.Fatal("event waiter did not make progress") + } + } + used, _, _ := quota.snapshot() + require.Zero(t, used) +} + +func TestMemoryQuotaWarmingScanUsesCurrentPressure(t *testing.T) { + quota := newMemoryQuotaController(100, 20) span := newTestQuotaSpan(1) currentTs := setTestQuotaSpanLag(span, lowLagRegionThreshold+time.Minute) - quota.addSubscription(span) region := newTestQuotaRegion(span) - lease1, admitted := quota.acquireScan(region, currentTs) - require.True(t, admitted) - lease2, admitted := quota.acquireScan(region, currentTs) + bytes1, _, admitted := quota.acquireScan(region, currentTs) require.True(t, admitted) + require.NotZero(t, bytes1) + scanUsed, _, _ := quota.scanSnapshot() + require.Greater(t, scanUsed, quota.pauseWarmingLimit) - lease3, admitted := quota.acquireScan(region, currentTs) + _, _, admitted = quota.acquireScan(region, currentTs) require.False(t, admitted) - require.Nil(t, lease3) - lease1.Release() - lease3, admitted = quota.acquireScan(region, currentTs) + quota.releaseScan(bytes1) + bytes2, _, admitted := quota.acquireScan(region, currentTs) require.True(t, admitted) - lease2.Release() - lease3.Release() + quota.releaseScan(bytes2) } func TestMemoryQuotaLowLagScanBypassesWarmingGate(t *testing.T) { quota := newMemoryQuotaController(100, 10) span := newTestQuotaSpan(1) currentTs := setTestQuotaSpanLag(span, lowLagRegionThreshold-time.Second) - quota.addSubscription(span) - pressureLease := quota.trackEvent(context.Background(), span, 20) - require.NotNil(t, pressureLease) - scanLease, admitted := quota.acquireScan(newTestQuotaRegion(span), currentTs) + require.True(t, quota.acquireEvent(context.Background(), span, 20)) + scanBytes, _, admitted := quota.acquireScan(newTestQuotaRegion(span), currentTs) require.True(t, admitted) - require.NotNil(t, scanLease) - _, warmingScanUsed, _, _, _ := quota.scanSnapshot() - require.Zero(t, warmingScanUsed) - - scanLease.Release() - pressureLease.Release() -} - -func TestMemoryQuotaRemovalNotifiesAdmissionWithoutLeases(t *testing.T) { - quota := newMemoryQuotaController(100, 10) - span := newTestQuotaSpan(1) - quota.addSubscription(span) + require.NotZero(t, scanBytes) + scanUsed, _, _ := quota.scanSnapshot() + require.NotZero(t, scanUsed) - notified := make(chan struct{}, 1) - quota.setOnAvailable(func() { - select { - case notified <- struct{}{}: - default: - } - }) - quota.removeSubscription(span) - select { - case <-notified: - case <-time.After(time.Second): - t.Fatal("subscription removal did not notify admission") - } + quota.releaseScan(scanBytes) + quota.releaseEvent(20) } -func TestAdmissionWaitsForMemoryAndReleasesScanLease(t *testing.T) { +func TestAdmissionWaitsForMemoryAndReleasesScanMemory(t *testing.T) { quota := newMemoryQuotaController(100, 10) span := newTestQuotaSpan(1) currentTs := setTestQuotaSpanLag(span, lowLagRegionThreshold+time.Minute) - quota.addSubscription(span) controller := newRegionAdmissionController(1, 1, quota, func() uint64 { return currentTs }) - quota.setOnAvailable(controller.notifyAvailable) - pressureLease := quota.trackEvent(context.Background(), span, 20) - require.NotNil(t, pressureLease) + require.True(t, quota.acquireEvent(context.Background(), span, 20)) region := newTestQuotaRegion(span) require.True(t, controller.submit(newRegionPriorityTask(region, currentTs, 1))) @@ -279,7 +300,7 @@ func TestAdmissionWaitsForMemoryAndReleasesScanLease(t *testing.T) { case <-time.After(100 * time.Millisecond): } - pressureLease.Release() + quota.releaseEvent(20) var resultValue popResult select { case resultValue = <-result: @@ -288,9 +309,63 @@ func TestAdmissionWaitsForMemoryAndReleasesScanLease(t *testing.T) { } require.NoError(t, resultValue.err) req := resultValue.req - scanUsed, _, _, _, _ := quota.scanSnapshot() + scanUsed, _, _ := quota.scanSnapshot() require.NotZero(t, scanUsed) require.True(t, req.abort()) - scanUsed, _, _, _, _ = quota.scanSnapshot() + scanUsed, _, _ = quota.scanSnapshot() require.Zero(t, scanUsed) } + +func TestAdmissionWakesWhenBlockedSpanStops(t *testing.T) { + quota := newMemoryQuotaController(100, 10) + span := newTestQuotaSpan(1) + currentTs := setTestQuotaSpanLag(span, lowLagRegionThreshold+time.Minute) + controller := newRegionAdmissionController(1, 1, quota, func() uint64 { + return currentTs + }) + + require.True(t, quota.acquireEvent(context.Background(), span, 20)) + require.True(t, controller.submit(newRegionPriorityTask( + newTestQuotaRegion(span), currentTs, 1))) + + type popResult struct { + req *regionReq + err error + } + result := make(chan popResult, 1) + go func() { + req, err := controller.pop(context.Background(), nil) + result <- popResult{req: req, err: err} + }() + select { + case <-result: + t.Fatal("warming scan should wait while memory is under pressure") + case <-time.After(100 * time.Millisecond): + } + + span.stopped.Store(true) + quota.wakeAll() + select { + case result := <-result: + require.NoError(t, result.err) + require.Zero(t, result.req.scanBytes) + require.True(t, result.req.abort()) + case <-time.After(time.Second): + t.Fatal("stopping the span did not wake scan admission") + } + quota.releaseEvent(20) +} + +func BenchmarkMemoryQuotaEventAccounting(b *testing.B) { + quota := newMemoryQuotaController(defaultLogPullerMemoryQuota, defaultScanBaseSize) + span := newTestQuotaSpan(1) + ctx := context.Background() + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + if !quota.acquireEvent(ctx, span, 1) { + b.Fatal("failed to acquire event memory") + } + quota.releaseEvent(1) + } +} diff --git a/logservice/logpuller/region_admission_controller.go b/logservice/logpuller/region_admission_controller.go index 87096bac49..a143ba1446 100644 --- a/logservice/logpuller/region_admission_controller.go +++ b/logservice/logpuller/region_admission_controller.go @@ -36,7 +36,7 @@ type regionReq struct { regionInfo regionInfo createTime time.Time controller *regionAdmissionController - scanQuota *memoryQuotaLease + scanBytes uint64 released atomic.Bool } @@ -73,7 +73,7 @@ func (r *regionReq) release() bool { if !r.released.CompareAndSwap(false, true) { return false } - r.scanQuota.Release() + r.controller.memoryQuota.releaseScan(r.scanBytes) r.controller.release() return true } @@ -151,11 +151,11 @@ func (c *regionAdmissionController) submit(task *regionPriorityTask) bool { return true } -// pop waits for an eligible request. If interrupt is signaled first, it returns -// nil without an error so the worker can handle its control queue. +// pop waits for an eligible request. If controlReady is signaled first, it +// returns nil without an error so the worker can handle its control queue. func (c *regionAdmissionController) pop( ctx context.Context, - interrupt <-chan struct{}, + controlReady <-chan struct{}, ) (*regionReq, error) { for { c.mu.Lock() @@ -163,7 +163,7 @@ func (c *regionAdmissionController) pop( c.mu.Unlock() return nil, context.Canceled } - request, scanQuota := c.popEligibleLocked() + request, scanBytes, memoryReady := c.popEligibleLocked() if request != nil { c.inflight++ c.mu.Unlock() @@ -171,14 +171,15 @@ func (c *regionAdmissionController) pop( regionInfo: request.regionInfo, createTime: time.Now(), controller: c, - scanQuota: scanQuota, + scanBytes: scanBytes, }, nil } c.mu.Unlock() select { case <-c.notify: - case <-interrupt: + case <-memoryReady: + case <-controlReady: return nil, nil case <-ctx.Done(): return nil, ctx.Err() @@ -188,23 +189,24 @@ func (c *regionAdmissionController) pop( func (c *regionAdmissionController) popEligibleLocked() ( *regionPriorityTask, - *memoryQuotaLease, + uint64, + <-chan struct{}, ) { request, ok := c.pending.PeekTop() if !ok { - return nil, nil + return nil, 0, nil } if c.inflight >= c.windowFor(request) { - return nil, nil + return nil, 0, nil } - scanQuota, admitted := c.memoryQuota.acquireScan( + scanBytes, memoryReady, admitted := c.memoryQuota.acquireScan( request.regionInfo, c.currentTs()) if !admitted { - return nil, nil + return nil, 0, memoryReady } request, _ = c.pending.PopTop() - return request, scanQuota + return request, scanBytes, nil } func (c *regionAdmissionController) windowFor(request *regionPriorityTask) int { @@ -223,12 +225,6 @@ func (c *regionAdmissionController) release() { c.mu.Unlock() } -func (c *regionAdmissionController) notifyAvailable() { - c.mu.Lock() - c.notifyOneLocked() - c.mu.Unlock() -} - func (c *regionAdmissionController) close() { c.mu.Lock() if !c.closed { diff --git a/logservice/logpuller/region_admission_controller_test.go b/logservice/logpuller/region_admission_controller_test.go index a6277f80de..0fdeb3d949 100644 --- a/logservice/logpuller/region_admission_controller_test.go +++ b/logservice/logpuller/region_admission_controller_test.go @@ -82,9 +82,9 @@ func TestRegionAdmissionControllerNormalWindow(t *testing.T) { req1, err := controller.pop(t.Context(), nil) require.NoError(t, err) require.Equal(t, 1, controller.stats().inflight) - interrupt := make(chan struct{}) - close(interrupt) - req2, err := controller.pop(t.Context(), interrupt) + controlReady := make(chan struct{}) + close(controlReady) + req2, err := controller.pop(t.Context(), controlReady) require.Nil(t, req2) require.NoError(t, err) @@ -117,9 +117,9 @@ func TestRegionAdmissionControllerLowLagUsesMaxWindow(t *testing.T) { req2, err := controller.pop(t.Context(), nil) require.NoError(t, err) require.Equal(t, uint64(3), req2.regionInfo.verID.GetID()) - interrupt := make(chan struct{}) - close(interrupt) - req3, err := controller.pop(t.Context(), interrupt) + controlReady := make(chan struct{}) + close(controlReady) + req3, err := controller.pop(t.Context(), controlReady) require.Nil(t, req3) require.NoError(t, err) require.Equal(t, 2, controller.stats().inflight) diff --git a/logservice/logpuller/region_event_handler.go b/logservice/logpuller/region_event_handler.go index 92a649349a..a30b349418 100644 --- a/logservice/logpuller/region_event_handler.go +++ b/logservice/logpuller/region_event_handler.go @@ -54,23 +54,17 @@ type regionEvent struct { // Resolved-ts events: `resolvedTs` is set and `states` contains all related regions. states []*regionFeedState - entries *cdcpb.Event_Entries_ - resolvedTs uint64 - memoryQuota *memoryQuotaLease + entries *cdcpb.Event_Entries_ + resolvedTs uint64 + // memoryBytes is released when this event is dropped or after downstream + // finishes consuming the entries derived from it. + memoryBytes uint64 } -func (event *regionEvent) needMemoryQuota() bool { +func (event *regionEvent) needsMemoryAccounting() bool { return event.entries != nil } -func (event *regionEvent) releaseMemoryQuota() { - if event.memoryQuota == nil { - return - } - event.memoryQuota.Release() - event.memoryQuota = nil -} - func (event *regionEvent) getSize() int { size := int(unsafe.Sizeof(*event)) if event.entries != nil { @@ -133,11 +127,9 @@ func (h *regionEventHandler) Handle(span *subscribedSpan, events ...regionEvent) newResolvedTs := uint64(0) wasInitialized := span.initialized.Load() - quotaLeases := make([]*memoryQuotaLease, 0, len(events)) + memoryBytes := uint64(0) for _, event := range events { - if event.needMemoryQuota() { - quotaLeases = append(quotaLeases, event.memoryQuota) - } + memoryBytes += event.memoryBytes if len(event.states) == 1 && event.states[0].isStale() { hasError = true h.handleRegionError(event.states[0]) @@ -159,7 +151,7 @@ func (h *regionEventHandler) Handle(span *subscribedSpan, events ...regionEvent) } } if !wasInitialized && span.initialized.Load() { - h.eventSink.memoryQuota.markSubscriptionInitialized() + h.eventSink.memoryQuota.notifyScanAdmission() } tryAdvanceResolvedTs := func() { if newResolvedTs != 0 { @@ -167,9 +159,7 @@ func (h *regionEventHandler) Handle(span *subscribedSpan, events ...regionEvent) } } releaseMemoryQuota := func() { - for _, lease := range quotaLeases { - lease.Release() - } + h.eventSink.memoryQuota.releaseEvent(memoryBytes) } if len(span.kvEventsCache) > 0 { metricsEventCount.Add(float64(len(span.kvEventsCache))) @@ -253,7 +243,7 @@ func (h *regionEventHandler) GetType(event regionEvent) dynstream.EventType { } func (h *regionEventHandler) OnDrop(event regionEvent) interface{} { - event.releaseMemoryQuota() + h.eventSink.memoryQuota.releaseEvent(event.memoryBytes) // TODO: Distinguish between drop events caused by "path not found" errors and memory control. state := event.mustFirstState() fields := []zap.Field{ diff --git a/logservice/logpuller/region_event_handler_test.go b/logservice/logpuller/region_event_handler_test.go index dd4fc1d77b..9227f2bb8e 100644 --- a/logservice/logpuller/region_event_handler_test.go +++ b/logservice/logpuller/region_event_handler_test.go @@ -107,9 +107,8 @@ func TestHandleEventEntryEventOutOfOrder(t *testing.T) { }, } regionEvent := regionEvent{ - states: []*regionFeedState{state}, - entries: events, - memoryQuota: newMemoryQuotaLease(func() {}), + states: []*regionFeedState{state}, + entries: events, } ds.Push(subID, regionEvent) } @@ -128,9 +127,8 @@ func TestHandleEventEntryEventOutOfOrder(t *testing.T) { }, } regionEvent := regionEvent{ - states: []*regionFeedState{state}, - entries: events, - memoryQuota: newMemoryQuotaLease(func() {}), + states: []*regionFeedState{state}, + entries: events, } ds.Push(subID, regionEvent) } @@ -159,9 +157,8 @@ func TestHandleEventEntryEventOutOfOrder(t *testing.T) { }, } regionEvent := regionEvent{ - states: []*regionFeedState{state}, - entries: events, - memoryQuota: newMemoryQuotaLease(func() {}), + states: []*regionFeedState{state}, + entries: events, } ds.Push(subID, regionEvent) } @@ -187,9 +184,8 @@ func TestHandleEventEntryEventOutOfOrder(t *testing.T) { }, } regionEvent := regionEvent{ - states: []*regionFeedState{state}, - entries: events, - memoryQuota: newMemoryQuotaLease(func() {}), + states: []*regionFeedState{state}, + entries: events, } ds.Push(subID, regionEvent) } @@ -384,7 +380,6 @@ func TestHandleResolvedTsThrottled(t *testing.T) { func TestHandleEntriesReleasesMemoryAfterDownstreamCallback(t *testing.T) { quota := newMemoryQuotaController(1024, 8) span := newTestQuotaSpan(1) - quota.addSubscription(span) callbackCh := make(chan func(), 1) span.consumeKVEvents = func(_ []common.RawKVEntry, callback func()) bool { callbackCh <- callback @@ -402,8 +397,7 @@ func TestHandleEntriesReleasesMemoryAfterDownstreamCallback(t *testing.T) { lockedRangeState: lockedState, }, } - lease := quota.trackEvent(context.Background(), span, 10) - require.NotNil(t, lease) + require.True(t, quota.acquireEvent(context.Background(), span, 10)) handler := ®ionEventHandler{eventSink: ®ionEventSink{ ds: newMockRegionEventSinkStream(), memoryQuota: quota, @@ -411,7 +405,7 @@ func TestHandleEntriesReleasesMemoryAfterDownstreamCallback(t *testing.T) { await := handler.Handle(span, regionEvent{ states: []*regionFeedState{state}, - memoryQuota: lease, + memoryBytes: 10, entries: &cdcpb.Event_Entries_{Entries: &cdcpb.Event_Entries{ Entries: []*cdcpb.Event_Row{{ Type: cdcpb.Event_COMMITTED, @@ -441,13 +435,9 @@ func TestTryMarkSpanInitializedByResolvedTs(t *testing.T) { func TestSpanInitializationNotifiesMemoryAdmission(t *testing.T) { quota := newMemoryQuotaController(1024, 8) - notified := make(chan struct{}, 1) - quota.setOnAvailable(func() { - select { - case notified <- struct{}{}: - default: - } - }) + quota.scanMu.Lock() + notified := quota.scanReady + quota.scanMu.Unlock() const startTs = 100 rangeLock := regionlock.NewRangeLock(1, []byte("a"), []byte("z"), startTs) @@ -463,7 +453,6 @@ func TestSpanInitializationNotifiesMemoryAdmission(t *testing.T) { advanceResolvedTs: func(uint64) {}, } span.resolvedTs.Store(startTs) - quota.addSubscription(span) state := newRegionFeedState(regionInfo{ verID: tikv.NewRegionVerID(1, 1, 1), subscribedSpan: span, diff --git a/logservice/logpuller/region_event_sink.go b/logservice/logpuller/region_event_sink.go index 25987ff997..4a5b5b0bf6 100644 --- a/logservice/logpuller/region_event_sink.go +++ b/logservice/logpuller/region_event_sink.go @@ -15,8 +15,6 @@ package logpuller import ( "context" - "sync" - "sync/atomic" "github.com/pingcap/log" "github.com/pingcap/ticdc/pkg/metrics" @@ -24,16 +22,12 @@ import ( "go.uber.org/zap" ) -// regionEventSink delivers region events to dynstream and owns push-side flow control. +// regionEventSink delivers region events to dynstream and accounts their memory. type regionEventSink struct { ctx context.Context ds dynstream.DynamicStream[int, SubscriptionID, regionEvent, *subscribedSpan, *regionEventHandler] memoryQuota *memoryQuotaController - // the following three fields are used to manage feedback from ds and notify other goroutines - mu sync.Mutex - cond *sync.Cond - paused atomic.Bool } func newRegionEventSink( @@ -49,7 +43,6 @@ func newRegionEventSink( // TODO: Set `UseBuffer` to true until we refactor the `regionEventHandler.Handle` method so that it doesn't call any method of the dynamic stream. Currently, if `UseBuffer` is set to false, there will be a deadlock: // ds.handleLoop fetch events from `ch` -> regionEventHandler.Handle -> ds.RemovePath -> send event to `ch` option.UseBuffer = true - option.EnableMemoryControl = false ds := dynstream.NewParallelDynamicStream( "log-puller", ®ionEventHandler{eventSink: sink, failureHandler: failureHandler}, @@ -57,7 +50,6 @@ func newRegionEventSink( ) ds.Start() sink.ds = ds - sink.cond = sync.NewCond(&sink.mu) return sink } @@ -78,94 +70,31 @@ func (s *regionEventSink) Wake(subID SubscriptionID) { } func (s *regionEventSink) Push(subID SubscriptionID, event regionEvent) { - if event.needMemoryQuota() { + if event.needsMemoryAccounting() { span := event.mustFirstState().region.subscribedSpan - event.memoryQuota = s.memoryQuota.trackEvent(s.ctx, span, uint64(event.getSize())) - if event.memoryQuota == nil { + event.memoryBytes = uint64(event.getSize()) + if !s.memoryQuota.acquireEvent(s.ctx, span, event.memoryBytes) { return } } - // fast path - if !s.paused.Load() { - s.ds.Push(subID, event) - return - } - - // slow path: wait until paused is false - s.mu.Lock() - for s.paused.Load() { - select { - case <-s.ctx.Done(): - s.mu.Unlock() - event.releaseMemoryQuota() - return - default: - s.cond.Wait() - } - } - s.mu.Unlock() s.ds.Push(subID, event) } -func (s *regionEventSink) Run(ctx context.Context) error { - for { - select { - case <-ctx.Done(): - return nil - case feedback := <-s.ds.Feedback(): - switch feedback.FeedbackType { - case dynstream.PauseArea: - s.mu.Lock() - s.paused.Store(true) - s.mu.Unlock() - log.Info("subscription client pause push region event") - case dynstream.ResumeArea: - s.mu.Lock() - s.paused.Store(false) - s.cond.Broadcast() - s.mu.Unlock() - log.Info("subscription client resume push region event") - case dynstream.ReleasePath, dynstream.ResumePath: - // Ignore it, because it is no need to pause and resume a path in puller. - } - } - } -} - func (s *regionEventSink) UpdateMetrics() { dsMetrics := s.ds.GetMetrics() metricSubscriptionClientDSChannelSize.Set(float64(dsMetrics.EventChanSize)) metricSubscriptionClientDSPendingQueueLen.Set(float64(dsMetrics.PendingQueueLen)) used, capacity, _ := s.memoryQuota.snapshot() - scanUsed, warmingScanUsed, warmingScanBudget, scanEstimate, hardLimit := - s.memoryQuota.scanSnapshot() + scanUsed, scanEstimate, hardLimit := s.memoryQuota.scanSnapshot() metrics.LogPullerMemoryQuota.WithLabelValues("max").Set(float64(capacity)) metrics.LogPullerMemoryQuota.WithLabelValues("used").Set(float64(used)) metrics.LogPullerMemoryQuota.WithLabelValues("scan_used").Set(float64(scanUsed)) - metrics.LogPullerMemoryQuota.WithLabelValues("warming_scan_used").Set(float64(warmingScanUsed)) - metrics.LogPullerMemoryQuota.WithLabelValues("warming_scan_budget").Set(float64(warmingScanBudget)) metrics.LogPullerMemoryQuota.WithLabelValues("scan_estimate").Set(float64(scanEstimate)) metrics.LogPullerMemoryQuota.WithLabelValues("hard_limit").Set(float64(hardLimit)) - metrics.DynamicStreamMemoryUsage.WithLabelValues( - "log-puller", - "max", - "default", - "default", - ).Set(float64(capacity)) - metrics.DynamicStreamMemoryUsage.WithLabelValues( - "log-puller", - "used", - "default", - "default", - ).Set(float64(used)) } func (s *regionEventSink) Close() { s.memoryQuota.wakeAll() - s.mu.Lock() - s.paused.Store(false) - s.cond.Broadcast() - s.mu.Unlock() s.ds.Close() } diff --git a/logservice/logpuller/region_event_sink_test.go b/logservice/logpuller/region_event_sink_test.go index 43fd6f088d..7786e4b524 100644 --- a/logservice/logpuller/region_event_sink_test.go +++ b/logservice/logpuller/region_event_sink_test.go @@ -15,10 +15,7 @@ package logpuller import ( "context" - "sync" - "sync/atomic" "testing" - "time" "github.com/pingcap/kvproto/pkg/cdcpb" "github.com/pingcap/ticdc/pkg/metrics" @@ -28,18 +25,13 @@ import ( ) type mockRegionEventSinkStream struct { - feedbackCh chan dynstream.Feedback[int, SubscriptionID, *subscribedSpan] - pushCount atomic.Int32 - pushCh chan struct{} - eventCh chan regionEvent - metrics dynstream.Metrics[int, SubscriptionID] + eventCh chan regionEvent + metrics dynstream.Metrics[int, SubscriptionID] } func newMockRegionEventSinkStream() *mockRegionEventSinkStream { return &mockRegionEventSinkStream{ - feedbackCh: make(chan dynstream.Feedback[int, SubscriptionID, *subscribedSpan], 2), - pushCh: make(chan struct{}, 1), - eventCh: make(chan regionEvent, 1), + eventCh: make(chan regionEvent, 1), } } @@ -48,15 +40,13 @@ func (s *mockRegionEventSinkStream) Start() {} func (s *mockRegionEventSinkStream) Close() {} func (s *mockRegionEventSinkStream) Push(_ SubscriptionID, event regionEvent) { - s.pushCount.Add(1) - s.pushCh <- struct{}{} s.eventCh <- event } func (s *mockRegionEventSinkStream) Wake(_ SubscriptionID) {} func (s *mockRegionEventSinkStream) Feedback() <-chan dynstream.Feedback[int, SubscriptionID, *subscribedSpan] { - return s.feedbackCh + return nil } func (s *mockRegionEventSinkStream) AddPath(_ SubscriptionID, _ *subscribedSpan, _ ...dynstream.AreaSettings) error { @@ -75,66 +65,6 @@ func (s *mockRegionEventSinkStream) GetMetrics() dynstream.Metrics[int, Subscrip return s.metrics } -func TestRegionEventSinkRunPausesAndResumesPush(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - ds := newMockRegionEventSinkStream() - sink := ®ionEventSink{ - ctx: ctx, - ds: ds, - memoryQuota: newMemoryQuotaController(0, 0), - } - sink.cond = sync.NewCond(&sink.mu) - - runErrCh := make(chan error, 1) - go func() { - runErrCh <- sink.Run(ctx) - }() - - ds.feedbackCh <- dynstream.Feedback[int, SubscriptionID, *subscribedSpan]{ - FeedbackType: dynstream.PauseArea, - } - require.Eventually(t, sink.paused.Load, time.Second, 10*time.Millisecond) - - pushDone := make(chan struct{}) - go func() { - sink.Push(SubscriptionID(1), regionEvent{resolvedTs: 100}) - close(pushDone) - }() - - select { - case <-pushDone: - t.Fatal("Push should block while the sink is paused") - case <-time.After(100 * time.Millisecond): - } - require.Equal(t, int32(0), ds.pushCount.Load()) - - ds.feedbackCh <- dynstream.Feedback[int, SubscriptionID, *subscribedSpan]{ - FeedbackType: dynstream.ResumeArea, - } - require.Eventually(t, func() bool { return !sink.paused.Load() }, time.Second, 10*time.Millisecond) - - select { - case <-ds.pushCh: - case <-time.After(time.Second): - t.Fatal("Push should resume after ResumeArea feedback") - } - select { - case <-pushDone: - case <-time.After(time.Second): - t.Fatal("Push should return after ResumeArea feedback") - } - - cancel() - select { - case err := <-runErrCh: - require.NoError(t, err) - case <-time.After(time.Second): - t.Fatal("Run should exit after context cancellation") - } -} - func TestRegionEventSinkUpdateMetrics(t *testing.T) { t.Run("quota updates memory gauges", func(t *testing.T) { ds := newMockRegionEventSinkStream() @@ -144,33 +74,18 @@ func TestRegionEventSinkUpdateMetrics(t *testing.T) { } quota := newMemoryQuotaController(66, 8) span := newTestQuotaSpan(1) - quota.addSubscription(span) - lease := quota.trackEvent(context.Background(), span, 55) - require.NotNil(t, lease) - t.Cleanup(lease.Release) + require.True(t, quota.acquireEvent(context.Background(), span, 55)) + t.Cleanup(func() { quota.releaseEvent(55) }) sink := ®ionEventSink{ ctx: context.Background(), ds: ds, memoryQuota: quota, } - sink.cond = sync.NewCond(&sink.mu) sink.UpdateMetrics() require.Equal(t, float64(33), testutil.ToFloat64(metricSubscriptionClientDSChannelSize)) require.Equal(t, float64(44), testutil.ToFloat64(metricSubscriptionClientDSPendingQueueLen)) - require.Equal(t, float64(66), testutil.ToFloat64(metrics.DynamicStreamMemoryUsage.WithLabelValues( - "log-puller", - "max", - "default", - "default", - ))) - require.Equal(t, float64(55), testutil.ToFloat64(metrics.DynamicStreamMemoryUsage.WithLabelValues( - "log-puller", - "used", - "default", - "default", - ))) require.Equal(t, float64(66), testutil.ToFloat64( metrics.LogPullerMemoryQuota.WithLabelValues("max"))) require.Equal(t, float64(55), testutil.ToFloat64( @@ -181,7 +96,6 @@ func TestRegionEventSinkUpdateMetrics(t *testing.T) { func TestRegionEventSinkTracksEntriesUntilDrop(t *testing.T) { quota := newMemoryQuotaController(1024, 8) span := newTestQuotaSpan(1) - quota.addSubscription(span) state := ®ionFeedState{ region: regionInfo{subscribedSpan: span}, worker: ®ionRequestWorker{}, @@ -192,7 +106,6 @@ func TestRegionEventSinkTracksEntriesUntilDrop(t *testing.T) { ds: ds, memoryQuota: quota, } - sink.cond = sync.NewCond(&sink.mu) sink.Push(span.subID, regionEvent{ states: []*regionFeedState{state}, @@ -201,11 +114,11 @@ func TestRegionEventSinkTracksEntriesUntilDrop(t *testing.T) { }}, }) pushed := <-ds.eventCh - require.NotNil(t, pushed.memoryQuota) + require.NotZero(t, pushed.memoryBytes) used, _, _ := quota.snapshot() require.NotZero(t, used) - (®ionEventHandler{}).OnDrop(pushed) + (®ionEventHandler{eventSink: sink}).OnDrop(pushed) used, _, _ = quota.snapshot() require.Zero(t, used) } diff --git a/logservice/logpuller/region_request_scheduler.go b/logservice/logpuller/region_request_scheduler.go index 4bf6dd3ea0..d71d5af62a 100644 --- a/logservice/logpuller/region_request_scheduler.go +++ b/logservice/logpuller/region_request_scheduler.go @@ -218,13 +218,6 @@ func (s *regionRequestScheduler) UpdateMetrics() { Set(float64(s.inflightCount())) } -func (s *regionRequestScheduler) notifyAvailable() { - s.stores.Range(func(_, value any) bool { - value.(*regionRequestStore).notifyAvailable() - return true - }) -} - func (s *regionRequestScheduler) Close() { s.taskQueue.Close() } diff --git a/logservice/logpuller/region_request_store.go b/logservice/logpuller/region_request_store.go index 509c7449df..38393b3acf 100644 --- a/logservice/logpuller/region_request_store.go +++ b/logservice/logpuller/region_request_store.go @@ -86,9 +86,3 @@ func (s *regionRequestStore) inflightCount() int { } return count } - -func (s *regionRequestStore) notifyAvailable() { - for _, worker := range s.workers { - worker.admission.notifyAvailable() - } -} diff --git a/logservice/logpuller/subscription_client.go b/logservice/logpuller/subscription_client.go index 28b26944c2..bfbe2f910b 100644 --- a/logservice/logpuller/subscription_client.go +++ b/logservice/logpuller/subscription_client.go @@ -190,7 +190,6 @@ func NewSubscriptionClient( subClient.failureHandler, subClient.memoryQuota, ) - subClient.memoryQuota.setOnAvailable(subClient.regionScheduler.notifyAvailable) return subClient } @@ -252,7 +251,6 @@ func (s *subscriptionClient) Subscribe( time.Duration(config.GetGlobalServerConfig().Debug.Puller.OldStartTsScanLowPriorityThreshold), ) s.spanRegistry.Add(rt) - s.memoryQuota.addSubscription(rt) s.eventSink.AddPath(rt) select { @@ -295,7 +293,6 @@ func (s *subscriptionClient) Run(ctx context.Context) error { // actual startup order. g.Go(func() error { return s.handleRangeTasks(ctx) }) g.Go(func() error { return s.regionScheduler.Run(ctx, g) }) - g.Go(func() error { return s.eventSink.Run(ctx) }) g.Go(func() error { return s.failureHandler.Run(ctx) }) g.Go(func() error { return s.spanRegistry.Run(ctx) }) g.Go(func() error { return s.handleResolveLockTasks(ctx) }) @@ -321,6 +318,8 @@ func (s *subscriptionClient) setTableStopped(rt *subscribedSpan) { // Set stopped to true so we can stop handling region events from the table, // then notify every existing worker to deregister the subscription. if rt.stopped.CompareAndSwap(false, true) { + // Wake event receivers and scan admission so they can observe stopped. + s.memoryQuota.wakeAll() s.regionScheduler.BroadcastDeregister(rt.subID, rt.filterLoop) if rt.rangeLock.Stop() { s.onTableDrained(rt) @@ -338,7 +337,6 @@ func (s *subscriptionClient) onTableDrained(rt *subscribedSpan) { zap.Uint64("subscriptionID", uint64(rt.subID)), zap.Error(err)) } - s.memoryQuota.removeSubscription(rt) s.spanRegistry.Remove(rt.subID) } From dfe8bfa821c88ead0fbd525334d19067861abe94 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Mon, 20 Jul 2026 10:57:30 +0800 Subject: [PATCH 53/71] refactor --- logservice/logpuller/memory_quota.go | 63 +++--- logservice/logpuller/memory_quota_test.go | 198 +++++++++++------- .../logpuller/region_admission_controller.go | 4 +- logservice/logpuller/region_event_handler.go | 6 +- .../logpuller/region_event_handler_test.go | 10 +- logservice/logpuller/region_event_sink.go | 13 +- .../logpuller/region_event_sink_test.go | 45 ++-- logservice/logpuller/subscription_client.go | 3 +- metrics/grafana/ticdc_new_arch.json | 127 ++++++++++- .../ticdc_new_arch_next_gen.json | 127 ++++++++++- pkg/metrics/log_puller.go | 25 +++ 11 files changed, 453 insertions(+), 168 deletions(-) diff --git a/logservice/logpuller/memory_quota.go b/logservice/logpuller/memory_quota.go index aa5396171e..9914b88ca3 100644 --- a/logservice/logpuller/memory_quota.go +++ b/logservice/logpuller/memory_quota.go @@ -20,6 +20,8 @@ import ( "sync" "sync/atomic" "time" + + "github.com/pingcap/ticdc/pkg/metrics" ) const ( @@ -193,28 +195,14 @@ func newMemoryQuotaController(capacity, scanBaseSize uint64) *memoryQuotaControl return c } -func (c *memoryQuotaController) wakeAll() { +// WakeAll wakes quota waiters so they can observe cancellation or a stopped span. +func (c *memoryQuotaController) WakeAll() { c.eventNotifier.notify() - c.notifyScanAdmission() -} - -func (c *memoryQuotaController) snapshot() (used, capacity uint64, level admissionLevel) { - c.scanMu.Lock() - defer c.scanMu.Unlock() - return c.used.Load(), c.capacity, c.level -} - -func (c *memoryQuotaController) scanSnapshot() ( - scanUsed uint64, - scanEstimate uint64, - hardLimit uint64, -) { - c.scanMu.Lock() - defer c.scanMu.Unlock() - return c.scanUsed, c.scanEstimate, c.hardLimit + c.NotifyScanAdmission() } -func (c *memoryQuotaController) acquireScan( +// AcquireScan admits one region scan and returns its memory estimate. +func (c *memoryQuotaController) AcquireScan( region regionInfo, currentTs uint64, ) (bytes uint64, retry <-chan struct{}, admitted bool) { @@ -244,7 +232,8 @@ func (c *memoryQuotaController) acquireScan( return bytes, nil, true } -func (c *memoryQuotaController) releaseScan(bytes uint64) { +// ReleaseScan releases the estimate owned by an admitted region scan. +func (c *memoryQuotaController) ReleaseScan(bytes uint64) { if bytes == 0 { return } @@ -258,10 +247,10 @@ func (c *memoryQuotaController) releaseScan(bytes uint64) { c.scanMu.Unlock() } -// acquireEvent accounts one event batch. Below the hard limit its hot path is +// AcquireEvent accounts one event batch. Below the hard limit its hot path is // a context check and an atomic compare-and-swap; it does not allocate or take // a mutex. -func (c *memoryQuotaController) acquireEvent( +func (c *memoryQuotaController) AcquireEvent( ctx context.Context, span *subscribedSpan, bytes uint64, @@ -273,9 +262,12 @@ func (c *memoryQuotaController) acquireEvent( return true } - return c.eventNotifier.wait(ctx, span, func() bool { + start := time.Now() + acquired := c.eventNotifier.wait(ctx, span, func() bool { return c.tryAcquireEvent(bytes) }) + metrics.LogPullerMemoryQuotaEventWaitDuration.Observe(time.Since(start).Seconds()) + return acquired } func (c *memoryQuotaController) tryAcquireEvent(bytes uint64) bool { @@ -293,7 +285,8 @@ func (c *memoryQuotaController) tryAcquireEvent(bytes uint64) bool { } } -func (c *memoryQuotaController) releaseEvent(bytes uint64) { +// ReleaseEvent releases event memory after downstream has consumed the event. +func (c *memoryQuotaController) ReleaseEvent(bytes uint64) { if bytes == 0 { return } @@ -306,12 +299,32 @@ func (c *memoryQuotaController) releaseEvent(bytes uint64) { c.eventNotifier.notify() } -func (c *memoryQuotaController) notifyScanAdmission() { +// NotifyScanAdmission wakes workers so they can recheck span state and admission. +func (c *memoryQuotaController) NotifyScanAdmission() { c.scanMu.Lock() c.notifyScanAdmissionLocked() c.scanMu.Unlock() } +// UpdateMetrics reports the current event-memory and scan-admission state. +func (c *memoryQuotaController) UpdateMetrics() { + c.scanMu.Lock() + used := c.used.Load() + scanUsed := c.scanUsed + level := c.level + scanEstimate := c.scanEstimate + c.scanMu.Unlock() + + metrics.LogPullerMemoryQuota.WithLabelValues("max").Set(float64(c.capacity)) + metrics.LogPullerMemoryQuota.WithLabelValues("used").Set(float64(used)) + metrics.LogPullerMemoryQuota.WithLabelValues("scan_used").Set(float64(scanUsed)) + metrics.LogPullerMemoryQuota.WithLabelValues("scan_estimate").Set(float64(scanEstimate)) + metrics.LogPullerMemoryQuota.WithLabelValues("hard_limit").Set(float64(c.hardLimit)) + metrics.LogPullerMemoryQuotaAdmissionLevel.Set(float64(level)) + metrics.LogPullerMemoryQuotaEventWaiterCount.Set( + float64(c.eventNotifier.waiters.Load())) +} + func (c *memoryQuotaController) notifyScanAdmissionLocked() { close(c.scanReady) c.scanReady = make(chan struct{}) diff --git a/logservice/logpuller/memory_quota_test.go b/logservice/logpuller/memory_quota_test.go index 1fac91945e..8b8b3cd1c6 100644 --- a/logservice/logpuller/memory_quota_test.go +++ b/logservice/logpuller/memory_quota_test.go @@ -20,10 +20,28 @@ import ( "time" "github.com/pingcap/ticdc/logservice/logpuller/regionlock" + "github.com/pingcap/ticdc/pkg/metrics" + "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/require" "github.com/tikv/client-go/v2/oracle" ) +type memoryQuotaTestState struct { + used uint64 + scanUsed uint64 + level admissionLevel +} + +func getMemoryQuotaTestState(quota *memoryQuotaController) memoryQuotaTestState { + quota.scanMu.Lock() + defer quota.scanMu.Unlock() + return memoryQuotaTestState{ + used: quota.used.Load(), + scanUsed: quota.scanUsed, + level: quota.level, + } +} + func newTestQuotaSpan(subID SubscriptionID) *subscribedSpan { span := &subscribedSpan{subID: subID} span.resolvedTs.Store(oracle.GoTimeToTS(time.Now())) @@ -45,6 +63,37 @@ func setTestQuotaSpanLag(span *subscribedSpan, lag time.Duration) uint64 { return oracle.GoTimeToTS(now) } +func TestMemoryQuotaUpdateMetrics(t *testing.T) { + quota := newMemoryQuotaController(66, 8) + span := newTestQuotaSpan(1) + require.True(t, quota.AcquireEvent(context.Background(), span, 55)) + t.Cleanup(func() { quota.ReleaseEvent(55) }) + + quota.scanMu.Lock() + quota.scanUsed = 7 + quota.level = admissionFreezeAllNewScans + quota.scanMu.Unlock() + quota.eventNotifier.waiters.Store(2) + t.Cleanup(func() { quota.eventNotifier.waiters.Store(0) }) + + quota.UpdateMetrics() + + require.Equal(t, float64(66), testutil.ToFloat64( + metrics.LogPullerMemoryQuota.WithLabelValues("max"))) + require.Equal(t, float64(55), testutil.ToFloat64( + metrics.LogPullerMemoryQuota.WithLabelValues("used"))) + require.Equal(t, float64(7), testutil.ToFloat64( + metrics.LogPullerMemoryQuota.WithLabelValues("scan_used"))) + require.Equal(t, float64(8), testutil.ToFloat64( + metrics.LogPullerMemoryQuota.WithLabelValues("scan_estimate"))) + require.Equal(t, float64(132), testutil.ToFloat64( + metrics.LogPullerMemoryQuota.WithLabelValues("hard_limit"))) + require.Equal(t, float64(admissionFreezeAllNewScans), + testutil.ToFloat64(metrics.LogPullerMemoryQuotaAdmissionLevel)) + require.Equal(t, float64(2), + testutil.ToFloat64(metrics.LogPullerMemoryQuotaEventWaiterCount)) +} + func TestMemoryQuotaAdmissionLevels(t *testing.T) { quota := newMemoryQuotaController(100, 10) warmingSpan := newTestQuotaSpan(1) @@ -53,32 +102,32 @@ func TestMemoryQuotaAdmissionLevels(t *testing.T) { warmingTs := setTestQuotaSpanLag(warmingSpan, lowLagRegionThreshold+time.Minute) initializedTs := setTestQuotaSpanLag(initializedSpan, lowLagRegionThreshold+time.Minute) - require.True(t, quota.acquireEvent(context.Background(), initializedSpan, 5)) - require.True(t, quota.acquireEvent(context.Background(), initializedSpan, 10)) - _, _, admitted := quota.acquireScan(newTestQuotaRegion(warmingSpan), warmingTs) + require.True(t, quota.AcquireEvent(context.Background(), initializedSpan, 5)) + require.True(t, quota.AcquireEvent(context.Background(), initializedSpan, 10)) + _, _, admitted := quota.AcquireScan(newTestQuotaRegion(warmingSpan), warmingTs) require.False(t, admitted) - scanBytes, _, admitted := quota.acquireScan( + scanBytes, _, admitted := quota.AcquireScan( newTestQuotaRegion(initializedSpan), initializedTs) require.True(t, admitted) - quota.releaseScan(scanBytes) + quota.ReleaseScan(scanBytes) - require.True(t, quota.acquireEvent(context.Background(), initializedSpan, 45)) - require.True(t, quota.acquireEvent(context.Background(), initializedSpan, 20)) - _, _, admitted = quota.acquireScan( + require.True(t, quota.AcquireEvent(context.Background(), initializedSpan, 45)) + require.True(t, quota.AcquireEvent(context.Background(), initializedSpan, 20)) + _, _, admitted = quota.AcquireScan( newTestQuotaRegion(initializedSpan), initializedTs) require.False(t, admitted) - quota.releaseEvent(20) - _, _, level := quota.snapshot() - require.Equal(t, admissionPauseWarming, level) - quota.releaseEvent(45) - _, _, level = quota.snapshot() - require.Equal(t, admissionPauseWarming, level) - quota.releaseEvent(10) - _, _, level = quota.snapshot() - require.Equal(t, admissionNormal, level) - quota.releaseEvent(5) + quota.ReleaseEvent(20) + state := getMemoryQuotaTestState(quota) + require.Equal(t, admissionPauseWarming, state.level) + quota.ReleaseEvent(45) + state = getMemoryQuotaTestState(quota) + require.Equal(t, admissionPauseWarming, state.level) + quota.ReleaseEvent(10) + state = getMemoryQuotaTestState(quota) + require.Equal(t, admissionNormal, state.level) + quota.ReleaseEvent(5) } func TestMemoryQuotaSpanStopKeepsOwnedMemoryUntilRelease(t *testing.T) { @@ -86,35 +135,34 @@ func TestMemoryQuotaSpanStopKeepsOwnedMemoryUntilRelease(t *testing.T) { span1 := newTestQuotaSpan(1) span2 := newTestQuotaSpan(2) - require.True(t, quota.acquireEvent(context.Background(), span1, 30)) - require.True(t, quota.acquireEvent(context.Background(), span2, 40)) - scanBytes, _, admitted := quota.acquireScan( + require.True(t, quota.AcquireEvent(context.Background(), span1, 30)) + require.True(t, quota.AcquireEvent(context.Background(), span2, 40)) + scanBytes, _, admitted := quota.AcquireScan( newTestQuotaRegion(span1), span1.resolvedTs.Load()) require.True(t, admitted) require.NotZero(t, scanBytes) span1.stopped.Store(true) - quota.wakeAll() - used, _, _ := quota.snapshot() - require.Equal(t, uint64(70), used) - scanUsed, _, _ := quota.scanSnapshot() - require.Equal(t, scanBytes, scanUsed) + quota.WakeAll() + state := getMemoryQuotaTestState(quota) + require.Equal(t, uint64(70), state.used) + require.Equal(t, scanBytes, state.scanUsed) - quota.releaseEvent(30) - quota.releaseScan(scanBytes) - used, _, _ = quota.snapshot() - require.Equal(t, uint64(40), used) + quota.ReleaseEvent(30) + quota.ReleaseScan(scanBytes) + state = getMemoryQuotaTestState(quota) + require.Equal(t, uint64(40), state.used) // Late tasks reach the stopped-subscription cleanup path without consuming // scan quota. - scanBytes, _, admitted = quota.acquireScan( + scanBytes, _, admitted = quota.AcquireScan( newTestQuotaRegion(span1), span1.resolvedTs.Load()) require.True(t, admitted) require.Zero(t, scanBytes) - quota.releaseEvent(40) - used, _, _ = quota.snapshot() - require.Zero(t, used) + quota.ReleaseEvent(40) + state = getMemoryQuotaTestState(quota) + require.Zero(t, state.used) } func TestMemoryQuotaBlockedEventStopsWhenSpanStops(t *testing.T) { @@ -122,10 +170,10 @@ func TestMemoryQuotaBlockedEventStopsWhenSpanStops(t *testing.T) { quota.hardLimit = 100 span := newTestQuotaSpan(1) - require.True(t, quota.acquireEvent(context.Background(), span, 100)) + require.True(t, quota.AcquireEvent(context.Background(), span, 100)) acquired := make(chan bool, 1) go func() { - acquired <- quota.acquireEvent(context.Background(), span, 1) + acquired <- quota.AcquireEvent(context.Background(), span, 1) }() select { @@ -135,24 +183,24 @@ func TestMemoryQuotaBlockedEventStopsWhenSpanStops(t *testing.T) { } span.stopped.Store(true) - quota.wakeAll() + quota.WakeAll() select { case ok := <-acquired: require.False(t, ok) case <-time.After(time.Second): t.Fatal("stopping the subscription did not wake the blocked event") } - quota.releaseEvent(100) + quota.ReleaseEvent(100) } func TestMemoryQuotaBlockedEventResumesAfterRelease(t *testing.T) { quota := newMemoryQuotaController(100, 10) span := newTestQuotaSpan(1) - require.True(t, quota.acquireEvent(context.Background(), span, 200)) + require.True(t, quota.AcquireEvent(context.Background(), span, 200)) acquired := make(chan bool, 1) go func() { - acquired <- quota.acquireEvent(context.Background(), span, 1) + acquired <- quota.AcquireEvent(context.Background(), span, 1) }() select { @@ -161,11 +209,11 @@ func TestMemoryQuotaBlockedEventResumesAfterRelease(t *testing.T) { case <-time.After(100 * time.Millisecond): } - quota.releaseEvent(200) + quota.ReleaseEvent(200) select { case ok := <-acquired: require.True(t, ok) - quota.releaseEvent(1) + quota.ReleaseEvent(1) case <-time.After(time.Second): t.Fatal("event memory did not resume after memory was released") } @@ -177,10 +225,10 @@ func TestMemoryQuotaBlockedEventStopsOnContextCancellation(t *testing.T) { span := newTestQuotaSpan(1) ctx, cancel := context.WithCancel(context.Background()) - require.True(t, quota.acquireEvent(context.Background(), span, 100)) + require.True(t, quota.AcquireEvent(context.Background(), span, 100)) acquired := make(chan bool, 1) go func() { - acquired <- quota.acquireEvent(ctx, span, 1) + acquired <- quota.AcquireEvent(ctx, span, 1) }() require.Eventually(t, func() bool { return quota.eventNotifier.waiters.Load() == 1 @@ -195,7 +243,7 @@ func TestMemoryQuotaBlockedEventStopsOnContextCancellation(t *testing.T) { case <-time.After(time.Second): t.Fatal("context cancellation did not stop the blocked event") } - quota.releaseEvent(100) + quota.ReleaseEvent(100) } func TestMemoryQuotaConcurrentWaitersDoNotLoseWakeups(t *testing.T) { @@ -208,13 +256,13 @@ func TestMemoryQuotaConcurrentWaitersDoNotLoseWakeups(t *testing.T) { // Hold the only available byte until every goroutine is waiting. Releasing // it wakes all waiters; each successful waiter then releases it for the next. - require.True(t, quota.acquireEvent(ctx, span, 1)) + require.True(t, quota.AcquireEvent(ctx, span, 1)) results := make(chan bool, waiterCount) for range waiterCount { go func() { - acquired := quota.acquireEvent(ctx, span, 1) + acquired := quota.AcquireEvent(ctx, span, 1) if acquired { - quota.releaseEvent(1) + quota.ReleaseEvent(1) } results <- acquired }() @@ -223,7 +271,7 @@ func TestMemoryQuotaConcurrentWaitersDoNotLoseWakeups(t *testing.T) { return quota.eventNotifier.waiters.Load() == waiterCount }, time.Second, time.Millisecond) - quota.releaseEvent(1) + quota.ReleaseEvent(1) for range waiterCount { select { case acquired := <-results: @@ -232,8 +280,8 @@ func TestMemoryQuotaConcurrentWaitersDoNotLoseWakeups(t *testing.T) { t.Fatal("event waiter did not make progress") } } - used, _, _ := quota.snapshot() - require.Zero(t, used) + state := getMemoryQuotaTestState(quota) + require.Zero(t, state.used) } func TestMemoryQuotaWarmingScanUsesCurrentPressure(t *testing.T) { @@ -242,19 +290,19 @@ func TestMemoryQuotaWarmingScanUsesCurrentPressure(t *testing.T) { currentTs := setTestQuotaSpanLag(span, lowLagRegionThreshold+time.Minute) region := newTestQuotaRegion(span) - bytes1, _, admitted := quota.acquireScan(region, currentTs) + bytes1, _, admitted := quota.AcquireScan(region, currentTs) require.True(t, admitted) require.NotZero(t, bytes1) - scanUsed, _, _ := quota.scanSnapshot() - require.Greater(t, scanUsed, quota.pauseWarmingLimit) + state := getMemoryQuotaTestState(quota) + require.Greater(t, state.scanUsed, quota.pauseWarmingLimit) - _, _, admitted = quota.acquireScan(region, currentTs) + _, _, admitted = quota.AcquireScan(region, currentTs) require.False(t, admitted) - quota.releaseScan(bytes1) - bytes2, _, admitted := quota.acquireScan(region, currentTs) + quota.ReleaseScan(bytes1) + bytes2, _, admitted := quota.AcquireScan(region, currentTs) require.True(t, admitted) - quota.releaseScan(bytes2) + quota.ReleaseScan(bytes2) } func TestMemoryQuotaLowLagScanBypassesWarmingGate(t *testing.T) { @@ -262,15 +310,15 @@ func TestMemoryQuotaLowLagScanBypassesWarmingGate(t *testing.T) { span := newTestQuotaSpan(1) currentTs := setTestQuotaSpanLag(span, lowLagRegionThreshold-time.Second) - require.True(t, quota.acquireEvent(context.Background(), span, 20)) - scanBytes, _, admitted := quota.acquireScan(newTestQuotaRegion(span), currentTs) + require.True(t, quota.AcquireEvent(context.Background(), span, 20)) + scanBytes, _, admitted := quota.AcquireScan(newTestQuotaRegion(span), currentTs) require.True(t, admitted) require.NotZero(t, scanBytes) - scanUsed, _, _ := quota.scanSnapshot() - require.NotZero(t, scanUsed) + state := getMemoryQuotaTestState(quota) + require.NotZero(t, state.scanUsed) - quota.releaseScan(scanBytes) - quota.releaseEvent(20) + quota.ReleaseScan(scanBytes) + quota.ReleaseEvent(20) } func TestAdmissionWaitsForMemoryAndReleasesScanMemory(t *testing.T) { @@ -281,7 +329,7 @@ func TestAdmissionWaitsForMemoryAndReleasesScanMemory(t *testing.T) { return currentTs }) - require.True(t, quota.acquireEvent(context.Background(), span, 20)) + require.True(t, quota.AcquireEvent(context.Background(), span, 20)) region := newTestQuotaRegion(span) require.True(t, controller.submit(newRegionPriorityTask(region, currentTs, 1))) @@ -300,7 +348,7 @@ func TestAdmissionWaitsForMemoryAndReleasesScanMemory(t *testing.T) { case <-time.After(100 * time.Millisecond): } - quota.releaseEvent(20) + quota.ReleaseEvent(20) var resultValue popResult select { case resultValue = <-result: @@ -309,11 +357,11 @@ func TestAdmissionWaitsForMemoryAndReleasesScanMemory(t *testing.T) { } require.NoError(t, resultValue.err) req := resultValue.req - scanUsed, _, _ := quota.scanSnapshot() - require.NotZero(t, scanUsed) + state := getMemoryQuotaTestState(quota) + require.NotZero(t, state.scanUsed) require.True(t, req.abort()) - scanUsed, _, _ = quota.scanSnapshot() - require.Zero(t, scanUsed) + state = getMemoryQuotaTestState(quota) + require.Zero(t, state.scanUsed) } func TestAdmissionWakesWhenBlockedSpanStops(t *testing.T) { @@ -324,7 +372,7 @@ func TestAdmissionWakesWhenBlockedSpanStops(t *testing.T) { return currentTs }) - require.True(t, quota.acquireEvent(context.Background(), span, 20)) + require.True(t, quota.AcquireEvent(context.Background(), span, 20)) require.True(t, controller.submit(newRegionPriorityTask( newTestQuotaRegion(span), currentTs, 1))) @@ -344,7 +392,7 @@ func TestAdmissionWakesWhenBlockedSpanStops(t *testing.T) { } span.stopped.Store(true) - quota.wakeAll() + quota.WakeAll() select { case result := <-result: require.NoError(t, result.err) @@ -353,7 +401,7 @@ func TestAdmissionWakesWhenBlockedSpanStops(t *testing.T) { case <-time.After(time.Second): t.Fatal("stopping the span did not wake scan admission") } - quota.releaseEvent(20) + quota.ReleaseEvent(20) } func BenchmarkMemoryQuotaEventAccounting(b *testing.B) { @@ -363,9 +411,9 @@ func BenchmarkMemoryQuotaEventAccounting(b *testing.B) { b.ReportAllocs() b.ResetTimer() for b.Loop() { - if !quota.acquireEvent(ctx, span, 1) { + if !quota.AcquireEvent(ctx, span, 1) { b.Fatal("failed to acquire event memory") } - quota.releaseEvent(1) + quota.ReleaseEvent(1) } } diff --git a/logservice/logpuller/region_admission_controller.go b/logservice/logpuller/region_admission_controller.go index a143ba1446..749946d4f6 100644 --- a/logservice/logpuller/region_admission_controller.go +++ b/logservice/logpuller/region_admission_controller.go @@ -73,7 +73,7 @@ func (r *regionReq) release() bool { if !r.released.CompareAndSwap(false, true) { return false } - r.controller.memoryQuota.releaseScan(r.scanBytes) + r.controller.memoryQuota.ReleaseScan(r.scanBytes) r.controller.release() return true } @@ -200,7 +200,7 @@ func (c *regionAdmissionController) popEligibleLocked() ( return nil, 0, nil } - scanBytes, memoryReady, admitted := c.memoryQuota.acquireScan( + scanBytes, memoryReady, admitted := c.memoryQuota.AcquireScan( request.regionInfo, c.currentTs()) if !admitted { return nil, 0, memoryReady diff --git a/logservice/logpuller/region_event_handler.go b/logservice/logpuller/region_event_handler.go index a30b349418..704e346505 100644 --- a/logservice/logpuller/region_event_handler.go +++ b/logservice/logpuller/region_event_handler.go @@ -151,7 +151,7 @@ func (h *regionEventHandler) Handle(span *subscribedSpan, events ...regionEvent) } } if !wasInitialized && span.initialized.Load() { - h.eventSink.memoryQuota.notifyScanAdmission() + h.eventSink.memoryQuota.NotifyScanAdmission() } tryAdvanceResolvedTs := func() { if newResolvedTs != 0 { @@ -159,7 +159,7 @@ func (h *regionEventHandler) Handle(span *subscribedSpan, events ...regionEvent) } } releaseMemoryQuota := func() { - h.eventSink.memoryQuota.releaseEvent(memoryBytes) + h.eventSink.memoryQuota.ReleaseEvent(memoryBytes) } if len(span.kvEventsCache) > 0 { metricsEventCount.Add(float64(len(span.kvEventsCache))) @@ -243,7 +243,7 @@ func (h *regionEventHandler) GetType(event regionEvent) dynstream.EventType { } func (h *regionEventHandler) OnDrop(event regionEvent) interface{} { - h.eventSink.memoryQuota.releaseEvent(event.memoryBytes) + h.eventSink.memoryQuota.ReleaseEvent(event.memoryBytes) // TODO: Distinguish between drop events caused by "path not found" errors and memory control. state := event.mustFirstState() fields := []zap.Field{ diff --git a/logservice/logpuller/region_event_handler_test.go b/logservice/logpuller/region_event_handler_test.go index 9227f2bb8e..91749cf028 100644 --- a/logservice/logpuller/region_event_handler_test.go +++ b/logservice/logpuller/region_event_handler_test.go @@ -397,7 +397,7 @@ func TestHandleEntriesReleasesMemoryAfterDownstreamCallback(t *testing.T) { lockedRangeState: lockedState, }, } - require.True(t, quota.acquireEvent(context.Background(), span, 10)) + require.True(t, quota.AcquireEvent(context.Background(), span, 10)) handler := ®ionEventHandler{eventSink: ®ionEventSink{ ds: newMockRegionEventSinkStream(), memoryQuota: quota, @@ -415,13 +415,13 @@ func TestHandleEntriesReleasesMemoryAfterDownstreamCallback(t *testing.T) { }}, }) require.True(t, await) - used, _, _ := quota.snapshot() - require.Equal(t, uint64(10), used) + quotaState := getMemoryQuotaTestState(quota) + require.Equal(t, uint64(10), quotaState.used) callback := <-callbackCh callback() - used, _, _ = quota.snapshot() - require.Zero(t, used) + quotaState = getMemoryQuotaTestState(quota) + require.Zero(t, quotaState.used) } func TestTryMarkSpanInitializedByResolvedTs(t *testing.T) { diff --git a/logservice/logpuller/region_event_sink.go b/logservice/logpuller/region_event_sink.go index 4a5b5b0bf6..0c015db743 100644 --- a/logservice/logpuller/region_event_sink.go +++ b/logservice/logpuller/region_event_sink.go @@ -17,7 +17,6 @@ import ( "context" "github.com/pingcap/log" - "github.com/pingcap/ticdc/pkg/metrics" "github.com/pingcap/ticdc/utils/dynstream" "go.uber.org/zap" ) @@ -73,7 +72,7 @@ func (s *regionEventSink) Push(subID SubscriptionID, event regionEvent) { if event.needsMemoryAccounting() { span := event.mustFirstState().region.subscribedSpan event.memoryBytes = uint64(event.getSize()) - if !s.memoryQuota.acquireEvent(s.ctx, span, event.memoryBytes) { + if !s.memoryQuota.AcquireEvent(s.ctx, span, event.memoryBytes) { return } } @@ -84,17 +83,9 @@ func (s *regionEventSink) UpdateMetrics() { dsMetrics := s.ds.GetMetrics() metricSubscriptionClientDSChannelSize.Set(float64(dsMetrics.EventChanSize)) metricSubscriptionClientDSPendingQueueLen.Set(float64(dsMetrics.PendingQueueLen)) - - used, capacity, _ := s.memoryQuota.snapshot() - scanUsed, scanEstimate, hardLimit := s.memoryQuota.scanSnapshot() - metrics.LogPullerMemoryQuota.WithLabelValues("max").Set(float64(capacity)) - metrics.LogPullerMemoryQuota.WithLabelValues("used").Set(float64(used)) - metrics.LogPullerMemoryQuota.WithLabelValues("scan_used").Set(float64(scanUsed)) - metrics.LogPullerMemoryQuota.WithLabelValues("scan_estimate").Set(float64(scanEstimate)) - metrics.LogPullerMemoryQuota.WithLabelValues("hard_limit").Set(float64(hardLimit)) } func (s *regionEventSink) Close() { - s.memoryQuota.wakeAll() + s.memoryQuota.WakeAll() s.ds.Close() } diff --git a/logservice/logpuller/region_event_sink_test.go b/logservice/logpuller/region_event_sink_test.go index 7786e4b524..291d4d867c 100644 --- a/logservice/logpuller/region_event_sink_test.go +++ b/logservice/logpuller/region_event_sink_test.go @@ -18,7 +18,6 @@ import ( "testing" "github.com/pingcap/kvproto/pkg/cdcpb" - "github.com/pingcap/ticdc/pkg/metrics" "github.com/pingcap/ticdc/utils/dynstream" "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/require" @@ -66,31 +65,17 @@ func (s *mockRegionEventSinkStream) GetMetrics() dynstream.Metrics[int, Subscrip } func TestRegionEventSinkUpdateMetrics(t *testing.T) { - t.Run("quota updates memory gauges", func(t *testing.T) { - ds := newMockRegionEventSinkStream() - ds.metrics = dynstream.Metrics[int, SubscriptionID]{ - EventChanSize: 33, - PendingQueueLen: 44, - } - quota := newMemoryQuotaController(66, 8) - span := newTestQuotaSpan(1) - require.True(t, quota.acquireEvent(context.Background(), span, 55)) - t.Cleanup(func() { quota.releaseEvent(55) }) - - sink := ®ionEventSink{ - ctx: context.Background(), - ds: ds, - memoryQuota: quota, - } - sink.UpdateMetrics() - - require.Equal(t, float64(33), testutil.ToFloat64(metricSubscriptionClientDSChannelSize)) - require.Equal(t, float64(44), testutil.ToFloat64(metricSubscriptionClientDSPendingQueueLen)) - require.Equal(t, float64(66), testutil.ToFloat64( - metrics.LogPullerMemoryQuota.WithLabelValues("max"))) - require.Equal(t, float64(55), testutil.ToFloat64( - metrics.LogPullerMemoryQuota.WithLabelValues("used"))) - }) + ds := newMockRegionEventSinkStream() + ds.metrics = dynstream.Metrics[int, SubscriptionID]{ + EventChanSize: 33, + PendingQueueLen: 44, + } + sink := ®ionEventSink{ds: ds} + + sink.UpdateMetrics() + + require.Equal(t, float64(33), testutil.ToFloat64(metricSubscriptionClientDSChannelSize)) + require.Equal(t, float64(44), testutil.ToFloat64(metricSubscriptionClientDSPendingQueueLen)) } func TestRegionEventSinkTracksEntriesUntilDrop(t *testing.T) { @@ -115,10 +100,10 @@ func TestRegionEventSinkTracksEntriesUntilDrop(t *testing.T) { }) pushed := <-ds.eventCh require.NotZero(t, pushed.memoryBytes) - used, _, _ := quota.snapshot() - require.NotZero(t, used) + quotaState := getMemoryQuotaTestState(quota) + require.NotZero(t, quotaState.used) (®ionEventHandler{eventSink: sink}).OnDrop(pushed) - used, _, _ = quota.snapshot() - require.Zero(t, used) + quotaState = getMemoryQuotaTestState(quota) + require.Zero(t, quotaState.used) } diff --git a/logservice/logpuller/subscription_client.go b/logservice/logpuller/subscription_client.go index bfbe2f910b..af039a9ebd 100644 --- a/logservice/logpuller/subscription_client.go +++ b/logservice/logpuller/subscription_client.go @@ -212,6 +212,7 @@ func (s *subscriptionClient) updateMetrics(ctx context.Context) error { case <-ticker.C: s.regionScheduler.UpdateMetrics() s.eventSink.UpdateMetrics() + s.memoryQuota.UpdateMetrics() s.spanRegistry.UpdateMetrics() } } @@ -319,7 +320,7 @@ func (s *subscriptionClient) setTableStopped(rt *subscribedSpan) { // then notify every existing worker to deregister the subscription. if rt.stopped.CompareAndSwap(false, true) { // Wake event receivers and scan admission so they can observe stopped. - s.memoryQuota.wakeAll() + s.memoryQuota.WakeAll() s.regionScheduler.BroadcastDeregister(rt.subID, rt.filterLoop) if rt.rangeLock.Stop() { s.onTableDrained(rt) diff --git a/metrics/grafana/ticdc_new_arch.json b/metrics/grafana/ticdc_new_arch.json index 29123f9715..52cda78859 100644 --- a/metrics/grafana/ticdc_new_arch.json +++ b/metrics/grafana/ticdc_new_arch.json @@ -8119,7 +8119,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Log puller memory quota", + "description": "Actual event memory, estimated in-flight scan memory, and the configured memory limits.", "fieldConfig": { "defaults": {}, "overrides": [] @@ -8162,7 +8162,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(ticdc_dynamic_stream_memory_usage{k8s_cluster=~\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", module=~\"log-puller\"}) by (instance, type)", + "expr": "sum(ticdc_log_puller_memory_quota{k8s_cluster=~\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance, type)", "interval": "", "legendFormat": "{{instance}}-{{type}}", "refId": "A" @@ -8172,7 +8172,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Memory Quota", + "title": "Memory Quota Usage", "tooltip": { "shared": true, "sort": 0, @@ -8822,7 +8822,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Log puller memory quota tracked by the local memory quota controller.", + "description": "Scan admission level (0: normal, 1: pause warming scans, 2: pause all scans) and event receivers currently blocked at the memory hard limit.", "fieldConfig": { "defaults": {}, "overrides": [] @@ -8858,6 +8858,110 @@ "pointradius": 2, "points": false, "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/event-waiters$/", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "ticdc_log_puller_memory_quota_admission_level{k8s_cluster=~\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}", + "interval": "", + "legendFormat": "{{instance}}-admission-level", + "refId": "A" + }, + { + "exemplar": true, + "expr": "ticdc_log_puller_memory_quota_event_waiter_count{k8s_cluster=~\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}", + "interval": "", + "legendFormat": "{{instance}}-event-waiters", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Quota Throttling", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": "2", + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Time spent waiting to receive more events after log puller memory reaches the hard limit.", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 57 + }, + "hiddenSeries": false, + "id": 26002, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, @@ -8865,17 +8969,24 @@ "targets": [ { "exemplar": true, - "expr": "sum(ticdc_log_puller_memory_quota{k8s_cluster=~\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance, type)", + "expr": "histogram_quantile(0.99, sum(rate(ticdc_log_puller_memory_quota_event_wait_duration_bucket{k8s_cluster=~\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", "interval": "", - "legendFormat": "{{instance}}-{{type}}", + "legendFormat": "{{instance}}-p99", "refId": "A" + }, + { + "exemplar": true, + "expr": "sum(rate(ticdc_log_puller_memory_quota_event_wait_duration_sum{k8s_cluster=~\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance) / sum(rate(ticdc_log_puller_memory_quota_event_wait_duration_count{k8s_cluster=~\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", + "interval": "", + "legendFormat": "{{instance}}-avg", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Memory Quota Controller", + "title": "Memory Quota Event Wait Duration", "tooltip": { "shared": true, "sort": 0, @@ -8891,7 +9002,7 @@ }, "yaxes": [ { - "format": "bytes", + "format": "s", "logBase": 1, "min": "0", "show": true diff --git a/metrics/nextgengrafana/ticdc_new_arch_next_gen.json b/metrics/nextgengrafana/ticdc_new_arch_next_gen.json index 2bfe4b8015..b8452acca1 100644 --- a/metrics/nextgengrafana/ticdc_new_arch_next_gen.json +++ b/metrics/nextgengrafana/ticdc_new_arch_next_gen.json @@ -8119,7 +8119,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Log puller memory quota", + "description": "Actual event memory, estimated in-flight scan memory, and the configured memory limits.", "fieldConfig": { "defaults": {}, "overrides": [] @@ -8162,7 +8162,7 @@ "targets": [ { "exemplar": true, - "expr": "sum(ticdc_dynamic_stream_memory_usage{k8s_cluster=~\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\", module=~\"log-puller\"}) by (instance, type)", + "expr": "sum(ticdc_log_puller_memory_quota{k8s_cluster=~\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance, type)", "interval": "", "legendFormat": "{{instance}}-{{type}}", "refId": "A" @@ -8172,7 +8172,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Memory Quota", + "title": "Memory Quota Usage", "tooltip": { "shared": true, "sort": 0, @@ -8822,7 +8822,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Log puller memory quota tracked by the local memory quota controller.", + "description": "Scan admission level (0: normal, 1: pause warming scans, 2: pause all scans) and event receivers currently blocked at the memory hard limit.", "fieldConfig": { "defaults": {}, "overrides": [] @@ -8858,6 +8858,110 @@ "pointradius": 2, "points": false, "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/event-waiters$/", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "ticdc_log_puller_memory_quota_admission_level{k8s_cluster=~\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}", + "interval": "", + "legendFormat": "{{instance}}-admission-level", + "refId": "A" + }, + { + "exemplar": true, + "expr": "ticdc_log_puller_memory_quota_event_waiter_count{k8s_cluster=~\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}", + "interval": "", + "legendFormat": "{{instance}}-event-waiters", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Quota Throttling", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": "2", + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Time spent waiting to receive more events after log puller memory reaches the hard limit.", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 57 + }, + "hiddenSeries": false, + "id": 26002, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, @@ -8865,17 +8969,24 @@ "targets": [ { "exemplar": true, - "expr": "sum(ticdc_log_puller_memory_quota{k8s_cluster=~\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance, type)", + "expr": "histogram_quantile(0.99, sum(rate(ticdc_log_puller_memory_quota_event_wait_duration_bucket{k8s_cluster=~\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", "interval": "", - "legendFormat": "{{instance}}-{{type}}", + "legendFormat": "{{instance}}-p99", "refId": "A" + }, + { + "exemplar": true, + "expr": "sum(rate(ticdc_log_puller_memory_quota_event_wait_duration_sum{k8s_cluster=~\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance) / sum(rate(ticdc_log_puller_memory_quota_event_wait_duration_count{k8s_cluster=~\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", + "interval": "", + "legendFormat": "{{instance}}-avg", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Memory Quota Controller", + "title": "Memory Quota Event Wait Duration", "tooltip": { "shared": true, "sort": 0, @@ -8891,7 +9002,7 @@ }, "yaxes": [ { - "format": "bytes", + "format": "s", "logBase": 1, "min": "0", "show": true diff --git a/pkg/metrics/log_puller.go b/pkg/metrics/log_puller.go index 0608d3e699..6b4e04cac2 100644 --- a/pkg/metrics/log_puller.go +++ b/pkg/metrics/log_puller.go @@ -71,6 +71,28 @@ var ( Name: "memory_quota", Help: "The log puller local memory quota usage.", }, []string{"type"}) + LogPullerMemoryQuotaAdmissionLevel = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: "ticdc", + Subsystem: "log_puller", + Name: "memory_quota_admission_level", + Help: "The log puller scan admission level: 0 normal, 1 pause warming scans, 2 pause all scans.", + }) + LogPullerMemoryQuotaEventWaiterCount = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: "ticdc", + Subsystem: "log_puller", + Name: "memory_quota_event_waiter_count", + Help: "The number of event receivers waiting at the log puller memory hard limit.", + }) + LogPullerMemoryQuotaEventWaitDuration = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Namespace: "ticdc", + Subsystem: "log_puller", + Name: "memory_quota_event_wait_duration", + Help: "The duration in seconds that an event receiver waits at the log puller memory hard limit.", + Buckets: prometheus.ExponentialBuckets(0.001, 2, 24), + }) SubscriptionClientResolvedTsLagGauge = prometheus.NewGauge( prometheus.GaugeOpts{ @@ -172,6 +194,9 @@ func initLogPullerMetrics(registry *prometheus.Registry) { registry.MustRegister(LogPullerMatcherCount) registry.MustRegister(LogPullerResolvedTsLag) registry.MustRegister(LogPullerMemoryQuota) + registry.MustRegister(LogPullerMemoryQuotaAdmissionLevel) + registry.MustRegister(LogPullerMemoryQuotaEventWaiterCount) + registry.MustRegister(LogPullerMemoryQuotaEventWaitDuration) registry.MustRegister(SubscriptionClientRequestedRegionCount) registry.MustRegister(SubscriptionClientAddRegionRequestDuration) registry.MustRegister(RegionRequestFinishScanDuration) From f315b89b88c94b09e4d44a257465a1c7413660e6 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Tue, 4 Aug 2026 17:59:12 +0800 Subject: [PATCH 54/71] more fix --- logservice/logpuller/memory_quota.go | 43 ++-------- logservice/logpuller/memory_quota_test.go | 9 +- logservice/logpuller/priority_task.go | 18 +++- logservice/logpuller/region_request_worker.go | 5 +- logservice/logpuller/requested_store.go | 85 ------------------- 5 files changed, 29 insertions(+), 131 deletions(-) delete mode 100644 logservice/logpuller/requested_store.go diff --git a/logservice/logpuller/memory_quota.go b/logservice/logpuller/memory_quota.go index 9914b88ca3..3698bf2085 100644 --- a/logservice/logpuller/memory_quota.go +++ b/logservice/logpuller/memory_quota.go @@ -40,14 +40,6 @@ const ( // falls to 5% of the soft capacity. defaultResumeWarmingRatio = 0.05 - // defaultFreezeAllRatio pauses every new scan when memory pressure reaches - // 80% of the soft capacity. - defaultFreezeAllRatio = 0.8 - - // defaultResumeAllRatio allows new scans again after memory pressure falls - // to 60% of the soft capacity. - defaultResumeAllRatio = 0.6 - // defaultHardLimitRatio blocks receiving more events when accounted event // memory reaches twice the soft capacity. defaultHardLimitRatio = 2.0 @@ -70,7 +62,6 @@ type admissionLevel uint8 const ( admissionNormal admissionLevel = iota admissionPauseWarming - admissionFreezeAllNewScans ) // eventMemoryNotifier wakes event receivers that are waiting for memory. Each @@ -140,8 +131,9 @@ func (n *eventMemoryNotifier) notify() { // memoryQuotaController tracks event memory retained by downstream callbacks // and estimated memory for admitted initial scans. Event memory is allowed to // exceed the soft capacity, but the receive path waits at the hard limit. Scan -// admission first pauses uninitialized high-lag spans and freezes all new scans -// only under heavier pressure; both transitions use hysteresis when resuming. +// admission pauses uninitialized high-lag spans under memory pressure and uses +// hysteresis when resuming them. Other scans continue to make progress and are +// bounded by the region request window and the event-memory hard limit. type memoryQuotaController struct { capacity uint64 // used tracks event bytes retained until downstream finishes consuming them. @@ -166,8 +158,6 @@ type memoryQuotaController struct { pauseWarmingLimit uint64 resumeWarmingLimit uint64 - freezeAllLimit uint64 - resumeAllLimit uint64 hardLimit uint64 scanEstimate uint64 @@ -185,8 +175,6 @@ func newMemoryQuotaController(capacity, scanBaseSize uint64) *memoryQuotaControl level: admissionNormal, pauseWarmingLimit: uint64(math.Ceil(float64(capacity) * defaultPauseWarmingRatio)), resumeWarmingLimit: uint64(float64(capacity) * defaultResumeWarmingRatio), - freezeAllLimit: uint64(math.Ceil(float64(capacity) * defaultFreezeAllRatio)), - resumeAllLimit: uint64(float64(capacity) * defaultResumeAllRatio), hardLimit: uint64(float64(capacity) * defaultHardLimitRatio), scanEstimate: scanBaseSize, eventNotifier: newEventMemoryNotifier(), @@ -216,10 +204,6 @@ func (c *memoryQuotaController) AcquireScan( c.scanMu.Lock() defer c.scanMu.Unlock() c.refreshLevelLocked() - if c.level == admissionFreezeAllNewScans { - return 0, c.scanReady, false - } - warming := isWarmingScan(region, currentTs) // Admission is based on the pressure before accounting this scan. This lets // one scan make progress even when its estimate alone exceeds the threshold. @@ -292,8 +276,7 @@ func (c *memoryQuotaController) ReleaseEvent(bytes uint64) { } used := c.used.Add(^(bytes - 1)) previousUsed := used + bytes - if crossesDown(previousUsed, used, c.resumeWarmingLimit) || - crossesDown(previousUsed, used, c.resumeAllLimit) { + if crossesDown(previousUsed, used, c.resumeWarmingLimit) { c.refreshAdmissionAndNotify() } c.eventNotifier.notify() @@ -381,26 +364,12 @@ func (c *memoryQuotaController) refreshLevelLocked() { // it to actual event bytes would count the same pressure twice. pressure := max(c.used.Load(), c.scanUsed) switch c.level { - case admissionFreezeAllNewScans: - if pressure <= c.resumeAllLimit { - if pressure >= c.pauseWarmingLimit { - c.level = admissionPauseWarming - } else { - c.level = admissionNormal - } - } case admissionPauseWarming: - switch { - case pressure >= c.freezeAllLimit: - c.level = admissionFreezeAllNewScans - case pressure <= c.resumeWarmingLimit: + if pressure <= c.resumeWarmingLimit { c.level = admissionNormal } default: - switch { - case pressure >= c.freezeAllLimit: - c.level = admissionFreezeAllNewScans - case pressure >= c.pauseWarmingLimit: + if pressure >= c.pauseWarmingLimit { c.level = admissionPauseWarming } } diff --git a/logservice/logpuller/memory_quota_test.go b/logservice/logpuller/memory_quota_test.go index 8b8b3cd1c6..02be6cf86f 100644 --- a/logservice/logpuller/memory_quota_test.go +++ b/logservice/logpuller/memory_quota_test.go @@ -71,7 +71,7 @@ func TestMemoryQuotaUpdateMetrics(t *testing.T) { quota.scanMu.Lock() quota.scanUsed = 7 - quota.level = admissionFreezeAllNewScans + quota.level = admissionPauseWarming quota.scanMu.Unlock() quota.eventNotifier.waiters.Store(2) t.Cleanup(func() { quota.eventNotifier.waiters.Store(0) }) @@ -88,7 +88,7 @@ func TestMemoryQuotaUpdateMetrics(t *testing.T) { metrics.LogPullerMemoryQuota.WithLabelValues("scan_estimate"))) require.Equal(t, float64(132), testutil.ToFloat64( metrics.LogPullerMemoryQuota.WithLabelValues("hard_limit"))) - require.Equal(t, float64(admissionFreezeAllNewScans), + require.Equal(t, float64(admissionPauseWarming), testutil.ToFloat64(metrics.LogPullerMemoryQuotaAdmissionLevel)) require.Equal(t, float64(2), testutil.ToFloat64(metrics.LogPullerMemoryQuotaEventWaiterCount)) @@ -114,9 +114,10 @@ func TestMemoryQuotaAdmissionLevels(t *testing.T) { require.True(t, quota.AcquireEvent(context.Background(), initializedSpan, 45)) require.True(t, quota.AcquireEvent(context.Background(), initializedSpan, 20)) - _, _, admitted = quota.AcquireScan( + scanBytes, _, admitted = quota.AcquireScan( newTestQuotaRegion(initializedSpan), initializedTs) - require.False(t, admitted) + require.True(t, admitted) + quota.ReleaseScan(scanBytes) quota.ReleaseEvent(20) state := getMemoryQuotaTestState(quota) diff --git a/logservice/logpuller/priority_task.go b/logservice/logpuller/priority_task.go index 38cd813c8b..aad3aab16b 100644 --- a/logservice/logpuller/priority_task.go +++ b/logservice/logpuller/priority_task.go @@ -13,7 +13,14 @@ package logpuller -import "github.com/pingcap/kvproto/pkg/cdcpb" +import ( + "time" + + "github.com/pingcap/kvproto/pkg/cdcpb" + "github.com/tikv/client-go/v2/oracle" +) + +const lowLagRegionThreshold = 30 * time.Minute func normalizeScanPriority(priority cdcpb.ScanPriority) cdcpb.ScanPriority { if priority == cdcpb.ScanPriority_SCAN_PRIORITY_HIGH { @@ -68,3 +75,12 @@ func (pt *regionPriorityTask) LessThan(other *regionPriorityTask) bool { } return pt.sequence < other.sequence } + +func regionScanLag(currentTs, checkpointTs uint64) time.Duration { + currentTime := oracle.GetTimeFromTS(currentTs) + checkpointTime := oracle.GetTimeFromTS(checkpointTs) + if !currentTime.After(checkpointTime) { + return 0 + } + return currentTime.Sub(checkpointTime) +} diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index 288bb384f7..ac3a35b787 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -39,10 +39,7 @@ const storeReconnectBackoff = time.Second // To generate a workerID in `newRegionRequestWorker`. var workerIDGen atomic.Uint64 -var ( - metricsResolvedTsCount = metrics.PullerEventCounter.WithLabelValues("resolved_ts") - metricBatchResolvedSize = metrics.BatchResolvedEventSize.WithLabelValues("event-store") -) +var metricBatchResolvedSize = metrics.BatchResolvedEventSize.WithLabelValues("event-store") type deregisterRequest struct { subID SubscriptionID diff --git a/logservice/logpuller/requested_store.go b/logservice/logpuller/requested_store.go deleted file mode 100644 index 97294a8e33..0000000000 --- a/logservice/logpuller/requested_store.go +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2026 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package logpuller - -import ( - "context" - "sync/atomic" - - "golang.org/x/sync/errgroup" -) - -// requestedStore owns the request workers connected to one TiKV store. The -// worker slice is complete before the store is published and is immutable -// afterwards, so task submission only needs an atomic round-robin counter. -type requestedStore struct { - storeAddr string - workers []*regionRequestWorker - nextWorker atomic.Uint64 -} - -func newRequestedStore( - upstream *upstreamHandle, - eventSink *regionEventSink, - failureHandler *regionFailureHandler, - storeAddr string, - workerCount int, - workerWindow int, - maxWindowMultiplier int, -) *requestedStore { - store := &requestedStore{ - storeAddr: storeAddr, - workers: make([]*regionRequestWorker, 0, workerCount), - } - for i := 0; i < workerCount; i++ { - store.workers = append(store.workers, newRegionRequestWorker( - upstream, eventSink, failureHandler, store, workerWindow, maxWindowMultiplier)) - } - return store -} - -func (s *requestedStore) run(ctx context.Context, group *errgroup.Group) { - for _, worker := range s.workers { - group.Go(func() error { return worker.Run(ctx) }) - } -} - -func (s *requestedStore) submit(task *regionPriorityTask) bool { - if len(s.workers) == 0 { - return false - } - index := (s.nextWorker.Add(1) - 1) % uint64(len(s.workers)) - return s.workers[index].admission.submit(task) -} - -func (s *requestedStore) broadcastDeregister(subID SubscriptionID, filterLoop bool) { - for _, worker := range s.workers { - worker.controlQueue.push(deregisterRequest{subID: subID, filterLoop: filterLoop}) - } -} - -func (s *requestedStore) close() { - for _, worker := range s.workers { - worker.admission.close() - } -} - -func (s *requestedStore) inflightCount() int { - count := 0 - for _, worker := range s.workers { - count += worker.admission.stats().inflight - } - return count -} From c476c509a90838d9982321364f97e2c571fcf6c8 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Tue, 4 Aug 2026 20:55:07 +0800 Subject: [PATCH 55/71] small fix --- logservice/eventstore/pebble.go | 2 +- logservice/logpuller/memory_quota.go | 32 ++++++++----------- logservice/logpuller/memory_quota_test.go | 2 +- .../region_admission_controller_test.go | 2 +- .../logpuller/region_event_handler_test.go | 4 +-- pkg/config/debug.go | 4 +-- 6 files changed, 20 insertions(+), 26 deletions(-) diff --git a/logservice/eventstore/pebble.go b/logservice/eventstore/pebble.go index 0d950c99d0..f5f48bf71b 100644 --- a/logservice/eventstore/pebble.go +++ b/logservice/eventstore/pebble.go @@ -42,7 +42,7 @@ func newPebbleOptions(dbNum int) *pebble.Options { MaxOpenFiles: maxOpenFilesPerDB, - MaxConcurrentCompactions: func() int { return 2 }, + MaxConcurrentCompactions: func() int { return 6 }, // Decrease compaction frequency L0CompactionThreshold: 20, diff --git a/logservice/logpuller/memory_quota.go b/logservice/logpuller/memory_quota.go index 3698bf2085..a499c1ab4e 100644 --- a/logservice/logpuller/memory_quota.go +++ b/logservice/logpuller/memory_quota.go @@ -25,10 +25,6 @@ import ( ) const ( - // defaultLogPullerMemoryQuota is the soft memory capacity shared by event - // accounting and initial-scan admission. - defaultLogPullerMemoryQuota uint64 = 1024 * 1024 * 1024 - // Admission ratios compare max(accounted event memory, estimated scan // memory) with the soft capacity. @@ -44,9 +40,6 @@ const ( // memory reaches twice the soft capacity. defaultHardLimitRatio = 2.0 - // defaultScanBaseSize is the minimum memory estimate for one admitted scan. - defaultScanBaseSize uint64 = 8 * 1024 * 1024 - // defaultScanLagUnit is the lag unit used by the logarithmic scan estimate. defaultScanLagUnit = 10 * time.Minute @@ -128,12 +121,19 @@ func (n *eventMemoryNotifier) notify() { n.mu.Unlock() } -// memoryQuotaController tracks event memory retained by downstream callbacks -// and estimated memory for admitted initial scans. Event memory is allowed to -// exceed the soft capacity, but the receive path waits at the hard limit. Scan -// admission pauses uninitialized high-lag spans under memory pressure and uses -// hysteresis when resuming them. Other scans continue to make progress and are -// bounded by the region request window and the event-memory hard limit. +// memoryQuotaController coordinates memory pressure from two sources: +// retained event memory and admitted initial scans. +// +// Event memory tracks bytes kept alive until downstream finishes consuming +// them. It may exceed the soft capacity temporarily, but the receive path +// blocks once it reaches the hard limit. +// +// Initial scans are charged by estimate instead of measured bytes. Each +// admitted scan starts from scanBaseSize, grows logarithmically with scan lag, +// and is capped at maxScanLagFactor times the base size. Scan admission +// compares max(event used, scan used) with the soft capacity: high-lag +// "warming" scans pause at pauseWarmingLimit and resume at +// resumeWarmingLimit, while lower-lag scans continue to make progress. type memoryQuotaController struct { capacity uint64 // used tracks event bytes retained until downstream finishes consuming them. @@ -164,12 +164,6 @@ type memoryQuotaController struct { } func newMemoryQuotaController(capacity, scanBaseSize uint64) *memoryQuotaController { - if capacity == 0 { - capacity = defaultLogPullerMemoryQuota - } - if scanBaseSize == 0 { - scanBaseSize = defaultScanBaseSize - } c := &memoryQuotaController{ capacity: capacity, level: admissionNormal, diff --git a/logservice/logpuller/memory_quota_test.go b/logservice/logpuller/memory_quota_test.go index 02be6cf86f..de409721a3 100644 --- a/logservice/logpuller/memory_quota_test.go +++ b/logservice/logpuller/memory_quota_test.go @@ -406,7 +406,7 @@ func TestAdmissionWakesWhenBlockedSpanStops(t *testing.T) { } func BenchmarkMemoryQuotaEventAccounting(b *testing.B) { - quota := newMemoryQuotaController(defaultLogPullerMemoryQuota, defaultScanBaseSize) + quota := newMemoryQuotaController(1024*1024*1024, 8*1024*1024) span := newTestQuotaSpan(1) ctx := context.Background() b.ReportAllocs() diff --git a/logservice/logpuller/region_admission_controller_test.go b/logservice/logpuller/region_admission_controller_test.go index 0fdeb3d949..770d849100 100644 --- a/logservice/logpuller/region_admission_controller_test.go +++ b/logservice/logpuller/region_admission_controller_test.go @@ -65,7 +65,7 @@ func newTestRegionAdmissionController( return newRegionAdmissionController( currentWindow, maxWindowMultiplier, - newMemoryQuotaController(0, 0), + newMemoryQuotaController(1024*1024*1024, 8*1024*1024), func() uint64 { return 0 }, ) } diff --git a/logservice/logpuller/region_event_handler_test.go b/logservice/logpuller/region_event_handler_test.go index 91749cf028..7c3db0c2cd 100644 --- a/logservice/logpuller/region_event_handler_test.go +++ b/logservice/logpuller/region_event_handler_test.go @@ -49,7 +49,7 @@ func TestHandleEventEntryEventOutOfOrder(t *testing.T) { // initialize option := dynstream.NewOption() handler := ®ionEventHandler{eventSink: ®ionEventSink{ - memoryQuota: newMemoryQuotaController(0, 0), + memoryQuota: newMemoryQuotaController(1024*1024*1024, 8*1024*1024), }} ds := dynstream.NewParallelDynamicStream("test", handler, option) ds.Start() @@ -208,7 +208,7 @@ func TestHandleResolvedTs(t *testing.T) { // initialize option := dynstream.NewOption() handler := ®ionEventHandler{eventSink: ®ionEventSink{ - memoryQuota: newMemoryQuotaController(0, 0), + memoryQuota: newMemoryQuotaController(1024*1024*1024, 8*1024*1024), }} ds := dynstream.NewParallelDynamicStream("test", handler, option) ds.Start() diff --git a/pkg/config/debug.go b/pkg/config/debug.go index c96527044e..eb7e45c472 100644 --- a/pkg/config/debug.go +++ b/pkg/config/debug.go @@ -24,9 +24,9 @@ import ( const ( // DefaultOldStartTsScanLowPriorityThreshold is the default lag threshold for // classifying scan tasks as low priority. - DefaultOldStartTsScanLowPriorityThreshold = 10 * time.Minute + DefaultOldStartTsScanLowPriorityThreshold = 10 * time.Minute defaultLogPullerMemoryQuota uint64 = 1024 * 1024 * 1024 - defaultLogPullerScanBaseSize uint64 = 8 * 1024 * 1024 + defaultLogPullerScanBaseSize uint64 = 8 * 1024 * 1024 ) // DebugConfig represents config for ticdc unexposed feature configurations From b1c67777c0b1d8d1b9e845cdabc8a8a7b0777f8d Mon Sep 17 00:00:00 2001 From: lidezhu Date: Tue, 4 Aug 2026 21:31:48 +0800 Subject: [PATCH 56/71] more fix --- logservice/logpuller/memory_quota.go | 24 +++--- logservice/logpuller/memory_quota_test.go | 76 +++++++++++-------- logservice/logpuller/priority_task.go | 14 ---- .../region_admission_controller_test.go | 17 +++-- logservice/logpuller/region_event_handler.go | 17 +++-- .../logpuller/region_event_handler_test.go | 47 ++++++++++-- .../region_request_scheduler_test.go | 10 +-- .../logpuller/region_request_store_test.go | 12 +-- .../logpuller/region_request_worker_test.go | 18 ++--- logservice/logpuller/span_registry.go | 12 --- .../logpuller/subscription_client_test.go | 34 +++++++-- pkg/metrics/log_puller.go | 8 -- 12 files changed, 165 insertions(+), 124 deletions(-) diff --git a/logservice/logpuller/memory_quota.go b/logservice/logpuller/memory_quota.go index a499c1ab4e..29b28874fa 100644 --- a/logservice/logpuller/memory_quota.go +++ b/logservice/logpuller/memory_quota.go @@ -22,12 +22,10 @@ import ( "time" "github.com/pingcap/ticdc/pkg/metrics" + "github.com/tikv/client-go/v2/oracle" ) const ( - // Admission ratios compare max(accounted event memory, estimated scan - // memory) with the soft capacity. - // defaultPauseWarmingRatio pauses new high-lag scans when memory pressure // reaches 15% of the soft capacity. defaultPauseWarmingRatio = 0.15 @@ -288,16 +286,11 @@ func (c *memoryQuotaController) UpdateMetrics() { c.scanMu.Lock() used := c.used.Load() scanUsed := c.scanUsed - level := c.level - scanEstimate := c.scanEstimate c.scanMu.Unlock() metrics.LogPullerMemoryQuota.WithLabelValues("max").Set(float64(c.capacity)) metrics.LogPullerMemoryQuota.WithLabelValues("used").Set(float64(used)) metrics.LogPullerMemoryQuota.WithLabelValues("scan_used").Set(float64(scanUsed)) - metrics.LogPullerMemoryQuota.WithLabelValues("scan_estimate").Set(float64(scanEstimate)) - metrics.LogPullerMemoryQuota.WithLabelValues("hard_limit").Set(float64(c.hardLimit)) - metrics.LogPullerMemoryQuotaAdmissionLevel.Set(float64(level)) metrics.LogPullerMemoryQuotaEventWaiterCount.Set( float64(c.eventNotifier.waiters.Load())) } @@ -345,12 +338,17 @@ func scanLagFactor(startTs, currentTs uint64) float64 { 1+defaultScanLagWeight*math.Log2(1+float64(lag)/float64(defaultScanLagUnit))) } -func isWarmingScan(region regionInfo, currentTs uint64) bool { - span := region.subscribedSpan - if span.initialized.Load() { - return false +func regionScanLag(currentTs, checkpointTs uint64) time.Duration { + currentTime := oracle.GetTimeFromTS(currentTs) + checkpointTime := oracle.GetTimeFromTS(checkpointTs) + if !currentTime.After(checkpointTime) { + return 0 } - return regionScanLag(currentTs, region.resolvedTs()) >= lowLagRegionThreshold + return currentTime.Sub(checkpointTime) +} + +func isWarmingScan(region regionInfo, _ uint64) bool { + return !isHighScanPriority(region.scanPriority) } func (c *memoryQuotaController) refreshLevelLocked() { diff --git a/logservice/logpuller/memory_quota_test.go b/logservice/logpuller/memory_quota_test.go index de409721a3..a808ec5be1 100644 --- a/logservice/logpuller/memory_quota_test.go +++ b/logservice/logpuller/memory_quota_test.go @@ -19,6 +19,7 @@ import ( "testing" "time" + "github.com/pingcap/kvproto/pkg/cdcpb" "github.com/pingcap/ticdc/logservice/logpuller/regionlock" "github.com/pingcap/ticdc/pkg/metrics" "github.com/prometheus/client_golang/prometheus/testutil" @@ -57,6 +58,15 @@ func newTestQuotaRegion(span *subscribedSpan) regionInfo { } } +func newTestQuotaRegionWithPriority( + span *subscribedSpan, + priority cdcpb.ScanPriority, +) regionInfo { + region := newTestQuotaRegion(span) + region.scanPriority = priority + return region +} + func setTestQuotaSpanLag(span *subscribedSpan, lag time.Duration) uint64 { now := time.Now() span.resolvedTs.Store(oracle.GoTimeToTS(now.Add(-lag))) @@ -71,7 +81,6 @@ func TestMemoryQuotaUpdateMetrics(t *testing.T) { quota.scanMu.Lock() quota.scanUsed = 7 - quota.level = admissionPauseWarming quota.scanMu.Unlock() quota.eventNotifier.waiters.Store(2) t.Cleanup(func() { quota.eventNotifier.waiters.Store(0) }) @@ -84,12 +93,6 @@ func TestMemoryQuotaUpdateMetrics(t *testing.T) { metrics.LogPullerMemoryQuota.WithLabelValues("used"))) require.Equal(t, float64(7), testutil.ToFloat64( metrics.LogPullerMemoryQuota.WithLabelValues("scan_used"))) - require.Equal(t, float64(8), testutil.ToFloat64( - metrics.LogPullerMemoryQuota.WithLabelValues("scan_estimate"))) - require.Equal(t, float64(132), testutil.ToFloat64( - metrics.LogPullerMemoryQuota.WithLabelValues("hard_limit"))) - require.Equal(t, float64(admissionPauseWarming), - testutil.ToFloat64(metrics.LogPullerMemoryQuotaAdmissionLevel)) require.Equal(t, float64(2), testutil.ToFloat64(metrics.LogPullerMemoryQuotaEventWaiterCount)) } @@ -97,25 +100,31 @@ func TestMemoryQuotaUpdateMetrics(t *testing.T) { func TestMemoryQuotaAdmissionLevels(t *testing.T) { quota := newMemoryQuotaController(100, 10) warmingSpan := newTestQuotaSpan(1) - initializedSpan := newTestQuotaSpan(2) - initializedSpan.initialized.Store(true) - warmingTs := setTestQuotaSpanLag(warmingSpan, lowLagRegionThreshold+time.Minute) - initializedTs := setTestQuotaSpanLag(initializedSpan, lowLagRegionThreshold+time.Minute) - - require.True(t, quota.AcquireEvent(context.Background(), initializedSpan, 5)) - require.True(t, quota.AcquireEvent(context.Background(), initializedSpan, 10)) - _, _, admitted := quota.AcquireScan(newTestQuotaRegion(warmingSpan), warmingTs) + highPrioritySpan := newTestQuotaSpan(2) + warmingTs := setTestQuotaSpanLag(warmingSpan, time.Hour) + highPriorityTs := setTestQuotaSpanLag(highPrioritySpan, time.Hour) + + require.True(t, quota.AcquireEvent(context.Background(), highPrioritySpan, 5)) + require.True(t, quota.AcquireEvent(context.Background(), highPrioritySpan, 10)) + _, _, admitted := quota.AcquireScan( + newTestQuotaRegionWithPriority(warmingSpan, cdcpb.ScanPriority_SCAN_PRIORITY_LOW), + warmingTs, + ) require.False(t, admitted) scanBytes, _, admitted := quota.AcquireScan( - newTestQuotaRegion(initializedSpan), initializedTs) + newTestQuotaRegionWithPriority(highPrioritySpan, cdcpb.ScanPriority_SCAN_PRIORITY_HIGH), + highPriorityTs, + ) require.True(t, admitted) quota.ReleaseScan(scanBytes) - require.True(t, quota.AcquireEvent(context.Background(), initializedSpan, 45)) - require.True(t, quota.AcquireEvent(context.Background(), initializedSpan, 20)) + require.True(t, quota.AcquireEvent(context.Background(), highPrioritySpan, 45)) + require.True(t, quota.AcquireEvent(context.Background(), highPrioritySpan, 20)) scanBytes, _, admitted = quota.AcquireScan( - newTestQuotaRegion(initializedSpan), initializedTs) + newTestQuotaRegionWithPriority(highPrioritySpan, cdcpb.ScanPriority_SCAN_PRIORITY_HIGH), + highPriorityTs, + ) require.True(t, admitted) quota.ReleaseScan(scanBytes) @@ -139,7 +148,9 @@ func TestMemoryQuotaSpanStopKeepsOwnedMemoryUntilRelease(t *testing.T) { require.True(t, quota.AcquireEvent(context.Background(), span1, 30)) require.True(t, quota.AcquireEvent(context.Background(), span2, 40)) scanBytes, _, admitted := quota.AcquireScan( - newTestQuotaRegion(span1), span1.resolvedTs.Load()) + newTestQuotaRegionWithPriority(span1, cdcpb.ScanPriority_SCAN_PRIORITY_HIGH), + span1.resolvedTs.Load(), + ) require.True(t, admitted) require.NotZero(t, scanBytes) @@ -157,7 +168,9 @@ func TestMemoryQuotaSpanStopKeepsOwnedMemoryUntilRelease(t *testing.T) { // Late tasks reach the stopped-subscription cleanup path without consuming // scan quota. scanBytes, _, admitted = quota.AcquireScan( - newTestQuotaRegion(span1), span1.resolvedTs.Load()) + newTestQuotaRegionWithPriority(span1, cdcpb.ScanPriority_SCAN_PRIORITY_HIGH), + span1.resolvedTs.Load(), + ) require.True(t, admitted) require.Zero(t, scanBytes) @@ -288,8 +301,8 @@ func TestMemoryQuotaConcurrentWaitersDoNotLoseWakeups(t *testing.T) { func TestMemoryQuotaWarmingScanUsesCurrentPressure(t *testing.T) { quota := newMemoryQuotaController(100, 20) span := newTestQuotaSpan(1) - currentTs := setTestQuotaSpanLag(span, lowLagRegionThreshold+time.Minute) - region := newTestQuotaRegion(span) + currentTs := setTestQuotaSpanLag(span, time.Hour) + region := newTestQuotaRegionWithPriority(span, cdcpb.ScanPriority_SCAN_PRIORITY_LOW) bytes1, _, admitted := quota.AcquireScan(region, currentTs) require.True(t, admitted) @@ -309,10 +322,13 @@ func TestMemoryQuotaWarmingScanUsesCurrentPressure(t *testing.T) { func TestMemoryQuotaLowLagScanBypassesWarmingGate(t *testing.T) { quota := newMemoryQuotaController(100, 10) span := newTestQuotaSpan(1) - currentTs := setTestQuotaSpanLag(span, lowLagRegionThreshold-time.Second) + currentTs := setTestQuotaSpanLag(span, time.Minute) require.True(t, quota.AcquireEvent(context.Background(), span, 20)) - scanBytes, _, admitted := quota.AcquireScan(newTestQuotaRegion(span), currentTs) + scanBytes, _, admitted := quota.AcquireScan( + newTestQuotaRegionWithPriority(span, cdcpb.ScanPriority_SCAN_PRIORITY_HIGH), + currentTs, + ) require.True(t, admitted) require.NotZero(t, scanBytes) state := getMemoryQuotaTestState(quota) @@ -325,14 +341,14 @@ func TestMemoryQuotaLowLagScanBypassesWarmingGate(t *testing.T) { func TestAdmissionWaitsForMemoryAndReleasesScanMemory(t *testing.T) { quota := newMemoryQuotaController(100, 10) span := newTestQuotaSpan(1) - currentTs := setTestQuotaSpanLag(span, lowLagRegionThreshold+time.Minute) + currentTs := setTestQuotaSpanLag(span, time.Hour) controller := newRegionAdmissionController(1, 1, quota, func() uint64 { return currentTs }) require.True(t, quota.AcquireEvent(context.Background(), span, 20)) - region := newTestQuotaRegion(span) - require.True(t, controller.submit(newRegionPriorityTask(region, currentTs, 1))) + region := newTestQuotaRegionWithPriority(span, cdcpb.ScanPriority_SCAN_PRIORITY_LOW) + require.True(t, controller.submit(newRegionPriorityTask(region, 1))) type popResult struct { req *regionReq @@ -368,14 +384,14 @@ func TestAdmissionWaitsForMemoryAndReleasesScanMemory(t *testing.T) { func TestAdmissionWakesWhenBlockedSpanStops(t *testing.T) { quota := newMemoryQuotaController(100, 10) span := newTestQuotaSpan(1) - currentTs := setTestQuotaSpanLag(span, lowLagRegionThreshold+time.Minute) + currentTs := setTestQuotaSpanLag(span, time.Hour) controller := newRegionAdmissionController(1, 1, quota, func() uint64 { return currentTs }) require.True(t, quota.AcquireEvent(context.Background(), span, 20)) require.True(t, controller.submit(newRegionPriorityTask( - newTestQuotaRegion(span), currentTs, 1))) + newTestQuotaRegionWithPriority(span, cdcpb.ScanPriority_SCAN_PRIORITY_LOW), 1))) type popResult struct { req *regionReq diff --git a/logservice/logpuller/priority_task.go b/logservice/logpuller/priority_task.go index aad3aab16b..d5a5b034e4 100644 --- a/logservice/logpuller/priority_task.go +++ b/logservice/logpuller/priority_task.go @@ -14,14 +14,9 @@ package logpuller import ( - "time" - "github.com/pingcap/kvproto/pkg/cdcpb" - "github.com/tikv/client-go/v2/oracle" ) -const lowLagRegionThreshold = 30 * time.Minute - func normalizeScanPriority(priority cdcpb.ScanPriority) cdcpb.ScanPriority { if priority == cdcpb.ScanPriority_SCAN_PRIORITY_HIGH { return cdcpb.ScanPriority_SCAN_PRIORITY_HIGH @@ -75,12 +70,3 @@ func (pt *regionPriorityTask) LessThan(other *regionPriorityTask) bool { } return pt.sequence < other.sequence } - -func regionScanLag(currentTs, checkpointTs uint64) time.Duration { - currentTime := oracle.GetTimeFromTS(currentTs) - checkpointTime := oracle.GetTimeFromTS(checkpointTs) - if !currentTime.After(checkpointTime) { - return 0 - } - return currentTime.Sub(checkpointTime) -} diff --git a/logservice/logpuller/region_admission_controller_test.go b/logservice/logpuller/region_admission_controller_test.go index 770d849100..226a1148cb 100644 --- a/logservice/logpuller/region_admission_controller_test.go +++ b/logservice/logpuller/region_admission_controller_test.go @@ -20,6 +20,7 @@ import ( "testing" "time" + "github.com/pingcap/kvproto/pkg/cdcpb" "github.com/pingcap/ticdc/heartbeatpb" "github.com/pingcap/ticdc/logservice/logpuller/regionlock" "github.com/stretchr/testify/require" @@ -38,6 +39,7 @@ func createTestRegionInfo(subID SubscriptionID, regionID uint64) regionInfo { span, nil, &subscribedSpan{subID: subID, startTs: 100, span: span}, + false, ) } @@ -54,7 +56,7 @@ func submitRegionForAdmission( currentTs uint64, ) { t.Helper() - task := newRegionPriorityTask(region, currentTs, region.verID.GetID()) + task := newRegionPriorityTask(region, region.verID.GetID()) require.True(t, controller.submit(task)) } @@ -132,11 +134,10 @@ func TestRegionAdmissionControllerLowLagUsesMaxWindow(t *testing.T) { require.True(t, req3.abort()) } -func TestRegionAdmissionControllerPrioritizesInitializedRegion(t *testing.T) { +func TestRegionAdmissionControllerPrioritizesHighPriorityRegion(t *testing.T) { controller := newTestRegionAdmissionController(1, 2) currentTs := oracle.GoTimeToTS(time.Now()) slowCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Hour)) - lowLagCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Minute)) submitRegionForAdmission(t, controller, prepareRegionForAdmission(createTestRegionInfo(1, 1), slowCheckpointTs), @@ -145,12 +146,12 @@ func TestRegionAdmissionControllerPrioritizesInitializedRegion(t *testing.T) { require.NoError(t, err) submitRegionForAdmission(t, controller, - prepareRegionForAdmission(createTestRegionInfo(1, 2), lowLagCheckpointTs), + prepareRegionForAdmission(createTestRegionInfo(1, 2), slowCheckpointTs), currentTs) - initializedRegion := prepareRegionForAdmission(createTestRegionInfo(1, 3), slowCheckpointTs) - initializedRegion.wasInitialized = true + highPriorityRegion := prepareRegionForAdmission(createTestRegionInfo(1, 3), slowCheckpointTs) + highPriorityRegion.scanPriority = cdcpb.ScanPriority_SCAN_PRIORITY_HIGH submitRegionForAdmission(t, controller, - initializedRegion, currentTs) + highPriorityRegion, currentTs) req2, err := controller.pop(t.Context(), nil) require.NoError(t, err) @@ -204,7 +205,7 @@ func TestRegionAdmissionControllerClose(t *testing.T) { controller := newTestRegionAdmissionController(1, 1) controller.close() region := prepareRegionForAdmission(createTestRegionInfo(1, 1), 1) - require.False(t, controller.submit(newRegionPriorityTask(region, 1, 1))) + require.False(t, controller.submit(newRegionPriorityTask(region, 1))) _, err := controller.pop(context.Background(), nil) require.ErrorIs(t, err, context.Canceled) diff --git a/logservice/logpuller/region_event_handler.go b/logservice/logpuller/region_event_handler.go index 704e346505..cd536847de 100644 --- a/logservice/logpuller/region_event_handler.go +++ b/logservice/logpuller/region_event_handler.go @@ -303,7 +303,7 @@ func handleEventEntries(span *subscribedSpan, state *regionFeedState, entries *c for _, entry := range entries.Entries.GetEntries() { switch entry.Type { case cdcpb.Event_INITIALIZED: - state.setInitialized() + span.markRegionInitialized(state) log.Debug("region is initialized", zap.Int64("tableID", span.span.TableID), zap.Uint64("regionID", regionID), @@ -410,7 +410,6 @@ func handleResolvedTs(span *subscribedSpan, state *regionFeedState, resolvedTs u } if shouldAdvance { - span.tryMarkInitialized(regionID, ts) lastResolvedTs := span.resolvedTs.Load() nextResolvedPhyTs := oracle.ExtractPhysical(ts) // Generally, we don't want to send duplicate resolved ts, @@ -418,7 +417,16 @@ func handleResolvedTs(span *subscribedSpan, state *regionFeedState, resolvedTs u // but when `ts` == `lastResolvedTs` == `span.startTs`, // the span may just be initialized and have not receive any resolved ts before, // so we also send ts in this case for quick notification to downstream. - if ts > lastResolvedTs || (ts == lastResolvedTs && lastResolvedTs == span.startTs) { + if ts > lastResolvedTs || + (span.initialized.Load() && ts == lastResolvedTs && lastResolvedTs == span.startTs) { + if lastResolvedTs == span.startTs && ts > span.startTs && !span.initialized.Load() { + log.Warn("should not happen: resolved ts advances before span is initialized", + zap.Uint64("subscriptionID", uint64(span.subID)), + zap.Int64("tableID", span.span.TableID), + zap.Uint64("regionID", regionID), + zap.Uint64("startTs", span.startTs), + zap.Uint64("resolvedTs", ts)) + } resolvedPhyTs := oracle.ExtractPhysical(lastResolvedTs) decreaseLag := float64(nextResolvedPhyTs-resolvedPhyTs) / 1e3 const largeResolvedTsAdvanceStepInSecs = 30 @@ -431,8 +439,7 @@ func handleResolvedTs(span *subscribedSpan, state *regionFeedState, resolvedTs u zap.Uint64("lastResolvedTs", lastResolvedTs), zap.Float64("decreaseLag(s)", decreaseLag)) } - span.resolvedTs.Store(ts) - span.resolvedTsUpdated.Store(time.Now().Unix()) + span.recordResolvedTs(ts) return ts } } diff --git a/logservice/logpuller/region_event_handler_test.go b/logservice/logpuller/region_event_handler_test.go index 7c3db0c2cd..1334acd2b6 100644 --- a/logservice/logpuller/region_event_handler_test.go +++ b/logservice/logpuller/region_event_handler_test.go @@ -88,6 +88,7 @@ func TestHandleEventEntryEventOutOfOrder(t *testing.T) { span, &tikv.RPCContext{}, subSpan, + false, ) region.lockedRangeState = ®ionlock.LockedRangeState{} state := newRegionFeedState(region, 1, worker, nil) @@ -424,13 +425,40 @@ func TestHandleEntriesReleasesMemoryAfterDownstreamCallback(t *testing.T) { require.Zero(t, quotaState.used) } -func TestTryMarkSpanInitializedByResolvedTs(t *testing.T) { - span := &subscribedSpan{subID: 1, startTs: 100} - require.False(t, span.tryMarkInitialized(1, 100)) +func TestSpanInitializedAfterFullRangeCoverage(t *testing.T) { + const startTs = 100 + span := &subscribedSpan{ + subID: 1, + startTs: startTs, + span: heartbeatpb.TableSpan{ + StartKey: []byte("a"), + EndKey: []byte("z"), + }, + } + firstState := newRegionFeedState(regionInfo{ + verID: tikv.NewRegionVerID(1, 1, 1), + span: heartbeatpb.TableSpan{ + StartKey: []byte("a"), + EndKey: []byte("m"), + }, + subscribedSpan: span, + lockedRangeState: ®ionlock.LockedRangeState{}, + }, uint64(span.subID), ®ionRequestWorker{}, nil) + secondState := newRegionFeedState(regionInfo{ + verID: tikv.NewRegionVerID(2, 1, 1), + span: heartbeatpb.TableSpan{ + StartKey: []byte("m"), + EndKey: []byte("z"), + }, + subscribedSpan: span, + lockedRangeState: ®ionlock.LockedRangeState{}, + }, uint64(span.subID), ®ionRequestWorker{}, nil) + + span.markRegionInitialized(firstState) require.False(t, span.initialized.Load()) - require.True(t, span.tryMarkInitialized(1, 101)) + + span.markRegionInitialized(secondState) require.True(t, span.initialized.Load()) - require.False(t, span.tryMarkInitialized(1, 102)) } func TestSpanInitializationNotifiesMemoryAdmission(t *testing.T) { @@ -443,11 +471,11 @@ func TestSpanInitializationNotifiesMemoryAdmission(t *testing.T) { rangeLock := regionlock.NewRangeLock(1, []byte("a"), []byte("z"), startTs) lockResult := rangeLock.LockRange(t.Context(), []byte("a"), []byte("z"), 1, 1) require.Equal(t, regionlock.LockRangeStatusSuccess, lockResult.Status) - lockResult.LockedRangeState.Initialized.Store(true) span := &subscribedSpan{ subID: 1, startTs: startTs, + span: heartbeatpb.TableSpan{StartKey: []byte("a"), EndKey: []byte("z")}, rangeLock: rangeLock, consumeKVEvents: func([]common.RawKVEntry, func()) bool { return false }, advanceResolvedTs: func(uint64) {}, @@ -455,14 +483,17 @@ func TestSpanInitializationNotifiesMemoryAdmission(t *testing.T) { span.resolvedTs.Store(startTs) state := newRegionFeedState(regionInfo{ verID: tikv.NewRegionVerID(1, 1, 1), + span: span.span, subscribedSpan: span, lockedRangeState: lockResult.LockedRangeState, }, uint64(span.subID), ®ionRequestWorker{}, nil) handler := ®ionEventHandler{eventSink: ®ionEventSink{memoryQuota: quota}} require.False(t, handler.Handle(span, regionEvent{ - states: []*regionFeedState{state}, - resolvedTs: startTs + 1, + states: []*regionFeedState{state}, + entries: &cdcpb.Event_Entries_{Entries: &cdcpb.Event_Entries{ + Entries: []*cdcpb.Event_Row{{Type: cdcpb.Event_INITIALIZED}}, + }}, })) require.True(t, span.initialized.Load()) select { diff --git a/logservice/logpuller/region_request_scheduler_test.go b/logservice/logpuller/region_request_scheduler_test.go index 59cc940651..6180d362df 100644 --- a/logservice/logpuller/region_request_scheduler_test.go +++ b/logservice/logpuller/region_request_scheduler_test.go @@ -35,12 +35,12 @@ func TestRegionRequestSchedulerBroadcastDeregisterUsesWorkerControlQueue(t *test worker1 := ®ionRequestWorker{ storeAddr: "store-1", - admission: newRegionAdmissionController(1, 1), + admission: newTestRegionAdmissionController(1, 1), controlQueue: newControlQueue(), } worker2 := ®ionRequestWorker{ storeAddr: "store-2", - admission: newRegionAdmissionController(1, 1), + admission: newTestRegionAdmissionController(1, 1), controlQueue: newControlQueue(), } store1 := ®ionRequestStore{workers: []*regionRequestWorker{worker1}} @@ -72,8 +72,8 @@ func TestRegionRequestSchedulerBroadcastDeregisterUsesWorkerControlQueue(t *test func TestRegionRequestSchedulerInflightCountAggregatesStores(t *testing.T) { scheduler := ®ionRequestScheduler{} - worker1 := ®ionRequestWorker{admission: newRegionAdmissionController(2, 1)} - worker2 := ®ionRequestWorker{admission: newRegionAdmissionController(2, 1)} + worker1 := ®ionRequestWorker{admission: newTestRegionAdmissionController(2, 1)} + worker2 := ®ionRequestWorker{admission: newTestRegionAdmissionController(2, 1)} scheduler.stores.Store("store-1", ®ionRequestStore{workers: []*regionRequestWorker{worker1}}) scheduler.stores.Store("store-2", ®ionRequestStore{workers: []*regionRequestWorker{worker2}}) @@ -121,7 +121,7 @@ func TestRegionRequestSchedulerReschedulesRegionWhenStoreSubmitFails(t *testing. context.Background(), rawSpan.StartKey, rawSpan.EndKey, location.Region.GetID(), location.Region.GetVer()) require.Equal(t, regionlock.LockRangeStatusSuccess, lockRes.Status) - admission := newRegionAdmissionController(1, 1) + admission := newTestRegionAdmissionController(1, 1) admission.close() store := ®ionRequestStore{workers: []*regionRequestWorker{{admission: admission}}} diff --git a/logservice/logpuller/region_request_store_test.go b/logservice/logpuller/region_request_store_test.go index c555a9441e..a433d47d23 100644 --- a/logservice/logpuller/region_request_store_test.go +++ b/logservice/logpuller/region_request_store_test.go @@ -22,8 +22,8 @@ import ( ) func TestRegionRequestStoreDistributesRegionsAcrossWorkers(t *testing.T) { - worker1 := ®ionRequestWorker{admission: newRegionAdmissionController(1, 1)} - worker2 := ®ionRequestWorker{admission: newRegionAdmissionController(1, 1)} + worker1 := ®ionRequestWorker{admission: newTestRegionAdmissionController(1, 1)} + worker2 := ®ionRequestWorker{admission: newTestRegionAdmissionController(1, 1)} store := ®ionRequestStore{ workers: []*regionRequestWorker{worker1, worker2}, } @@ -39,8 +39,8 @@ func TestRegionRequestStoreDistributesRegionsAcrossWorkers(t *testing.T) { } func TestRegionRequestStoreInflightCountAggregatesWorkers(t *testing.T) { - worker1 := ®ionRequestWorker{admission: newRegionAdmissionController(2, 1)} - worker2 := ®ionRequestWorker{admission: newRegionAdmissionController(2, 1)} + worker1 := ®ionRequestWorker{admission: newTestRegionAdmissionController(2, 1)} + worker2 := ®ionRequestWorker{admission: newTestRegionAdmissionController(2, 1)} store := ®ionRequestStore{ workers: []*regionRequestWorker{worker1, worker2}, } @@ -55,8 +55,8 @@ func TestRegionRequestStoreInflightCountAggregatesWorkers(t *testing.T) { } func TestRegionRequestStoreCloseClosesWorkerAdmissions(t *testing.T) { - worker1 := ®ionRequestWorker{admission: newRegionAdmissionController(1, 1)} - worker2 := ®ionRequestWorker{admission: newRegionAdmissionController(1, 1)} + worker1 := ®ionRequestWorker{admission: newTestRegionAdmissionController(1, 1)} + worker2 := ®ionRequestWorker{admission: newTestRegionAdmissionController(1, 1)} store := ®ionRequestStore{ workers: []*regionRequestWorker{worker1, worker2}, } diff --git a/logservice/logpuller/region_request_worker_test.go b/logservice/logpuller/region_request_worker_test.go index 8735f3021c..59da548746 100644 --- a/logservice/logpuller/region_request_worker_test.go +++ b/logservice/logpuller/region_request_worker_test.go @@ -207,7 +207,7 @@ func errCacheLen(handler *regionFailureHandler) int { } func TestRegionRequestWorkerIgnoresDuplicateActiveRegion(t *testing.T) { - admission := newRegionAdmissionController(10, 1) + admission := newTestRegionAdmissionController(10, 1) worker := ®ionRequestWorker{ admission: admission, storeAddr: "store-1", @@ -366,7 +366,7 @@ func benchmarkDispatchResolvedTsEvent(b *testing.B, regionCount int, useLegacy b } func TestWaitForRegionRequestDrainsIdleControlQueue(t *testing.T) { - admission := newRegionAdmissionController(1, 1) + admission := newTestRegionAdmissionController(1, 1) worker := ®ionRequestWorker{ admission: admission, controlQueue: newControlQueue(), @@ -448,7 +448,7 @@ func BenchmarkDispatchResolvedTsEventSmallBatchCurrent(b *testing.B) { } func TestStoppedStateRemovesSentRequest(t *testing.T) { - admission := newRegionAdmissionController(10, 1) + admission := newTestRegionAdmissionController(10, 1) worker := ®ionRequestWorker{ admission: admission, tracker: newRegionTracker(), @@ -474,7 +474,7 @@ func TestRunStreamFailurePushesTrackedRegionToEventSink(t *testing.T) { upstream: &upstreamHandle{pd: pdClient, credential: &security.Credential{}}, eventSink: ®ionEventSink{ds: ds}, failureHandler: handler, - admission: newRegionAdmissionController(10, 1), + admission: newTestRegionAdmissionController(10, 1), controlQueue: newControlQueue(), tracker: newRegionTracker(), storeAddr: "127.0.0.1:1", @@ -522,7 +522,7 @@ func TestRunStreamFailureReportsPendingRegionsToFailureHandler(t *testing.T) { upstream: &upstreamHandle{pd: pdClient, credential: &security.Credential{}}, eventSink: ®ionEventSink{ds: &mockDynamicStream{}}, failureHandler: handler, - admission: newRegionAdmissionController(10, 1), + admission: newTestRegionAdmissionController(10, 1), controlQueue: newControlQueue(), tracker: newRegionTracker(), storeAddr: "127.0.0.1:1", @@ -551,7 +551,7 @@ func TestRunStreamFailureReportsPendingRegionsToFailureHandler(t *testing.T) { } func TestProcessRegionSendTaskSendFailureCleansSentRequest(t *testing.T) { - admission := newRegionAdmissionController(10, 1) + admission := newTestRegionAdmissionController(10, 1) worker := ®ionRequestWorker{ admission: admission, controlQueue: newControlQueue(), @@ -582,7 +582,7 @@ func TestProcessRegionSendTaskSendFailureCleansSentRequest(t *testing.T) { } func TestProcessRegionSendTaskDoesNotSendRemovedRequest(t *testing.T) { - admission := newRegionAdmissionController(1, 1) + admission := newTestRegionAdmissionController(1, 1) worker := ®ionRequestWorker{ admission: admission, controlQueue: newControlQueue(), @@ -630,7 +630,7 @@ func TestProcessRegionSendTaskSendEOFIsRetriable(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - admission := newRegionAdmissionController(10, 1) + admission := newTestRegionAdmissionController(10, 1) worker := ®ionRequestWorker{ admission: admission, controlQueue: newControlQueue(), @@ -665,7 +665,7 @@ func TestProcessRegionSendTaskSendEOFIsRetriable(t *testing.T) { func TestProcessRegionSendTaskHandlesDeregisterFromControlQueue(t *testing.T) { ds := &mockRegionEventDynamicStream{} worker := ®ionRequestWorker{ - admission: newRegionAdmissionController(1, 1), + admission: newTestRegionAdmissionController(1, 1), controlQueue: newControlQueue(), storeAddr: "store-1", upstream: &upstreamHandle{clusterID: 42}, diff --git a/logservice/logpuller/span_registry.go b/logservice/logpuller/span_registry.go index 5c8f70fba8..797e42d833 100644 --- a/logservice/logpuller/span_registry.go +++ b/logservice/logpuller/span_registry.go @@ -190,18 +190,6 @@ func (span *subscribedSpan) resolveStaleLocks(targetTs uint64) { zap.Any("ranges", res)) } -func (span *subscribedSpan) tryMarkInitialized(regionID, resolvedTs uint64) bool { - if resolvedTs <= span.startTs || !span.initialized.CompareAndSwap(false, true) { - return false - } - log.Info("subscription client is initialized", - zap.Uint64("subscriptionID", uint64(span.subID)), - zap.Uint64("regionID", regionID), - zap.Uint64("resolvedTs", resolvedTs), - zap.Uint64("startTs", span.startTs)) - return true -} - func newSpanRegistry(pd pd.Client, pdClock pdutil.Clock) *spanRegistry { return &spanRegistry{ spans: make(map[SubscriptionID]*subscribedSpan), diff --git a/logservice/logpuller/subscription_client_test.go b/logservice/logpuller/subscription_client_test.go index 99600666ea..c7e4591f92 100644 --- a/logservice/logpuller/subscription_client_test.go +++ b/logservice/logpuller/subscription_client_test.go @@ -424,27 +424,49 @@ func (s *mockDynamicStream) GetMetrics() dynstream.Metrics[int, SubscriptionID] } func TestRegionEventSinkPushUnblocksOnClientClose(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + quota := newMemoryQuotaController(10, 8) + span := &subscribedSpan{subID: 1} + require.True(t, quota.AcquireEvent(ctx, span, 20)) + t.Cleanup(func() { quota.ReleaseEvent(20) }) + sink := ®ionEventSink{ - ds: &mockDynamicStream{}, + ctx: ctx, + ds: &mockDynamicStream{}, + memoryQuota: quota, } - sink.cond = sync.NewCond(&sink.mu) client := &subscriptionClient{eventSink: sink} client.regionScheduler = ®ionRequestScheduler{ taskQueue: priorityqueue.New[*regionPriorityTask](), } - client.ctx, client.cancel = context.WithCancel(context.Background()) + client.ctx = ctx + client.cancel = cancel - sink.paused.Store(true) + event := regionEvent{ + states: []*regionFeedState{{ + region: regionInfo{subscribedSpan: span}, + }}, + entries: &cdcpb.Event_Entries_{ + Entries: &cdcpb.Event_Entries{ + Entries: []*cdcpb.Event_Row{{ + Key: []byte("key"), + Value: []byte("value"), + }}, + }, + }, + } done := make(chan struct{}) go func() { - sink.Push(SubscriptionID(1), regionEvent{}) + sink.Push(SubscriptionID(1), event) close(done) }() select { case <-done: - t.Fatal("pushRegionEventToDS should block when paused") + t.Fatal("pushRegionEventToDS should block when event memory is exhausted") case <-time.After(100 * time.Millisecond): } diff --git a/pkg/metrics/log_puller.go b/pkg/metrics/log_puller.go index 6b4e04cac2..ec99e11c78 100644 --- a/pkg/metrics/log_puller.go +++ b/pkg/metrics/log_puller.go @@ -71,13 +71,6 @@ var ( Name: "memory_quota", Help: "The log puller local memory quota usage.", }, []string{"type"}) - LogPullerMemoryQuotaAdmissionLevel = prometheus.NewGauge( - prometheus.GaugeOpts{ - Namespace: "ticdc", - Subsystem: "log_puller", - Name: "memory_quota_admission_level", - Help: "The log puller scan admission level: 0 normal, 1 pause warming scans, 2 pause all scans.", - }) LogPullerMemoryQuotaEventWaiterCount = prometheus.NewGauge( prometheus.GaugeOpts{ Namespace: "ticdc", @@ -194,7 +187,6 @@ func initLogPullerMetrics(registry *prometheus.Registry) { registry.MustRegister(LogPullerMatcherCount) registry.MustRegister(LogPullerResolvedTsLag) registry.MustRegister(LogPullerMemoryQuota) - registry.MustRegister(LogPullerMemoryQuotaAdmissionLevel) registry.MustRegister(LogPullerMemoryQuotaEventWaiterCount) registry.MustRegister(LogPullerMemoryQuotaEventWaitDuration) registry.MustRegister(SubscriptionClientRequestedRegionCount) From 43ccf97ed4fa18724ea7b311ccdac282a06ee2be Mon Sep 17 00:00:00 2001 From: lidezhu Date: Tue, 4 Aug 2026 21:42:53 +0800 Subject: [PATCH 57/71] small fix --- logservice/logpuller/region_event_handler.go | 1 - logservice/logpuller/region_request_worker.go | 5 ++++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/logservice/logpuller/region_event_handler.go b/logservice/logpuller/region_event_handler.go index cd536847de..0ae78d097c 100644 --- a/logservice/logpuller/region_event_handler.go +++ b/logservice/logpuller/region_event_handler.go @@ -29,7 +29,6 @@ import ( ) var ( - metricsResolvedTsCount = metrics.PullerEventCounter.WithLabelValues("resolved_ts") metricsEventCount = metrics.PullerEventCounter.WithLabelValues("event") metricRegionEventHandleDurationEntries = metrics.SubscriptionClientRegionEventHandleDuration.WithLabelValues("entries") diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index ac3a35b787..baf388d107 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -39,7 +39,10 @@ const storeReconnectBackoff = time.Second // To generate a workerID in `newRegionRequestWorker`. var workerIDGen atomic.Uint64 -var metricBatchResolvedSize = metrics.BatchResolvedEventSize.WithLabelValues("event-store") +var ( + metricsResolvedTsCount = metrics.PullerEventCounter.WithLabelValues("resolved_ts") + metricBatchResolvedSize = metrics.BatchResolvedEventSize.WithLabelValues("event-store") +) type deregisterRequest struct { subID SubscriptionID From e625a55fbc228e3e062d4b7f5dd0d2b7b2c60b6e Mon Sep 17 00:00:00 2001 From: lidezhu Date: Tue, 4 Aug 2026 22:05:59 +0800 Subject: [PATCH 58/71] more fix --- logservice/logpuller/memory_quota_test.go | 13 ++--- .../logpuller/region_admission_controller.go | 11 +++-- .../region_admission_controller_test.go | 4 +- logservice/logpuller/region_event_handler.go | 6 +-- .../logpuller/region_event_handler_test.go | 47 ++----------------- logservice/logpuller/region_request_worker.go | 4 +- 6 files changed, 24 insertions(+), 61 deletions(-) diff --git a/logservice/logpuller/memory_quota_test.go b/logservice/logpuller/memory_quota_test.go index a808ec5be1..1fe8226e51 100644 --- a/logservice/logpuller/memory_quota_test.go +++ b/logservice/logpuller/memory_quota_test.go @@ -22,6 +22,7 @@ import ( "github.com/pingcap/kvproto/pkg/cdcpb" "github.com/pingcap/ticdc/logservice/logpuller/regionlock" "github.com/pingcap/ticdc/pkg/metrics" + "github.com/pingcap/ticdc/pkg/pdutil" "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/require" "github.com/tikv/client-go/v2/oracle" @@ -342,9 +343,9 @@ func TestAdmissionWaitsForMemoryAndReleasesScanMemory(t *testing.T) { quota := newMemoryQuotaController(100, 10) span := newTestQuotaSpan(1) currentTs := setTestQuotaSpanLag(span, time.Hour) - controller := newRegionAdmissionController(1, 1, quota, func() uint64 { - return currentTs - }) + clock := pdutil.NewClock4Test().(*pdutil.Clock4Test) + clock.SetTS(currentTs) + controller := newRegionAdmissionController(1, 1, quota, clock) require.True(t, quota.AcquireEvent(context.Background(), span, 20)) region := newTestQuotaRegionWithPriority(span, cdcpb.ScanPriority_SCAN_PRIORITY_LOW) @@ -385,9 +386,9 @@ func TestAdmissionWakesWhenBlockedSpanStops(t *testing.T) { quota := newMemoryQuotaController(100, 10) span := newTestQuotaSpan(1) currentTs := setTestQuotaSpanLag(span, time.Hour) - controller := newRegionAdmissionController(1, 1, quota, func() uint64 { - return currentTs - }) + clock := pdutil.NewClock4Test().(*pdutil.Clock4Test) + clock.SetTS(currentTs) + controller := newRegionAdmissionController(1, 1, quota, clock) require.True(t, quota.AcquireEvent(context.Background(), span, 20)) require.True(t, controller.submit(newRegionPriorityTask( diff --git a/logservice/logpuller/region_admission_controller.go b/logservice/logpuller/region_admission_controller.go index 749946d4f6..4003f93fd3 100644 --- a/logservice/logpuller/region_admission_controller.go +++ b/logservice/logpuller/region_admission_controller.go @@ -22,6 +22,7 @@ import ( "github.com/pingcap/log" "github.com/pingcap/ticdc/pkg/metrics" + "github.com/pingcap/ticdc/pkg/pdutil" "github.com/pingcap/ticdc/utils/heap" "go.uber.org/zap" ) @@ -95,9 +96,9 @@ type regionAdmissionController struct { // It is guarded by mu. pending *heap.Heap[*regionPriorityTask] // memoryQuota gates initial scans using the log puller's global memory - // pressure. currentTs is sampled when a request is admitted. + // pressure. pdClock is sampled when a request is admitted. memoryQuota *memoryQuotaController - currentTs func() uint64 + pdClock pdutil.Clock // notify wakes workers when a request is submitted or an admission slot is // released. The one-element buffer prevents a wakeup from being lost between // checking the admission condition and waiting on this channel. Notifications @@ -117,7 +118,7 @@ func newRegionAdmissionController( currentWindow int, maxWindowMultiplier int, memoryQuota *memoryQuotaController, - currentTs func() uint64, + pdClock pdutil.Clock, ) *regionAdmissionController { if currentWindow <= 0 { currentWindow = 1 @@ -134,7 +135,7 @@ func newRegionAdmissionController( maxWindow: maxWindow, pending: heap.NewHeap[*regionPriorityTask](), memoryQuota: memoryQuota, - currentTs: currentTs, + pdClock: pdClock, notify: make(chan struct{}, 1), } } @@ -201,7 +202,7 @@ func (c *regionAdmissionController) popEligibleLocked() ( } scanBytes, memoryReady, admitted := c.memoryQuota.AcquireScan( - request.regionInfo, c.currentTs()) + request.regionInfo, c.pdClock.CurrentTS()) if !admitted { return nil, 0, memoryReady } diff --git a/logservice/logpuller/region_admission_controller_test.go b/logservice/logpuller/region_admission_controller_test.go index 226a1148cb..485b527c63 100644 --- a/logservice/logpuller/region_admission_controller_test.go +++ b/logservice/logpuller/region_admission_controller_test.go @@ -23,6 +23,7 @@ import ( "github.com/pingcap/kvproto/pkg/cdcpb" "github.com/pingcap/ticdc/heartbeatpb" "github.com/pingcap/ticdc/logservice/logpuller/regionlock" + "github.com/pingcap/ticdc/pkg/pdutil" "github.com/stretchr/testify/require" "github.com/tikv/client-go/v2/oracle" "github.com/tikv/client-go/v2/tikv" @@ -64,11 +65,12 @@ func newTestRegionAdmissionController( currentWindow int, maxWindowMultiplier int, ) *regionAdmissionController { + clock := pdutil.NewClock4Test() return newRegionAdmissionController( currentWindow, maxWindowMultiplier, newMemoryQuotaController(1024*1024*1024, 8*1024*1024), - func() uint64 { return 0 }, + clock, ) } diff --git a/logservice/logpuller/region_event_handler.go b/logservice/logpuller/region_event_handler.go index 0ae78d097c..359e915675 100644 --- a/logservice/logpuller/region_event_handler.go +++ b/logservice/logpuller/region_event_handler.go @@ -29,7 +29,7 @@ import ( ) var ( - metricsEventCount = metrics.PullerEventCounter.WithLabelValues("event") + metricsEventCount = metrics.PullerEventCounter.WithLabelValues("event") metricRegionEventHandleDurationEntries = metrics.SubscriptionClientRegionEventHandleDuration.WithLabelValues("entries") metricRegionEventHandleDurationResolved = metrics.SubscriptionClientRegionEventHandleDuration.WithLabelValues("resolved") @@ -125,7 +125,6 @@ func (h *regionEventHandler) Handle(span *subscribedSpan, events ...regionEvent) } newResolvedTs := uint64(0) - wasInitialized := span.initialized.Load() memoryBytes := uint64(0) for _, event := range events { memoryBytes += event.memoryBytes @@ -149,9 +148,6 @@ func (h *regionEventHandler) Handle(span *subscribedSpan, events ...regionEvent) log.Panic("should not reach", zap.Any("event", event), zap.Any("events", events)) } } - if !wasInitialized && span.initialized.Load() { - h.eventSink.memoryQuota.NotifyScanAdmission() - } tryAdvanceResolvedTs := func() { if newResolvedTs != 0 { span.advanceResolvedTs(newResolvedTs) diff --git a/logservice/logpuller/region_event_handler_test.go b/logservice/logpuller/region_event_handler_test.go index 1334acd2b6..38a2b50f2e 100644 --- a/logservice/logpuller/region_event_handler_test.go +++ b/logservice/logpuller/region_event_handler_test.go @@ -23,6 +23,7 @@ import ( "github.com/pingcap/ticdc/heartbeatpb" "github.com/pingcap/ticdc/logservice/logpuller/regionlock" "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/pdutil" "github.com/pingcap/ticdc/utils/dynstream" "github.com/stretchr/testify/require" "github.com/tikv/client-go/v2/tikv" @@ -238,6 +239,7 @@ func TestHandleResolvedTs(t *testing.T) { consumeKVEvents: consumeKVEvents, advanceResolvedTs: advanceResolvedTs, advanceInterval: 0, + priorityPolicy: newScanPriorityPolicy(pdutil.NewClock4Test(), 30*time.Minute), } ds.AddPath(subID1, subSpan, dynstream.AreaSettings{}) state1.region.subscribedSpan = subSpan @@ -261,6 +263,7 @@ func TestHandleResolvedTs(t *testing.T) { consumeKVEvents: consumeKVEvents, advanceResolvedTs: advanceResolvedTs, advanceInterval: 0, + priorityPolicy: newScanPriorityPolicy(pdutil.NewClock4Test(), 30*time.Minute), } ds.AddPath(subID2, subSpan, dynstream.AreaSettings{}) state2.region.subscribedSpan = subSpan @@ -284,6 +287,7 @@ func TestHandleResolvedTs(t *testing.T) { consumeKVEvents: consumeKVEvents, advanceResolvedTs: advanceResolvedTs, advanceInterval: 0, + priorityPolicy: newScanPriorityPolicy(pdutil.NewClock4Test(), 30*time.Minute), } ds.AddPath(subID3, subSpan, dynstream.AreaSettings{}) state3.region.subscribedSpan = subSpan @@ -361,6 +365,7 @@ func TestHandleResolvedTsThrottled(t *testing.T) { subID: SubscriptionID(1), rangeLock: l, advanceInterval: 100, + priorityPolicy: newScanPriorityPolicy(pdutil.NewClock4Test(), 30*time.Minute), } span.lastAdvanceTime.Store(0) worker := ®ionRequestWorker{tracker: newRegionTracker()} @@ -460,45 +465,3 @@ func TestSpanInitializedAfterFullRangeCoverage(t *testing.T) { span.markRegionInitialized(secondState) require.True(t, span.initialized.Load()) } - -func TestSpanInitializationNotifiesMemoryAdmission(t *testing.T) { - quota := newMemoryQuotaController(1024, 8) - quota.scanMu.Lock() - notified := quota.scanReady - quota.scanMu.Unlock() - - const startTs = 100 - rangeLock := regionlock.NewRangeLock(1, []byte("a"), []byte("z"), startTs) - lockResult := rangeLock.LockRange(t.Context(), []byte("a"), []byte("z"), 1, 1) - require.Equal(t, regionlock.LockRangeStatusSuccess, lockResult.Status) - - span := &subscribedSpan{ - subID: 1, - startTs: startTs, - span: heartbeatpb.TableSpan{StartKey: []byte("a"), EndKey: []byte("z")}, - rangeLock: rangeLock, - consumeKVEvents: func([]common.RawKVEntry, func()) bool { return false }, - advanceResolvedTs: func(uint64) {}, - } - span.resolvedTs.Store(startTs) - state := newRegionFeedState(regionInfo{ - verID: tikv.NewRegionVerID(1, 1, 1), - span: span.span, - subscribedSpan: span, - lockedRangeState: lockResult.LockedRangeState, - }, uint64(span.subID), ®ionRequestWorker{}, nil) - handler := ®ionEventHandler{eventSink: ®ionEventSink{memoryQuota: quota}} - - require.False(t, handler.Handle(span, regionEvent{ - states: []*regionFeedState{state}, - entries: &cdcpb.Event_Entries_{Entries: &cdcpb.Event_Entries{ - Entries: []*cdcpb.Event_Row{{Type: cdcpb.Event_INITIALIZED}}, - }}, - })) - require.True(t, span.initialized.Load()) - select { - case <-notified: - case <-time.After(time.Second): - t.Fatal("span initialization did not notify memory admission") - } -} diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index baf388d107..3f954168d0 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -40,7 +40,7 @@ const storeReconnectBackoff = time.Second var workerIDGen atomic.Uint64 var ( - metricsResolvedTsCount = metrics.PullerEventCounter.WithLabelValues("resolved_ts") + metricsResolvedTsCount = metrics.PullerEventCounter.WithLabelValues("resolved_ts") metricBatchResolvedSize = metrics.BatchResolvedEventSize.WithLabelValues("event-store") ) @@ -125,7 +125,7 @@ func newRegionRequestWorker( currentWindow, maxWindowMultiplier, memoryQuota, - upstream.pdClock.CurrentTS, + upstream.pdClock, ), controlQueue: newControlQueue(), tracker: newRegionTracker(), From bbd6b4c0a3e106f95e8ba87a3fc4a0a0176dee45 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Tue, 4 Aug 2026 22:15:28 +0800 Subject: [PATCH 59/71] more refactor --- logservice/logpuller/memory_quota.go | 44 +++++++++++------------ logservice/logpuller/memory_quota_test.go | 20 +++++------ 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/logservice/logpuller/memory_quota.go b/logservice/logpuller/memory_quota.go index 29b28874fa..88dcf170f5 100644 --- a/logservice/logpuller/memory_quota.go +++ b/logservice/logpuller/memory_quota.go @@ -26,13 +26,13 @@ import ( ) const ( - // defaultPauseWarmingRatio pauses new high-lag scans when memory pressure - // reaches 15% of the soft capacity. - defaultPauseWarmingRatio = 0.15 + // defaultPauseLowPriorityRatio pauses new low-priority scans when memory + // pressure reaches 15% of the soft capacity. + defaultPauseLowPriorityRatio = 0.15 - // defaultResumeWarmingRatio resumes high-lag scans after memory pressure - // falls to 5% of the soft capacity. - defaultResumeWarmingRatio = 0.05 + // defaultResumeLowPriorityRatio resumes low-priority scans after memory + // pressure falls to 5% of the soft capacity. + defaultResumeLowPriorityRatio = 0.05 // defaultHardLimitRatio blocks receiving more events when accounted event // memory reaches twice the soft capacity. @@ -52,7 +52,7 @@ type admissionLevel uint8 const ( admissionNormal admissionLevel = iota - admissionPauseWarming + admissionPauseLowPriority ) // eventMemoryNotifier wakes event receivers that are waiting for memory. Each @@ -129,9 +129,9 @@ func (n *eventMemoryNotifier) notify() { // Initial scans are charged by estimate instead of measured bytes. Each // admitted scan starts from scanBaseSize, grows logarithmically with scan lag, // and is capped at maxScanLagFactor times the base size. Scan admission -// compares max(event used, scan used) with the soft capacity: high-lag -// "warming" scans pause at pauseWarmingLimit and resume at -// resumeWarmingLimit, while lower-lag scans continue to make progress. +// compares max(event used, scan used) with the soft capacity: low-priority +// scans pause at pauseLowPriorityLimit and resume at +// resumeLowPriorityLimit, while high-priority scans continue to make progress. type memoryQuotaController struct { capacity uint64 // used tracks event bytes retained until downstream finishes consuming them. @@ -154,8 +154,8 @@ type memoryQuotaController struct { // synchronous broadcast to every store and request worker. scanReady chan struct{} - pauseWarmingLimit uint64 - resumeWarmingLimit uint64 + pauseLowPriorityLimit uint64 + resumeLowPriorityLimit uint64 hardLimit uint64 scanEstimate uint64 @@ -165,8 +165,8 @@ func newMemoryQuotaController(capacity, scanBaseSize uint64) *memoryQuotaControl c := &memoryQuotaController{ capacity: capacity, level: admissionNormal, - pauseWarmingLimit: uint64(math.Ceil(float64(capacity) * defaultPauseWarmingRatio)), - resumeWarmingLimit: uint64(float64(capacity) * defaultResumeWarmingRatio), + pauseLowPriorityLimit: uint64(math.Ceil(float64(capacity) * defaultPauseLowPriorityRatio)), + resumeLowPriorityLimit: uint64(float64(capacity) * defaultResumeLowPriorityRatio), hardLimit: uint64(float64(capacity) * defaultHardLimitRatio), scanEstimate: scanBaseSize, eventNotifier: newEventMemoryNotifier(), @@ -196,10 +196,10 @@ func (c *memoryQuotaController) AcquireScan( c.scanMu.Lock() defer c.scanMu.Unlock() c.refreshLevelLocked() - warming := isWarmingScan(region, currentTs) + lowPriority := isLowPriorityScan(region, currentTs) // Admission is based on the pressure before accounting this scan. This lets // one scan make progress even when its estimate alone exceeds the threshold. - if warming && c.level == admissionPauseWarming { + if lowPriority && c.level == admissionPauseLowPriority { return 0, c.scanReady, false } bytes = c.estimateScanSizeLocked(region, currentTs) @@ -268,7 +268,7 @@ func (c *memoryQuotaController) ReleaseEvent(bytes uint64) { } used := c.used.Add(^(bytes - 1)) previousUsed := used + bytes - if crossesDown(previousUsed, used, c.resumeWarmingLimit) { + if crossesDown(previousUsed, used, c.resumeLowPriorityLimit) { c.refreshAdmissionAndNotify() } c.eventNotifier.notify() @@ -347,7 +347,7 @@ func regionScanLag(currentTs, checkpointTs uint64) time.Duration { return currentTime.Sub(checkpointTime) } -func isWarmingScan(region regionInfo, _ uint64) bool { +func isLowPriorityScan(region regionInfo, _ uint64) bool { return !isHighScanPriority(region.scanPriority) } @@ -356,13 +356,13 @@ func (c *memoryQuotaController) refreshLevelLocked() { // it to actual event bytes would count the same pressure twice. pressure := max(c.used.Load(), c.scanUsed) switch c.level { - case admissionPauseWarming: - if pressure <= c.resumeWarmingLimit { + case admissionPauseLowPriority: + if pressure <= c.resumeLowPriorityLimit { c.level = admissionNormal } default: - if pressure >= c.pauseWarmingLimit { - c.level = admissionPauseWarming + if pressure >= c.pauseLowPriorityLimit { + c.level = admissionPauseLowPriority } } } diff --git a/logservice/logpuller/memory_quota_test.go b/logservice/logpuller/memory_quota_test.go index 1fe8226e51..4f43ccf565 100644 --- a/logservice/logpuller/memory_quota_test.go +++ b/logservice/logpuller/memory_quota_test.go @@ -100,16 +100,16 @@ func TestMemoryQuotaUpdateMetrics(t *testing.T) { func TestMemoryQuotaAdmissionLevels(t *testing.T) { quota := newMemoryQuotaController(100, 10) - warmingSpan := newTestQuotaSpan(1) + lowPrioritySpan := newTestQuotaSpan(1) highPrioritySpan := newTestQuotaSpan(2) - warmingTs := setTestQuotaSpanLag(warmingSpan, time.Hour) + lowPriorityTs := setTestQuotaSpanLag(lowPrioritySpan, time.Hour) highPriorityTs := setTestQuotaSpanLag(highPrioritySpan, time.Hour) require.True(t, quota.AcquireEvent(context.Background(), highPrioritySpan, 5)) require.True(t, quota.AcquireEvent(context.Background(), highPrioritySpan, 10)) _, _, admitted := quota.AcquireScan( - newTestQuotaRegionWithPriority(warmingSpan, cdcpb.ScanPriority_SCAN_PRIORITY_LOW), - warmingTs, + newTestQuotaRegionWithPriority(lowPrioritySpan, cdcpb.ScanPriority_SCAN_PRIORITY_LOW), + lowPriorityTs, ) require.False(t, admitted) @@ -131,10 +131,10 @@ func TestMemoryQuotaAdmissionLevels(t *testing.T) { quota.ReleaseEvent(20) state := getMemoryQuotaTestState(quota) - require.Equal(t, admissionPauseWarming, state.level) + require.Equal(t, admissionPauseLowPriority, state.level) quota.ReleaseEvent(45) state = getMemoryQuotaTestState(quota) - require.Equal(t, admissionPauseWarming, state.level) + require.Equal(t, admissionPauseLowPriority, state.level) quota.ReleaseEvent(10) state = getMemoryQuotaTestState(quota) require.Equal(t, admissionNormal, state.level) @@ -299,7 +299,7 @@ func TestMemoryQuotaConcurrentWaitersDoNotLoseWakeups(t *testing.T) { require.Zero(t, state.used) } -func TestMemoryQuotaWarmingScanUsesCurrentPressure(t *testing.T) { +func TestMemoryQuotaLowPriorityScanUsesCurrentPressure(t *testing.T) { quota := newMemoryQuotaController(100, 20) span := newTestQuotaSpan(1) currentTs := setTestQuotaSpanLag(span, time.Hour) @@ -309,7 +309,7 @@ func TestMemoryQuotaWarmingScanUsesCurrentPressure(t *testing.T) { require.True(t, admitted) require.NotZero(t, bytes1) state := getMemoryQuotaTestState(quota) - require.Greater(t, state.scanUsed, quota.pauseWarmingLimit) + require.Greater(t, state.scanUsed, quota.pauseLowPriorityLimit) _, _, admitted = quota.AcquireScan(region, currentTs) require.False(t, admitted) @@ -362,7 +362,7 @@ func TestAdmissionWaitsForMemoryAndReleasesScanMemory(t *testing.T) { }() select { case <-result: - t.Fatal("warming scan should wait while memory is under pressure") + t.Fatal("low-priority scan should wait while memory is under pressure") case <-time.After(100 * time.Millisecond): } @@ -405,7 +405,7 @@ func TestAdmissionWakesWhenBlockedSpanStops(t *testing.T) { }() select { case <-result: - t.Fatal("warming scan should wait while memory is under pressure") + t.Fatal("low-priority scan should wait while memory is under pressure") case <-time.After(100 * time.Millisecond): } From 3b421bba16b10913373e10f7d743b8566ff9dd6a Mon Sep 17 00:00:00 2001 From: lidezhu Date: Wed, 5 Aug 2026 09:32:55 +0800 Subject: [PATCH 60/71] small fix --- logservice/logpuller/priority_task.go | 4 +--- .../logpuller/region_admission_controller.go | 16 ++++++++-------- .../region_admission_controller_test.go | 12 ++++++------ 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/logservice/logpuller/priority_task.go b/logservice/logpuller/priority_task.go index d5a5b034e4..38cd813c8b 100644 --- a/logservice/logpuller/priority_task.go +++ b/logservice/logpuller/priority_task.go @@ -13,9 +13,7 @@ package logpuller -import ( - "github.com/pingcap/kvproto/pkg/cdcpb" -) +import "github.com/pingcap/kvproto/pkg/cdcpb" func normalizeScanPriority(priority cdcpb.ScanPriority) cdcpb.ScanPriority { if priority == cdcpb.ScanPriority_SCAN_PRIORITY_HIGH { diff --git a/logservice/logpuller/region_admission_controller.go b/logservice/logpuller/region_admission_controller.go index 4003f93fd3..443e159c3a 100644 --- a/logservice/logpuller/region_admission_controller.go +++ b/logservice/logpuller/region_admission_controller.go @@ -74,8 +74,7 @@ func (r *regionReq) release() bool { if !r.released.CompareAndSwap(false, true) { return false } - r.controller.memoryQuota.ReleaseScan(r.scanBytes) - r.controller.release() + r.controller.release(r.scanBytes) return true } @@ -96,7 +95,7 @@ type regionAdmissionController struct { // It is guarded by mu. pending *heap.Heap[*regionPriorityTask] // memoryQuota gates initial scans using the log puller's global memory - // pressure. pdClock is sampled when a request is admitted. + // pressure. pdClock provides the current TS used for scan estimation. memoryQuota *memoryQuotaController pdClock pdutil.Clock // notify wakes workers when a request is submitted or an admission slot is @@ -152,11 +151,11 @@ func (c *regionAdmissionController) submit(task *regionPriorityTask) bool { return true } -// pop waits for an eligible request. If controlReady is signaled first, it -// returns nil without an error so the worker can handle its control queue. +// pop waits for an eligible request. If interrupt is signaled first, it +// returns nil without an error so the worker can handle the interrupt source. func (c *regionAdmissionController) pop( ctx context.Context, - controlReady <-chan struct{}, + interrupt <-chan struct{}, ) (*regionReq, error) { for { c.mu.Lock() @@ -180,7 +179,7 @@ func (c *regionAdmissionController) pop( select { case <-c.notify: case <-memoryReady: - case <-controlReady: + case <-interrupt: return nil, nil case <-ctx.Done(): return nil, ctx.Err() @@ -217,7 +216,8 @@ func (c *regionAdmissionController) windowFor(request *regionPriorityTask) int { return c.currentWindow } -func (c *regionAdmissionController) release() { +func (c *regionAdmissionController) release(scanBytes uint64) { + c.memoryQuota.ReleaseScan(scanBytes) c.mu.Lock() if c.inflight > 0 { c.inflight-- diff --git a/logservice/logpuller/region_admission_controller_test.go b/logservice/logpuller/region_admission_controller_test.go index 485b527c63..c1c393916b 100644 --- a/logservice/logpuller/region_admission_controller_test.go +++ b/logservice/logpuller/region_admission_controller_test.go @@ -86,9 +86,9 @@ func TestRegionAdmissionControllerNormalWindow(t *testing.T) { req1, err := controller.pop(t.Context(), nil) require.NoError(t, err) require.Equal(t, 1, controller.stats().inflight) - controlReady := make(chan struct{}) - close(controlReady) - req2, err := controller.pop(t.Context(), controlReady) + interrupt := make(chan struct{}) + close(interrupt) + req2, err := controller.pop(t.Context(), interrupt) require.Nil(t, req2) require.NoError(t, err) @@ -121,9 +121,9 @@ func TestRegionAdmissionControllerLowLagUsesMaxWindow(t *testing.T) { req2, err := controller.pop(t.Context(), nil) require.NoError(t, err) require.Equal(t, uint64(3), req2.regionInfo.verID.GetID()) - controlReady := make(chan struct{}) - close(controlReady) - req3, err := controller.pop(t.Context(), controlReady) + interrupt := make(chan struct{}) + close(interrupt) + req3, err := controller.pop(t.Context(), interrupt) require.Nil(t, req3) require.NoError(t, err) require.Equal(t, 2, controller.stats().inflight) From 7482dc779c20ec4ad73d8a009853a296ed4094f3 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Wed, 5 Aug 2026 10:32:24 +0800 Subject: [PATCH 61/71] more fix and metrics --- logservice/logpuller/memory_quota.go | 17 ++++--- logservice/logpuller/memory_quota_test.go | 4 ++ .../logpuller/region_admission_controller.go | 20 +++++++++ logservice/logpuller/region_event_handler.go | 3 ++ metrics/grafana/ticdc_new_arch.json | 45 +++++++++++-------- .../ticdc_new_arch_next_gen.json | 45 +++++++++++-------- pkg/metrics/log_puller.go | 17 +++++++ 7 files changed, 106 insertions(+), 45 deletions(-) diff --git a/logservice/logpuller/memory_quota.go b/logservice/logpuller/memory_quota.go index 88dcf170f5..fbc0bc6c86 100644 --- a/logservice/logpuller/memory_quota.go +++ b/logservice/logpuller/memory_quota.go @@ -141,6 +141,7 @@ type memoryQuotaController struct { // eventNotifier owns the wait protocol used after the hard limit is reached. eventNotifier *eventMemoryNotifier + scanWaiters atomic.Int64 // scanMu guards scan admission state and scanReady. Scan admission happens // once per region rather than once per event batch, so it is intentionally @@ -156,21 +157,21 @@ type memoryQuotaController struct { pauseLowPriorityLimit uint64 resumeLowPriorityLimit uint64 - hardLimit uint64 + hardLimit uint64 scanEstimate uint64 } func newMemoryQuotaController(capacity, scanBaseSize uint64) *memoryQuotaController { c := &memoryQuotaController{ - capacity: capacity, - level: admissionNormal, + capacity: capacity, + level: admissionNormal, pauseLowPriorityLimit: uint64(math.Ceil(float64(capacity) * defaultPauseLowPriorityRatio)), resumeLowPriorityLimit: uint64(float64(capacity) * defaultResumeLowPriorityRatio), - hardLimit: uint64(float64(capacity) * defaultHardLimitRatio), - scanEstimate: scanBaseSize, - eventNotifier: newEventMemoryNotifier(), - scanReady: make(chan struct{}), + hardLimit: uint64(float64(capacity) * defaultHardLimitRatio), + scanEstimate: scanBaseSize, + eventNotifier: newEventMemoryNotifier(), + scanReady: make(chan struct{}), } return c } @@ -293,6 +294,8 @@ func (c *memoryQuotaController) UpdateMetrics() { metrics.LogPullerMemoryQuota.WithLabelValues("scan_used").Set(float64(scanUsed)) metrics.LogPullerMemoryQuotaEventWaiterCount.Set( float64(c.eventNotifier.waiters.Load())) + metrics.LogPullerMemoryQuotaScanWaiterCount.Set( + float64(c.scanWaiters.Load())) } func (c *memoryQuotaController) notifyScanAdmissionLocked() { diff --git a/logservice/logpuller/memory_quota_test.go b/logservice/logpuller/memory_quota_test.go index 4f43ccf565..41525f7b1e 100644 --- a/logservice/logpuller/memory_quota_test.go +++ b/logservice/logpuller/memory_quota_test.go @@ -85,6 +85,8 @@ func TestMemoryQuotaUpdateMetrics(t *testing.T) { quota.scanMu.Unlock() quota.eventNotifier.waiters.Store(2) t.Cleanup(func() { quota.eventNotifier.waiters.Store(0) }) + quota.scanWaiters.Store(3) + t.Cleanup(func() { quota.scanWaiters.Store(0) }) quota.UpdateMetrics() @@ -96,6 +98,8 @@ func TestMemoryQuotaUpdateMetrics(t *testing.T) { metrics.LogPullerMemoryQuota.WithLabelValues("scan_used"))) require.Equal(t, float64(2), testutil.ToFloat64(metrics.LogPullerMemoryQuotaEventWaiterCount)) + require.Equal(t, float64(3), + testutil.ToFloat64(metrics.LogPullerMemoryQuotaScanWaiterCount)) } func TestMemoryQuotaAdmissionLevels(t *testing.T) { diff --git a/logservice/logpuller/region_admission_controller.go b/logservice/logpuller/region_admission_controller.go index 443e159c3a..7ebf3d1ff1 100644 --- a/logservice/logpuller/region_admission_controller.go +++ b/logservice/logpuller/region_admission_controller.go @@ -176,14 +176,34 @@ func (c *regionAdmissionController) pop( } c.mu.Unlock() + waitingForMemory := memoryReady != nil + if waitingForMemory { + c.memoryQuota.scanWaiters.Add(1) + } + waitStart := time.Time{} + if waitingForMemory { + waitStart = time.Now() + } select { case <-c.notify: case <-memoryReady: case <-interrupt: + if waitingForMemory { + c.memoryQuota.scanWaiters.Add(-1) + metrics.LogPullerMemoryQuotaScanWaitDuration.Observe(time.Since(waitStart).Seconds()) + } return nil, nil case <-ctx.Done(): + if waitingForMemory { + c.memoryQuota.scanWaiters.Add(-1) + metrics.LogPullerMemoryQuotaScanWaitDuration.Observe(time.Since(waitStart).Seconds()) + } return nil, ctx.Err() } + if waitingForMemory { + c.memoryQuota.scanWaiters.Add(-1) + metrics.LogPullerMemoryQuotaScanWaitDuration.Observe(time.Since(waitStart).Seconds()) + } } } diff --git a/logservice/logpuller/region_event_handler.go b/logservice/logpuller/region_event_handler.go index 359e915675..c8024b7020 100644 --- a/logservice/logpuller/region_event_handler.go +++ b/logservice/logpuller/region_event_handler.go @@ -65,6 +65,9 @@ func (event *regionEvent) needsMemoryAccounting() bool { } func (event *regionEvent) getSize() int { + if event == nil { + return 0 + } size := int(unsafe.Sizeof(*event)) if event.entries != nil { size += int(unsafe.Sizeof(*event.entries)) diff --git a/metrics/grafana/ticdc_new_arch.json b/metrics/grafana/ticdc_new_arch.json index 52cda78859..c92a6c9381 100644 --- a/metrics/grafana/ticdc_new_arch.json +++ b/metrics/grafana/ticdc_new_arch.json @@ -8822,7 +8822,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Scan admission level (0: normal, 1: pause warming scans, 2: pause all scans) and event receivers currently blocked at the memory hard limit.", + "description": "Event receivers blocked at the memory hard limit and region scans blocked at the scan admission gate.", "fieldConfig": { "defaults": {}, "overrides": [] @@ -8858,28 +8858,23 @@ "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/event-waiters$/", - "yaxis": 2 - } - ], + "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, - "expr": "ticdc_log_puller_memory_quota_admission_level{k8s_cluster=~\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}", + "expr": "ticdc_log_puller_memory_quota_event_waiter_count{k8s_cluster=~\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}", "interval": "", - "legendFormat": "{{instance}}-admission-level", + "legendFormat": "{{instance}}-event-waiters", "refId": "A" }, { "exemplar": true, - "expr": "ticdc_log_puller_memory_quota_event_waiter_count{k8s_cluster=~\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}", + "expr": "ticdc_log_puller_memory_quota_scan_waiter_count{k8s_cluster=~\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}", "interval": "", - "legendFormat": "{{instance}}-event-waiters", + "legendFormat": "{{instance}}-scan-waiters", "refId": "B" } ], @@ -8887,7 +8882,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Memory Quota Throttling", + "title": "Memory Quota Waiters", "tooltip": { "shared": true, "sort": 0, @@ -8905,15 +8900,13 @@ { "format": "short", "logBase": 1, - "max": "2", "min": "0", "show": true }, { "format": "short", "logBase": 1, - "min": "0", - "show": true + "show": false } ], "yaxis": { @@ -8926,7 +8919,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Time spent waiting to receive more events after log puller memory reaches the hard limit.", + "description": "Time spent waiting at the event hard limit or the scan admission gate.", "fieldConfig": { "defaults": {}, "overrides": [] @@ -8971,22 +8964,36 @@ "exemplar": true, "expr": "histogram_quantile(0.99, sum(rate(ticdc_log_puller_memory_quota_event_wait_duration_bucket{k8s_cluster=~\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", "interval": "", - "legendFormat": "{{instance}}-p99", + "legendFormat": "{{instance}}-event-p99", "refId": "A" }, { "exemplar": true, "expr": "sum(rate(ticdc_log_puller_memory_quota_event_wait_duration_sum{k8s_cluster=~\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance) / sum(rate(ticdc_log_puller_memory_quota_event_wait_duration_count{k8s_cluster=~\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", "interval": "", - "legendFormat": "{{instance}}-avg", + "legendFormat": "{{instance}}-event-avg", "refId": "B" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_log_puller_memory_quota_scan_wait_duration_bucket{k8s_cluster=~\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", + "interval": "", + "legendFormat": "{{instance}}-scan-p99", + "refId": "C" + }, + { + "exemplar": true, + "expr": "sum(rate(ticdc_log_puller_memory_quota_scan_wait_duration_sum{k8s_cluster=~\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance) / sum(rate(ticdc_log_puller_memory_quota_scan_wait_duration_count{k8s_cluster=~\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", + "interval": "", + "legendFormat": "{{instance}}-scan-avg", + "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Memory Quota Event Wait Duration", + "title": "Memory Quota Wait Duration", "tooltip": { "shared": true, "sort": 0, diff --git a/metrics/nextgengrafana/ticdc_new_arch_next_gen.json b/metrics/nextgengrafana/ticdc_new_arch_next_gen.json index b8452acca1..b684f1d915 100644 --- a/metrics/nextgengrafana/ticdc_new_arch_next_gen.json +++ b/metrics/nextgengrafana/ticdc_new_arch_next_gen.json @@ -8822,7 +8822,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Scan admission level (0: normal, 1: pause warming scans, 2: pause all scans) and event receivers currently blocked at the memory hard limit.", + "description": "Event receivers blocked at the memory hard limit and region scans blocked at the scan admission gate.", "fieldConfig": { "defaults": {}, "overrides": [] @@ -8858,28 +8858,23 @@ "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/event-waiters$/", - "yaxis": 2 - } - ], + "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, - "expr": "ticdc_log_puller_memory_quota_admission_level{k8s_cluster=~\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}", + "expr": "ticdc_log_puller_memory_quota_event_waiter_count{k8s_cluster=~\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}", "interval": "", - "legendFormat": "{{instance}}-admission-level", + "legendFormat": "{{instance}}-event-waiters", "refId": "A" }, { "exemplar": true, - "expr": "ticdc_log_puller_memory_quota_event_waiter_count{k8s_cluster=~\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}", + "expr": "ticdc_log_puller_memory_quota_scan_waiter_count{k8s_cluster=~\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}", "interval": "", - "legendFormat": "{{instance}}-event-waiters", + "legendFormat": "{{instance}}-scan-waiters", "refId": "B" } ], @@ -8887,7 +8882,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Memory Quota Throttling", + "title": "Memory Quota Waiters", "tooltip": { "shared": true, "sort": 0, @@ -8905,15 +8900,13 @@ { "format": "short", "logBase": 1, - "max": "2", "min": "0", "show": true }, { "format": "short", "logBase": 1, - "min": "0", - "show": true + "show": false } ], "yaxis": { @@ -8926,7 +8919,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "Time spent waiting to receive more events after log puller memory reaches the hard limit.", + "description": "Time spent waiting at the event hard limit or the scan admission gate.", "fieldConfig": { "defaults": {}, "overrides": [] @@ -8971,22 +8964,36 @@ "exemplar": true, "expr": "histogram_quantile(0.99, sum(rate(ticdc_log_puller_memory_quota_event_wait_duration_bucket{k8s_cluster=~\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", "interval": "", - "legendFormat": "{{instance}}-p99", + "legendFormat": "{{instance}}-event-p99", "refId": "A" }, { "exemplar": true, "expr": "sum(rate(ticdc_log_puller_memory_quota_event_wait_duration_sum{k8s_cluster=~\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance) / sum(rate(ticdc_log_puller_memory_quota_event_wait_duration_count{k8s_cluster=~\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", "interval": "", - "legendFormat": "{{instance}}-avg", + "legendFormat": "{{instance}}-event-avg", "refId": "B" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_log_puller_memory_quota_scan_wait_duration_bucket{k8s_cluster=~\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", + "interval": "", + "legendFormat": "{{instance}}-scan-p99", + "refId": "C" + }, + { + "exemplar": true, + "expr": "sum(rate(ticdc_log_puller_memory_quota_scan_wait_duration_sum{k8s_cluster=~\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance) / sum(rate(ticdc_log_puller_memory_quota_scan_wait_duration_count{k8s_cluster=~\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", + "interval": "", + "legendFormat": "{{instance}}-scan-avg", + "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Memory Quota Event Wait Duration", + "title": "Memory Quota Wait Duration", "tooltip": { "shared": true, "sort": 0, diff --git a/pkg/metrics/log_puller.go b/pkg/metrics/log_puller.go index ec99e11c78..08f3ecb137 100644 --- a/pkg/metrics/log_puller.go +++ b/pkg/metrics/log_puller.go @@ -86,6 +86,21 @@ var ( Help: "The duration in seconds that an event receiver waits at the log puller memory hard limit.", Buckets: prometheus.ExponentialBuckets(0.001, 2, 24), }) + LogPullerMemoryQuotaScanWaiterCount = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: "ticdc", + Subsystem: "log_puller", + Name: "memory_quota_scan_waiter_count", + Help: "The number of region scans waiting at the log puller memory quota gate.", + }) + LogPullerMemoryQuotaScanWaitDuration = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Namespace: "ticdc", + Subsystem: "log_puller", + Name: "memory_quota_scan_wait_duration", + Help: "The duration in seconds that a region scan waits at the log puller memory quota gate.", + Buckets: prometheus.ExponentialBuckets(0.001, 2, 24), + }) SubscriptionClientResolvedTsLagGauge = prometheus.NewGauge( prometheus.GaugeOpts{ @@ -189,6 +204,8 @@ func initLogPullerMetrics(registry *prometheus.Registry) { registry.MustRegister(LogPullerMemoryQuota) registry.MustRegister(LogPullerMemoryQuotaEventWaiterCount) registry.MustRegister(LogPullerMemoryQuotaEventWaitDuration) + registry.MustRegister(LogPullerMemoryQuotaScanWaiterCount) + registry.MustRegister(LogPullerMemoryQuotaScanWaitDuration) registry.MustRegister(SubscriptionClientRequestedRegionCount) registry.MustRegister(SubscriptionClientAddRegionRequestDuration) registry.MustRegister(RegionRequestFinishScanDuration) From 17631b1626953fe1f87e0122f206747c0cd99aba Mon Sep 17 00:00:00 2001 From: lidezhu Date: Wed, 5 Aug 2026 11:54:38 +0800 Subject: [PATCH 62/71] more fix --- logservice/logpuller/region_event_handler.go | 4 ++-- logservice/logpuller/region_event_sink.go | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/logservice/logpuller/region_event_handler.go b/logservice/logpuller/region_event_handler.go index c8024b7020..49dab3e46c 100644 --- a/logservice/logpuller/region_event_handler.go +++ b/logservice/logpuller/region_event_handler.go @@ -55,8 +55,8 @@ type regionEvent struct { entries *cdcpb.Event_Entries_ resolvedTs uint64 - // memoryBytes is released when this event is dropped or after downstream - // finishes consuming the entries derived from it. + // memoryBytes is released when this event is dropped or when the derived KV + // events no longer need to be retained by the log puller. memoryBytes uint64 } diff --git a/logservice/logpuller/region_event_sink.go b/logservice/logpuller/region_event_sink.go index 0c015db743..64c837f320 100644 --- a/logservice/logpuller/region_event_sink.go +++ b/logservice/logpuller/region_event_sink.go @@ -72,6 +72,8 @@ func (s *regionEventSink) Push(subID SubscriptionID, event regionEvent) { if event.needsMemoryAccounting() { span := event.mustFirstState().region.subscribedSpan event.memoryBytes = uint64(event.getSize()) + // AcquireEvent only returns false after shutdown or when the + // subscription has already been stopped. if !s.memoryQuota.AcquireEvent(s.ctx, span, event.memoryBytes) { return } From 148fc599a842ed58285a3829bf61a451f7d7b85f Mon Sep 17 00:00:00 2001 From: lidezhu Date: Tue, 4 Aug 2026 18:41:16 +0800 Subject: [PATCH 63/71] update compaction --- logservice/eventstore/pebble.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logservice/eventstore/pebble.go b/logservice/eventstore/pebble.go index f5f48bf71b..96caabc3cb 100644 --- a/logservice/eventstore/pebble.go +++ b/logservice/eventstore/pebble.go @@ -42,7 +42,7 @@ func newPebbleOptions(dbNum int) *pebble.Options { MaxOpenFiles: maxOpenFilesPerDB, - MaxConcurrentCompactions: func() int { return 6 }, + MaxConcurrentCompactions: func() int { return 3 }, // Decrease compaction frequency L0CompactionThreshold: 20, From 13ba9afffe377ed6b6aa57af63f51e300bc8b29c Mon Sep 17 00:00:00 2001 From: lidezhu Date: Tue, 4 Aug 2026 20:16:49 +0800 Subject: [PATCH 64/71] fix --- .../region_admission_controller_test.go | 12 +- logservice/logpuller/region_event_handler.go | 7 +- .../logpuller/region_event_handler_test.go | 34 +++++ .../logpuller/region_failure_handler.go | 142 +++++++++++++++++- .../logpuller/region_failure_handler_test.go | 78 ++++++++++ .../logpuller/subscription_client_test.go | 6 +- 6 files changed, 267 insertions(+), 12 deletions(-) diff --git a/logservice/logpuller/region_admission_controller_test.go b/logservice/logpuller/region_admission_controller_test.go index c1c393916b..2633e99681 100644 --- a/logservice/logpuller/region_admission_controller_test.go +++ b/logservice/logpuller/region_admission_controller_test.go @@ -99,11 +99,10 @@ func TestRegionAdmissionControllerNormalWindow(t *testing.T) { require.True(t, req2.abort()) } -func TestRegionAdmissionControllerLowLagUsesMaxWindow(t *testing.T) { +func TestRegionAdmissionControllerHighPriorityUsesMaxWindow(t *testing.T) { controller := newTestRegionAdmissionController(1, 2) currentTs := oracle.GoTimeToTS(time.Now()) slowCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Hour)) - lowLagCheckpointTs := oracle.GoTimeToTS(time.Now().Add(-time.Minute)) submitRegionForAdmission(t, controller, prepareRegionForAdmission(createTestRegionInfo(1, 1), slowCheckpointTs), @@ -114,9 +113,9 @@ func TestRegionAdmissionControllerLowLagUsesMaxWindow(t *testing.T) { submitRegionForAdmission(t, controller, prepareRegionForAdmission(createTestRegionInfo(1, 2), slowCheckpointTs), currentTs) - submitRegionForAdmission(t, controller, - prepareRegionForAdmission(createTestRegionInfo(1, 3), lowLagCheckpointTs), - currentTs) + highPriorityRegion := prepareRegionForAdmission(createTestRegionInfo(1, 3), slowCheckpointTs) + highPriorityRegion.scanPriority = cdcpb.ScanPriority_SCAN_PRIORITY_HIGH + submitRegionForAdmission(t, controller, highPriorityRegion, currentTs) req2, err := controller.pop(t.Context(), nil) require.NoError(t, err) @@ -152,8 +151,7 @@ func TestRegionAdmissionControllerPrioritizesHighPriorityRegion(t *testing.T) { currentTs) highPriorityRegion := prepareRegionForAdmission(createTestRegionInfo(1, 3), slowCheckpointTs) highPriorityRegion.scanPriority = cdcpb.ScanPriority_SCAN_PRIORITY_HIGH - submitRegionForAdmission(t, controller, - highPriorityRegion, currentTs) + submitRegionForAdmission(t, controller, highPriorityRegion, currentTs) req2, err := controller.pop(t.Context(), nil) require.NoError(t, err) diff --git a/logservice/logpuller/region_event_handler.go b/logservice/logpuller/region_event_handler.go index 49dab3e46c..afc8b95630 100644 --- a/logservice/logpuller/region_event_handler.go +++ b/logservice/logpuller/region_event_handler.go @@ -138,7 +138,12 @@ func (h *regionEventHandler) Handle(span *subscribedSpan, events ...regionEvent) } if event.entries != nil { hasEntries = true - handleEventEntries(span, event.mustFirstState(), event.entries) + state := event.mustFirstState() + wasInitialized := state.isInitialized() + handleEventEntries(span, state, event.entries) + if !wasInitialized && state.isInitialized() && h.failureHandler != nil { + h.failureHandler.resetRegionRetry(span.subID, state.getRegionID()) + } } else if event.resolvedTs != 0 { hasResolved = true for _, state := range event.states { diff --git a/logservice/logpuller/region_event_handler_test.go b/logservice/logpuller/region_event_handler_test.go index 38a2b50f2e..01ad6f9264 100644 --- a/logservice/logpuller/region_event_handler_test.go +++ b/logservice/logpuller/region_event_handler_test.go @@ -430,6 +430,40 @@ func TestHandleEntriesReleasesMemoryAfterDownstreamCallback(t *testing.T) { require.Zero(t, quotaState.used) } +func TestRegionEventHandlerInitializedResetsRetryState(t *testing.T) { + span := &subscribedSpan{ + subID: 1, + span: heartbeatpb.TableSpan{TableID: 1}, + advanceResolvedTs: func(uint64) {}, + } + failureHandler := newRegionFailureHandler(nil, func(*subscribedSpan) {}, func(context.Context, regionInfo) {}, func(context.Context, rangeTask) {}) + key := regionRetryKey{subscriptionID: span.subID, regionID: 1} + failureHandler.retries[key] = ®ionRetryState{} + + region := newRegionInfo(tikv.NewRegionVerID(1, 1, 1), span.span, nil, span, false) + region.lockedRangeState = ®ionlock.LockedRangeState{} + region.rpcCtx = &tikv.RPCContext{Addr: "store-1"} + state := newRegionFeedState(region, uint64(span.subID), ®ionRequestWorker{tracker: newRegionTracker()}, nil) + + handler := ®ionEventHandler{ + eventSink: ®ionEventSink{memoryQuota: newMemoryQuotaController(0, 0)}, + failureHandler: failureHandler, + } + handler.Handle(span, regionEvent{ + states: []*regionFeedState{state}, + entries: &cdcpb.Event_Entries_{ + Entries: &cdcpb.Event_Entries{ + Entries: []*cdcpb.Event_Row{{Type: cdcpb.Event_INITIALIZED}}, + }, + }, + }) + + failureHandler.retryMu.Lock() + _, ok := failureHandler.retries[key] + failureHandler.retryMu.Unlock() + require.False(t, ok) +} + func TestSpanInitializedAfterFullRangeCoverage(t *testing.T) { const startTs = 100 span := &subscribedSpan{ diff --git a/logservice/logpuller/region_failure_handler.go b/logservice/logpuller/region_failure_handler.go index 3622d927d9..6f1fc7ed00 100644 --- a/logservice/logpuller/region_failure_handler.go +++ b/logservice/logpuller/region_failure_handler.go @@ -15,6 +15,7 @@ package logpuller import ( "context" + "math/rand/v2" "sync" "time" @@ -43,12 +44,49 @@ var ( type regionFailureHandler struct { cache *errCache regionCache *tikv.RegionCache + retryMu sync.Mutex + retries map[regionRetryKey]*regionRetryState + retryDelay func(uint32) time.Duration onTableDrained func(*subscribedSpan) scheduleRegionRequest func(context.Context, regionInfo) scheduleRangeRequest func(context.Context, rangeTask) } +const ( + regionRetryBaseDelay = 50 * time.Millisecond + regionRetryMaxDelay = 2 * time.Second + regionRetryStateTTL = 5 * time.Minute +) + +type regionRetryKey struct { + subscriptionID SubscriptionID + regionID uint64 +} + +type regionRetryState struct { + attempt uint32 + generation uint64 + pending bool + timer *time.Timer +} + +func notLeaderRetryDelay(attempt uint32) time.Duration { + if attempt == 0 { + attempt = 1 + } + exponent := attempt - 1 + if exponent > 16 { + exponent = 16 + } + delay := regionRetryBaseDelay << exponent + if delay > regionRetryMaxDelay { + delay = regionRetryMaxDelay + } + half := delay / 2 + return half + time.Duration(rand.Int64N(int64(delay-half)+1)) +} + func newRegionFailureHandler( regionCache *tikv.RegionCache, onTableDrained func(*subscribedSpan), @@ -58,12 +96,100 @@ func newRegionFailureHandler( return ®ionFailureHandler{ cache: newErrCache(), regionCache: regionCache, + retries: make(map[regionRetryKey]*regionRetryState), + retryDelay: notLeaderRetryDelay, onTableDrained: onTableDrained, scheduleRegionRequest: scheduleRegionRequest, scheduleRangeRequest: scheduleRangeRequest, } } +func (r *regionFailureHandler) scheduleRegionRetry( + ctx context.Context, + region regionInfo, + retry func(), +) { + if region.subscribedSpan == nil || region.subscribedSpan.stopped.Load() { + return + } + key := regionRetryKey{ + subscriptionID: region.subscribedSpan.subID, + regionID: region.verID.GetID(), + } + + r.retryMu.Lock() + defer r.retryMu.Unlock() + state := r.retries[key] + if state == nil { + state = ®ionRetryState{} + r.retries[key] = state + } + if state.pending { + return + } + if state.timer != nil { + state.timer.Stop() + } + if state.attempt < 32 { + state.attempt++ + } + state.generation++ + generation := state.generation + state.pending = true + delay := r.retryDelay(state.attempt) + state.timer = time.AfterFunc(delay, func() { + r.retryMu.Lock() + current := r.retries[key] + if current == nil || current.generation != generation || !current.pending { + r.retryMu.Unlock() + return + } + current.pending = false + current.timer = time.AfterFunc(regionRetryStateTTL, func() { + r.expireRegionRetry(key, generation) + }) + r.retryMu.Unlock() + + if ctx.Err() != nil || region.subscribedSpan.stopped.Load() { + r.resetRegionRetry(key.subscriptionID, key.regionID) + return + } + retry() + }) +} + +func (r *regionFailureHandler) expireRegionRetry(key regionRetryKey, generation uint64) { + r.retryMu.Lock() + defer r.retryMu.Unlock() + state := r.retries[key] + if state != nil && state.generation == generation && !state.pending { + delete(r.retries, key) + } +} + +func (r *regionFailureHandler) resetRegionRetry(subscriptionID SubscriptionID, regionID uint64) { + key := regionRetryKey{subscriptionID: subscriptionID, regionID: regionID} + r.retryMu.Lock() + defer r.retryMu.Unlock() + if state := r.retries[key]; state != nil { + if state.timer != nil { + state.timer.Stop() + } + delete(r.retries, key) + } +} + +func (r *regionFailureHandler) cancelRegionRetries() { + r.retryMu.Lock() + defer r.retryMu.Unlock() + for key, state := range r.retries { + if state.timer != nil { + state.timer.Stop() + } + delete(r.retries, key) + } +} + // Report admits a region failure into the recovery pipeline. It releases the // corresponding range lock before enqueueing the failure so new range tasks are // not blocked by stale region ownership. @@ -80,6 +206,7 @@ func (r *regionFailureHandler) Report(errInfo regionErrorInfo) { func (r *regionFailureHandler) Run(ctx context.Context) error { log.Info("region failure handler starts") defer log.Info("region failure handler exits") + defer r.cancelRegionRetries() handleCachedErrors := func() error { for { @@ -153,8 +280,16 @@ func (r *regionFailureHandler) handleError(ctx context.Context, errInfo regionEr innerErr := eerr.err if notLeader := innerErr.GetNotLeader(); notLeader != nil { metricFeedNotLeaderCounter.Inc() - r.regionCache.UpdateLeader(errInfo.verID, notLeader.GetLeader(), errInfo.rpcCtx.AccessIdx) - r.scheduleRegionRequest(ctx, errInfo.regionInfo) + leader := notLeader.GetLeader() + if leader == nil || leader.GetId() == 0 || leader.GetStoreId() == 0 || errInfo.rpcCtx == nil { + r.regionCache.InvalidateCachedRegion(errInfo.verID) + r.scheduleRegionRetry(ctx, errInfo.regionInfo, rescheduleRange) + return nil + } + r.regionCache.UpdateLeader(errInfo.verID, leader, errInfo.rpcCtx.AccessIdx) + r.scheduleRegionRetry(ctx, errInfo.regionInfo, func() { + r.scheduleRegionRequest(ctx, errInfo.regionInfo) + }) return nil } if innerErr.GetEpochNotMatch() != nil { @@ -214,6 +349,9 @@ func (r *regionFailureHandler) handleError(ctx context.Context, errInfo regionEr return nil case *requestCancelledErr: // the corresponding subscription has been unsubscribed, just ignore. + if errInfo.subscribedSpan != nil { + r.resetRegionRetry(errInfo.subscribedSpan.subID, errInfo.verID.GetID()) + } return nil default: // TODO(qupeng): for some errors it's better to just deregister the region from TiKVs. diff --git a/logservice/logpuller/region_failure_handler_test.go b/logservice/logpuller/region_failure_handler_test.go index f47fca3509..6b34192de3 100644 --- a/logservice/logpuller/region_failure_handler_test.go +++ b/logservice/logpuller/region_failure_handler_test.go @@ -19,7 +19,11 @@ import ( "time" "github.com/pingcap/errors" + "github.com/pingcap/kvproto/pkg/cdcpb" + "github.com/pingcap/kvproto/pkg/errorpb" "github.com/pingcap/ticdc/heartbeatpb" + "github.com/pingcap/ticdc/pkg/pdutil" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/tikv/client-go/v2/tikv" ) @@ -142,3 +146,77 @@ func TestRegionFailureHandlerRunDrainsErrCacheWithoutDispatcher(t *testing.T) { t.Fatal("failure handler did not exit after context cancellation") } } + +func TestRegionFailureHandlerDelaysNotLeaderRangeRetry(t *testing.T) { + pdClient := newFailureRecoveryTestPDClient(t) + defer pdClient.Close() + + regionCache := tikv.NewRegionCache(pdClient) + defer regionCache.Close() + + region := createFailureRecoveryTestRegion(t, SubscriptionID(1), 1) + region.subscribedSpan.priorityPolicy = newScanPriorityPolicy(pdutil.NewClock4Test(), 30*time.Minute) + + rangeRetryCh := make(chan rangeTask, 2) + handler := newRegionFailureHandler( + regionCache, + func(*subscribedSpan) {}, + func(context.Context, regionInfo) { + t.Fatal("unexpected region retry") + }, + func(_ context.Context, task rangeTask) { + rangeRetryCh <- task + }, + ) + handler.retryDelay = func(uint32) time.Duration { return 50 * time.Millisecond } + + errInfo := newRegionErrorInfo(region, &eventError{ + err: &cdcpb.Error{NotLeader: &errorpb.NotLeader{}}, + }) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + require.NoError(t, handler.handleError(ctx, errInfo)) + require.NoError(t, handler.handleError(ctx, errInfo)) + + select { + case <-rangeRetryCh: + t.Fatal("not leader retry should be delayed") + case <-time.After(20 * time.Millisecond): + } + + select { + case task := <-rangeRetryCh: + require.Equal(t, region.span, task.span) + require.Same(t, region.subscribedSpan, task.subscribedSpan) + case <-time.After(time.Second): + t.Fatal("not leader retry was not scheduled") + } + + select { + case <-rangeRetryCh: + t.Fatal("pending not leader retry should be deduplicated") + case <-time.After(20 * time.Millisecond): + } +} + +func TestRegionFailureHandlerRequestCancelledResetsRetryState(t *testing.T) { + handler := newRegionFailureHandler(nil, func(*subscribedSpan) {}, func(context.Context, regionInfo) {}, func(context.Context, rangeTask) {}) + region := createFailureRecoveryTestRegion(t, SubscriptionID(1), 1) + key := regionRetryKey{ + subscriptionID: region.subscribedSpan.subID, + regionID: region.verID.GetID(), + } + timer := time.NewTimer(time.Hour) + t.Cleanup(func() { timer.Stop() }) + handler.retries[key] = ®ionRetryState{pending: true, timer: timer} + + err := handler.handleError(context.Background(), newRegionErrorInfo(region, &requestCancelledErr{})) + require.NoError(t, err) + + handler.retryMu.Lock() + _, ok := handler.retries[key] + handler.retryMu.Unlock() + assert.False(t, ok) +} diff --git a/logservice/logpuller/subscription_client_test.go b/logservice/logpuller/subscription_client_test.go index c7e4591f92..cec978ddb6 100644 --- a/logservice/logpuller/subscription_client_test.go +++ b/logservice/logpuller/subscription_client_test.go @@ -57,6 +57,7 @@ func TestGenerateResolveLockTask(t *testing.T) { client := &subscriptionClient{ resolveLockTaskCh: make(chan resolveLockTask, 10), resolveLockRateLimiter: newResolveLockRateLimiter(), + memoryQuota: newMemoryQuotaController(0, 0), } client.ctx, client.cancel = context.WithCancel(context.Background()) rawSpan := heartbeatpb.TableSpan{ @@ -305,7 +306,9 @@ func TestResolveLockTaskDroppedWhenChannelFull(t *testing.T) { func TestStopTaskUsesSubscribedSpanFilterLoop(t *testing.T) { client := &subscriptionClient{ - resolveLockTaskCh: make(chan resolveLockTask, 1), + resolveLockTaskCh: make(chan resolveLockTask, 1), + resolveLockRateLimiter: newResolveLockRateLimiter(), + memoryQuota: newMemoryQuotaController(0, 0), } client.ctx, client.cancel = context.WithCancel(context.Background()) defer client.cancel() @@ -457,7 +460,6 @@ func TestRegionEventSinkPushUnblocksOnClientClose(t *testing.T) { }, }, } - done := make(chan struct{}) go func() { sink.Push(SubscriptionID(1), event) From 143077c3c453df507f67bc6feb605a93e7b3cf3b Mon Sep 17 00:00:00 2001 From: lidezhu Date: Wed, 5 Aug 2026 13:45:04 +0800 Subject: [PATCH 65/71] refactor --- logservice/logpuller/region_event_handler.go | 7 +------ .../logpuller/region_event_handler_test.go | 17 +++++++++------- logservice/logpuller/region_request_worker.go | 6 +++++- .../logpuller/region_request_worker_test.go | 6 +++--- logservice/logpuller/region_state.go | 20 ++++++++++++++----- .../logpuller/subscription_client_test.go | 2 +- 6 files changed, 35 insertions(+), 23 deletions(-) diff --git a/logservice/logpuller/region_event_handler.go b/logservice/logpuller/region_event_handler.go index afc8b95630..49dab3e46c 100644 --- a/logservice/logpuller/region_event_handler.go +++ b/logservice/logpuller/region_event_handler.go @@ -138,12 +138,7 @@ func (h *regionEventHandler) Handle(span *subscribedSpan, events ...regionEvent) } if event.entries != nil { hasEntries = true - state := event.mustFirstState() - wasInitialized := state.isInitialized() - handleEventEntries(span, state, event.entries) - if !wasInitialized && state.isInitialized() && h.failureHandler != nil { - h.failureHandler.resetRegionRetry(span.subID, state.getRegionID()) - } + handleEventEntries(span, event.mustFirstState(), event.entries) } else if event.resolvedTs != 0 { hasResolved = true for _, state := range event.states { diff --git a/logservice/logpuller/region_event_handler_test.go b/logservice/logpuller/region_event_handler_test.go index 01ad6f9264..19b5d2a03e 100644 --- a/logservice/logpuller/region_event_handler_test.go +++ b/logservice/logpuller/region_event_handler_test.go @@ -92,7 +92,7 @@ func TestHandleEventEntryEventOutOfOrder(t *testing.T) { false, ) region.lockedRangeState = ®ionlock.LockedRangeState{} - state := newRegionFeedState(region, 1, worker, nil) + state := newRegionFeedState(region, 1, worker, nil, nil) // Receive prewrite2 with empty value. { @@ -225,7 +225,7 @@ func TestHandleResolvedTs(t *testing.T) { worker := ®ionRequestWorker{ tracker: newRegionTracker(), } - state1 := newRegionFeedState(regionInfo{verID: tikv.NewRegionVerID(1, 1, 1)}, uint64(subID1), worker, nil) + state1 := newRegionFeedState(regionInfo{verID: tikv.NewRegionVerID(1, 1, 1)}, uint64(subID1), worker, nil, nil) { span := heartbeatpb.TableSpan{ TableID: 100, @@ -249,7 +249,7 @@ func TestHandleResolvedTs(t *testing.T) { } subID2 := SubscriptionID(2) - state2 := newRegionFeedState(regionInfo{verID: tikv.NewRegionVerID(2, 2, 2)}, uint64(subID2), worker, nil) + state2 := newRegionFeedState(regionInfo{verID: tikv.NewRegionVerID(2, 2, 2)}, uint64(subID2), worker, nil, nil) { span := heartbeatpb.TableSpan{ TableID: 100, @@ -273,7 +273,7 @@ func TestHandleResolvedTs(t *testing.T) { } subID3 := SubscriptionID(3) - state3 := newRegionFeedState(regionInfo{verID: tikv.NewRegionVerID(3, 3, 3)}, uint64(subID3), worker, nil) + state3 := newRegionFeedState(regionInfo{verID: tikv.NewRegionVerID(3, 3, 3)}, uint64(subID3), worker, nil, nil) { span := heartbeatpb.TableSpan{ TableID: 100, @@ -378,6 +378,7 @@ func TestHandleResolvedTsThrottled(t *testing.T) { 1, worker, nil, + nil, ) require.Equal(t, uint64(200), handleResolvedTs(span, state, 300)) @@ -443,7 +444,9 @@ func TestRegionEventHandlerInitializedResetsRetryState(t *testing.T) { region := newRegionInfo(tikv.NewRegionVerID(1, 1, 1), span.span, nil, span, false) region.lockedRangeState = ®ionlock.LockedRangeState{} region.rpcCtx = &tikv.RPCContext{Addr: "store-1"} - state := newRegionFeedState(region, uint64(span.subID), ®ionRequestWorker{tracker: newRegionTracker()}, nil) + state := newRegionFeedState(region, uint64(span.subID), ®ionRequestWorker{tracker: newRegionTracker()}, nil, func(state *regionFeedState) { + failureHandler.resetRegionRetry(span.subID, state.getRegionID()) + }) handler := ®ionEventHandler{ eventSink: ®ionEventSink{memoryQuota: newMemoryQuotaController(0, 0)}, @@ -482,7 +485,7 @@ func TestSpanInitializedAfterFullRangeCoverage(t *testing.T) { }, subscribedSpan: span, lockedRangeState: ®ionlock.LockedRangeState{}, - }, uint64(span.subID), ®ionRequestWorker{}, nil) + }, uint64(span.subID), ®ionRequestWorker{}, nil, nil) secondState := newRegionFeedState(regionInfo{ verID: tikv.NewRegionVerID(2, 1, 1), span: heartbeatpb.TableSpan{ @@ -491,7 +494,7 @@ func TestSpanInitializedAfterFullRangeCoverage(t *testing.T) { }, subscribedSpan: span, lockedRangeState: ®ionlock.LockedRangeState{}, - }, uint64(span.subID), ®ionRequestWorker{}, nil) + }, uint64(span.subID), ®ionRequestWorker{}, nil, nil) span.markRegionInitialized(firstState) require.False(t, span.initialized.Load()) diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index 3f954168d0..1bcbc0faea 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -454,7 +454,11 @@ func (s *regionRequestWorker) sendRegionRequest(conn *ConnAndClient, req *region // Publish the state before Send so a fast response observes its owner and // admission lease. - state := newRegionFeedState(region, uint64(subID), s, req) + state := newRegionFeedState(region, uint64(subID), s, req, func(state *regionFeedState) { + if s.failureHandler != nil { + s.failureHandler.resetRegionRetry(subID, state.getRegionID()) + } + }) if !s.tracker.Add(subID, region.verID.GetID(), state) { // RangeLock normally prevents duplicate active regions. Keep the existing // owner, including its range-lock ownership, if that invariant is ever diff --git a/logservice/logpuller/region_request_worker_test.go b/logservice/logpuller/region_request_worker_test.go index 59da548746..4f429925ee 100644 --- a/logservice/logpuller/region_request_worker_test.go +++ b/logservice/logpuller/region_request_worker_test.go @@ -217,7 +217,7 @@ func TestRegionRequestWorkerIgnoresDuplicateActiveRegion(t *testing.T) { region := prepareRegionForSendTest(createTestRegionInfo(1, 1)) req1 := admitRegionRequest(t, admission, region) - state1 := newRegionFeedState(region, uint64(region.subscribedSpan.subID), worker, req1) + state1 := newRegionFeedState(region, uint64(region.subscribedSpan.subID), worker, req1, nil) require.True(t, worker.tracker.Add(region.subscribedSpan.subID, region.verID.GetID(), state1)) req2 := admitRegionRequest(t, admission, region) @@ -456,7 +456,7 @@ func TestStoppedStateRemovesSentRequest(t *testing.T) { region := prepareRegionForSendTest(createTestRegionInfo(1, 1)) req := admitRegionRequest(t, admission, region) - state := newRegionFeedState(req.regionInfo, uint64(req.regionInfo.subscribedSpan.subID), worker, req) + state := newRegionFeedState(req.regionInfo, uint64(req.regionInfo.subscribedSpan.subID), worker, req, nil) require.True(t, worker.tracker.Add(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID(), state)) state.markStopped(errors.New("send request to store error")) worker.tracker.RemoveIf(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID(), state) @@ -482,7 +482,7 @@ func TestRunStreamFailurePushesTrackedRegionToEventSink(t *testing.T) { sentRegion := createFailureRecoveryTestRegion(t, 1, 1) sentReq := admitRegionRequest(t, worker.admission, sentRegion) - sentState := newRegionFeedState(sentRegion, uint64(sentRegion.subscribedSpan.subID), worker, sentReq) + sentState := newRegionFeedState(sentRegion, uint64(sentRegion.subscribedSpan.subID), worker, sentReq, nil) require.True(t, worker.tracker.Add(sentRegion.subscribedSpan.subID, sentRegion.verID.GetID(), sentState)) firstRegion := createFailureRecoveryTestRegion(t, 2, 2) diff --git a/logservice/logpuller/region_state.go b/logservice/logpuller/region_state.go index 0f2f84ce3e..422c3bb39c 100644 --- a/logservice/logpuller/region_state.go +++ b/logservice/logpuller/region_state.go @@ -90,6 +90,9 @@ type regionFeedState struct { region regionInfo requestID uint64 // It is also the subscription ID matcher *matcher + // onInitialized runs once when the region finishes its first successful + // initialization for this request lifecycle. + onInitialized func(*regionFeedState) // Transform: normal -> stopped -> removed. // normal: the region is in replicating. @@ -113,12 +116,14 @@ func newRegionFeedState( requestID uint64, worker *regionRequestWorker, request *regionReq, + onInitialized func(*regionFeedState), ) *regionFeedState { state := ®ionFeedState{ - region: region, - requestID: requestID, - matcher: newMatcher(), - worker: worker, + region: region, + requestID: requestID, + matcher: newMatcher(), + onInitialized: onInitialized, + worker: worker, } state.regionReq.Store(request) return state @@ -167,8 +172,13 @@ func (s *regionFeedState) isInitialized() bool { } func (s *regionFeedState) setInitialized() { - s.region.lockedRangeState.Initialized.Store(true) + if !s.region.lockedRangeState.Initialized.CompareAndSwap(false, true) { + return + } s.finishScan() + if s.onInitialized != nil { + s.onInitialized(s) + } } func (s *regionFeedState) finishScan() { diff --git a/logservice/logpuller/subscription_client_test.go b/logservice/logpuller/subscription_client_test.go index cec978ddb6..fa42a096c1 100644 --- a/logservice/logpuller/subscription_client_test.go +++ b/logservice/logpuller/subscription_client_test.go @@ -112,7 +112,7 @@ func TestGenerateResolveLockTask(t *testing.T) { // Lock another range, no task will be triggered before initialized. res = span.rangeLock.LockRange(context.Background(), []byte{'c'}, []byte{'d'}, 2, 100) require.Equal(t, regionlock.LockRangeStatusSuccess, res.Status) - state := newRegionFeedState(regionInfo{lockedRangeState: res.LockedRangeState, subscribedSpan: span}, 1, worker, nil) + state := newRegionFeedState(regionInfo{lockedRangeState: res.LockedRangeState, subscribedSpan: span}, 1, worker, nil, nil) span.resolveStaleLocks(200) select { case <-client.resolveLockTaskCh: From 7f0bc6e258b712621bbe24aa260dd6aef16721f7 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Wed, 5 Aug 2026 14:38:08 +0800 Subject: [PATCH 66/71] more refactor --- .../logpuller/region_event_handler_test.go | 14 +- .../logpuller/region_failure_handler.go | 222 ++++++++++++------ .../logpuller/region_failure_handler_test.go | 94 +++++++- logservice/logpuller/region_request_worker.go | 2 +- logservice/logpuller/subscription_client.go | 31 ++- 5 files changed, 255 insertions(+), 108 deletions(-) diff --git a/logservice/logpuller/region_event_handler_test.go b/logservice/logpuller/region_event_handler_test.go index 19b5d2a03e..bbdd5cbee4 100644 --- a/logservice/logpuller/region_event_handler_test.go +++ b/logservice/logpuller/region_event_handler_test.go @@ -431,21 +431,21 @@ func TestHandleEntriesReleasesMemoryAfterDownstreamCallback(t *testing.T) { require.Zero(t, quotaState.used) } -func TestRegionEventHandlerInitializedResetsRetryState(t *testing.T) { +func TestRegionEventHandlerInitializedResetsRecoveryState(t *testing.T) { span := &subscribedSpan{ subID: 1, span: heartbeatpb.TableSpan{TableID: 1}, advanceResolvedTs: func(uint64) {}, } failureHandler := newRegionFailureHandler(nil, func(*subscribedSpan) {}, func(context.Context, regionInfo) {}, func(context.Context, rangeTask) {}) - key := regionRetryKey{subscriptionID: span.subID, regionID: 1} - failureHandler.retries[key] = ®ionRetryState{} + key := newRegionRecoveryKey(span.subID, span.span) + failureHandler.recoveries[key] = ®ionRecoveryState{} region := newRegionInfo(tikv.NewRegionVerID(1, 1, 1), span.span, nil, span, false) region.lockedRangeState = ®ionlock.LockedRangeState{} region.rpcCtx = &tikv.RPCContext{Addr: "store-1"} state := newRegionFeedState(region, uint64(span.subID), ®ionRequestWorker{tracker: newRegionTracker()}, nil, func(state *regionFeedState) { - failureHandler.resetRegionRetry(span.subID, state.getRegionID()) + failureHandler.resetRegionRecovery(state.region) }) handler := ®ionEventHandler{ @@ -461,9 +461,9 @@ func TestRegionEventHandlerInitializedResetsRetryState(t *testing.T) { }, }) - failureHandler.retryMu.Lock() - _, ok := failureHandler.retries[key] - failureHandler.retryMu.Unlock() + failureHandler.recoveryMu.Lock() + _, ok := failureHandler.recoveries[key] + failureHandler.recoveryMu.Unlock() require.False(t, ok) } diff --git a/logservice/logpuller/region_failure_handler.go b/logservice/logpuller/region_failure_handler.go index 6f1fc7ed00..47a4524335 100644 --- a/logservice/logpuller/region_failure_handler.go +++ b/logservice/logpuller/region_failure_handler.go @@ -21,6 +21,7 @@ import ( "github.com/pingcap/kvproto/pkg/cdcpb" "github.com/pingcap/log" + "github.com/pingcap/ticdc/heartbeatpb" "github.com/pingcap/ticdc/pkg/errors" "github.com/pingcap/ticdc/pkg/metrics" "github.com/tikv/client-go/v2/tikv" @@ -42,11 +43,11 @@ var ( // regionFailureHandler handles failed regions and owns retry and reschedule decisions. type regionFailureHandler struct { - cache *errCache - regionCache *tikv.RegionCache - retryMu sync.Mutex - retries map[regionRetryKey]*regionRetryState - retryDelay func(uint32) time.Duration + cache *errCache + regionCache *tikv.RegionCache + recoveryMu sync.Mutex + recoveries map[regionRecoveryKey]*regionRecoveryState + recoveryDelay func(uint32) time.Duration onTableDrained func(*subscribedSpan) scheduleRegionRequest func(context.Context, regionInfo) @@ -54,24 +55,67 @@ type regionFailureHandler struct { } const ( - regionRetryBaseDelay = 50 * time.Millisecond - regionRetryMaxDelay = 2 * time.Second - regionRetryStateTTL = 5 * time.Minute + regionRecoveryBaseDelay = 50 * time.Millisecond + regionRecoveryMaxDelay = 2 * time.Second + regionRecoveryStateTTL = 5 * time.Minute ) -type regionRetryKey struct { +// regionRecoveryKey keeps backoff state across region ID and epoch changes for +// the same logical range. +type regionRecoveryKey struct { subscriptionID SubscriptionID - regionID uint64 + startKey string + endKey string } -type regionRetryState struct { +type regionRecoveryAction int + +const ( + retryRegionRequest regionRecoveryAction = iota + retryRangeRequest +) + +type regionRecovery struct { + action regionRecoveryAction + region regionInfo + rangeTask rangeTask + minDelay time.Duration +} + +func (r regionRecovery) key() regionRecoveryKey { + if r.action == retryRegionRequest { + return newRegionRecoveryKey(r.region.subscribedSpan.subID, r.region.span) + } + return newRegionRecoveryKey(r.rangeTask.subscribedSpan.subID, r.rangeTask.span) +} + +func (r regionRecovery) subscribedSpan() *subscribedSpan { + if r.action == retryRegionRequest { + return r.region.subscribedSpan + } + return r.rangeTask.subscribedSpan +} + +type regionRecoveryState struct { attempt uint32 generation uint64 pending bool + recovery regionRecovery timer *time.Timer } -func notLeaderRetryDelay(attempt uint32) time.Duration { +func newRegionRecoveryKey( + subscriptionID SubscriptionID, + span heartbeatpb.TableSpan, +) regionRecoveryKey { + return regionRecoveryKey{ + subscriptionID: subscriptionID, + startKey: string(span.StartKey), + endKey: string(span.EndKey), + } +} + +func regionRecoveryDelay(attempt uint32) time.Duration { if attempt == 0 { attempt = 1 } @@ -79,9 +123,9 @@ func notLeaderRetryDelay(attempt uint32) time.Duration { if exponent > 16 { exponent = 16 } - delay := regionRetryBaseDelay << exponent - if delay > regionRetryMaxDelay { - delay = regionRetryMaxDelay + delay := regionRecoveryBaseDelay << exponent + if delay > regionRecoveryMaxDelay { + delay = regionRecoveryMaxDelay } half := delay / 2 return half + time.Duration(rand.Int64N(int64(delay-half)+1)) @@ -96,35 +140,36 @@ func newRegionFailureHandler( return ®ionFailureHandler{ cache: newErrCache(), regionCache: regionCache, - retries: make(map[regionRetryKey]*regionRetryState), - retryDelay: notLeaderRetryDelay, + recoveries: make(map[regionRecoveryKey]*regionRecoveryState), + recoveryDelay: regionRecoveryDelay, onTableDrained: onTableDrained, scheduleRegionRequest: scheduleRegionRequest, scheduleRangeRequest: scheduleRangeRequest, } } -func (r *regionFailureHandler) scheduleRegionRetry( +func (r *regionFailureHandler) scheduleRecovery( ctx context.Context, - region regionInfo, - retry func(), + recovery regionRecovery, ) { - if region.subscribedSpan == nil || region.subscribedSpan.stopped.Load() { + subscribedSpan := recovery.subscribedSpan() + if subscribedSpan == nil || subscribedSpan.stopped.Load() { return } - key := regionRetryKey{ - subscriptionID: region.subscribedSpan.subID, - regionID: region.verID.GetID(), - } + key := recovery.key() - r.retryMu.Lock() - defer r.retryMu.Unlock() - state := r.retries[key] + r.recoveryMu.Lock() + defer r.recoveryMu.Unlock() + state := r.recoveries[key] if state == nil { - state = ®ionRetryState{} - r.retries[key] = state + state = ®ionRecoveryState{} + r.recoveries[key] = state } if state.pending { + // Reloading the range supersedes retrying a possibly stale region. + if recovery.action >= state.recovery.action { + state.recovery = recovery + } return } if state.timer != nil { @@ -136,57 +181,72 @@ func (r *regionFailureHandler) scheduleRegionRetry( state.generation++ generation := state.generation state.pending = true - delay := r.retryDelay(state.attempt) + state.recovery = recovery + delay := r.recoveryDelay(state.attempt) + if recovery.minDelay > delay { + delay = recovery.minDelay + } state.timer = time.AfterFunc(delay, func() { - r.retryMu.Lock() - current := r.retries[key] + r.recoveryMu.Lock() + current := r.recoveries[key] if current == nil || current.generation != generation || !current.pending { - r.retryMu.Unlock() + r.recoveryMu.Unlock() return } + recovery := current.recovery + // Keep the state after dispatch so the next failure of this range advances + // the backoff attempt. Successful initialization resets it. current.pending = false - current.timer = time.AfterFunc(regionRetryStateTTL, func() { - r.expireRegionRetry(key, generation) + current.timer = time.AfterFunc(regionRecoveryStateTTL, func() { + r.expireRecovery(key, generation) }) - r.retryMu.Unlock() + r.recoveryMu.Unlock() - if ctx.Err() != nil || region.subscribedSpan.stopped.Load() { - r.resetRegionRetry(key.subscriptionID, key.regionID) + if ctx.Err() != nil || recovery.subscribedSpan().stopped.Load() { + r.resetRecovery(key) return } - retry() + switch recovery.action { + case retryRegionRequest: + r.scheduleRegionRequest(ctx, recovery.region) + case retryRangeRequest: + r.scheduleRangeRequest(ctx, recovery.rangeTask) + } }) } -func (r *regionFailureHandler) expireRegionRetry(key regionRetryKey, generation uint64) { - r.retryMu.Lock() - defer r.retryMu.Unlock() - state := r.retries[key] +func (r *regionFailureHandler) expireRecovery(key regionRecoveryKey, generation uint64) { + r.recoveryMu.Lock() + defer r.recoveryMu.Unlock() + state := r.recoveries[key] if state != nil && state.generation == generation && !state.pending { - delete(r.retries, key) + delete(r.recoveries, key) } } -func (r *regionFailureHandler) resetRegionRetry(subscriptionID SubscriptionID, regionID uint64) { - key := regionRetryKey{subscriptionID: subscriptionID, regionID: regionID} - r.retryMu.Lock() - defer r.retryMu.Unlock() - if state := r.retries[key]; state != nil { +func (r *regionFailureHandler) resetRecovery(key regionRecoveryKey) { + r.recoveryMu.Lock() + defer r.recoveryMu.Unlock() + if state := r.recoveries[key]; state != nil { if state.timer != nil { state.timer.Stop() } - delete(r.retries, key) + delete(r.recoveries, key) } } -func (r *regionFailureHandler) cancelRegionRetries() { - r.retryMu.Lock() - defer r.retryMu.Unlock() - for key, state := range r.retries { +func (r *regionFailureHandler) resetRegionRecovery(region regionInfo) { + r.resetRecovery(newRegionRecoveryKey(region.subscribedSpan.subID, region.span)) +} + +func (r *regionFailureHandler) cancelRecoveries() { + r.recoveryMu.Lock() + defer r.recoveryMu.Unlock() + for key, state := range r.recoveries { if state.timer != nil { state.timer.Stop() } - delete(r.retries, key) + delete(r.recoveries, key) } } @@ -206,7 +266,7 @@ func (r *regionFailureHandler) Report(errInfo regionErrorInfo) { func (r *regionFailureHandler) Run(ctx context.Context) error { log.Info("region failure handler starts") defer log.Info("region failure handler exits") - defer r.cancelRegionRetries() + defer r.cancelRecoveries() handleCachedErrors := func() error { for { @@ -249,7 +309,14 @@ func (r *regionFailureHandler) Run(ctx context.Context) error { func (r *regionFailureHandler) handleError(ctx context.Context, errInfo regionErrorInfo) error { err := errors.Cause(errInfo.err) - rescheduleRange := func() { + retryRegion := func(minDelay time.Duration) { + r.scheduleRecovery(ctx, regionRecovery{ + action: retryRegionRequest, + region: errInfo.regionInfo, + minDelay: minDelay, + }) + } + retryRange := func() { priority := normalizeScanPriority(errInfo.scanPriority) if priority == cdcpb.ScanPriority_SCAN_PRIORITY_LOW { priority = errInfo.subscribedSpan.priorityPolicy.resolve( @@ -258,11 +325,14 @@ func (r *regionFailureHandler) handleError(ctx context.Context, errInfo regionEr errInfo.subscribedSpan.priorityPolicy.pdClock.CurrentTime(), ) } - r.scheduleRangeRequest(ctx, rangeTask{ - span: errInfo.span, - subscribedSpan: errInfo.subscribedSpan, - filterLoop: errInfo.filterLoop, - priority: priority, + r.scheduleRecovery(ctx, regionRecovery{ + action: retryRangeRequest, + rangeTask: rangeTask{ + span: errInfo.span, + subscribedSpan: errInfo.subscribedSpan, + filterLoop: errInfo.filterLoop, + priority: priority, + }, }) } @@ -283,33 +353,31 @@ func (r *regionFailureHandler) handleError(ctx context.Context, errInfo regionEr leader := notLeader.GetLeader() if leader == nil || leader.GetId() == 0 || leader.GetStoreId() == 0 || errInfo.rpcCtx == nil { r.regionCache.InvalidateCachedRegion(errInfo.verID) - r.scheduleRegionRetry(ctx, errInfo.regionInfo, rescheduleRange) + retryRange() return nil } r.regionCache.UpdateLeader(errInfo.verID, leader, errInfo.rpcCtx.AccessIdx) - r.scheduleRegionRetry(ctx, errInfo.regionInfo, func() { - r.scheduleRegionRequest(ctx, errInfo.regionInfo) - }) + retryRegion(0) return nil } if innerErr.GetEpochNotMatch() != nil { metricFeedEpochNotMatchCounter.Inc() - rescheduleRange() + retryRange() return nil } if innerErr.GetRegionNotFound() != nil { metricFeedRegionNotFoundCounter.Inc() - rescheduleRange() + retryRange() return nil } if innerErr.GetCongested() != nil { metricKvCongestedCounter.Inc() - r.scheduleRegionRequest(ctx, errInfo.regionInfo) + retryRegion(0) return nil } - if innerErr.GetServerIsBusy() != nil { + if busy := innerErr.GetServerIsBusy(); busy != nil { metricKvIsBusyCounter.Inc() - r.scheduleRegionRequest(ctx, errInfo.regionInfo) + retryRegion(time.Duration(busy.GetBackoffMs()) * time.Millisecond) return nil } if duplicated := innerErr.GetDuplicateRequest(); duplicated != nil { @@ -328,29 +396,29 @@ func (r *regionFailureHandler) handleError(ctx context.Context, errInfo regionEr zap.Uint64("subscriptionID", uint64(errInfo.subscribedSpan.subID)), zap.Stringer("error", innerErr)) metricFeedUnknownErrorCounter.Inc() - r.scheduleRegionRequest(ctx, errInfo.regionInfo) + retryRegion(0) return nil case *rpcCtxUnavailableErr: metricFeedRPCCtxUnavailable.Inc() - rescheduleRange() + retryRange() return nil case *getStoreErr: metricGetStoreErr.Inc() bo := tikv.NewBackoffer(ctx, tikvRequestMaxBackoff) // cannot get the store the region belongs to, so we need to reload the region. r.regionCache.OnSendFail(bo, errInfo.rpcCtx, true, err) - rescheduleRange() + retryRange() return nil case *storeStreamErr: metricStoreSendRequestErr.Inc() bo := tikv.NewBackoffer(ctx, tikvRequestMaxBackoff) r.regionCache.OnSendFail(bo, errInfo.rpcCtx, regionScheduleReload, err) - r.scheduleRegionRequest(ctx, errInfo.regionInfo) + retryRegion(0) return nil case *requestCancelledErr: // the corresponding subscription has been unsubscribed, just ignore. if errInfo.subscribedSpan != nil { - r.resetRegionRetry(errInfo.subscribedSpan.subID, errInfo.verID.GetID()) + r.resetRegionRecovery(errInfo.regionInfo) } return nil default: diff --git a/logservice/logpuller/region_failure_handler_test.go b/logservice/logpuller/region_failure_handler_test.go index 6b34192de3..8f57f672d8 100644 --- a/logservice/logpuller/region_failure_handler_test.go +++ b/logservice/logpuller/region_failure_handler_test.go @@ -168,7 +168,7 @@ func TestRegionFailureHandlerDelaysNotLeaderRangeRetry(t *testing.T) { rangeRetryCh <- task }, ) - handler.retryDelay = func(uint32) time.Duration { return 50 * time.Millisecond } + handler.recoveryDelay = func(uint32) time.Duration { return 50 * time.Millisecond } errInfo := newRegionErrorInfo(region, &eventError{ err: &cdcpb.Error{NotLeader: &errorpb.NotLeader{}}, @@ -201,22 +201,96 @@ func TestRegionFailureHandlerDelaysNotLeaderRangeRetry(t *testing.T) { } } -func TestRegionFailureHandlerRequestCancelledResetsRetryState(t *testing.T) { - handler := newRegionFailureHandler(nil, func(*subscribedSpan) {}, func(context.Context, regionInfo) {}, func(context.Context, rangeTask) {}) +func TestRegionRecoveryBackoffFollowsRangeAcrossRegionChanges(t *testing.T) { + regionRetryCh := make(chan regionInfo, 2) + handler := newRegionFailureHandler( + nil, + func(*subscribedSpan) {}, + func(_ context.Context, region regionInfo) { + regionRetryCh <- region + }, + func(context.Context, rangeTask) {}, + ) + t.Cleanup(handler.cancelRecoveries) + + attempts := make([]uint32, 0, 2) + handler.recoveryDelay = func(attempt uint32) time.Duration { + attempts = append(attempts, attempt) + return time.Millisecond + } + region := createFailureRecoveryTestRegion(t, SubscriptionID(1), 1) - key := regionRetryKey{ - subscriptionID: region.subscribedSpan.subID, - regionID: region.verID.GetID(), + errInfo := newRegionErrorInfo(region, &eventError{ + err: &cdcpb.Error{Congested: &cdcpb.Congested{}}, + }) + require.NoError(t, handler.handleError(context.Background(), errInfo)) + select { + case retried := <-regionRetryCh: + require.Equal(t, uint64(1), retried.verID.GetID()) + case <-time.After(time.Second): + t.Fatal("first region recovery was not scheduled") + } + + region.verID = tikv.NewRegionVerID(2, 1, 1) + errInfo = newRegionErrorInfo(region, &eventError{ + err: &cdcpb.Error{Congested: &cdcpb.Congested{}}, + }) + require.NoError(t, handler.handleError(context.Background(), errInfo)) + select { + case retried := <-regionRetryCh: + require.Equal(t, uint64(2), retried.verID.GetID()) + case <-time.After(time.Second): + t.Fatal("second region recovery was not scheduled") } + + require.Equal(t, []uint32{1, 2}, attempts) +} + +func TestRegionRecoveryPendingRangeRetrySupersedesRegionRetry(t *testing.T) { + handler := newRegionFailureHandler( + nil, + func(*subscribedSpan) {}, + func(context.Context, regionInfo) {}, + func(context.Context, rangeTask) {}, + ) + t.Cleanup(handler.cancelRecoveries) + handler.recoveryDelay = func(uint32) time.Duration { return time.Hour } + + region := createFailureRecoveryTestRegion(t, SubscriptionID(1), 1) + handler.scheduleRecovery(context.Background(), regionRecovery{ + action: retryRegionRequest, + region: region, + }) + handler.scheduleRecovery(context.Background(), regionRecovery{ + action: retryRangeRequest, + rangeTask: rangeTask{ + span: region.span, + subscribedSpan: region.subscribedSpan, + }, + }) + + key := newRegionRecoveryKey(region.subscribedSpan.subID, region.span) + handler.recoveryMu.Lock() + state := handler.recoveries[key] + handler.recoveryMu.Unlock() + require.NotNil(t, state) + require.Equal(t, retryRangeRequest, state.recovery.action) + require.Equal(t, uint32(1), state.attempt) +} + +func TestRegionFailureHandlerRequestCancelledResetsRecoveryState(t *testing.T) { + handler := newRegionFailureHandler(nil, func(*subscribedSpan) {}, func(context.Context, regionInfo) {}, func(context.Context, rangeTask) {}) + region := createFailureRecoveryTestRegion(t, SubscriptionID(1), 1) + key := newRegionRecoveryKey(region.subscribedSpan.subID, region.span) timer := time.NewTimer(time.Hour) t.Cleanup(func() { timer.Stop() }) - handler.retries[key] = ®ionRetryState{pending: true, timer: timer} + handler.recoveries[key] = ®ionRecoveryState{pending: true, timer: timer} err := handler.handleError(context.Background(), newRegionErrorInfo(region, &requestCancelledErr{})) require.NoError(t, err) - handler.retryMu.Lock() - _, ok := handler.retries[key] - handler.retryMu.Unlock() + handler.recoveryMu.Lock() + _, ok := handler.recoveries[key] + handler.recoveryMu.Unlock() assert.False(t, ok) } diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index 1bcbc0faea..581f2bfea3 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -456,7 +456,7 @@ func (s *regionRequestWorker) sendRegionRequest(conn *ConnAndClient, req *region // admission lease. state := newRegionFeedState(region, uint64(subID), s, req, func(state *regionFeedState) { if s.failureHandler != nil { - s.failureHandler.resetRegionRetry(subID, state.getRegionID()) + s.failureHandler.resetRegionRecovery(state.region) } }) if !s.tracker.Add(subID, region.verID.GetID(), state) { diff --git a/logservice/logpuller/subscription_client.go b/logservice/logpuller/subscription_client.go index af039a9ebd..7a6616625d 100644 --- a/logservice/logpuller/subscription_client.go +++ b/logservice/logpuller/subscription_client.go @@ -31,7 +31,6 @@ import ( "github.com/pingcap/ticdc/pkg/pdutil" "github.com/pingcap/ticdc/pkg/security" "github.com/pingcap/ticdc/pkg/spanz" - "github.com/pingcap/ticdc/pkg/util" "github.com/tikv/client-go/v2/tikv" pd "github.com/tikv/pd/client" "go.uber.org/zap" @@ -47,7 +46,6 @@ const ( // don't need to force reload region anymore. regionScheduleReload = false - loadRegionRetryInterval time.Duration = 100 * time.Millisecond resolveLockMinInterval time.Duration = 10 * time.Second resolveLockTickInterval time.Duration = 2 * time.Second resolveLockFence time.Duration = 4 * time.Second @@ -372,14 +370,18 @@ func (s *subscriptionClient) divideSpanAndScheduleRegionRequests( // Limit the number of regions loaded at a time to make the load more stable. limit := 1024 nextSpan := span - backoffBeforeLoad := false + retryRange := func() { + s.failureHandler.scheduleRecovery(ctx, regionRecovery{ + action: retryRangeRequest, + rangeTask: rangeTask{ + span: nextSpan, + subscribedSpan: subscribedSpan, + filterLoop: task.filterLoop, + priority: task.priority, + }, + }) + } for { - if backoffBeforeLoad { - if err := util.Hang(ctx, loadRegionRetryInterval); err != nil { - return err - } - backoffBeforeLoad = false - } log.Debug("subscription client is going to load regions", zap.Uint64("subscriptionID", uint64(subscribedSpan.subID)), zap.Any("span", common.FormatTableSpan(&nextSpan))) @@ -388,12 +390,15 @@ func (s *subscriptionClient) divideSpanAndScheduleRegionRequests( regions, err := s.upstream.regionCache.BatchLoadRegionsWithKeyRange( backoff, nextSpan.StartKey, nextSpan.EndKey, limit) if err != nil { + if ctx.Err() != nil { + return ctx.Err() + } log.Warn("subscription client load regions failed", zap.Uint64("subscriptionID", uint64(subscribedSpan.subID)), zap.Any("span", common.FormatTableSpan(&nextSpan)), zap.Error(err)) - backoffBeforeLoad = true - continue + retryRange() + return nil } regionMetas := make([]*metapb.Region, 0, len(regions)) for _, region := range regions { @@ -406,8 +411,8 @@ func (s *subscriptionClient) divideSpanAndScheduleRegionRequests( log.Warn("subscription client load regions with holes", zap.Uint64("subscriptionID", uint64(subscribedSpan.subID)), zap.Any("span", common.FormatTableSpan(&nextSpan))) - backoffBeforeLoad = true - continue + retryRange() + return nil } for _, regionMeta := range regionMetas { From b21fa6788691af83552eff7d46bc5aa40cfc722a Mon Sep 17 00:00:00 2001 From: lidezhu Date: Wed, 5 Aug 2026 15:42:52 +0800 Subject: [PATCH 67/71] more refactor --- .../logpuller/region_failure_handler.go | 92 +++++++------------ .../logpuller/region_failure_handler_test.go | 32 ------- logservice/logpuller/subscription_client.go | 22 +++-- 3 files changed, 46 insertions(+), 100 deletions(-) diff --git a/logservice/logpuller/region_failure_handler.go b/logservice/logpuller/region_failure_handler.go index 47a4524335..97986905fa 100644 --- a/logservice/logpuller/region_failure_handler.go +++ b/logservice/logpuller/region_failure_handler.go @@ -68,39 +68,10 @@ type regionRecoveryKey struct { endKey string } -type regionRecoveryAction int - -const ( - retryRegionRequest regionRecoveryAction = iota - retryRangeRequest -) - -type regionRecovery struct { - action regionRecoveryAction - region regionInfo - rangeTask rangeTask - minDelay time.Duration -} - -func (r regionRecovery) key() regionRecoveryKey { - if r.action == retryRegionRequest { - return newRegionRecoveryKey(r.region.subscribedSpan.subID, r.region.span) - } - return newRegionRecoveryKey(r.rangeTask.subscribedSpan.subID, r.rangeTask.span) -} - -func (r regionRecovery) subscribedSpan() *subscribedSpan { - if r.action == retryRegionRequest { - return r.region.subscribedSpan - } - return r.rangeTask.subscribedSpan -} - type regionRecoveryState struct { attempt uint32 generation uint64 pending bool - recovery regionRecovery timer *time.Timer } @@ -150,13 +121,15 @@ func newRegionFailureHandler( func (r *regionFailureHandler) scheduleRecovery( ctx context.Context, - recovery regionRecovery, + subscribedSpan *subscribedSpan, + span heartbeatpb.TableSpan, + minDelay time.Duration, + retry func(), ) { - subscribedSpan := recovery.subscribedSpan() if subscribedSpan == nil || subscribedSpan.stopped.Load() { return } - key := recovery.key() + key := newRegionRecoveryKey(subscribedSpan.subID, span) r.recoveryMu.Lock() defer r.recoveryMu.Unlock() @@ -166,10 +139,6 @@ func (r *regionFailureHandler) scheduleRecovery( r.recoveries[key] = state } if state.pending { - // Reloading the range supersedes retrying a possibly stale region. - if recovery.action >= state.recovery.action { - state.recovery = recovery - } return } if state.timer != nil { @@ -181,10 +150,9 @@ func (r *regionFailureHandler) scheduleRecovery( state.generation++ generation := state.generation state.pending = true - state.recovery = recovery delay := r.recoveryDelay(state.attempt) - if recovery.minDelay > delay { - delay = recovery.minDelay + if minDelay > delay { + delay = minDelay } state.timer = time.AfterFunc(delay, func() { r.recoveryMu.Lock() @@ -193,7 +161,6 @@ func (r *regionFailureHandler) scheduleRecovery( r.recoveryMu.Unlock() return } - recovery := current.recovery // Keep the state after dispatch so the next failure of this range advances // the backoff attempt. Successful initialization resets it. current.pending = false @@ -202,16 +169,11 @@ func (r *regionFailureHandler) scheduleRecovery( }) r.recoveryMu.Unlock() - if ctx.Err() != nil || recovery.subscribedSpan().stopped.Load() { + if ctx.Err() != nil || subscribedSpan.stopped.Load() { r.resetRecovery(key) return } - switch recovery.action { - case retryRegionRequest: - r.scheduleRegionRequest(ctx, recovery.region) - case retryRangeRequest: - r.scheduleRangeRequest(ctx, recovery.rangeTask) - } + retry() }) } @@ -310,11 +272,15 @@ func (r *regionFailureHandler) Run(ctx context.Context) error { func (r *regionFailureHandler) handleError(ctx context.Context, errInfo regionErrorInfo) error { err := errors.Cause(errInfo.err) retryRegion := func(minDelay time.Duration) { - r.scheduleRecovery(ctx, regionRecovery{ - action: retryRegionRequest, - region: errInfo.regionInfo, - minDelay: minDelay, - }) + r.scheduleRecovery( + ctx, + errInfo.subscribedSpan, + errInfo.span, + minDelay, + func() { + r.scheduleRegionRequest(ctx, errInfo.regionInfo) + }, + ) } retryRange := func() { priority := normalizeScanPriority(errInfo.scanPriority) @@ -325,15 +291,21 @@ func (r *regionFailureHandler) handleError(ctx context.Context, errInfo regionEr errInfo.subscribedSpan.priorityPolicy.pdClock.CurrentTime(), ) } - r.scheduleRecovery(ctx, regionRecovery{ - action: retryRangeRequest, - rangeTask: rangeTask{ - span: errInfo.span, - subscribedSpan: errInfo.subscribedSpan, - filterLoop: errInfo.filterLoop, - priority: priority, + task := rangeTask{ + span: errInfo.span, + subscribedSpan: errInfo.subscribedSpan, + filterLoop: errInfo.filterLoop, + priority: priority, + } + r.scheduleRecovery( + ctx, + task.subscribedSpan, + task.span, + 0, + func() { + r.scheduleRangeRequest(ctx, task) }, - }) + ) } //nolint:errorlint // converting large type switch to errors.As is a significant refactor diff --git a/logservice/logpuller/region_failure_handler_test.go b/logservice/logpuller/region_failure_handler_test.go index 8f57f672d8..5c7f1456de 100644 --- a/logservice/logpuller/region_failure_handler_test.go +++ b/logservice/logpuller/region_failure_handler_test.go @@ -246,38 +246,6 @@ func TestRegionRecoveryBackoffFollowsRangeAcrossRegionChanges(t *testing.T) { require.Equal(t, []uint32{1, 2}, attempts) } -func TestRegionRecoveryPendingRangeRetrySupersedesRegionRetry(t *testing.T) { - handler := newRegionFailureHandler( - nil, - func(*subscribedSpan) {}, - func(context.Context, regionInfo) {}, - func(context.Context, rangeTask) {}, - ) - t.Cleanup(handler.cancelRecoveries) - handler.recoveryDelay = func(uint32) time.Duration { return time.Hour } - - region := createFailureRecoveryTestRegion(t, SubscriptionID(1), 1) - handler.scheduleRecovery(context.Background(), regionRecovery{ - action: retryRegionRequest, - region: region, - }) - handler.scheduleRecovery(context.Background(), regionRecovery{ - action: retryRangeRequest, - rangeTask: rangeTask{ - span: region.span, - subscribedSpan: region.subscribedSpan, - }, - }) - - key := newRegionRecoveryKey(region.subscribedSpan.subID, region.span) - handler.recoveryMu.Lock() - state := handler.recoveries[key] - handler.recoveryMu.Unlock() - require.NotNil(t, state) - require.Equal(t, retryRangeRequest, state.recovery.action) - require.Equal(t, uint32(1), state.attempt) -} - func TestRegionFailureHandlerRequestCancelledResetsRecoveryState(t *testing.T) { handler := newRegionFailureHandler(nil, func(*subscribedSpan) {}, func(context.Context, regionInfo) {}, func(context.Context, rangeTask) {}) region := createFailureRecoveryTestRegion(t, SubscriptionID(1), 1) diff --git a/logservice/logpuller/subscription_client.go b/logservice/logpuller/subscription_client.go index 7a6616625d..331243a4a7 100644 --- a/logservice/logpuller/subscription_client.go +++ b/logservice/logpuller/subscription_client.go @@ -371,15 +371,21 @@ func (s *subscriptionClient) divideSpanAndScheduleRegionRequests( limit := 1024 nextSpan := span retryRange := func() { - s.failureHandler.scheduleRecovery(ctx, regionRecovery{ - action: retryRangeRequest, - rangeTask: rangeTask{ - span: nextSpan, - subscribedSpan: subscribedSpan, - filterLoop: task.filterLoop, - priority: task.priority, + retryTask := rangeTask{ + span: nextSpan, + subscribedSpan: subscribedSpan, + filterLoop: task.filterLoop, + priority: task.priority, + } + s.failureHandler.scheduleRecovery( + ctx, + retryTask.subscribedSpan, + retryTask.span, + 0, + func() { + s.scheduleRangeRequest(ctx, retryTask) }, - }) + ) } for { log.Debug("subscription client is going to load regions", From 976517492854aaf20a5de15ae0d1b54b04b55934 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Wed, 5 Aug 2026 16:29:20 +0800 Subject: [PATCH 68/71] fix --- .../logpuller/region_failure_handler.go | 77 +++++++------------ .../logpuller/region_failure_handler_test.go | 51 ++++++------ 2 files changed, 53 insertions(+), 75 deletions(-) diff --git a/logservice/logpuller/region_failure_handler.go b/logservice/logpuller/region_failure_handler.go index 97986905fa..64568df09c 100644 --- a/logservice/logpuller/region_failure_handler.go +++ b/logservice/logpuller/region_failure_handler.go @@ -43,11 +43,10 @@ var ( // regionFailureHandler handles failed regions and owns retry and reschedule decisions. type regionFailureHandler struct { - cache *errCache - regionCache *tikv.RegionCache - recoveryMu sync.Mutex - recoveries map[regionRecoveryKey]*regionRecoveryState - recoveryDelay func(uint32) time.Duration + cache *errCache + regionCache *tikv.RegionCache + recoveryMu sync.Mutex + recoveries map[regionRecoveryKey]*regionRecoveryState onTableDrained func(*subscribedSpan) scheduleRegionRequest func(context.Context, regionInfo) @@ -69,10 +68,8 @@ type regionRecoveryKey struct { } type regionRecoveryState struct { - attempt uint32 - generation uint64 - pending bool - timer *time.Timer + attempt uint32 + expiresAt time.Time } func newRegionRecoveryKey( @@ -112,7 +109,6 @@ func newRegionFailureHandler( cache: newErrCache(), regionCache: regionCache, recoveries: make(map[regionRecoveryKey]*regionRecoveryState), - recoveryDelay: regionRecoveryDelay, onTableDrained: onTableDrained, scheduleRegionRequest: scheduleRegionRequest, scheduleRangeRequest: scheduleRangeRequest, @@ -132,41 +128,29 @@ func (r *regionFailureHandler) scheduleRecovery( key := newRegionRecoveryKey(subscribedSpan.subID, span) r.recoveryMu.Lock() - defer r.recoveryMu.Unlock() state := r.recoveries[key] if state == nil { state = ®ionRecoveryState{} r.recoveries[key] = state } - if state.pending { - return - } - if state.timer != nil { - state.timer.Stop() - } if state.attempt < 32 { state.attempt++ } - state.generation++ - generation := state.generation - state.pending = true - delay := r.recoveryDelay(state.attempt) + delay := regionRecoveryDelay(state.attempt) if minDelay > delay { delay = minDelay } - state.timer = time.AfterFunc(delay, func() { + state.expiresAt = time.Now().Add(delay + regionRecoveryStateTTL) + r.recoveryMu.Unlock() + + time.AfterFunc(delay, func() { r.recoveryMu.Lock() - current := r.recoveries[key] - if current == nil || current.generation != generation || !current.pending { + if r.recoveries[key] != state { r.recoveryMu.Unlock() return } - // Keep the state after dispatch so the next failure of this range advances - // the backoff attempt. Successful initialization resets it. - current.pending = false - current.timer = time.AfterFunc(regionRecoveryStateTTL, func() { - r.expireRecovery(key, generation) - }) + // Keep the attempt until the retry succeeds or the state expires. + state.expiresAt = time.Now().Add(regionRecoveryStateTTL) r.recoveryMu.Unlock() if ctx.Err() != nil || subscribedSpan.stopped.Load() { @@ -177,24 +161,20 @@ func (r *regionFailureHandler) scheduleRecovery( }) } -func (r *regionFailureHandler) expireRecovery(key regionRecoveryKey, generation uint64) { +func (r *regionFailureHandler) expireRecoveries(now time.Time) { r.recoveryMu.Lock() defer r.recoveryMu.Unlock() - state := r.recoveries[key] - if state != nil && state.generation == generation && !state.pending { - delete(r.recoveries, key) + for key, state := range r.recoveries { + if !state.expiresAt.After(now) { + delete(r.recoveries, key) + } } } func (r *regionFailureHandler) resetRecovery(key regionRecoveryKey) { r.recoveryMu.Lock() defer r.recoveryMu.Unlock() - if state := r.recoveries[key]; state != nil { - if state.timer != nil { - state.timer.Stop() - } - delete(r.recoveries, key) - } + delete(r.recoveries, key) } func (r *regionFailureHandler) resetRegionRecovery(region regionInfo) { @@ -204,12 +184,7 @@ func (r *regionFailureHandler) resetRegionRecovery(region regionInfo) { func (r *regionFailureHandler) cancelRecoveries() { r.recoveryMu.Lock() defer r.recoveryMu.Unlock() - for key, state := range r.recoveries { - if state.timer != nil { - state.timer.Stop() - } - delete(r.recoveries, key) - } + clear(r.recoveries) } // Report admits a region failure into the recovery pipeline. It releases the @@ -251,13 +226,17 @@ func (r *regionFailureHandler) Run(ctx context.Context) error { // r.cache.ready() should handle failures promptly in normal flow. The ticker is only a // fallback scan and is not expected to be needed in practice. - ticker := time.NewTicker(200 * time.Millisecond) - defer ticker.Stop() + fallbackTicker := time.NewTicker(200 * time.Millisecond) + defer fallbackTicker.Stop() + cleanupTicker := time.NewTicker(regionRecoveryStateTTL) + defer cleanupTicker.Stop() for { select { case <-ctx.Done(): return ctx.Err() - case <-ticker.C: + case now := <-cleanupTicker.C: + r.expireRecoveries(now) + case <-fallbackTicker.C: if err := handleCachedErrors(); err != nil { return err } diff --git a/logservice/logpuller/region_failure_handler_test.go b/logservice/logpuller/region_failure_handler_test.go index 5c7f1456de..0ce52a44a5 100644 --- a/logservice/logpuller/region_failure_handler_test.go +++ b/logservice/logpuller/region_failure_handler_test.go @@ -147,7 +147,7 @@ func TestRegionFailureHandlerRunDrainsErrCacheWithoutDispatcher(t *testing.T) { } } -func TestRegionFailureHandlerDelaysNotLeaderRangeRetry(t *testing.T) { +func TestRegionFailureHandlerSchedulesNotLeaderRangeRetry(t *testing.T) { pdClient := newFailureRecoveryTestPDClient(t) defer pdClient.Close() @@ -168,8 +168,6 @@ func TestRegionFailureHandlerDelaysNotLeaderRangeRetry(t *testing.T) { rangeRetryCh <- task }, ) - handler.recoveryDelay = func(uint32) time.Duration { return 50 * time.Millisecond } - errInfo := newRegionErrorInfo(region, &eventError{ err: &cdcpb.Error{NotLeader: &errorpb.NotLeader{}}, }) @@ -178,13 +176,6 @@ func TestRegionFailureHandlerDelaysNotLeaderRangeRetry(t *testing.T) { defer cancel() require.NoError(t, handler.handleError(ctx, errInfo)) - require.NoError(t, handler.handleError(ctx, errInfo)) - - select { - case <-rangeRetryCh: - t.Fatal("not leader retry should be delayed") - case <-time.After(20 * time.Millisecond): - } select { case task := <-rangeRetryCh: @@ -193,12 +184,6 @@ func TestRegionFailureHandlerDelaysNotLeaderRangeRetry(t *testing.T) { case <-time.After(time.Second): t.Fatal("not leader retry was not scheduled") } - - select { - case <-rangeRetryCh: - t.Fatal("pending not leader retry should be deduplicated") - case <-time.After(20 * time.Millisecond): - } } func TestRegionRecoveryBackoffFollowsRangeAcrossRegionChanges(t *testing.T) { @@ -213,12 +198,6 @@ func TestRegionRecoveryBackoffFollowsRangeAcrossRegionChanges(t *testing.T) { ) t.Cleanup(handler.cancelRecoveries) - attempts := make([]uint32, 0, 2) - handler.recoveryDelay = func(attempt uint32) time.Duration { - attempts = append(attempts, attempt) - return time.Millisecond - } - region := createFailureRecoveryTestRegion(t, SubscriptionID(1), 1) errInfo := newRegionErrorInfo(region, &eventError{ err: &cdcpb.Error{Congested: &cdcpb.Congested{}}, @@ -243,16 +222,18 @@ func TestRegionRecoveryBackoffFollowsRangeAcrossRegionChanges(t *testing.T) { t.Fatal("second region recovery was not scheduled") } - require.Equal(t, []uint32{1, 2}, attempts) + key := newRegionRecoveryKey(region.subscribedSpan.subID, region.span) + handler.recoveryMu.Lock() + attempt := handler.recoveries[key].attempt + handler.recoveryMu.Unlock() + require.Equal(t, uint32(2), attempt) } func TestRegionFailureHandlerRequestCancelledResetsRecoveryState(t *testing.T) { handler := newRegionFailureHandler(nil, func(*subscribedSpan) {}, func(context.Context, regionInfo) {}, func(context.Context, rangeTask) {}) region := createFailureRecoveryTestRegion(t, SubscriptionID(1), 1) key := newRegionRecoveryKey(region.subscribedSpan.subID, region.span) - timer := time.NewTimer(time.Hour) - t.Cleanup(func() { timer.Stop() }) - handler.recoveries[key] = ®ionRecoveryState{pending: true, timer: timer} + handler.recoveries[key] = ®ionRecoveryState{} err := handler.handleError(context.Background(), newRegionErrorInfo(region, &requestCancelledErr{})) require.NoError(t, err) @@ -262,3 +243,21 @@ func TestRegionFailureHandlerRequestCancelledResetsRecoveryState(t *testing.T) { handler.recoveryMu.Unlock() assert.False(t, ok) } + +func TestRegionFailureHandlerExpiresRecoveryStates(t *testing.T) { + handler := newRegionFailureHandler(nil, func(*subscribedSpan) {}, func(context.Context, regionInfo) {}, func(context.Context, rangeTask) {}) + now := time.Now() + expiredKey := newRegionRecoveryKey(1, heartbeatpb.TableSpan{StartKey: []byte("a"), EndKey: []byte("b")}) + activeKey := newRegionRecoveryKey(1, heartbeatpb.TableSpan{StartKey: []byte("b"), EndKey: []byte("c")}) + handler.recoveries[expiredKey] = ®ionRecoveryState{expiresAt: now.Add(-time.Second)} + handler.recoveries[activeKey] = ®ionRecoveryState{expiresAt: now.Add(time.Second)} + + handler.expireRecoveries(now) + + handler.recoveryMu.Lock() + _, expiredExists := handler.recoveries[expiredKey] + _, activeExists := handler.recoveries[activeKey] + handler.recoveryMu.Unlock() + require.False(t, expiredExists) + require.True(t, activeExists) +} From ade797bd3d11f3497514de9954201e6017faae28 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Wed, 5 Aug 2026 16:36:57 +0800 Subject: [PATCH 69/71] more refactor --- logservice/logpuller/subscription_client.go | 37 ++++++++------------- 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/logservice/logpuller/subscription_client.go b/logservice/logpuller/subscription_client.go index 331243a4a7..af039a9ebd 100644 --- a/logservice/logpuller/subscription_client.go +++ b/logservice/logpuller/subscription_client.go @@ -31,6 +31,7 @@ import ( "github.com/pingcap/ticdc/pkg/pdutil" "github.com/pingcap/ticdc/pkg/security" "github.com/pingcap/ticdc/pkg/spanz" + "github.com/pingcap/ticdc/pkg/util" "github.com/tikv/client-go/v2/tikv" pd "github.com/tikv/pd/client" "go.uber.org/zap" @@ -46,6 +47,7 @@ const ( // don't need to force reload region anymore. regionScheduleReload = false + loadRegionRetryInterval time.Duration = 100 * time.Millisecond resolveLockMinInterval time.Duration = 10 * time.Second resolveLockTickInterval time.Duration = 2 * time.Second resolveLockFence time.Duration = 4 * time.Second @@ -370,24 +372,14 @@ func (s *subscriptionClient) divideSpanAndScheduleRegionRequests( // Limit the number of regions loaded at a time to make the load more stable. limit := 1024 nextSpan := span - retryRange := func() { - retryTask := rangeTask{ - span: nextSpan, - subscribedSpan: subscribedSpan, - filterLoop: task.filterLoop, - priority: task.priority, - } - s.failureHandler.scheduleRecovery( - ctx, - retryTask.subscribedSpan, - retryTask.span, - 0, - func() { - s.scheduleRangeRequest(ctx, retryTask) - }, - ) - } + backoffBeforeLoad := false for { + if backoffBeforeLoad { + if err := util.Hang(ctx, loadRegionRetryInterval); err != nil { + return err + } + backoffBeforeLoad = false + } log.Debug("subscription client is going to load regions", zap.Uint64("subscriptionID", uint64(subscribedSpan.subID)), zap.Any("span", common.FormatTableSpan(&nextSpan))) @@ -396,15 +388,12 @@ func (s *subscriptionClient) divideSpanAndScheduleRegionRequests( regions, err := s.upstream.regionCache.BatchLoadRegionsWithKeyRange( backoff, nextSpan.StartKey, nextSpan.EndKey, limit) if err != nil { - if ctx.Err() != nil { - return ctx.Err() - } log.Warn("subscription client load regions failed", zap.Uint64("subscriptionID", uint64(subscribedSpan.subID)), zap.Any("span", common.FormatTableSpan(&nextSpan)), zap.Error(err)) - retryRange() - return nil + backoffBeforeLoad = true + continue } regionMetas := make([]*metapb.Region, 0, len(regions)) for _, region := range regions { @@ -417,8 +406,8 @@ func (s *subscriptionClient) divideSpanAndScheduleRegionRequests( log.Warn("subscription client load regions with holes", zap.Uint64("subscriptionID", uint64(subscribedSpan.subID)), zap.Any("span", common.FormatTableSpan(&nextSpan))) - retryRange() - return nil + backoffBeforeLoad = true + continue } for _, regionMeta := range regionMetas { From 5bfe1d80e7104d58812003b9f1857d50c7ae7fd8 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Wed, 5 Aug 2026 16:38:30 +0800 Subject: [PATCH 70/71] fix --- logservice/logpuller/region_request_worker.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index 581f2bfea3..614b67d7f0 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -455,9 +455,7 @@ func (s *regionRequestWorker) sendRegionRequest(conn *ConnAndClient, req *region // Publish the state before Send so a fast response observes its owner and // admission lease. state := newRegionFeedState(region, uint64(subID), s, req, func(state *regionFeedState) { - if s.failureHandler != nil { - s.failureHandler.resetRegionRecovery(state.region) - } + s.failureHandler.resetRegionRecovery(state.region) }) if !s.tracker.Add(subID, region.verID.GetID(), state) { // RangeLock normally prevents duplicate active regions. Keep the existing From 421803329403419bd2beb2bb9337a654e17487a0 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Sun, 9 Aug 2026 14:50:06 +0800 Subject: [PATCH 71/71] remove unnecessary change --- logservice/eventstore/pebble.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logservice/eventstore/pebble.go b/logservice/eventstore/pebble.go index 96caabc3cb..f5f48bf71b 100644 --- a/logservice/eventstore/pebble.go +++ b/logservice/eventstore/pebble.go @@ -42,7 +42,7 @@ func newPebbleOptions(dbNum int) *pebble.Options { MaxOpenFiles: maxOpenFilesPerDB, - MaxConcurrentCompactions: func() int { return 3 }, + MaxConcurrentCompactions: func() int { return 6 }, // Decrease compaction frequency L0CompactionThreshold: 20,