From 14e1fb5ba8fb465191947dfac522c8d30327ab6c Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Fri, 26 Jun 2026 17:14:33 +0800 Subject: [PATCH 01/35] eventstore: support row-level scan resume --- logservice/eventstore/event_store.go | 31 +++++- logservice/eventstore/event_store_test.go | 118 ++++++++++++++++++++++ logservice/eventstore/format.go | 14 +++ logservice/eventstore/pebble_test.go | 13 +++ pkg/common/table_span.go | 11 +- 5 files changed, 183 insertions(+), 4 deletions(-) diff --git a/logservice/eventstore/event_store.go b/logservice/eventstore/event_store.go index f5b3e78986..77c2e92817 100644 --- a/logservice/eventstore/event_store.go +++ b/logservice/eventstore/event_store.go @@ -115,6 +115,14 @@ type EventIterator interface { Close() (eventCnt int64, err error) } +type EventIteratorWithScanPosition interface { + EventIterator + + // NextWithScanPosition returns the next event, the opaque position of the + // returned event, and whether this event is from a new txn. + NextWithScanPosition() (*common.RawKVEntry, common.ScanPosition, bool) +} + type dispatcherStat struct { dispatcherID common.DispatcherID // data span of this dispatcher @@ -918,6 +926,9 @@ func (e *eventStore) GetIterator(dispatcherID common.DispatcherID, dataRange com // Pebble uses [LowerBound, UpperBound), so end is always encoded as // CommitTsEnd+1. // + // If RowLevelScanPosition is present, continue scanning from the next + // eventstore key after that opaque position. + // // If LastScannedTxnStartTs is zero, scan commit ts in // (CommitTsStart, CommitTsEnd], and use CommitTsStart+1 as LowerBound. // @@ -926,7 +937,13 @@ func (e *eventStore) GetIterator(dispatcherID common.DispatcherID, dataRange com // later commit ts up to CommitTsEnd. // var start []byte - if dataRange.LastScannedTxnStartTs != 0 { + if len(dataRange.RowLevelScanPosition) != 0 { + start = encodeRowLevelScanPositionLowerBound( + uint64(subStat.subID), + stat.tableSpan.TableID, + dataRange.RowLevelScanPosition, + ) + } else if dataRange.LastScannedTxnStartTs != 0 { start = encodeScanLowerBound( uint64(subStat.subID), stat.tableSpan.TableID, @@ -1563,10 +1580,16 @@ type eventStoreIter struct { } func (iter *eventStoreIter) Next() (*common.RawKVEntry, bool) { + rawKV, _, isNewTxn := iter.NextWithScanPosition() + return rawKV, isNewTxn +} + +func (iter *eventStoreIter) NextWithScanPosition() (*common.RawKVEntry, common.ScanPosition, bool) { rawKV := &common.RawKVEntry{} + var scanPosition common.ScanPosition for { if !iter.innerIter.Valid() { - return nil, false + return nil, nil, false } key := iter.innerIter.Key() value := iter.innerIter.Value() @@ -1605,11 +1628,13 @@ func (iter *eventStoreIter) Next() (*common.RawKVEntry, bool) { } scannedBytesMetrics.Add(float64(len(value))) if !iter.needCheckSpan { + scanPosition = encodeRowLevelScanPosition(key) break } comparableKey := common.ToComparableKey(rawKV.Key) if bytes.Compare(comparableKey, iter.tableSpan.StartKey) >= 0 && bytes.Compare(comparableKey, iter.tableSpan.EndKey) < 0 { + scanPosition = encodeRowLevelScanPosition(key) break } log.Debug("event store iter skip kv not in table span", @@ -1635,7 +1660,7 @@ func (iter *eventStoreIter) Next() (*common.RawKVEntry, bool) { startTime := time.Now() iter.innerIter.Next() metricEventStoreNextReadDurationHistogram.Observe(time.Since(startTime).Seconds()) - return rawKV, isNewTxn + return rawKV, scanPosition, isNewTxn } func (iter *eventStoreIter) Close() (int64, error) { diff --git a/logservice/eventstore/event_store_test.go b/logservice/eventstore/event_store_test.go index 4ac22e504d..f07e7ff86e 100644 --- a/logservice/eventstore/event_store_test.go +++ b/logservice/eventstore/event_store_test.go @@ -1608,6 +1608,124 @@ func TestEventStoreGetIteratorConcurrently(t *testing.T) { wg.Wait() } +func TestEventStoreResumeTokenSupportsRowLevelResume(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + _, storeInt := newEventStoreForTest(dir) + store := storeInt.(*eventStore) + defer store.Close(ctx) + + const ( + tableID int64 = 1 + txnStartTs uint64 = 120 + txnCommitTs uint64 = 200 + nextStartTs uint64 = 130 + nextCommitTs uint64 = 201 + ) + + dispatcherID := common.NewDispatcherID() + cfID := common.NewChangefeedID4Test("default", "test-cf") + span := &heartbeatpb.TableSpan{TableID: tableID, StartKey: []byte("a"), EndKey: []byte("z")} + ok := store.RegisterDispatcher(cfID, dispatcherID, span, 100, func(watermark, latestCommitTs uint64) {}, false, false) + require.True(t, ok) + + dispatcherStat := store.dispatcherMeta.dispatcherStats[dispatcherID] + require.NotNil(t, dispatcherStat) + subStat := dispatcherStat.subStat + require.NotNil(t, subStat) + + events := []eventWithCallback{ + { + subID: subStat.subID, + tableID: tableID, + kvs: []common.RawKVEntry{ + {OpType: common.OpTypePut, StartTs: txnStartTs, CRTs: txnCommitTs, Key: []byte("row-1"), Value: []byte("value-1")}, + {OpType: common.OpTypePut, StartTs: txnStartTs, CRTs: txnCommitTs, Key: []byte("row-2"), Value: []byte("value-2")}, + {OpType: common.OpTypePut, StartTs: nextStartTs, CRTs: nextCommitTs, Key: []byte("next-row"), Value: []byte("value-3")}, + }, + callback: func() {}, + }, + } + encoder, err := zstd.NewWriter(nil) + require.NoError(t, err) + defer encoder.Close() + var compressionBuf []byte + var rawValueBuf []byte + err = store.writeEvents(store.dbs[subStat.dbIndex], events, encoder, &compressionBuf, &rawValueBuf) + require.NoError(t, err) + subStat.resolvedTs.Store(nextCommitTs) + + type scannedEvent struct { + key string + position common.ScanPosition + } + collectEvents := func(dataRange common.DataRange) []scannedEvent { + iter, err := store.GetIterator(dispatcherID, dataRange) + require.NoError(t, err) + if iter == nil { + return nil + } + positionIter, ok := iter.(EventIteratorWithScanPosition) + require.True(t, ok) + + events := make([]scannedEvent, 0) + for { + rawKV, position, _ := positionIter.NextWithScanPosition() + if rawKV == nil { + break + } + require.NotEmpty(t, position) + events = append(events, scannedEvent{ + key: string(rawKV.Key), + position: position, + }) + } + rowCount, err := iter.Close() + require.NoError(t, err) + require.Equal(t, int64(len(events)), rowCount) + return events + } + + fullRange := common.DataRange{ + Span: span, + CommitTsStart: txnCommitTs - 1, + CommitTsEnd: nextCommitTs, + } + fullEvents := collectEvents(fullRange) + require.Len(t, fullEvents, 3) + require.Equal(t, []string{"row-1", "row-2", "next-row"}, []string{ + fullEvents[0].key, + fullEvents[1].key, + fullEvents[2].key, + }) + + resumeAfterTxnStart := common.DataRange{ + Span: span, + CommitTsStart: txnCommitTs, + CommitTsEnd: nextCommitTs, + LastScannedTxnStartTs: txnStartTs, + } + // LastScannedTxnStartTs can resume after a txn start-ts, but it cannot + // identify a specific row inside the same txn. Once set to txnStartTs, all + // rows in that txn are skipped, including row-2. + txnLevelEvents := collectEvents(resumeAfterTxnStart) + require.Len(t, txnLevelEvents, 1) + require.Equal(t, []string{"next-row"}, []string{txnLevelEvents[0].key}) + + resumeAfterRow1 := common.DataRange{ + Span: span, + CommitTsStart: txnCommitTs, + CommitTsEnd: nextCommitTs, + RowLevelScanPosition: fullEvents[0].position, + } + rowLevelEvents := collectEvents(resumeAfterRow1) + require.Len(t, rowLevelEvents, 2) + require.Equal(t, []string{"row-2", "next-row"}, []string{ + rowLevelEvents[0].key, + rowLevelEvents[1].key, + }) +} + func TestEventWithCallbackSizerUsesCurrentKVBytes(t *testing.T) { event := eventWithCallback{ kvs: []common.RawKVEntry{{ diff --git a/logservice/eventstore/format.go b/logservice/eventstore/format.go index 29942031ef..c75dbc3be8 100644 --- a/logservice/eventstore/format.go +++ b/logservice/eventstore/format.go @@ -93,6 +93,20 @@ func encodeScanLowerBound(uniqueID uint64, tableID int64, txnCommitTs uint64, tx return buf } +func encodeRowLevelScanPosition(key []byte) common.ScanPosition { + position := make(common.ScanPosition, len(key)-encodedKeyTxnCommitTsOffset) + copy(position, key[encodedKeyTxnCommitTsOffset:]) + return position +} + +func encodeRowLevelScanPositionLowerBound(uniqueID uint64, tableID int64, position common.ScanPosition) []byte { + buf := make([]byte, encodedKeyTxnCommitTsOffset, encodedKeyTxnCommitTsOffset+len(position)+1) + binary.BigEndian.PutUint64(buf[encodedKeyUniqueIDOffset:encodedKeyUniqueIDOffset+encodedKeyUniqueIDLen], uniqueID) + binary.BigEndian.PutUint64(buf[encodedKeyTableIDOffset:encodedKeyTableIDOffset+encodedKeyTableIDLen], uint64(tableID)) + buf = append(buf, position...) + return append(buf, 0) +} + func encodeTxnCommitTsBoundaryKeyTo(buf []byte, uniqueID uint64, tableID int64, txnCommitTs uint64) { binary.BigEndian.PutUint64(buf[encodedKeyUniqueIDOffset:encodedKeyUniqueIDOffset+encodedKeyUniqueIDLen], uniqueID) binary.BigEndian.PutUint64(buf[encodedKeyTableIDOffset:encodedKeyTableIDOffset+encodedKeyTableIDLen], uint64(tableID)) diff --git a/logservice/eventstore/pebble_test.go b/logservice/eventstore/pebble_test.go index d143f9904b..ad8d64722f 100644 --- a/logservice/eventstore/pebble_test.go +++ b/logservice/eventstore/pebble_test.go @@ -170,6 +170,11 @@ func TestEventStoreKeyBounds(t *testing.T) { require.Len(t, lowerBound, encodedKeyTxnStartTsOffset+encodedKeyTxnStartTsLen) require.True(t, bytes.HasPrefix(key, lowerBound)) + rowLevelPosition := encodeRowLevelScanPosition(key) + require.Equal(t, common.ScanPosition(key[encodedKeyTxnCommitTsOffset:]), rowLevelPosition) + rowLevelLowerBound := encodeRowLevelScanPositionLowerBound(1, 1, rowLevelPosition) + require.Less(t, bytes.Compare(key, rowLevelLowerBound), 0) + previousEvent := &common.RawKVEntry{ OpType: common.OpTypePut, StartTs: event.StartTs - 1, @@ -177,4 +182,12 @@ func TestEventStoreKeyBounds(t *testing.T) { Key: []byte("key"), } require.Less(t, bytes.Compare(EncodeKey(1, 1, previousEvent, CompressionNone), lowerBound), 0) + + nextRowSameTxn := &common.RawKVEntry{ + OpType: common.OpTypePut, + StartTs: event.StartTs, + CRTs: event.CRTs, + Key: []byte("key\x00"), + } + require.LessOrEqual(t, bytes.Compare(rowLevelLowerBound, EncodeKey(1, 1, nextRowSameTxn, CompressionNone)), 0) } diff --git a/pkg/common/table_span.go b/pkg/common/table_span.go index fd40bd5f15..73661e7f5b 100644 --- a/pkg/common/table_span.go +++ b/pkg/common/table_span.go @@ -26,11 +26,17 @@ type DataRange struct { CommitTsStart uint64 CommitTsEnd uint64 + // RowLevelScanPosition is an opaque eventstore-owned token used to resume + // scanning after a specific row inside the scan window. + RowLevelScanPosition ScanPosition + // LastScannedTxnStartTs is the start-ts of the last scanned DML event. // it should less than the CommitTsStart LastScannedTxnStartTs uint64 } +type ScanPosition []byte + func NewDataRange(clusterID uint64, span *heartbeatpb.TableSpan, startTs, endTs uint64) *DataRange { return &DataRange{ ClusterID: clusterID, @@ -53,7 +59,10 @@ func (d *DataRange) String() string { } func (d *DataRange) Equal(other *DataRange) bool { - return d.Span.Equal(other.Span) && d.CommitTsStart == other.CommitTsStart && d.CommitTsEnd == other.CommitTsEnd + return d.Span.Equal(other.Span) && + d.CommitTsStart == other.CommitTsStart && + d.CommitTsEnd == other.CommitTsEnd && + bytes.Equal(d.RowLevelScanPosition, other.RowLevelScanPosition) } // Merge merges two DataRange, if the two DataRange have different Span, return nil. From a0779ff64d7286f958432b1d9b39bd32d48affef Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Fri, 26 Jun 2026 17:58:07 +0800 Subject: [PATCH 02/35] eventservice: split large scanned transactions --- pkg/eventservice/dispatcher_stat.go | 46 ++- pkg/eventservice/dispatcher_stat_test.go | 17 +- pkg/eventservice/event_broker.go | 34 +- pkg/eventservice/event_broker_test.go | 92 ++++++ pkg/eventservice/event_scanner.go | 378 ++++++++++++++++++++++- pkg/eventservice/event_scanner_test.go | 344 +++++++++++++++++++-- pkg/eventservice/event_service_test.go | 60 +++- pkg/eventservice/large_txn_spill.go | 187 +++++++++++ pkg/eventservice/large_txn_spill_test.go | 102 ++++++ pkg/eventservice/large_txn_state.go | 200 ++++++++++++ 10 files changed, 1415 insertions(+), 45 deletions(-) create mode 100644 pkg/eventservice/large_txn_spill.go create mode 100644 pkg/eventservice/large_txn_spill_test.go create mode 100644 pkg/eventservice/large_txn_state.go diff --git a/pkg/eventservice/dispatcher_stat.go b/pkg/eventservice/dispatcher_stat.go index 6132055271..16c26a1941 100644 --- a/pkg/eventservice/dispatcher_stat.go +++ b/pkg/eventservice/dispatcher_stat.go @@ -108,6 +108,10 @@ type dispatcherStat struct { // These two values are used to construct the scan range for the next scan task. lastScannedCommitTs atomic.Uint64 lastScannedStartTs atomic.Uint64 + lastScannedPosition atomic.Value + + largeTxnStateMu sync.Mutex + largeTxnState *largeTxnScanState // isRemoved is used to indicate whether the dispatcher is removed. // it is set to true in the following two cases: @@ -211,8 +215,41 @@ func (a *dispatcherStat) updateSentResolvedTs(resolvedTs uint64) { } func (a *dispatcherStat) updateScanRange(txnCommitTs, txnStartTs uint64) { + a.updateScanRangeWithPosition(txnCommitTs, txnStartTs, nil) +} + +func (a *dispatcherStat) updateScanRangeWithPosition( + txnCommitTs uint64, + txnStartTs uint64, + position common.ScanPosition, +) { a.lastScannedCommitTs.Store(txnCommitTs) a.lastScannedStartTs.Store(txnStartTs) + a.storeLastScannedPosition(position) +} + +func (a *dispatcherStat) storeLastScannedPosition(position common.ScanPosition) { + if len(position) == 0 { + a.lastScannedPosition.Store(common.ScanPosition{}) + return + } + positionCopy := make(common.ScanPosition, len(position)) + copy(positionCopy, position) + a.lastScannedPosition.Store(positionCopy) +} + +func (a *dispatcherStat) getLastScannedPosition() common.ScanPosition { + value := a.lastScannedPosition.Load() + if value == nil { + return nil + } + position := value.(common.ScanPosition) + if len(position) == 0 { + return nil + } + positionCopy := make(common.ScanPosition, len(position)) + copy(positionCopy, position) + return positionCopy } // onResolvedTs try to update the resolved ts of the dispatcher. @@ -239,10 +276,16 @@ func (a *dispatcherStat) onLatestCommitTs(latestCommitTs uint64) bool { func (a *dispatcherStat) getDataRange() (common.DataRange, bool) { lastTxnCommitTs := a.lastScannedCommitTs.Load() lastTxnStartTs := a.lastScannedStartTs.Load() + lastPosition := a.getLastScannedPosition() + hasPendingLargeTxn := a.hasPendingLargeTxnState() // the data not received by the event store yet, so just skip it. resolvedTs := a.receivedResolvedTs.Load() - if lastTxnCommitTs >= resolvedTs { + if lastTxnCommitTs > resolvedTs { + return common.DataRange{}, false + } + if lastTxnCommitTs == resolvedTs && lastTxnStartTs == 0 && + len(lastPosition) == 0 && !hasPendingLargeTxn { return common.DataRange{}, false } // Range: (CommitTsStart-lastScannedStartTs, CommitTsEnd], @@ -251,6 +294,7 @@ func (a *dispatcherStat) getDataRange() (common.DataRange, bool) { Span: a.info.GetTableSpan(), CommitTsStart: lastTxnCommitTs, CommitTsEnd: resolvedTs, + RowLevelScanPosition: lastPosition, LastScannedTxnStartTs: lastTxnStartTs, } return r, true diff --git a/pkg/eventservice/dispatcher_stat_test.go b/pkg/eventservice/dispatcher_stat_test.go index e58e175217..08ccb7dd91 100644 --- a/pkg/eventservice/dispatcher_stat_test.go +++ b/pkg/eventservice/dispatcher_stat_test.go @@ -108,14 +108,27 @@ func TestDispatcherStatGetDataRange(t *testing.T) { r, ok = stat.getDataRange() require.False(t, ok) - // case 4: get range after resolved ts update again + // case 4: same commit-ts may still have later transactions when the + // transaction-level resume start-ts is set. + stat.updateScanRange(200, 150) + r, ok = stat.getDataRange() + require.True(t, ok) + require.Equal(t, uint64(200), r.CommitTsStart) + require.Equal(t, uint64(150), r.LastScannedTxnStartTs) + require.Equal(t, uint64(200), r.CommitTsEnd) + + // case 5: get range after resolved ts update again stat.onResolvedTs(300) r, ok = stat.getDataRange() require.True(t, ok) require.Equal(t, uint64(200), r.CommitTsStart) - require.Equal(t, uint64(0), r.LastScannedTxnStartTs) + require.Equal(t, uint64(150), r.LastScannedTxnStartTs) require.Equal(t, uint64(300), r.CommitTsEnd) require.Equal(t, info.GetTableSpan(), r.Span) + + stat.updateSentResolvedTs(300) + r, ok = stat.getDataRange() + require.False(t, ok) } func TestDispatcherStatUpdateWatermark(t *testing.T) { diff --git a/pkg/eventservice/event_broker.go b/pkg/eventservice/event_broker.go index e21c3aeb26..cae9772012 100644 --- a/pkg/eventservice/event_broker.go +++ b/pkg/eventservice/event_broker.go @@ -471,6 +471,13 @@ func (c *eventBroker) getScanTaskDataRange(task scanTask) (bool, common.DataRang } if dataRange.CommitTsEnd <= dataRange.CommitTsStart { + hasSameCommitTxnResume := dataRange.LastScannedTxnStartTs != 0 && + dataRange.CommitTsEnd == dataRange.CommitTsStart + if len(dataRange.RowLevelScanPosition) != 0 || + hasSameCommitTxnResume || + task.hasPendingLargeTxnState() { + return true, dataRange + } updateMetricEventServiceSkipResolvedTsCount(task.info.GetMode()) // Scan range can become empty after applying capping (for example, scan window). // Send a signal resolved-ts event (rate limited) to keep downstream responsive, @@ -698,7 +705,7 @@ func (c *eventBroker) doScan(ctx context.Context, task scanTask) { } scanner := newEventScanner(c.eventStore, c.schemaStore, c.mounter, task.info.GetMode()) - scannedBytes, events, interrupted, err := scanner.scan(ctx, task, dataRange, sl) + scannedBytes, events, progress, interrupted, err := scanner.scan(ctx, task, dataRange, sl) if scannedBytes < 0 { releaseQuota(available, uint64(sl.maxDMLBytes)) } else if scannedBytes >= 0 && scannedBytes < sl.maxDMLBytes { @@ -758,6 +765,13 @@ func (c *eventBroker) doScan(ctx context.Context, task scanTask) { log.Panic("unknown event type", zap.Any("event", e)) } } + if progress.valid { + task.updateScanRangeWithPosition( + progress.txnCommitTs, + progress.txnStartTs, + progress.rowLevelScanPosition, + ) + } task.info.GetMode() // Update metrics metricEventBrokerScanTaskCount.Inc() @@ -1095,6 +1109,12 @@ func (c *eventBroker) removeDispatcher(dispatcherInfo DispatcherInfo) { stat := statPtr.(*atomic.Pointer[dispatcherStat]).Load() stat.isRemoved.Store(true) + if err := stat.cleanupLargeTxnState(); err != nil { + log.Warn("cleanup large txn state failed when removing dispatcher", + zap.Stringer("changefeedID", dispatcherInfo.GetChangefeedID()), + zap.Stringer("dispatcherID", id), + zap.Error(err)) + } if isTableTriggerDispatcher { c.tableTriggerDispatchers.Delete(id) @@ -1168,6 +1188,12 @@ func (c *eventBroker) resetDispatcher(dispatcherInfo DispatcherInfo) error { // No need to worry that the old dispatcher is still scanning, // because its data will be filtered by event collector because of stale epoch. oldStat.isRemoved.Store(true) + if err := oldStat.cleanupLargeTxnState(); err != nil { + log.Warn("cleanup large txn state failed when resetting dispatcher", + zap.Stringer("changefeedID", dispatcherInfo.GetChangefeedID()), + zap.Stringer("dispatcherID", dispatcherID), + zap.Error(err)) + } // Create a new dispatcherStat and replace the old one. // The new dispatcherStat will be used for all future operations. @@ -1218,6 +1244,12 @@ func (c *eventBroker) resetDispatcher(dispatcherInfo DispatcherInfo) error { return nil } oldStat.isRemoved.Store(true) + if err := oldStat.cleanupLargeTxnState(); err != nil { + log.Warn("cleanup large txn state failed when retrying dispatcher reset", + zap.Stringer("changefeedID", changefeedID), + zap.Stringer("dispatcherID", dispatcherID), + zap.Error(err)) + } } log.Info("reset dispatcher", diff --git a/pkg/eventservice/event_broker_test.go b/pkg/eventservice/event_broker_test.go index da6a03b9cf..e7ffc97705 100644 --- a/pkg/eventservice/event_broker_test.go +++ b/pkg/eventservice/event_broker_test.go @@ -16,6 +16,7 @@ package eventservice import ( "context" "errors" + "os" "sync" "testing" "time" @@ -423,6 +424,46 @@ func TestDoScanSkipWhenChangefeedStatusNotFound(t *testing.T) { require.False(t, disp.isTaskScanning.Load()) } +func TestDoScanKeepsRowLevelProgressAfterSendingFragment(t *testing.T) { + broker, mockStore, mockSchemaStore, _ := newEventBrokerForTest() + broker.close() + + helper := event.NewEventTestHelper(t) + defer helper.Close() + ddlEvent, kvEvents := genEvents(helper, `create table test.t_do_scan_split(id int primary key, c char(50))`, []string{ + `insert into test.t_do_scan_split(id,c) values (0, "c0")`, + `insert into test.t_do_scan_split(id,c) values (1, "c1")`, + }...) + require.Len(t, kvEvents, 2) + kvEvents[1].StartTs = kvEvents[0].StartTs + kvEvents[1].CRTs = kvEvents[0].CRTs + resolvedTs := kvEvents[0].CRTs + + dispInfo := newMockDispatcherInfoForTest(t) + dispInfo.startTs = ddlEvent.FinishedTs + require.NoError(t, broker.addDispatcher(dispInfo)) + + disp := broker.getDispatcher(dispInfo.GetID()).Load() + require.NotNil(t, disp) + disp.setHandshaked() + disp.currentScanLimitInBytes.Store(1) + disp.receivedResolvedTs.Store(resolvedTs) + disp.eventStoreCommitTs.Store(resolvedTs) + + status := broker.getOrSetChangefeedStatus(dispInfo) + status.availableMemoryQuota.Store(node.ID(dispInfo.GetServerID()), atomic.NewUint64(broker.scanLimitInBytes)) + + mockSchemaStore.AppendDDLEvent(dispInfo.GetTableSpan().TableID, ddlEvent) + require.NoError(t, mockStore.AppendEvents(dispInfo.GetID(), resolvedTs, kvEvents...)) + + broker.doScan(context.Background(), disp) + + require.Equal(t, resolvedTs, disp.lastScannedCommitTs.Load()) + require.Equal(t, kvEvents[0].StartTs, disp.lastScannedStartTs.Load()) + require.NotEmpty(t, disp.getLastScannedPosition()) + require.True(t, disp.isTaskScanning.Load()) +} + func TestCURDDispatcher(t *testing.T) { broker, _, _, _ := newEventBrokerForTest() defer broker.close() @@ -544,6 +585,57 @@ func TestResetDispatcher(t *testing.T) { require.Equal(t, dispInfo.GetID(), newStat.id) } +func TestDispatcherLifecycleCleansLargeTxnState(t *testing.T) { + t.Run("reset", func(t *testing.T) { + broker, _, _, _ := newEventBrokerForTest() + defer broker.close() + + dispInfo := newMockDispatcherInfoForTest(t) + require.NoError(t, broker.addDispatcher(dispInfo)) + + dispPtr := broker.getDispatcher(dispInfo.GetID()) + require.NotNil(t, dispPtr) + oldStat := dispPtr.Load() + spillPath := mustCreateLargeTxnState(t, oldStat, dispInfo.GetTableSpan().TableID) + + resetInfo := newMockDispatcherInfo(t, 500, dispInfo.GetID(), dispInfo.GetTableSpan().TableID, eventpb.ActionType_ACTION_TYPE_RESET) + resetInfo.epoch = oldStat.epoch + 1 + require.NoError(t, broker.resetDispatcher(resetInfo)) + + require.Nil(t, oldStat.getLargeTxnState()) + _, err := os.Stat(spillPath) + require.True(t, os.IsNotExist(err)) + }) + + t.Run("remove", func(t *testing.T) { + broker, _, _, _ := newEventBrokerForTest() + defer broker.close() + + dispInfo := newMockDispatcherInfoForTest(t) + require.NoError(t, broker.addDispatcher(dispInfo)) + + dispPtr := broker.getDispatcher(dispInfo.GetID()) + require.NotNil(t, dispPtr) + stat := dispPtr.Load() + spillPath := mustCreateLargeTxnState(t, stat, dispInfo.GetTableSpan().TableID) + + broker.removeDispatcher(dispInfo) + + require.Nil(t, stat.getLargeTxnState()) + _, err := os.Stat(spillPath) + require.True(t, os.IsNotExist(err)) + }) +} + +func mustCreateLargeTxnState(t *testing.T, stat *dispatcherStat, tableID int64) string { + t.Helper() + + state, err := stat.getOrCreateLargeTxnState(t.TempDir(), tableID, nil, 90, 100) + require.NoError(t, err) + require.NoError(t, state.appendInsert(newTestSpillRawKVEntry(1))) + return state.spill.path +} + func TestResetDispatcherConcurrently(t *testing.T) { broker, _, _, _ := newEventBrokerForTest() defer broker.close() diff --git a/pkg/eventservice/event_scanner.go b/pkg/eventservice/event_scanner.go index 974751502c..be34c25674 100644 --- a/pkg/eventservice/event_scanner.go +++ b/pkg/eventservice/event_scanner.go @@ -15,6 +15,8 @@ package eventservice import ( "context" + "io" + "os" "time" "github.com/pingcap/log" @@ -63,6 +65,30 @@ type eventScanner struct { mode int64 } +type scanProgress struct { + valid bool + txnCommitTs uint64 + txnStartTs uint64 + rowLevelScanPosition common.ScanPosition +} + +func newTxnScanProgress(commitTs uint64, startTs uint64) scanProgress { + return scanProgress{ + valid: true, + txnCommitTs: commitTs, + txnStartTs: startTs, + } +} + +func newRowLevelScanProgress(commitTs uint64, startTs uint64, position common.ScanPosition) scanProgress { + progress := newTxnScanProgress(commitTs, startTs) + if len(position) > 0 { + progress.rowLevelScanPosition = make(common.ScanPosition, len(position)) + copy(progress.rowLevelScanPosition, position) + } + return progress +} + // newEventScanner creates a new EventScanner func newEventScanner( eventStore eventGetter, @@ -116,28 +142,44 @@ func (s *eventScanner) scan( dispatcherStat *dispatcherStat, dataRange common.DataRange, limit scanLimit, -) (int64, []event.Event, bool, error) { +) (int64, []event.Event, scanProgress, bool, error) { // Initialize scan session sess := newSession(ctx, dispatcherStat, dataRange, limit) defer sess.recordMetrics() + if state := dispatcherStat.getLargeTxnState(); state != nil && + state.getPhase() == largeTxnScanPhaseDrainInserts { + interrupted, err := s.drainLargeTxnInserts(sess, state) + if err != nil { + _ = dispatcherStat.cleanupLargeTxnState() + } + return sess.eventBytes, sess.events, sess.progress, interrupted, err + } + // Fetch DDL events start := time.Now() events, err := s.fetchDDLEvents(dispatcherStat, dataRange) if err != nil { - return 0, nil, false, err + return 0, nil, scanProgress{}, false, err } metrics.EventServiceGetDDLEventDuration.Observe(time.Since(start).Seconds()) iter, err := s.eventGetter.GetIterator(dispatcherStat.info.GetID(), dataRange) if err != nil { - return 0, nil, false, err + return 0, nil, scanProgress{}, false, err } if iter == nil { + if state := dispatcherStat.getLargeTxnState(); state != nil && + state.getPhase() == largeTxnScanPhaseOriginal { + dispatcherStat.markLargeTxnDrainInserts(state.startTs, state.commitTs, false, 0) + sess.progress = newTxnScanProgress(state.commitTs, state.startTs) + return 0, sess.events, sess.progress, true, nil + } resolved := event.NewResolvedEvent(dataRange.CommitTsEnd, dispatcherStat.id, dispatcherStat.epoch) events = append(events, resolved) sess.appendEvents(events) - return 0, sess.events, false, nil + sess.progress = newTxnScanProgress(dataRange.CommitTsEnd, 0) + return 0, sess.events, sess.progress, false, nil } // Execute event scanning and merging @@ -150,12 +192,14 @@ func (s *eventScanner) scan( zap.Stringer("dispatcherID", dispatcherStat.info.GetID()), zap.Error(closeErr)) } - return sess.eventBytes, sess.events, interrupted, scanErr + _ = dispatcherStat.cleanupLargeTxnState() + return sess.eventBytes, sess.events, sess.progress, interrupted, scanErr } if closeErr != nil { - return 0, nil, false, closeErr + _ = dispatcherStat.cleanupLargeTxnState() + return 0, nil, scanProgress{}, false, closeErr } - return sess.eventBytes, sess.events, interrupted, nil + return sess.eventBytes, sess.events, sess.progress, interrupted, nil } // fetchDDLEvents retrieves DDL events which finishedTs are within the range (start, end] @@ -213,6 +257,7 @@ func (s *eventScanner) scanAndMergeEvents( dispatcher.info.IsOutputRawChangeEvent(), s.mode, dispatcher.info.EnableIgnoreUpdateOnlyColumns()) + processor.dispatcherStat = dispatcher for { shouldStop, err := s.checkScanConditions(session) @@ -223,18 +268,55 @@ func (s *eventScanner) scanAndMergeEvents( return false, nil } - rawEvent, isNewTxn := iter.Next() + rawEvent, position, isNewTxn := nextEventWithScanPosition(iter) if rawEvent == nil { + if state := dispatcher.getLargeTxnState(); processor.currentTxn == nil && + state != nil && + state.getPhase() == largeTxnScanPhaseOriginal { + dispatcher.markLargeTxnDrainInserts(state.startTs, state.commitTs, false, 0) + session.progress = newTxnScanProgress(state.commitTs, state.startTs) + return true, nil + } + interrupted, err := s.finishCurrentTxn( + session, + merger, + processor, + 0, + 0, + false, + ) + if err != nil || interrupted { + return interrupted, err + } err = finalizeScan(merger, processor, session, session.dataRange.CommitTsEnd) return false, err } - session.observeRawEntry(rawEvent) + if state := dispatcher.getLargeTxnState(); processor.currentTxn == nil && + state != nil && + state.getPhase() == largeTxnScanPhaseOriginal && + (rawEvent.StartTs != state.startTs || rawEvent.CRTs != state.commitTs) { + dispatcher.markLargeTxnDrainInserts(state.startTs, state.commitTs, true, rawEvent.CRTs) + session.progress = newTxnScanProgress(state.commitTs, state.startTs) + return true, nil + } + if isNewTxn { tableInfo, err := s.getTableInfo4Txn(dispatcher, tableID, rawEvent.CRTs-1) if err != nil { return false, err } + interrupted, err := s.finishCurrentTxn( + session, + merger, + processor, + rawEvent.CRTs, + getTableInfoUpdateTs(tableInfo), + tableInfo == nil, + ) + if err != nil || interrupted { + return interrupted, err + } // The table has been deleted, so the current raw event cannot be // decoded as DML. Resolve to its commit ts to skip it; resolving to // rawEvent.CRTs-1 can equal the scan start and cause a no-progress loop. @@ -243,10 +325,6 @@ func (s *eventScanner) scanAndMergeEvents( return false, err } - if err = s.commitTxn(session, merger, processor, rawEvent.CRTs, tableInfo.GetUpdateTS()); err != nil { - return false, err - } - if session.exceedLimit(processor.batchDML.GetSize(), processor.batchDML) && merger.canInterrupt(rawEvent.CRTs, processor.batchDML) { interruptScan(session, merger, processor, rawEvent.CRTs, rawEvent.StartTs) @@ -259,6 +337,7 @@ func (s *eventScanner) scanAndMergeEvents( } } + session.observeRawEntry(rawEvent, position) if err = processor.appendRow(rawEvent); err != nil { log.Error("append row failed", zap.Error(err), zap.Stringer("dispatcherID", session.dispatcherStat.id), @@ -268,7 +347,28 @@ func (s *eventScanner) scanAndMergeEvents( zap.Int64("mode", s.mode)) return false, err } + if s.canInterruptCurrentTxn(session, merger, processor, rawEvent, position) { + interruptCurrentTxn(session, merger, processor, rawEvent.CRTs, rawEvent.StartTs, position) + return true, nil + } + } +} + +func nextEventWithScanPosition( + iter eventstore.EventIterator, +) (*common.RawKVEntry, common.ScanPosition, bool) { + if positionIter, ok := iter.(eventstore.EventIteratorWithScanPosition); ok { + return positionIter.NextWithScanPosition() } + rawEvent, isNewTxn := iter.Next() + return rawEvent, nil, isNewTxn +} + +func getTableInfoUpdateTs(tableInfo *common.TableInfo) uint64 { + if tableInfo == nil { + return 0 + } + return tableInfo.GetUpdateTS() } // checkScanConditions checks context cancellation and dispatcher status @@ -355,6 +455,68 @@ func (s *eventScanner) commitTxn( return nil } +func (s *eventScanner) finishCurrentTxn( + session *session, + merger *eventMerger, + processor *dmlProcessor, + nextCommitTs uint64, + nextTableInfoUpdateTs uint64, + nextTableDeleted bool, +) (bool, error) { + if processor.currentTxn == nil { + return false, nil + } + currentStartTs := processor.currentTxn.CurrentDMLEvent.GetStartTs() + currentCommitTs := processor.currentTxn.CurrentDMLEvent.GetCommitTs() + + if processor.hasSpilledInsertsForCurrentTxn() && merger.hasDDLAtCommitTs(currentCommitTs) { + if err := processor.flushSpilledInsertsIntoCurrentTxn(); err != nil { + return false, err + } + } + + if err := s.commitTxn(session, merger, processor, nextCommitTs, nextTableInfoUpdateTs); err != nil { + return false, err + } + if !processor.hasLargeTxnState(currentStartTs, currentCommitTs) { + return false, nil + } + + events := merger.mergeWithPrecedingDDLs(processor.getCurrentBatchDML()) + session.appendEvents(events) + processor.resetBatchDML() + + hasFollowingTxn := nextCommitTs != 0 && !nextTableDeleted + session.dispatcherStat.markLargeTxnDrainInserts( + currentStartTs, + currentCommitTs, + hasFollowingTxn, + nextCommitTs, + ) + if len(session.lastRowPosition) > 0 { + session.progress = newRowLevelScanProgress(currentCommitTs, currentStartTs, session.lastRowPosition) + } else { + session.progress = newTxnScanProgress(currentCommitTs, currentStartTs) + } + return true, nil +} + +func (s *eventScanner) canInterruptCurrentTxn( + session *session, + merger *eventMerger, + processor *dmlProcessor, + rawEvent *common.RawKVEntry, + position common.ScanPosition, +) bool { + if len(position) == 0 || !session.dispatcherStat.txnAtomicity.ShouldSplitTxn() { + return false + } + if !merger.canInterrupt(rawEvent.CRTs, processor.batchDML) { + return false + } + return session.exceedLimit(processor.batchDML.GetSize(), processor.batchDML) +} + // finalizeScan finalizes the scan when all events have been processed // it's called when the iterator is nil, always indicates that all entries // with the same commit-ts is processed, so it's ok to append resolved-ts event @@ -375,6 +537,7 @@ func finalizeScan( resolveTs := event.NewResolvedEvent(endTs, sess.dispatcherStat.id, sess.dispatcherStat.epoch) events = append(events, resolveTs) sess.appendEvents(events) + sess.progress = newTxnScanProgress(endTs, 0) return nil } @@ -421,6 +584,106 @@ func interruptScan( session.appendEvents(events) } +func interruptCurrentTxn( + session *session, + merger *eventMerger, + processor *dmlProcessor, + commitTs uint64, + startTs uint64, + position common.ScanPosition, +) { + events := merger.mergeWithPrecedingDDLs(processor.getCurrentBatchDML()) + session.appendEvents(events) + session.progress = newRowLevelScanProgress(commitTs, startTs, position) + log.Debug("scan interrupted inside a large txn", + zap.Stringer("dispatcherID", session.dispatcherStat.id), + zap.Uint64("startTs", startTs), + zap.Uint64("commitTs", commitTs), + zap.Int("scannedEntryCount", session.scannedEntryCount), + zap.Duration("duration", time.Since(session.startTime))) +} + +func (s *eventScanner) drainLargeTxnInserts( + session *session, + state *largeTxnScanState, +) (bool, error) { + processor := newDMLProcessor( + s.mounter, + s.schemaGetter, + session.dispatcherStat.filter, + session.dispatcherStat.info.IsOutputRawChangeEvent(), + s.mode, + session.dispatcherStat.info.EnableIgnoreUpdateOnlyColumns()) + processor.dispatcherStat = session.dispatcherStat + + for { + shouldStop, err := s.checkScanConditions(session) + if err != nil { + return false, err + } + if shouldStop { + return false, nil + } + + entry, err := state.nextInsert() + if err != nil { + if session.dispatcherStat.isRemoved.Load() { + return false, nil + } + if errors.Is(err, io.EOF) { + if processor.currentTxn != nil { + if err := processor.commitTxn(); err != nil { + return false, err + } + if processor.getCurrentBatchDML().DMLCount() != 0 { + session.appendEvents([]event.Event{processor.getCurrentBatchDML()}) + } + } + session.progress = newTxnScanProgress(state.commitTs, state.startTs) + hasFollowingTxn, followingCommitTs := state.snapshotDrainInfo() + shouldResolveCommitTs := (!hasFollowingTxn || followingCommitTs > state.commitTs) && + session.dataRange.CommitTsEnd == state.commitTs + if err := session.dispatcherStat.cleanupLargeTxnState(); err != nil { + return false, err + } + if shouldResolveCommitTs { + resolved := event.NewResolvedEvent(state.commitTs, session.dispatcherStat.id, session.dispatcherStat.epoch) + session.appendEvents([]event.Event{resolved}) + session.progress = newTxnScanProgress(state.commitTs, 0) + return false, nil + } + return true, nil + } + return false, err + } + + if processor.currentTxn == nil { + if err := processor.startTxn( + session.dispatcherStat.id, + state.tableID, + state.tableInfo, + state.startTs, + state.commitTs, + true, + ); err != nil { + return false, err + } + } + session.observeRawEntry(entry, nil) + if err := processor.appendInsertRow(entry); err != nil { + return false, err + } + if session.exceedLimit(processor.batchDML.GetSize(), processor.batchDML) { + if err := processor.commitTxn(); err != nil { + return false, err + } + session.appendEvents([]event.Event{processor.getCurrentBatchDML()}) + session.progress = newTxnScanProgress(state.commitTs, state.startTs) + return true, nil + } + } +} + // session manages the state and context of a scan operation type session struct { ctx context.Context @@ -433,12 +696,14 @@ type session struct { scannedBytes int64 scannedEntryCount int + lastRowPosition common.ScanPosition // dmlCount is the count of transactions. dmlCount int // Result collection, including DDL, BatchedDML, ResolvedTs events in the timestamp order. events []event.Event eventBytes int64 + progress scanProgress } // newSession creates a new scan session @@ -459,9 +724,15 @@ func newSession( } // observeRawEntry adds to the total bytes scanned -func (s *session) observeRawEntry(entry *common.RawKVEntry) { +func (s *session) observeRawEntry(entry *common.RawKVEntry, position common.ScanPosition) { s.scannedBytes += entry.GetSize() s.scannedEntryCount++ + if len(position) == 0 { + s.lastRowPosition = nil + return + } + s.lastRowPosition = make(common.ScanPosition, len(position)) + copy(s.lastRowPosition, position) } // isContextDone checks if the context is cancelled @@ -689,8 +960,10 @@ type dmlProcessor struct { mounter event.Mounter schemaGetter schemaGetter - filter filter.Filter - filterContext filter.DMLFilterContext + filter filter.Filter + filterContext filter.DMLFilterContext + dispatcherStat *dispatcherStat + spillDir string // insertRowCache is used to cache the split update event's insert part of the current transaction. // It will be used to append to the current DML event when the transaction is finished. @@ -722,6 +995,7 @@ func newDMLProcessor( filterContext: filterContext, batchDML: event.NewBatchDMLEvent(), insertRowCache: make([]*common.RawKVEntry, 0), + spillDir: os.TempDir(), outputRawChangeEvent: outputRawChangeEvent, mode: mode, } @@ -757,6 +1031,56 @@ func (p *dmlProcessor) commitTxn() error { return nil } +func (p *dmlProcessor) hasSpilledInsertsForCurrentTxn() bool { + if p.currentTxn == nil || p.dispatcherStat == nil { + return false + } + current := p.currentTxn.CurrentDMLEvent + return p.hasLargeTxnState(current.GetStartTs(), current.GetCommitTs()) +} + +func (p *dmlProcessor) hasLargeTxnState(startTs uint64, commitTs uint64) bool { + if p.dispatcherStat == nil { + return false + } + state := p.dispatcherStat.getLargeTxnState() + return state != nil && + state.startTs == startTs && + state.commitTs == commitTs && + state.getPhase() == largeTxnScanPhaseOriginal +} + +func (p *dmlProcessor) flushSpilledInsertsIntoCurrentTxn() error { + if p.currentTxn == nil || p.dispatcherStat == nil { + return nil + } + state := p.dispatcherStat.getLargeTxnState() + if state == nil || state.getPhase() != largeTxnScanPhaseOriginal { + return nil + } + for { + insertRow, err := state.nextInsert() + if err != nil { + if errors.Is(err, io.EOF) { + return p.dispatcherStat.cleanupLargeTxnState() + } + return err + } + if err := p.appendInsertRow(insertRow); err != nil { + return err + } + } +} + +func (p *dmlProcessor) appendInsertRow(rawEvent *common.RawKVEntry) error { + if p.currentTxn == nil { + log.Panic("no current DML event to append to") + } + rawEvent.Key = event.RemoveKeyspacePrefix(rawEvent.Key) + updateMetricEventServiceSendDMLTypeCount(p.mode, rawEvent.GetType(), false) + return p.currentTxn.AppendRow(rawEvent, p.mounter.DecodeToChunk, p.filter, p.filterContext) +} + // appendRow appends a row to the current DML event. // // This method processes a raw KV entry and appends it to the current DML event. It handles @@ -819,7 +1143,27 @@ func (p *dmlProcessor) appendRow(rawEvent *common.RawKVEntry) error { if err != nil { return err } - p.insertRowCache = append(p.insertRowCache, insertRow) + if p.currentTxn.shouldSplitTxn { + if p.dispatcherStat == nil { + return errors.New("dispatcher stat is required for large txn update spill") + } + currentDML := p.currentTxn.CurrentDMLEvent + state, err := p.dispatcherStat.getOrCreateLargeTxnState( + p.spillDir, + currentDML.GetTableID(), + currentDML.TableInfo, + currentDML.GetStartTs(), + currentDML.GetCommitTs(), + ) + if err != nil { + return err + } + if err := state.appendInsert(insertRow); err != nil { + return err + } + } else { + p.insertRowCache = append(p.insertRowCache, insertRow) + } return p.currentTxn.AppendRow(deleteRow, p.mounter.DecodeToChunk, p.filter, p.filterContext) } diff --git a/pkg/eventservice/event_scanner_test.go b/pkg/eventservice/event_scanner_test.go index 451c9cb39d..99874f9d29 100644 --- a/pkg/eventservice/event_scanner_test.go +++ b/pkg/eventservice/event_scanner_test.go @@ -25,6 +25,7 @@ import ( "github.com/pingcap/ticdc/logservice/schemastore" "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/common/event" + "github.com/pingcap/ticdc/pkg/config" "github.com/pingcap/ticdc/pkg/filter" "github.com/pingcap/ticdc/pkg/integrity" "github.com/pingcap/tidb/pkg/util/chunk" @@ -78,7 +79,7 @@ func TestEventScannerReturnsIteratorErrors(t *testing.T) { &mockMounter{}, 0, ) - _, events, interrupted, err := scanner.scan(context.Background(), disp, dataRange, scanLimit{}) + _, events, _, interrupted, err := scanner.scan(context.Background(), disp, dataRange, scanLimit{}) require.ErrorIs(t, err, getIterErr) require.Nil(t, events) require.False(t, interrupted) @@ -90,7 +91,7 @@ func TestEventScannerReturnsIteratorErrors(t *testing.T) { &mockMounter{}, 0, ) - _, events, interrupted, err = scanner.scan(context.Background(), disp, dataRange, scanLimit{}) + _, events, _, interrupted, err = scanner.scan(context.Background(), disp, dataRange, scanLimit{}) require.ErrorIs(t, err, closeErr) require.Nil(t, events) require.False(t, interrupted) @@ -118,6 +119,7 @@ func TestEventScanner(t *testing.T) { ctx := context.Background() disp := newDispatcherStat(disInfo, 1, 1, nil, changefeedStatus) + disp.txnAtomicity = config.AtomicityLevel("table") makeDispatcherReady(disp) err := broker.addDispatcher(disp.info) require.NoError(t, err) @@ -134,7 +136,7 @@ func TestEventScanner(t *testing.T) { ok, dataRange := broker.getScanTaskDataRange(disp) require.True(t, ok) - _, events, isInterrupted, err := scanner.scan(ctx, disp, dataRange, sl) + _, events, _, isInterrupted, err := scanner.scan(ctx, disp, dataRange, sl) require.NoError(t, err) require.False(t, isInterrupted) require.Equal(t, 1, len(events)) @@ -155,7 +157,7 @@ func TestEventScanner(t *testing.T) { } scanner = newEventScanner(broker.eventStore, broker.schemaStore, &mockMounter{}, 0) - _, events, isInterrupted, err = scanner.scan(ctx, disp, dataRange, sl) + _, events, _, isInterrupted, err = scanner.scan(ctx, disp, dataRange, sl) require.NoError(t, err) require.False(t, isInterrupted) require.Equal(t, 2, len(events)) @@ -185,7 +187,7 @@ func TestEventScanner(t *testing.T) { } scanner = newEventScanner(broker.eventStore, broker.schemaStore, &mockMounter{}, 0) - _, events, isInterrupted, err = scanner.scan(ctx, disp, dataRange, sl) + _, events, _, isInterrupted, err = scanner.scan(ctx, disp, dataRange, sl) require.NoError(t, err) require.False(t, isInterrupted) require.Equal(t, 2, len(events)) @@ -224,7 +226,7 @@ func TestEventScanner(t *testing.T) { maxDMLBytes: 1000, } scanner = newEventScanner(broker.eventStore, broker.schemaStore, &mockMounter{}, 0) - _, events, isInterrupted, err = scanner.scan(ctx, disp, dataRange, sl) + _, events, _, isInterrupted, err = scanner.scan(ctx, disp, dataRange, sl) require.NoError(t, err) require.False(t, isInterrupted) require.Equal(t, 4, len(events)) @@ -241,6 +243,7 @@ func TestEventScanner(t *testing.T) { require.Equal(t, batchDML2.DMLEvents[1].GetCommitTs(), kvEvents[2].CRTs) require.Equal(t, batchDML2.DMLEvents[2].GetCommitTs(), kvEvents[3].CRTs) + disp.txnAtomicity = config.AtomicityLevel("table") // case 5: Reaches scan limit, only 1 DDL and 1 DML event scanned // Tests that when MaxBytes limit is reached, the scanner returns partial events with isInterrupted=true // Event sequence: @@ -254,7 +257,7 @@ func TestEventScanner(t *testing.T) { } scanner = newEventScanner(broker.eventStore, broker.schemaStore, &mockMounter{}, 0) - _, events, isInterrupted, err = scanner.scan(ctx, disp, dataRange, sl) + _, events, _, isInterrupted, err = scanner.scan(ctx, disp, dataRange, sl) require.NoError(t, err) require.True(t, isInterrupted) require.Equal(t, 3, len(events)) @@ -298,7 +301,7 @@ func TestEventScanner(t *testing.T) { require.True(t, ok) scanner = newEventScanner(broker.eventStore, broker.schemaStore, &mockMounter{}, 0) - _, events, isInterrupted, err = scanner.scan(ctx, disp, dataRange, sl) + _, events, _, isInterrupted, err = scanner.scan(ctx, disp, dataRange, sl) require.NoError(t, err) require.True(t, isInterrupted) require.Equal(t, 2, len(events)) @@ -337,7 +340,7 @@ func TestEventScanner(t *testing.T) { } scanner = newEventScanner(broker.eventStore, broker.schemaStore, &mockMounter{}, 0) - _, events, isInterrupted, err = scanner.scan(ctx, disp, dataRange, sl) + _, events, _, isInterrupted, err = scanner.scan(ctx, disp, dataRange, sl) require.NoError(t, err) require.True(t, isInterrupted) require.Equal(t, 3, len(events)) @@ -376,7 +379,7 @@ func TestEventScanner(t *testing.T) { maxDMLBytes: 1000, } scanner = newEventScanner(broker.eventStore, broker.schemaStore, &mockMounter{}, 0) - _, events, isInterrupted, err = scanner.scan(ctx, disp, dataRange, sl) + _, events, _, isInterrupted, err = scanner.scan(ctx, disp, dataRange, sl) require.NoError(t, err) require.False(t, isInterrupted) require.Equal(t, 5, len(events)) @@ -411,6 +414,277 @@ func TestEventScanner(t *testing.T) { require.Equal(t, resolvedTs, e.GetCommitTs()) } +func TestEventScannerSplitsLargeTxnWithRowLevelProgress(t *testing.T) { + helper := event.NewEventTestHelper(t) + defer helper.Close() + ddlEvent, kvEvents := genEvents(helper, `create table test.t_split(id int primary key, c char(50))`, []string{ + `insert into test.t_split(id,c) values (0, "c0")`, + `insert into test.t_split(id,c) values (1, "c1")`, + }...) + require.Len(t, kvEvents, 2) + + kvEvents[1].StartTs = kvEvents[0].StartTs + kvEvents[1].CRTs = kvEvents[0].CRTs + resolvedTs := kvEvents[0].CRTs + + broker, _, _, _ := newEventBrokerForTest() + broker.close() + mockStore := broker.eventStore.(*mockEventStore) + mockSchemaStore := broker.schemaStore.(*mockSchemaStore) + + disInfo := newMockDispatcherInfoForTest(t) + disInfo.startTs = ddlEvent.FinishedTs + changefeedStatus := broker.getOrSetChangefeedStatus(disInfo) + disp := newDispatcherStat(disInfo, 1, 1, nil, changefeedStatus) + makeDispatcherReady(disp) + require.NoError(t, broker.addDispatcher(disp.info)) + + mockSchemaStore.AppendDDLEvent(disInfo.GetTableSpan().TableID, ddlEvent) + require.NoError(t, mockStore.AppendEvents(disInfo.GetID(), resolvedTs, kvEvents...)) + disp.receivedResolvedTs.Store(resolvedTs) + disp.eventStoreCommitTs.Store(resolvedTs) + + scanner := newEventScanner(broker.eventStore, broker.schemaStore, &mockMounter{}, 0) + sl := scanLimit{maxDMLBytes: 1, isInUnitTest: true} + + dataRange, ok := disp.getDataRange() + require.True(t, ok) + _, events, progress, interrupted, err := scanner.scan(context.Background(), disp, dataRange, sl) + require.NoError(t, err) + require.True(t, interrupted) + require.True(t, progress.valid) + require.NotEmpty(t, progress.rowLevelScanPosition) + require.Len(t, events, 1) + firstBatch := events[0].(*event.BatchDMLEvent) + require.Equal(t, int32(1), firstBatch.Len()) + require.Equal(t, kvEvents[0].CRTs, firstBatch.GetCommitTs()) + + disp.updateScanRangeWithPosition(progress.txnCommitTs, progress.txnStartTs, progress.rowLevelScanPosition) + dataRange, ok = disp.getDataRange() + require.True(t, ok) + require.Equal(t, progress.rowLevelScanPosition, dataRange.RowLevelScanPosition) + _, events, progress, interrupted, err = scanner.scan(context.Background(), disp, dataRange, sl) + require.NoError(t, err) + require.True(t, interrupted) + require.True(t, progress.valid) + require.NotEmpty(t, progress.rowLevelScanPosition) + require.Len(t, events, 1) + secondBatch := events[0].(*event.BatchDMLEvent) + require.Equal(t, int32(1), secondBatch.Len()) + require.Equal(t, kvEvents[1].CRTs, secondBatch.GetCommitTs()) + + disp.updateScanRangeWithPosition(progress.txnCommitTs, progress.txnStartTs, progress.rowLevelScanPosition) + dataRange, ok = disp.getDataRange() + require.True(t, ok) + _, events, progress, interrupted, err = scanner.scan(context.Background(), disp, dataRange, sl) + require.NoError(t, err) + require.False(t, interrupted) + require.True(t, progress.valid) + require.Len(t, events, 1) + resolved, ok := events[0].(event.ResolvedEvent) + require.True(t, ok) + require.Equal(t, resolvedTs, resolved.ResolvedTs) +} + +func TestEventScannerSpillsSplitUKUpdateInLargeTxn(t *testing.T) { + helper := event.NewEventTestHelper(t) + defer helper.Close() + helper.Tk().MustExec("use test") + ddlEvent := helper.DDL2Event("create table t_uk_split (id int primary key, a int, b char(50), unique key uk_a(a))") + _, updateEvent := helper.DML2UpdateEvent("test", "t_uk_split", + "insert into test.t_uk_split(id,a,b) values (1, 10, 'old_b')", + "update test.t_uk_split set a = 20 where id = 1") + resolvedTs := updateEvent.CRTs + + broker, _, _, _ := newEventBrokerForTest() + broker.close() + mockStore := broker.eventStore.(*mockEventStore) + mockSchemaStore := broker.schemaStore.(*mockSchemaStore) + + disInfo := newMockDispatcherInfoForTest(t) + disInfo.startTs = updateEvent.StartTs + changefeedStatus := broker.getOrSetChangefeedStatus(disInfo) + disp := newDispatcherStat(disInfo, 1, 1, nil, changefeedStatus) + makeDispatcherReady(disp) + require.NoError(t, broker.addDispatcher(disp.info)) + + mockSchemaStore.AppendDDLEvent(disInfo.GetTableSpan().TableID, *ddlEvent) + require.NoError(t, mockStore.AppendEvents(disInfo.GetID(), resolvedTs, updateEvent)) + disp.receivedResolvedTs.Store(resolvedTs) + disp.eventStoreCommitTs.Store(resolvedTs) + + scanner := newEventScanner(broker.eventStore, broker.schemaStore, event.NewMounter(time.UTC, &integrity.Config{}), 0) + sl := scanLimit{maxDMLBytes: 1, isInUnitTest: true} + + dataRange, ok := disp.getDataRange() + require.True(t, ok) + _, events, progress, interrupted, err := scanner.scan(context.Background(), disp, dataRange, sl) + require.NoError(t, err) + require.True(t, interrupted) + require.True(t, progress.valid) + require.NotEmpty(t, progress.rowLevelScanPosition) + require.Len(t, events, 1) + deleteBatch := events[0].(*event.BatchDMLEvent) + require.Equal(t, int32(1), deleteBatch.Len()) + deleteRow, ok := deleteBatch.DMLEvents[0].GetNextRow() + require.True(t, ok) + require.Equal(t, common.RowTypeDelete, deleteRow.RowType) + require.NotNil(t, disp.getLargeTxnState()) + + disp.updateScanRangeWithPosition(progress.txnCommitTs, progress.txnStartTs, progress.rowLevelScanPosition) + dataRange, ok = disp.getDataRange() + require.True(t, ok) + _, events, progress, interrupted, err = scanner.scan(context.Background(), disp, dataRange, sl) + require.NoError(t, err) + require.True(t, interrupted) + require.Empty(t, events) + require.True(t, progress.valid) + + disp.updateScanRangeWithPosition(progress.txnCommitTs, progress.txnStartTs, progress.rowLevelScanPosition) + dataRange, ok = disp.getDataRange() + require.True(t, ok) + _, events, progress, interrupted, err = scanner.scan(context.Background(), disp, dataRange, sl) + require.NoError(t, err) + require.True(t, interrupted) + require.True(t, progress.valid) + require.Len(t, events, 1) + insertBatch := events[0].(*event.BatchDMLEvent) + require.Equal(t, int32(1), insertBatch.Len()) + insertRow, ok := insertBatch.DMLEvents[0].GetNextRow() + require.True(t, ok) + require.Equal(t, common.RowTypeInsert, insertRow.RowType) + + disp.updateScanRangeWithPosition(progress.txnCommitTs, progress.txnStartTs, progress.rowLevelScanPosition) + dataRange, ok = disp.getDataRange() + require.True(t, ok) + _, events, progress, interrupted, err = scanner.scan(context.Background(), disp, dataRange, sl) + require.NoError(t, err) + require.False(t, interrupted) + require.True(t, progress.valid) + require.Len(t, events, 1) + resolved, ok := events[0].(event.ResolvedEvent) + require.True(t, ok) + require.Equal(t, resolvedTs, resolved.ResolvedTs) + require.Nil(t, disp.getLargeTxnState()) +} + +func TestEventScannerDrainsSpillBeforeFollowingSameCommitTxn(t *testing.T) { + helper := event.NewEventTestHelper(t) + defer helper.Close() + helper.Tk().MustExec("use test") + ddlEvent := helper.DDL2Event("create table t_uk_follow (id int primary key, a int, b char(50), unique key uk_a(a))") + _, updateEvent := helper.DML2UpdateEvent("test", "t_uk_follow", + "insert into test.t_uk_follow(id,a,b) values (1, 10, 'old_b')", + "update test.t_uk_follow set a = 20 where id = 1") + followingEvents := helper.DML2RawKv( + ddlEvent.GetTableID(), + ddlEvent.FinishedTs, + "insert into test.t_uk_follow(id,a,b) values (2, 30, 'new_b')") + require.Len(t, followingEvents, 1) + followingEvent := followingEvents[0] + + resolvedTs := updateEvent.CRTs + updateEvent.StartTs = resolvedTs - 2 + followingEvent.StartTs = resolvedTs - 1 + followingEvent.CRTs = resolvedTs + + broker, _, _, _ := newEventBrokerForTest() + broker.close() + mockStore := broker.eventStore.(*mockEventStore) + mockSchemaStore := broker.schemaStore.(*mockSchemaStore) + + disInfo := newMockDispatcherInfoForTest(t) + disInfo.startTs = updateEvent.StartTs - 1 + changefeedStatus := broker.getOrSetChangefeedStatus(disInfo) + disp := newDispatcherStat(disInfo, 1, 1, nil, changefeedStatus) + makeDispatcherReady(disp) + require.NoError(t, broker.addDispatcher(disp.info)) + + mockSchemaStore.AppendDDLEvent(disInfo.GetTableSpan().TableID, *ddlEvent) + require.NoError(t, mockStore.AppendEvents(disInfo.GetID(), resolvedTs, updateEvent, followingEvent)) + disp.receivedResolvedTs.Store(resolvedTs) + disp.eventStoreCommitTs.Store(resolvedTs) + + scanner := newEventScanner(broker.eventStore, broker.schemaStore, event.NewMounter(time.UTC, &integrity.Config{}), 0) + smallLimit := scanLimit{maxDMLBytes: 1, isInUnitTest: true} + + scanAndAdvance := func(limit scanLimit) ([]event.Event, scanProgress, bool) { + ok, dataRange := broker.getScanTaskDataRange(disp) + require.True(t, ok) + _, events, progress, interrupted, err := scanner.scan(context.Background(), disp, dataRange, limit) + require.NoError(t, err) + require.True(t, progress.valid) + disp.updateScanRangeWithPosition(progress.txnCommitTs, progress.txnStartTs, progress.rowLevelScanPosition) + return events, progress, interrupted + } + + events, progress, interrupted := scanAndAdvance(smallLimit) + require.True(t, interrupted) + require.NotEmpty(t, progress.rowLevelScanPosition) + require.Len(t, events, 1) + deleteBatch := events[0].(*event.BatchDMLEvent) + deleteRow, ok := deleteBatch.DMLEvents[0].GetNextRow() + require.True(t, ok) + require.Equal(t, common.RowTypeDelete, deleteRow.RowType) + + events, _, interrupted = scanAndAdvance(smallLimit) + require.True(t, interrupted) + require.Empty(t, events) + state := disp.getLargeTxnState() + require.NotNil(t, state) + require.Equal(t, largeTxnScanPhaseDrainInserts, state.getPhase()) + + events, _, interrupted = scanAndAdvance(smallLimit) + require.True(t, interrupted) + require.Len(t, events, 1) + insertBatch := events[0].(*event.BatchDMLEvent) + insertRow, ok := insertBatch.DMLEvents[0].GetNextRow() + require.True(t, ok) + require.Equal(t, common.RowTypeInsert, insertRow.RowType) + + events, _, interrupted = scanAndAdvance(smallLimit) + require.True(t, interrupted) + require.Empty(t, events) + require.Nil(t, disp.getLargeTxnState()) + + events, _, interrupted = scanAndAdvance(scanLimit{maxDMLBytes: 100, isInUnitTest: true}) + require.False(t, interrupted) + require.Len(t, events, 2) + followingBatch := events[0].(*event.BatchDMLEvent) + require.Equal(t, followingEvent.StartTs, followingBatch.DMLEvents[0].GetStartTs()) + require.Equal(t, followingEvent.CRTs, followingBatch.GetCommitTs()) + followingRow, ok := followingBatch.DMLEvents[0].GetNextRow() + require.True(t, ok) + require.Equal(t, common.RowTypeInsert, followingRow.RowType) + resolved, ok := events[1].(event.ResolvedEvent) + require.True(t, ok) + require.Equal(t, resolvedTs, resolved.ResolvedTs) +} + +func TestDrainLargeTxnInsertsStopsWhenDispatcherRemoved(t *testing.T) { + info := newMockDispatcherInfoForTest(t) + status := newChangefeedStatusForTest(t, info) + disp := newDispatcherStat(info, 1, 1, nil, status) + state, err := disp.getOrCreateLargeTxnState(t.TempDir(), info.GetTableSpan().TableID, nil, 90, 100) + require.NoError(t, err) + require.NoError(t, state.appendInsert(newTestSpillRawKVEntry(1))) + disp.markLargeTxnDrainInserts(90, 100, false, 0) + disp.isRemoved.Store(true) + + scanner := newEventScanner(nil, NewMockSchemaStore(), &mockMounter{}, 0) + sess := newSession(context.Background(), disp, common.DataRange{ + Span: info.GetTableSpan(), + CommitTsStart: 100, + CommitTsEnd: 100, + }, scanLimit{maxDMLBytes: 1, isInUnitTest: true}) + + interrupted, err := scanner.drainLargeTxnInserts(sess, state) + require.NoError(t, err) + require.False(t, interrupted) + require.Empty(t, sess.events) + require.NoError(t, disp.cleanupLargeTxnState()) +} + // Test the case where some DMLs have commit timestamps newer than the table's delete version func TestEventScannerWithDeleteTable(t *testing.T) { broker, _, _, _ := newEventBrokerForTest() @@ -426,6 +700,7 @@ func TestEventScannerWithDeleteTable(t *testing.T) { dispatcherID := disInfo.GetID() disp := newDispatcherStat(disInfo, 1, 1, nil, changefeedStatus) + disp.txnAtomicity = config.AtomicityLevel("table") makeDispatcherReady(disp) err := broker.addDispatcher(disp.info) require.NoError(t, err) @@ -459,7 +734,7 @@ func TestEventScannerWithDeleteTable(t *testing.T) { sl := scanLimit{ maxDMLBytes: 10000, } - _, events, isInterrupted, err := scanner.scan(context.Background(), disp, dataRange, sl) + _, events, _, isInterrupted, err := scanner.scan(context.Background(), disp, dataRange, sl) require.NoError(t, err) require.False(t, isInterrupted) require.Equal(t, 4, len(events)) @@ -507,6 +782,7 @@ func TestEventScannerWithDDL(t *testing.T) { dispatcherID := disInfo.GetID() disp := newDispatcherStat(disInfo, 1, 1, nil, changefeedStatus) + disp.txnAtomicity = config.AtomicityLevel("table") makeDispatcherReady(disp) err := broker.addDispatcher(disp.info) @@ -561,7 +837,7 @@ func TestEventScannerWithDDL(t *testing.T) { } ctx := context.Background() - _, events, isInterrupted, err := scanner.scan(ctx, disp, dataRange, sl) + _, events, _, isInterrupted, err := scanner.scan(ctx, disp, dataRange, sl) require.NoError(t, err) require.True(t, isInterrupted) require.Equal(t, 3, len(events)) @@ -596,7 +872,7 @@ func TestEventScannerWithDDL(t *testing.T) { maxDMLBytes: 2, isInUnitTest: true, } - _, events, isInterrupted, err = scanner.scan(ctx, disp, dataRange, sl) + _, events, _, isInterrupted, err = scanner.scan(ctx, disp, dataRange, sl) require.NoError(t, err) require.True(t, isInterrupted) require.Equal(t, 3, len(events)) @@ -635,7 +911,7 @@ func TestEventScannerWithDDL(t *testing.T) { maxDMLBytes: 3, // Event if we set 3, it should not be interrupted at DML2 isInUnitTest: true, } - _, events, isInterrupted, err = scanner.scan(ctx, disp, dataRange, sl) + _, events, _, isInterrupted, err = scanner.scan(ctx, disp, dataRange, sl) require.NoError(t, err) require.True(t, isInterrupted) @@ -702,7 +978,7 @@ func TestEventScannerWithDDL(t *testing.T) { ok, dataRange = broker.getScanTaskDataRange(disp) require.True(t, ok) - _, events, isInterrupted, err = scanner.scan(ctx, disp, dataRange, sl) + _, events, _, isInterrupted, err = scanner.scan(ctx, disp, dataRange, sl) require.NoError(t, err) require.False(t, isInterrupted) @@ -948,6 +1224,33 @@ func TestDMLProcessor(t *testing.T) { require.Equal(t, 1, len(processor.batchDML.DMLEvents)) require.Equal(t, int32(2), processor.batchDML.Len()) }) + + t.Run("UpdateThatChangesUKSpillsInsertWhenSplitTxn", func(t *testing.T) { + processor := newDMLProcessor(mockMounter, mockSchemaGetter, nil, false, common.DefaultMode, false) + disp := &dispatcherStat{id: dispatcherID} + processor.dispatcherStat = disp + processor.spillDir = t.TempDir() + + helper.Tk().MustExec("use test") + ddlEvent := helper.DDL2Event("create table t3 (id int primary key, a int(50), b char(50), unique key uk_a(a))") + tableInfo := ddlEvent.TableInfo + tableID := ddlEvent.GetTableID() + + _, updateEvent := helper.DML2UpdateEvent("test", "t3", + "insert into test.t3(id, a, b) values (0, 1, 'b0')", + "update test.t3 set a = 2 where id = 0") + require.NoError(t, processor.startTxn(dispatcherID, tableID, tableInfo, updateEvent.StartTs, updateEvent.CRTs, true)) + require.NoError(t, processor.appendRow(updateEvent)) + + require.Empty(t, processor.insertRowCache) + state := disp.getLargeTxnState() + require.NotNil(t, state) + insertRow, err := state.nextInsert() + require.NoError(t, err) + require.Equal(t, common.OpTypePut, insertRow.OpType) + require.False(t, insertRow.IsUpdate()) + require.NoError(t, disp.cleanupLargeTxnState()) + }) } // TestDMLProcessorAppendRow tests the appendRow method of dmlProcessor @@ -1141,12 +1444,12 @@ func TestScanSession(t *testing.T) { Key: []byte("insert_key_1"), Value: []byte("insert_value_1"), } - sess.observeRawEntry(entry) + sess.observeRawEntry(entry, nil) require.Equal(t, entry.GetSize(), sess.scannedBytes) require.Equal(t, 1, sess.scannedEntryCount) // Test adding more bytes - sess.observeRawEntry(entry) + sess.observeRawEntry(entry, nil) require.Equal(t, 2*entry.GetSize(), sess.scannedBytes) require.Equal(t, 2, sess.scannedEntryCount) }) @@ -1497,9 +1800,10 @@ func TestScanAndMergeEventsSingleUKUpdate(t *testing.T) { disInfo := newMockDispatcherInfoForTest(t) stat := &dispatcherStat{ - info: disInfo, - id: dispatcherID, - isRemoved: atomic.Bool{}, + info: disInfo, + id: dispatcherID, + txnAtomicity: config.AtomicityLevel("table"), + isRemoved: atomic.Bool{}, } dataRange := common.DataRange{ diff --git a/pkg/eventservice/event_service_test.go b/pkg/eventservice/event_service_test.go index 166df9429e..8f39f7c5e4 100644 --- a/pkg/eventservice/event_service_test.go +++ b/pkg/eventservice/event_service_test.go @@ -15,6 +15,7 @@ package eventservice import ( "context" + "encoding/binary" "fmt" "sync" "sync/atomic" @@ -269,15 +270,38 @@ func (m *mockEventStore) GetIterator( events := spanStats.getAllEvents() entries := make([]*common.RawKVEntry, 0) - for _, e := range events { + positions := make([]common.ScanPosition, 0) + rowLevelStart := decodeMockScanPosition(dataRange.RowLevelScanPosition) + for i, e := range events { + if rowLevelStart >= 0 && i <= rowLevelStart { + continue + } + if len(dataRange.RowLevelScanPosition) != 0 { + if e.CRTs >= dataRange.CommitTsStart && e.CRTs <= dataRange.CommitTsEnd { + entries = append(entries, e) + positions = append(positions, encodeMockScanPosition(i)) + } + continue + } + if dataRange.LastScannedTxnStartTs != 0 { + if e.CRTs == dataRange.CommitTsStart && e.StartTs <= dataRange.LastScannedTxnStartTs { + continue + } + if e.CRTs >= dataRange.CommitTsStart && e.CRTs <= dataRange.CommitTsEnd { + entries = append(entries, e) + positions = append(positions, encodeMockScanPosition(i)) + } + continue + } if e.CRTs > dataRange.CommitTsStart && e.CRTs <= dataRange.CommitTsEnd { entries = append(entries, e) + positions = append(positions, encodeMockScanPosition(i)) } } var iter eventstore.EventIterator if len(entries) != 0 { - iter = &mockEventIterator{events: entries} + iter = &mockEventIterator{events: entries, positions: positions} } return iter, nil } @@ -311,6 +335,7 @@ func (m *mockEventStore) RegisterDispatcher( type mockEventIterator struct { events []*common.RawKVEntry + positions []common.ScanPosition prevStartTS uint64 prevCommitTS uint64 rowCount int @@ -318,24 +343,51 @@ type mockEventIterator struct { } func (iter *mockEventIterator) Next() (*common.RawKVEntry, bool) { + row, _, isNewTxn := iter.NextWithScanPosition() + return row, isNewTxn +} + +func (iter *mockEventIterator) NextWithScanPosition() (*common.RawKVEntry, common.ScanPosition, bool) { if len(iter.events) == 0 { - return nil, false + return nil, nil, false } row := iter.events[0] iter.events = iter.events[1:] + var position common.ScanPosition + if len(iter.positions) > 0 { + position = iter.positions[0] + iter.positions = iter.positions[1:] + } else { + position = encodeMockScanPosition(iter.rowCount) + } isNewTxn := iter.prevCommitTS == 0 || row.StartTs != iter.prevStartTS || row.CRTs != iter.prevCommitTS iter.prevStartTS = row.StartTs iter.prevCommitTS = row.CRTs iter.rowCount++ - return row, isNewTxn + return row, position, isNewTxn } func (m *mockEventIterator) Close() (int64, error) { return int64(m.rowCount), m.closeErr } +func encodeMockScanPosition(index int) common.ScanPosition { + var buf [8]byte + binary.BigEndian.PutUint64(buf[:], uint64(index)) + position := make(common.ScanPosition, len(buf)) + copy(position, buf[:]) + return position +} + +func decodeMockScanPosition(position common.ScanPosition) int { + if len(position) == 0 { + return -1 + } + return int(binary.BigEndian.Uint64(position)) +} + var _ schemastore.SchemaStore = &mockSchemaStore{} type mockSpanStats struct { diff --git a/pkg/eventservice/large_txn_spill.go b/pkg/eventservice/large_txn_spill.go new file mode 100644 index 0000000000..83092db996 --- /dev/null +++ b/pkg/eventservice/large_txn_spill.go @@ -0,0 +1,187 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package eventservice + +import ( + "encoding/binary" + "io" + "os" + + "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/errors" +) + +const largeTxnSpillRecordLenSize = 8 + +// largeTxnInsertSpill stores deferred insert rows for a single large transaction. +type largeTxnInsertSpill struct { + path string + file *os.File + closed bool + cleaned bool +} + +func newLargeTxnInsertSpill(dir string) (*largeTxnInsertSpill, error) { + if dir == "" { + return nil, errors.New("large txn spill dir is empty") + } + + file, err := os.CreateTemp(dir, "eventservice-large-txn-insert-*.spill") + if err != nil { + return nil, errors.Trace(err) + } + + return &largeTxnInsertSpill{ + path: file.Name(), + file: file, + }, nil +} + +func (s *largeTxnInsertSpill) Append(entry *common.RawKVEntry) error { + if s.cleaned { + return errors.New("large txn spill has been cleaned up") + } + if s.closed { + return errors.New("large txn spill is closed") + } + if entry == nil { + return errors.New("cannot append nil RawKVEntry to large txn spill") + } + + payload := entry.Encode() + var lenBuf [largeTxnSpillRecordLenSize]byte + binary.LittleEndian.PutUint64(lenBuf[:], uint64(len(payload))) + if err := writeLargeTxnSpillRecord(s.file, lenBuf[:]); err != nil { + return err + } + return writeLargeTxnSpillRecord(s.file, payload) +} + +func (s *largeTxnInsertSpill) NewReader() (*largeTxnInsertSpillReader, error) { + if s.cleaned { + return nil, errors.New("large txn spill has been cleaned up") + } + if err := s.Close(); err != nil { + return nil, err + } + + file, err := os.Open(s.path) + if err != nil { + return nil, errors.Trace(err) + } + return &largeTxnInsertSpillReader{file: file}, nil +} + +func (s *largeTxnInsertSpill) Close() error { + if s == nil || s.closed { + return nil + } + s.closed = true + if s.file == nil { + return nil + } + + err := s.file.Close() + s.file = nil + return errors.Trace(err) +} + +func (s *largeTxnInsertSpill) Cleanup() error { + if s == nil { + return nil + } + if err := s.Close(); err != nil { + return err + } + if s.cleaned { + return nil + } + s.cleaned = true + if s.path == "" { + return nil + } + + err := os.Remove(s.path) + if err != nil && !os.IsNotExist(err) { + return errors.Trace(err) + } + return nil +} + +type largeTxnInsertSpillReader struct { + file *os.File + closed bool +} + +func (r *largeTxnInsertSpillReader) Next() (*common.RawKVEntry, error) { + if r.closed { + return nil, errors.New("large txn spill reader is closed") + } + + var lenBuf [largeTxnSpillRecordLenSize]byte + _, err := io.ReadFull(r.file, lenBuf[:]) + if err != nil { + if errors.Is(err, io.EOF) { + return nil, io.EOF + } + return nil, errors.Trace(err) + } + + recordLen := binary.LittleEndian.Uint64(lenBuf[:]) + if recordLen == 0 { + return nil, errors.New("large txn spill contains empty record") + } + if recordLen > uint64(int(^uint(0)>>1)) { + return nil, errors.Errorf("large txn spill record is too large: %d", recordLen) + } + + data := make([]byte, int(recordLen)) + if _, err := io.ReadFull(r.file, data); err != nil { + return nil, errors.Trace(err) + } + + entry := &common.RawKVEntry{} + if err := entry.Decode(data); err != nil { + return nil, errors.Trace(err) + } + return entry, nil +} + +func (r *largeTxnInsertSpillReader) Close() error { + if r == nil || r.closed { + return nil + } + r.closed = true + if r.file == nil { + return nil + } + + err := r.file.Close() + r.file = nil + return errors.Trace(err) +} + +func writeLargeTxnSpillRecord(writer io.Writer, data []byte) error { + for len(data) > 0 { + n, err := writer.Write(data) + if err != nil { + return errors.Trace(err) + } + if n == 0 { + return errors.New("failed to write large txn spill record") + } + data = data[n:] + } + return nil +} diff --git a/pkg/eventservice/large_txn_spill_test.go b/pkg/eventservice/large_txn_spill_test.go new file mode 100644 index 0000000000..3e67468798 --- /dev/null +++ b/pkg/eventservice/large_txn_spill_test.go @@ -0,0 +1,102 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package eventservice + +import ( + "fmt" + "io" + "os" + "testing" + + "github.com/pingcap/ticdc/pkg/common" + "github.com/stretchr/testify/require" +) + +func TestLargeTxnInsertSpillReadOrder(t *testing.T) { + spill, err := newLargeTxnInsertSpill(t.TempDir()) + require.NoError(t, err) + defer func() { + require.NoError(t, spill.Cleanup()) + }() + + entries := []*common.RawKVEntry{ + newTestSpillRawKVEntry(1), + newTestSpillRawKVEntry(2), + newTestSpillRawKVEntry(3), + } + for _, entry := range entries { + require.NoError(t, spill.Append(entry)) + } + + reader, err := spill.NewReader() + require.NoError(t, err) + defer func() { + require.NoError(t, reader.Close()) + }() + + for _, expected := range entries { + actual, err := reader.Next() + require.NoError(t, err) + require.Equal(t, expected, actual) + } + actual, err := reader.Next() + require.ErrorIs(t, err, io.EOF) + require.Nil(t, actual) +} + +func TestLargeTxnInsertSpillCleanup(t *testing.T) { + spill, err := newLargeTxnInsertSpill(t.TempDir()) + require.NoError(t, err) + require.NoError(t, spill.Append(newTestSpillRawKVEntry(1))) + + path := spill.path + require.NoError(t, spill.Cleanup()) + require.NoError(t, spill.Cleanup()) + + _, err = os.Stat(path) + require.True(t, os.IsNotExist(err)) + + reader, err := spill.NewReader() + require.Error(t, err) + require.Nil(t, reader) +} + +func TestLargeTxnInsertSpillEmpty(t *testing.T) { + spill, err := newLargeTxnInsertSpill(t.TempDir()) + require.NoError(t, err) + defer func() { + require.NoError(t, spill.Cleanup()) + }() + + reader, err := spill.NewReader() + require.NoError(t, err) + defer func() { + require.NoError(t, reader.Close()) + }() + + entry, err := reader.Next() + require.ErrorIs(t, err, io.EOF) + require.Nil(t, entry) +} + +func newTestSpillRawKVEntry(index int) *common.RawKVEntry { + return &common.RawKVEntry{ + OpType: common.OpTypePut, + CRTs: 100, + StartTs: 90, + RegionID: uint64(index), + Key: []byte(fmt.Sprintf("key-%02d", index)), + Value: []byte(fmt.Sprintf("value-%02d", index)), + } +} diff --git a/pkg/eventservice/large_txn_state.go b/pkg/eventservice/large_txn_state.go new file mode 100644 index 0000000000..780c7d0b1a --- /dev/null +++ b/pkg/eventservice/large_txn_state.go @@ -0,0 +1,200 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package eventservice + +import ( + "io" + "sync" + + "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/errors" +) + +type largeTxnScanPhase int + +const ( + largeTxnScanPhaseOriginal largeTxnScanPhase = iota + largeTxnScanPhaseDrainInserts +) + +type largeTxnScanState struct { + mu sync.Mutex + + startTs uint64 + commitTs uint64 + tableID int64 + + tableInfo *common.TableInfo + + phase largeTxnScanPhase + hasFollowingTxn bool + followingCommitTs uint64 + + spill *largeTxnInsertSpill + reader *largeTxnInsertSpillReader + cleaned bool +} + +func (a *dispatcherStat) getOrCreateLargeTxnState( + spillDir string, + tableID int64, + tableInfo *common.TableInfo, + startTs uint64, + commitTs uint64, +) (*largeTxnScanState, error) { + a.largeTxnStateMu.Lock() + defer a.largeTxnStateMu.Unlock() + + if a.largeTxnState != nil { + state := a.largeTxnState + if state.startTs != startTs || state.commitTs != commitTs || state.tableID != tableID { + return nil, errors.Errorf( + "large txn spill state mismatch, existing start-ts: %d, commit-ts: %d, table-id: %d, new start-ts: %d, commit-ts: %d, table-id: %d", + state.startTs, state.commitTs, state.tableID, startTs, commitTs, tableID) + } + return state, nil + } + + spill, err := newLargeTxnInsertSpill(spillDir) + if err != nil { + return nil, err + } + state := &largeTxnScanState{ + startTs: startTs, + commitTs: commitTs, + tableID: tableID, + tableInfo: tableInfo, + spill: spill, + } + a.largeTxnState = state + return state, nil +} + +func (a *dispatcherStat) getLargeTxnState() *largeTxnScanState { + a.largeTxnStateMu.Lock() + defer a.largeTxnStateMu.Unlock() + return a.largeTxnState +} + +func (a *dispatcherStat) hasPendingLargeTxnState() bool { + a.largeTxnStateMu.Lock() + defer a.largeTxnStateMu.Unlock() + return a.largeTxnState != nil +} + +func (a *dispatcherStat) markLargeTxnDrainInserts( + startTs uint64, + commitTs uint64, + hasFollowingTxn bool, + followingCommitTs uint64, +) { + a.largeTxnStateMu.Lock() + defer a.largeTxnStateMu.Unlock() + if a.largeTxnState == nil || + a.largeTxnState.startTs != startTs || + a.largeTxnState.commitTs != commitTs { + return + } + a.largeTxnState.markDrainInserts(hasFollowingTxn, followingCommitTs) +} + +func (a *dispatcherStat) cleanupLargeTxnState() error { + a.largeTxnStateMu.Lock() + state := a.largeTxnState + a.largeTxnState = nil + a.largeTxnStateMu.Unlock() + + if state == nil { + return nil + } + return state.cleanup() +} + +func (s *largeTxnScanState) appendInsert(entry *common.RawKVEntry) error { + s.mu.Lock() + defer s.mu.Unlock() + if s.cleaned { + return errors.New("large txn state has been cleaned up") + } + if s.phase != largeTxnScanPhaseOriginal { + return errors.New("large txn spill is no longer accepting original txn rows") + } + return s.spill.Append(entry) +} + +func (s *largeTxnScanState) nextInsert() (*common.RawKVEntry, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.cleaned { + return nil, errors.New("large txn state has been cleaned up") + } + if s.reader == nil { + reader, err := s.spill.NewReader() + if err != nil { + return nil, err + } + s.reader = reader + } + + entry, err := s.reader.Next() + if err != nil { + if errors.Is(err, io.EOF) { + return nil, io.EOF + } + return nil, err + } + return entry, nil +} + +func (s *largeTxnScanState) markDrainInserts(hasFollowingTxn bool, followingCommitTs uint64) { + s.mu.Lock() + defer s.mu.Unlock() + if s.cleaned { + return + } + s.phase = largeTxnScanPhaseDrainInserts + s.hasFollowingTxn = hasFollowingTxn + s.followingCommitTs = followingCommitTs +} + +func (s *largeTxnScanState) getPhase() largeTxnScanPhase { + s.mu.Lock() + defer s.mu.Unlock() + return s.phase +} + +func (s *largeTxnScanState) snapshotDrainInfo() (bool, uint64) { + s.mu.Lock() + defer s.mu.Unlock() + return s.hasFollowingTxn, s.followingCommitTs +} + +func (s *largeTxnScanState) cleanup() error { + s.mu.Lock() + defer s.mu.Unlock() + if s.cleaned { + return nil + } + s.cleaned = true + var closeErr error + if s.reader != nil { + closeErr = s.reader.Close() + s.reader = nil + } + cleanupErr := s.spill.Cleanup() + if closeErr != nil { + return closeErr + } + return cleanupErr +} From 9605655338a06dbe5fa475ebde7280222631e5c9 Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Sat, 27 Jun 2026 16:50:05 +0800 Subject: [PATCH 03/35] eventservice: address large txn spill review comments --- pkg/eventservice/event_scanner.go | 110 ++++++++++++++++++----- pkg/eventservice/event_scanner_test.go | 92 ++++++++++++++++++- pkg/eventservice/large_txn_spill.go | 3 + pkg/eventservice/large_txn_spill_test.go | 14 +++ 4 files changed, 197 insertions(+), 22 deletions(-) diff --git a/pkg/eventservice/event_scanner.go b/pkg/eventservice/event_scanner.go index be34c25674..0e1907836f 100644 --- a/pkg/eventservice/event_scanner.go +++ b/pkg/eventservice/event_scanner.go @@ -16,7 +16,7 @@ package eventservice import ( "context" "io" - "os" + "path/filepath" "time" "github.com/pingcap/log" @@ -470,6 +470,9 @@ func (s *eventScanner) finishCurrentTxn( currentCommitTs := processor.currentTxn.CurrentDMLEvent.GetCommitTs() if processor.hasSpilledInsertsForCurrentTxn() && merger.hasDDLAtCommitTs(currentCommitTs) { + if err := processor.flushCachedInsertRows(); err != nil { + return false, err + } if err := processor.flushSpilledInsertsIntoCurrentTxn(); err != nil { return false, err } @@ -511,6 +514,9 @@ func (s *eventScanner) canInterruptCurrentTxn( if len(position) == 0 || !session.dispatcherStat.txnAtomicity.ShouldSplitTxn() { return false } + if len(processor.insertRowCache) > 0 { + return false + } if !merger.canInterrupt(rawEvent.CRTs, processor.batchDML) { return false } @@ -906,9 +912,21 @@ type TxnEvent struct { CurrentDMLEvent *event.DMLEvent DMLEventMaxRows int32 DMLEventMaxBytes int64 + rawKVBytes int64 shouldSplitTxn bool } +const ( + largeTxnInsertSpillDirName = "eventservice" + defaultLargeTxnInsertSpillThreshold = int64(128 * 1024 * 1024) +) + +var largeTxnInsertSpillThreshold = defaultLargeTxnInsertSpillThreshold + +func getLargeTxnInsertSpillDir() string { + return filepath.Join(config.GetGlobalServerConfig().DataDir, largeTxnInsertSpillDirName) +} + func newTxnEvent( batchDML *event.BatchDMLEvent, dispatcherID common.DispatcherID, @@ -955,6 +973,14 @@ func (t *TxnEvent) AppendRow( return t.CurrentDMLEvent.AppendRow(rawEvent, decode, filter, filterContext) } +func (t *TxnEvent) observeRawKVBytes(rawEvent *common.RawKVEntry) { + t.rawKVBytes += rawEvent.GetSize() +} + +func (t *TxnEvent) shouldSpillSplitUpdateInsert() bool { + return t.shouldSplitTxn && t.rawKVBytes > largeTxnInsertSpillThreshold +} + // dmlProcessor handles DML event processing and batching type dmlProcessor struct { mounter event.Mounter @@ -995,7 +1021,7 @@ func newDMLProcessor( filterContext: filterContext, batchDML: event.NewBatchDMLEvent(), insertRowCache: make([]*common.RawKVEntry, 0), - spillDir: os.TempDir(), + spillDir: getLargeTxnInsertSpillDir(), outputRawChangeEvent: outputRawChangeEvent, mode: mode, } @@ -1019,18 +1045,50 @@ func (p *dmlProcessor) startTxn( } func (p *dmlProcessor) commitTxn() error { - if p.currentTxn != nil && len(p.insertRowCache) > 0 { - for _, insertRow := range p.insertRowCache { - if err := p.currentTxn.AppendRow(insertRow, p.mounter.DecodeToChunk, p.filter, p.filterContext); err != nil { - return err - } - } - p.insertRowCache = make([]*common.RawKVEntry, 0) + if err := p.flushCachedInsertRows(); err != nil { + return err } p.currentTxn = nil return nil } +func (p *dmlProcessor) flushCachedInsertRows() error { + if p.currentTxn == nil || len(p.insertRowCache) == 0 { + return nil + } + for _, insertRow := range p.insertRowCache { + if err := p.currentTxn.AppendRow(insertRow, p.mounter.DecodeToChunk, p.filter, p.filterContext); err != nil { + return err + } + } + p.insertRowCache = make([]*common.RawKVEntry, 0) + return nil +} + +func (p *dmlProcessor) spillCachedInsertRows() error { + if len(p.insertRowCache) == 0 { + return nil + } + state, err := p.getOrCreateLargeTxnState() + if err != nil { + return err + } + for _, insertRow := range p.insertRowCache { + if err := state.appendInsert(insertRow); err != nil { + return err + } + } + p.insertRowCache = make([]*common.RawKVEntry, 0) + return nil +} + +func (p *dmlProcessor) shouldSpillSplitUpdateInsert() bool { + if p.currentTxn == nil { + return false + } + return p.currentTxn.shouldSpillSplitUpdateInsert() || p.hasSpilledInsertsForCurrentTxn() +} + func (p *dmlProcessor) hasSpilledInsertsForCurrentTxn() bool { if p.currentTxn == nil || p.dispatcherStat == nil { return false @@ -1072,6 +1130,20 @@ func (p *dmlProcessor) flushSpilledInsertsIntoCurrentTxn() error { } } +func (p *dmlProcessor) getOrCreateLargeTxnState() (*largeTxnScanState, error) { + if p.dispatcherStat == nil { + return nil, errors.New("dispatcher stat is required for large txn update spill") + } + currentDML := p.currentTxn.CurrentDMLEvent + return p.dispatcherStat.getOrCreateLargeTxnState( + p.spillDir, + currentDML.GetTableID(), + currentDML.TableInfo, + currentDML.GetStartTs(), + currentDML.GetCommitTs(), + ) +} + func (p *dmlProcessor) appendInsertRow(rawEvent *common.RawKVEntry) error { if p.currentTxn == nil { log.Panic("no current DML event to append to") @@ -1112,6 +1184,12 @@ func (p *dmlProcessor) appendRow(rawEvent *common.RawKVEntry) error { } rawEvent.Key = event.RemoveKeyspacePrefix(rawEvent.Key) + p.currentTxn.observeRawKVBytes(rawEvent) + if p.shouldSpillSplitUpdateInsert() { + if err := p.spillCachedInsertRows(); err != nil { + return err + } + } rawType := rawEvent.GetType() if !rawEvent.IsUpdate() { @@ -1143,18 +1221,8 @@ func (p *dmlProcessor) appendRow(rawEvent *common.RawKVEntry) error { if err != nil { return err } - if p.currentTxn.shouldSplitTxn { - if p.dispatcherStat == nil { - return errors.New("dispatcher stat is required for large txn update spill") - } - currentDML := p.currentTxn.CurrentDMLEvent - state, err := p.dispatcherStat.getOrCreateLargeTxnState( - p.spillDir, - currentDML.GetTableID(), - currentDML.TableInfo, - currentDML.GetStartTs(), - currentDML.GetCommitTs(), - ) + if p.shouldSpillSplitUpdateInsert() { + state, err := p.getOrCreateLargeTxnState() if err != nil { return err } diff --git a/pkg/eventservice/event_scanner_test.go b/pkg/eventservice/event_scanner_test.go index 99874f9d29..5cbfef31e4 100644 --- a/pkg/eventservice/event_scanner_test.go +++ b/pkg/eventservice/event_scanner_test.go @@ -16,6 +16,7 @@ package eventservice import ( "context" "errors" + "path/filepath" "testing" "time" @@ -52,6 +53,14 @@ func makeDispatcherReady(disp *dispatcherStat) { disp.setHandshaked() } +func setLargeTxnInsertSpillThresholdForTest(t *testing.T, threshold int64) { + original := largeTxnInsertSpillThreshold + largeTxnInsertSpillThreshold = threshold + t.Cleanup(func() { + largeTxnInsertSpillThreshold = original + }) +} + func (m *mockMounter) DecodeToChunk(rawKV *common.RawKVEntry, tableInfo *common.TableInfo, chk *chunk.Chunk) (int, *integrity.Checksum, error) { if rawKV.IsUpdate() { return 2, nil, nil @@ -487,6 +496,8 @@ func TestEventScannerSplitsLargeTxnWithRowLevelProgress(t *testing.T) { } func TestEventScannerSpillsSplitUKUpdateInLargeTxn(t *testing.T) { + setLargeTxnInsertSpillThresholdForTest(t, 0) + helper := event.NewEventTestHelper(t) defer helper.Close() helper.Tk().MustExec("use test") @@ -569,6 +580,8 @@ func TestEventScannerSpillsSplitUKUpdateInLargeTxn(t *testing.T) { } func TestEventScannerDrainsSpillBeforeFollowingSameCommitTxn(t *testing.T) { + setLargeTxnInsertSpillThresholdForTest(t, 0) + helper := event.NewEventTestHelper(t) defer helper.Close() helper.Tk().MustExec("use test") @@ -1025,11 +1038,20 @@ func TestDMLProcessor(t *testing.T) { // Test case 0: create a new DML processor t.Run("CreateNewDMLProcessor", func(t *testing.T) { + originalConfig := config.GetGlobalServerConfig().Clone() + cfg := originalConfig.Clone() + cfg.DataDir = t.TempDir() + config.StoreGlobalServerConfig(cfg) + t.Cleanup(func() { + config.StoreGlobalServerConfig(originalConfig) + }) + processor := newDMLProcessor(mockMounter, mockSchemaGetter, nil, false, common.DefaultMode, false) require.NotNil(t, processor) require.NotNil(t, processor.batchDML) require.Nil(t, processor.currentTxn) require.Empty(t, processor.insertRowCache) + require.Equal(t, filepath.Join(cfg.DataDir, largeTxnInsertSpillDirName), processor.spillDir) }) // Test case 1: commitTxn with no current DML, happens when the iter is nil. @@ -1225,7 +1247,32 @@ func TestDMLProcessor(t *testing.T) { require.Equal(t, int32(2), processor.batchDML.Len()) }) - t.Run("UpdateThatChangesUKSpillsInsertWhenSplitTxn", func(t *testing.T) { + t.Run("UpdateThatChangesUKCachesInsertWhenSplitTxnIsBelowSpillThreshold", func(t *testing.T) { + processor := newDMLProcessor(mockMounter, mockSchemaGetter, nil, false, common.DefaultMode, false) + disp := &dispatcherStat{id: dispatcherID} + processor.dispatcherStat = disp + processor.spillDir = t.TempDir() + + helper.Tk().MustExec("use test") + ddlEvent := helper.DDL2Event("create table t4 (id int primary key, a int(50), b char(50), unique key uk_a(a))") + tableInfo := ddlEvent.TableInfo + tableID := ddlEvent.GetTableID() + + _, updateEvent := helper.DML2UpdateEvent("test", "t4", + "insert into test.t4(id, a, b) values (0, 1, 'b0')", + "update test.t4 set a = 2 where id = 0") + require.NoError(t, processor.startTxn(dispatcherID, tableID, tableInfo, updateEvent.StartTs, updateEvent.CRTs, true)) + require.NoError(t, processor.appendRow(updateEvent)) + + require.Equal(t, 1, len(processor.insertRowCache)) + require.Nil(t, disp.getLargeTxnState()) + + require.NoError(t, processor.commitTxn()) + require.Empty(t, processor.insertRowCache) + require.Equal(t, int32(2), processor.batchDML.Len()) + }) + + t.Run("UpdateThatChangesUKSpillsInsertWhenSplitTxnExceedsThreshold", func(t *testing.T) { processor := newDMLProcessor(mockMounter, mockSchemaGetter, nil, false, common.DefaultMode, false) disp := &dispatcherStat{id: dispatcherID} processor.dispatcherStat = disp @@ -1241,6 +1288,11 @@ func TestDMLProcessor(t *testing.T) { "update test.t3 set a = 2 where id = 0") require.NoError(t, processor.startTxn(dispatcherID, tableID, tableInfo, updateEvent.StartTs, updateEvent.CRTs, true)) require.NoError(t, processor.appendRow(updateEvent)) + require.Equal(t, 1, len(processor.insertRowCache)) + require.Nil(t, disp.getLargeTxnState()) + + processor.currentTxn.rawKVBytes = largeTxnInsertSpillThreshold + require.NoError(t, processor.appendRow(updateEvent)) require.Empty(t, processor.insertRowCache) state := disp.getLargeTxnState() @@ -1249,6 +1301,44 @@ func TestDMLProcessor(t *testing.T) { require.NoError(t, err) require.Equal(t, common.OpTypePut, insertRow.OpType) require.False(t, insertRow.IsUpdate()) + insertRow, err = state.nextInsert() + require.NoError(t, err) + require.Equal(t, common.OpTypePut, insertRow.OpType) + require.False(t, insertRow.IsUpdate()) + require.NoError(t, disp.cleanupLargeTxnState()) + }) + + t.Run("UpdateThatChangesUKKeepsSpillingAfterLargeTxnResume", func(t *testing.T) { + processor := newDMLProcessor(mockMounter, mockSchemaGetter, nil, false, common.DefaultMode, false) + disp := &dispatcherStat{id: dispatcherID} + processor.dispatcherStat = disp + processor.spillDir = t.TempDir() + + helper.Tk().MustExec("use test") + ddlEvent := helper.DDL2Event("create table t5 (id int primary key, a int(50), b char(50), unique key uk_a(a))") + tableInfo := ddlEvent.TableInfo + tableID := ddlEvent.GetTableID() + + _, updateEvent := helper.DML2UpdateEvent("test", "t5", + "insert into test.t5(id, a, b) values (0, 1, 'b0')", + "update test.t5 set a = 2 where id = 0") + require.NoError(t, processor.startTxn(dispatcherID, tableID, tableInfo, updateEvent.StartTs, updateEvent.CRTs, true)) + + state, err := disp.getOrCreateLargeTxnState( + processor.spillDir, + tableID, + tableInfo, + updateEvent.StartTs, + updateEvent.CRTs, + ) + require.NoError(t, err) + require.NoError(t, processor.appendRow(updateEvent)) + + require.Empty(t, processor.insertRowCache) + insertRow, err := state.nextInsert() + require.NoError(t, err) + require.Equal(t, common.OpTypePut, insertRow.OpType) + require.False(t, insertRow.IsUpdate()) require.NoError(t, disp.cleanupLargeTxnState()) }) } diff --git a/pkg/eventservice/large_txn_spill.go b/pkg/eventservice/large_txn_spill.go index 83092db996..7cb2da25f1 100644 --- a/pkg/eventservice/large_txn_spill.go +++ b/pkg/eventservice/large_txn_spill.go @@ -36,6 +36,9 @@ func newLargeTxnInsertSpill(dir string) (*largeTxnInsertSpill, error) { if dir == "" { return nil, errors.New("large txn spill dir is empty") } + if err := os.MkdirAll(dir, 0o700); err != nil { + return nil, errors.Trace(err) + } file, err := os.CreateTemp(dir, "eventservice-large-txn-insert-*.spill") if err != nil { diff --git a/pkg/eventservice/large_txn_spill_test.go b/pkg/eventservice/large_txn_spill_test.go index 3e67468798..17b7c59236 100644 --- a/pkg/eventservice/large_txn_spill_test.go +++ b/pkg/eventservice/large_txn_spill_test.go @@ -17,6 +17,7 @@ import ( "fmt" "io" "os" + "path/filepath" "testing" "github.com/pingcap/ticdc/pkg/common" @@ -55,6 +56,19 @@ func TestLargeTxnInsertSpillReadOrder(t *testing.T) { require.Nil(t, actual) } +func TestLargeTxnInsertSpillCreatesDir(t *testing.T) { + dir := filepath.Join(t.TempDir(), "data-dir", largeTxnInsertSpillDirName) + + spill, err := newLargeTxnInsertSpill(dir) + require.NoError(t, err) + defer func() { + require.NoError(t, spill.Cleanup()) + }() + + require.DirExists(t, dir) + require.Equal(t, dir, filepath.Dir(spill.path)) +} + func TestLargeTxnInsertSpillCleanup(t *testing.T) { spill, err := newLargeTxnInsertSpill(t.TempDir()) require.NoError(t, err) From f7771ad43a2275666cb80fb9c1f5718332d0fcdd Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Mon, 29 Jun 2026 10:29:46 +0800 Subject: [PATCH 04/35] eventservice: gate txn split by large txn threshold --- pkg/config/debug.go | 7 ++- pkg/config/server_config_test.go | 1 + pkg/eventservice/event_broker_test.go | 2 + pkg/eventservice/event_scanner.go | 42 +++++++-------- pkg/eventservice/event_scanner_test.go | 71 +++++++++++++++++++++++--- 5 files changed, 94 insertions(+), 29 deletions(-) diff --git a/pkg/config/debug.go b/pkg/config/debug.go index 1aa6f9b924..5afc77eff4 100644 --- a/pkg/config/debug.go +++ b/pkg/config/debug.go @@ -123,6 +123,10 @@ type EventServiceConfig struct { // DMLEventMaxBytes is the maximum size of a DML event in bytes when split txn is enabled. DMLEventMaxBytes int64 `toml:"dml-event-max-bytes" json:"dml_event_max_bytes"` + // LargeTxnThresholdInBytes is the raw KV size threshold for row-level large transaction scan interrupt + // and split update insert spill. + LargeTxnThresholdInBytes int64 `toml:"large-txn-threshold-in-bytes" json:"large_txn_threshold_in_bytes"` + EnableScanWindow bool `toml:"enable-scan-window" json:"enable_scan_window"` // FIXME: For now we found cdc may OOM when there is a large amount of events to be sent to event collector from a remote event service. @@ -138,7 +142,8 @@ func NewDefaultEventServiceConfig() *EventServiceConfig { ScanTaskQueueSize: 1024 * 8, ScanLimitInBytes: 1024 * 1024 * 256, // 256MB DMLEventMaxRows: 256, - DMLEventMaxBytes: 1024 * 1024 * 1, // 1MB + DMLEventMaxBytes: 1024 * 1024 * 1, // 1MB + LargeTxnThresholdInBytes: 1024 * 1024 * 32, // 32MB EnableScanWindow: true, EnableRemoteEventService: true, } diff --git a/pkg/config/server_config_test.go b/pkg/config/server_config_test.go index daa49cb642..0199960121 100644 --- a/pkg/config/server_config_test.go +++ b/pkg/config/server_config_test.go @@ -58,6 +58,7 @@ enable-legacy-safepoint = true func TestServerConfigClone(t *testing.T) { t.Parallel() conf := GetDefaultServerConfig() + require.Equal(t, int64(32*1024*1024), conf.Debug.EventService.LargeTxnThresholdInBytes) conf.Addr = "192.155.22.33:8887" conf.Sorter.SortDir = "/tmp" conf2 := conf.Clone() diff --git a/pkg/eventservice/event_broker_test.go b/pkg/eventservice/event_broker_test.go index e7ffc97705..fe7c7fcb95 100644 --- a/pkg/eventservice/event_broker_test.go +++ b/pkg/eventservice/event_broker_test.go @@ -425,6 +425,8 @@ func TestDoScanSkipWhenChangefeedStatusNotFound(t *testing.T) { } func TestDoScanKeepsRowLevelProgressAfterSendingFragment(t *testing.T) { + setLargeTxnThresholdForTest(t, 0) + broker, mockStore, mockSchemaStore, _ := newEventBrokerForTest() broker.close() diff --git a/pkg/eventservice/event_scanner.go b/pkg/eventservice/event_scanner.go index 0e1907836f..47fd1abc99 100644 --- a/pkg/eventservice/event_scanner.go +++ b/pkg/eventservice/event_scanner.go @@ -517,10 +517,13 @@ func (s *eventScanner) canInterruptCurrentTxn( if len(processor.insertRowCache) > 0 { return false } + if processor.currentTxn == nil || !processor.currentTxn.exceedsLargeTxnThreshold() { + return false + } if !merger.canInterrupt(rawEvent.CRTs, processor.batchDML) { return false } - return session.exceedLimit(processor.batchDML.GetSize(), processor.batchDML) + return true } // finalizeScan finalizes the scan when all events have been processed @@ -908,20 +911,16 @@ func (m *eventMerger) canInterrupt(newCommitTs uint64, currentBatchDML *event.Ba // TxnEvent represents a transaction, it may generates one or multiple DMLEvents type TxnEvent struct { - BatchDML *event.BatchDMLEvent - CurrentDMLEvent *event.DMLEvent - DMLEventMaxRows int32 - DMLEventMaxBytes int64 - rawKVBytes int64 - shouldSplitTxn bool + BatchDML *event.BatchDMLEvent + CurrentDMLEvent *event.DMLEvent + DMLEventMaxRows int32 + DMLEventMaxBytes int64 + rawKVBytes int64 + largeTxnThresholdInBytes int64 + shouldSplitTxn bool } -const ( - largeTxnInsertSpillDirName = "eventservice" - defaultLargeTxnInsertSpillThreshold = int64(128 * 1024 * 1024) -) - -var largeTxnInsertSpillThreshold = defaultLargeTxnInsertSpillThreshold +const largeTxnInsertSpillDirName = "eventservice" func getLargeTxnInsertSpillDir() string { return filepath.Join(config.GetGlobalServerConfig().DataDir, largeTxnInsertSpillDirName) @@ -938,11 +937,12 @@ func newTxnEvent( ) (*TxnEvent, error) { serverConfig := config.GetGlobalServerConfig() txn := &TxnEvent{ - BatchDML: batchDML, - CurrentDMLEvent: event.NewDMLEvent(dispatcherID, tableID, startTs, commitTs, tableInfo), - DMLEventMaxRows: serverConfig.Debug.EventService.DMLEventMaxRows, - DMLEventMaxBytes: serverConfig.Debug.EventService.DMLEventMaxBytes, - shouldSplitTxn: shouldSplitTxn, + BatchDML: batchDML, + CurrentDMLEvent: event.NewDMLEvent(dispatcherID, tableID, startTs, commitTs, tableInfo), + DMLEventMaxRows: serverConfig.Debug.EventService.DMLEventMaxRows, + DMLEventMaxBytes: serverConfig.Debug.EventService.DMLEventMaxBytes, + largeTxnThresholdInBytes: serverConfig.Debug.EventService.LargeTxnThresholdInBytes, + shouldSplitTxn: shouldSplitTxn, } return txn, txn.BatchDML.AppendDMLEvent(txn.CurrentDMLEvent) } @@ -977,8 +977,8 @@ func (t *TxnEvent) observeRawKVBytes(rawEvent *common.RawKVEntry) { t.rawKVBytes += rawEvent.GetSize() } -func (t *TxnEvent) shouldSpillSplitUpdateInsert() bool { - return t.shouldSplitTxn && t.rawKVBytes > largeTxnInsertSpillThreshold +func (t *TxnEvent) exceedsLargeTxnThreshold() bool { + return t.shouldSplitTxn && t.rawKVBytes > t.largeTxnThresholdInBytes } // dmlProcessor handles DML event processing and batching @@ -1086,7 +1086,7 @@ func (p *dmlProcessor) shouldSpillSplitUpdateInsert() bool { if p.currentTxn == nil { return false } - return p.currentTxn.shouldSpillSplitUpdateInsert() || p.hasSpilledInsertsForCurrentTxn() + return p.currentTxn.exceedsLargeTxnThreshold() || p.hasSpilledInsertsForCurrentTxn() } func (p *dmlProcessor) hasSpilledInsertsForCurrentTxn() bool { diff --git a/pkg/eventservice/event_scanner_test.go b/pkg/eventservice/event_scanner_test.go index 5cbfef31e4..50ab9ed87e 100644 --- a/pkg/eventservice/event_scanner_test.go +++ b/pkg/eventservice/event_scanner_test.go @@ -53,11 +53,13 @@ func makeDispatcherReady(disp *dispatcherStat) { disp.setHandshaked() } -func setLargeTxnInsertSpillThresholdForTest(t *testing.T, threshold int64) { - original := largeTxnInsertSpillThreshold - largeTxnInsertSpillThreshold = threshold +func setLargeTxnThresholdForTest(t *testing.T, threshold int64) { + original := config.GetGlobalServerConfig().Clone() + cfg := original.Clone() + cfg.Debug.EventService.LargeTxnThresholdInBytes = threshold + config.StoreGlobalServerConfig(cfg) t.Cleanup(func() { - largeTxnInsertSpillThreshold = original + config.StoreGlobalServerConfig(original) }) } @@ -424,6 +426,8 @@ func TestEventScanner(t *testing.T) { } func TestEventScannerSplitsLargeTxnWithRowLevelProgress(t *testing.T) { + setLargeTxnThresholdForTest(t, 0) + helper := event.NewEventTestHelper(t) defer helper.Close() ddlEvent, kvEvents := genEvents(helper, `create table test.t_split(id int primary key, c char(50))`, []string{ @@ -495,8 +499,60 @@ func TestEventScannerSplitsLargeTxnWithRowLevelProgress(t *testing.T) { require.Equal(t, resolvedTs, resolved.ResolvedTs) } +func TestEventScannerDoesNotSplitCurrentTxnBelowLargeTxnThreshold(t *testing.T) { + setLargeTxnThresholdForTest(t, 1<<30) + + helper := event.NewEventTestHelper(t) + defer helper.Close() + ddlEvent, kvEvents := genEvents(helper, `create table test.t_no_split(id int primary key, c char(50))`, []string{ + `insert into test.t_no_split(id,c) values (0, "c0")`, + `insert into test.t_no_split(id,c) values (1, "c1")`, + }...) + require.Len(t, kvEvents, 2) + + kvEvents[1].StartTs = kvEvents[0].StartTs + kvEvents[1].CRTs = kvEvents[0].CRTs + resolvedTs := kvEvents[0].CRTs + + broker, _, _, _ := newEventBrokerForTest() + broker.close() + mockStore := broker.eventStore.(*mockEventStore) + mockSchemaStore := broker.schemaStore.(*mockSchemaStore) + + disInfo := newMockDispatcherInfoForTest(t) + disInfo.startTs = ddlEvent.FinishedTs + changefeedStatus := broker.getOrSetChangefeedStatus(disInfo) + disp := newDispatcherStat(disInfo, 1, 1, nil, changefeedStatus) + makeDispatcherReady(disp) + require.NoError(t, broker.addDispatcher(disp.info)) + + mockSchemaStore.AppendDDLEvent(disInfo.GetTableSpan().TableID, ddlEvent) + require.NoError(t, mockStore.AppendEvents(disInfo.GetID(), resolvedTs, kvEvents...)) + disp.receivedResolvedTs.Store(resolvedTs) + disp.eventStoreCommitTs.Store(resolvedTs) + + scanner := newEventScanner(broker.eventStore, broker.schemaStore, &mockMounter{}, 0) + sl := scanLimit{maxDMLBytes: 1, isInUnitTest: true} + + dataRange, ok := disp.getDataRange() + require.True(t, ok) + _, events, progress, interrupted, err := scanner.scan(context.Background(), disp, dataRange, sl) + require.NoError(t, err) + require.False(t, interrupted) + require.True(t, progress.valid) + require.Empty(t, progress.rowLevelScanPosition) + require.Len(t, events, 2) + + batch := events[0].(*event.BatchDMLEvent) + require.Equal(t, int32(2), batch.Len()) + require.Equal(t, kvEvents[0].CRTs, batch.GetCommitTs()) + resolved, ok := events[1].(event.ResolvedEvent) + require.True(t, ok) + require.Equal(t, resolvedTs, resolved.ResolvedTs) +} + func TestEventScannerSpillsSplitUKUpdateInLargeTxn(t *testing.T) { - setLargeTxnInsertSpillThresholdForTest(t, 0) + setLargeTxnThresholdForTest(t, 0) helper := event.NewEventTestHelper(t) defer helper.Close() @@ -580,7 +636,7 @@ func TestEventScannerSpillsSplitUKUpdateInLargeTxn(t *testing.T) { } func TestEventScannerDrainsSpillBeforeFollowingSameCommitTxn(t *testing.T) { - setLargeTxnInsertSpillThresholdForTest(t, 0) + setLargeTxnThresholdForTest(t, 0) helper := event.NewEventTestHelper(t) defer helper.Close() @@ -660,6 +716,7 @@ func TestEventScannerDrainsSpillBeforeFollowingSameCommitTxn(t *testing.T) { require.Empty(t, events) require.Nil(t, disp.getLargeTxnState()) + setLargeTxnThresholdForTest(t, 1<<30) events, _, interrupted = scanAndAdvance(scanLimit{maxDMLBytes: 100, isInUnitTest: true}) require.False(t, interrupted) require.Len(t, events, 2) @@ -1291,7 +1348,7 @@ func TestDMLProcessor(t *testing.T) { require.Equal(t, 1, len(processor.insertRowCache)) require.Nil(t, disp.getLargeTxnState()) - processor.currentTxn.rawKVBytes = largeTxnInsertSpillThreshold + processor.currentTxn.rawKVBytes = processor.currentTxn.largeTxnThresholdInBytes require.NoError(t, processor.appendRow(updateEvent)) require.Empty(t, processor.insertRowCache) From e159706b587406d7f7c456b401474ef0c94ad331 Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Mon, 29 Jun 2026 10:33:28 +0800 Subject: [PATCH 05/35] tests: add large txn split integration case --- pkg/eventservice/event_scanner.go | 11 ++-- .../large_txn_split/conf/diff_config.toml | 23 +++++++++ .../large_txn_split/conf/server.toml | 2 + .../integration_tests/large_txn_split/run.sh | 50 +++++++++++++++++++ tests/integration_tests/run_light_it_in_ci.sh | 2 +- 5 files changed, 84 insertions(+), 4 deletions(-) create mode 100644 tests/integration_tests/large_txn_split/conf/diff_config.toml create mode 100644 tests/integration_tests/large_txn_split/conf/server.toml create mode 100755 tests/integration_tests/large_txn_split/run.sh diff --git a/pkg/eventservice/event_scanner.go b/pkg/eventservice/event_scanner.go index 47fd1abc99..7eb8374a8a 100644 --- a/pkg/eventservice/event_scanner.go +++ b/pkg/eventservice/event_scanner.go @@ -126,13 +126,18 @@ func newEventScanner( // - At TS40, DML4 is processed first, then DML5, then DDL2 (same timestamp) // // The scan operation may be interrupted when ANY of these limits are reached: -// - Maximum bytes processed (limit.MaxBytes) -// - Timeout duration (limit.Timeout) +// - Maximum bytes processed (limit.MaxBytes) at a transaction boundary +// - Timeout duration (limit.Timeout) at a transaction boundary +// - Large transaction threshold inside the current transaction // -// A scan interruption is ONLY allowed when both conditions are met: +// A transaction-boundary scan interruption is ONLY allowed when both conditions are met: // 1. The current event's commit timestamp is greater than the lastCommitTs (a commit TS boundary is reached) // 2. At least one DML event has been successfully scanned // +// A current-transaction interruption is ONLY allowed when split transaction is enabled, +// the eventstore iterator provides a row-level scan position, and the current +// transaction fragment exceeds the large transaction threshold. +// // Returns: // - events: The scanned events in commitTs order // - isBroken: true if the scan was interrupted due to reaching a limit, false otherwise diff --git a/tests/integration_tests/large_txn_split/conf/diff_config.toml b/tests/integration_tests/large_txn_split/conf/diff_config.toml new file mode 100644 index 0000000000..ddf7f7d633 --- /dev/null +++ b/tests/integration_tests/large_txn_split/conf/diff_config.toml @@ -0,0 +1,23 @@ +tz = "Asia/Shanghai" +check-thread-count = 4 +export-fix-sql = true +check-struct-only = false + +[data-sources] +[data-sources.mysql1] + host = "127.0.0.1" + port = 3306 + user = "root" + password = "" + +[data-sources.tidb0] + host = "127.0.0.1" + port = 4000 + user = "root" + password = "" + +[task] + output-dir = "./output" + source-instances = ["mysql1"] + target-instance = "tidb0" + target-check-tables = ["large_txn_split.large_txn_table"] diff --git a/tests/integration_tests/large_txn_split/conf/server.toml b/tests/integration_tests/large_txn_split/conf/server.toml new file mode 100644 index 0000000000..4275104e69 --- /dev/null +++ b/tests/integration_tests/large_txn_split/conf/server.toml @@ -0,0 +1,2 @@ +[debug.event-service] +large-txn-threshold-in-bytes = 1048576 diff --git a/tests/integration_tests/large_txn_split/run.sh b/tests/integration_tests/large_txn_split/run.sh new file mode 100755 index 0000000000..ef9a090b6a --- /dev/null +++ b/tests/integration_tests/large_txn_split/run.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +set -eu + +CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +source $CUR/../_utils/test_prepare +WORK_DIR=$OUT_DIR/$TEST_NAME +CDC_BINARY=cdc.test +SINK_TYPE=$1 + +function prepare() { + rm -rf $WORK_DIR && mkdir -p $WORK_DIR + start_tidb_cluster --workdir $WORK_DIR + + run_sql "CREATE DATABASE large_txn_split" ${UP_TIDB_HOST} ${UP_TIDB_PORT} + run_sql "CREATE DATABASE large_txn_split" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} + + pd_addr="http://$UP_PD_HOST_1:$UP_PD_PORT_1" + run_cdc_server \ + --workdir $WORK_DIR \ + --binary $CDC_BINARY \ + --addr "127.0.0.1:8300" \ + --pd $pd_addr \ + --config "$CUR/conf/server.toml" + + cdc_cli_changefeed create --sink-uri="mysql://root@${DOWN_TIDB_HOST}:${DOWN_TIDB_PORT}/?transaction-atomicity=none" +} + +trap 'stop_test $WORK_DIR' EXIT + +if [ "$SINK_TYPE" == "mysql" ]; then + prepare $* + + set -euxo pipefail + + echo "[$(date)] Starting large transaction split workload..." + + GO111MODULE=on go run "$CUR/../large_txn/main.go" \ + -dsn "root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/large_txn_split" \ + --rows=2048 \ + --txns=1 + + echo "[$(date)] Workload completed, verifying split path and data consistency..." + + $CUR/../_utils/check_logs_contains $WORK_DIR "scan interrupted inside a large txn" + check_sync_diff $WORK_DIR $CUR/conf/diff_config.toml 200 3 + + cleanup_process $CDC_BINARY + echo "[$(date)] <<<<<< run test case $TEST_NAME success! >>>>>>" +fi diff --git a/tests/integration_tests/run_light_it_in_ci.sh b/tests/integration_tests/run_light_it_in_ci.sh index 1dd75a780f..f4afa3a090 100755 --- a/tests/integration_tests/run_light_it_in_ci.sh +++ b/tests/integration_tests/run_light_it_in_ci.sh @@ -34,7 +34,7 @@ mysql_groups=( # G00 'event_filter charset_gbk changefeed_finish sql_mode changefeed_reconstruct fail_over_ddl_A' # G01 - 'common_1 large_txn foreign_key changefeed_pause_resume fail_over_ddl_B' + 'common_1 large_txn large_txn_split foreign_key changefeed_pause_resume fail_over_ddl_B' # G02 'new_ci_collation safe_mode savepoint fail_over_ddl_C unsplittable_tables' # G03 From 8c0c29cadcc045b92260cfe5a5fd03fcbbee276f Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Mon, 29 Jun 2026 10:36:57 +0800 Subject: [PATCH 06/35] tests: avoid go dependency in large txn split case --- .../integration_tests/large_txn_split/run.sh | 38 +++++++++++++++++-- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/tests/integration_tests/large_txn_split/run.sh b/tests/integration_tests/large_txn_split/run.sh index ef9a090b6a..84a02de201 100755 --- a/tests/integration_tests/large_txn_split/run.sh +++ b/tests/integration_tests/large_txn_split/run.sh @@ -26,6 +26,37 @@ function prepare() { cdc_cli_changefeed create --sink-uri="mysql://root@${DOWN_TIDB_HOST}:${DOWN_TIDB_PORT}/?transaction-atomicity=none" } +function generate_workload() { + local sql_file=$1 + local rows=2048 + + { + echo "USE large_txn_split;" + echo "CREATE TABLE IF NOT EXISTS large_txn_table (id INT AUTO_INCREMENT PRIMARY KEY, batch_id INT, data LONGTEXT);" + + echo "BEGIN;" + for i in $(seq 1 "$rows"); do + echo "INSERT INTO large_txn_table (batch_id, data) VALUES (0, REPEAT('a', 1024));" + done + echo "COMMIT;" + + echo "BEGIN;" + echo "UPDATE large_txn_table SET data = REPEAT('b', 1024) WHERE batch_id = 0;" + echo "COMMIT;" + + echo "BEGIN;" + echo "DELETE FROM large_txn_table WHERE batch_id = 0;" + echo "COMMIT;" + + echo "TRUNCATE TABLE large_txn_table;" + echo "BEGIN;" + for i in $(seq 1 "$rows"); do + echo "INSERT INTO large_txn_table (batch_id, data) VALUES (1, REPEAT('c', 1024));" + done + echo "COMMIT;" + } >"$sql_file" +} + trap 'stop_test $WORK_DIR' EXIT if [ "$SINK_TYPE" == "mysql" ]; then @@ -35,10 +66,9 @@ if [ "$SINK_TYPE" == "mysql" ]; then echo "[$(date)] Starting large transaction split workload..." - GO111MODULE=on go run "$CUR/../large_txn/main.go" \ - -dsn "root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/large_txn_split" \ - --rows=2048 \ - --txns=1 + workload_sql=$WORK_DIR/large_txn_split_workload.sql + generate_workload "$workload_sql" + run_sql_file "$workload_sql" ${UP_TIDB_HOST} ${UP_TIDB_PORT} large_txn_split echo "[$(date)] Workload completed, verifying split path and data consistency..." From 51a44b903189bb111d65b99e038757be582bd224 Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Mon, 29 Jun 2026 10:38:36 +0800 Subject: [PATCH 07/35] tests: isolate large txn split cdc port --- tests/integration_tests/large_txn_split/run.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/integration_tests/large_txn_split/run.sh b/tests/integration_tests/large_txn_split/run.sh index 84a02de201..927f70a4c5 100755 --- a/tests/integration_tests/large_txn_split/run.sh +++ b/tests/integration_tests/large_txn_split/run.sh @@ -7,6 +7,7 @@ source $CUR/../_utils/test_prepare WORK_DIR=$OUT_DIR/$TEST_NAME CDC_BINARY=cdc.test SINK_TYPE=$1 +CDC_ADDR="127.0.0.1:18300" function prepare() { rm -rf $WORK_DIR && mkdir -p $WORK_DIR @@ -19,11 +20,13 @@ function prepare() { run_cdc_server \ --workdir $WORK_DIR \ --binary $CDC_BINARY \ - --addr "127.0.0.1:8300" \ + --addr "$CDC_ADDR" \ --pd $pd_addr \ --config "$CUR/conf/server.toml" - cdc_cli_changefeed create --sink-uri="mysql://root@${DOWN_TIDB_HOST}:${DOWN_TIDB_PORT}/?transaction-atomicity=none" + cdc_cli_changefeed create \ + --server "$CDC_ADDR" \ + --sink-uri="mysql://root@${DOWN_TIDB_HOST}:${DOWN_TIDB_PORT}/?transaction-atomicity=none" } function generate_workload() { From 07088cb1b189df8e5ce8030b3363ba2ee282e00e Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Mon, 29 Jun 2026 10:48:04 +0800 Subject: [PATCH 08/35] tests: reduce large txn split workload logs --- tests/integration_tests/large_txn_split/run.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/integration_tests/large_txn_split/run.sh b/tests/integration_tests/large_txn_split/run.sh index 927f70a4c5..01b72eb7ed 100755 --- a/tests/integration_tests/large_txn_split/run.sh +++ b/tests/integration_tests/large_txn_split/run.sh @@ -60,6 +60,16 @@ function generate_workload() { } >"$sql_file" } +function run_workload() { + local sql_file=$1 + local workload_log=$WORK_DIR/workload.log + + if ! mysql -uroot -h${UP_TIDB_HOST} -P${UP_TIDB_PORT} --default-character-set utf8mb4 <"$sql_file" >"$workload_log" 2>&1; then + cat "$workload_log" + return 1 + fi +} + trap 'stop_test $WORK_DIR' EXIT if [ "$SINK_TYPE" == "mysql" ]; then @@ -70,8 +80,10 @@ if [ "$SINK_TYPE" == "mysql" ]; then echo "[$(date)] Starting large transaction split workload..." workload_sql=$WORK_DIR/large_txn_split_workload.sql + set +x generate_workload "$workload_sql" - run_sql_file "$workload_sql" ${UP_TIDB_HOST} ${UP_TIDB_PORT} large_txn_split + set -x + run_workload "$workload_sql" echo "[$(date)] Workload completed, verifying split path and data consistency..." From f5c2ee923f36c26544ec27c7046abc81b58930a9 Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Mon, 29 Jun 2026 10:50:27 +0800 Subject: [PATCH 09/35] tests: wait for sync before checking split log --- tests/integration_tests/large_txn_split/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration_tests/large_txn_split/run.sh b/tests/integration_tests/large_txn_split/run.sh index 01b72eb7ed..67e70a616a 100755 --- a/tests/integration_tests/large_txn_split/run.sh +++ b/tests/integration_tests/large_txn_split/run.sh @@ -87,8 +87,8 @@ if [ "$SINK_TYPE" == "mysql" ]; then echo "[$(date)] Workload completed, verifying split path and data consistency..." - $CUR/../_utils/check_logs_contains $WORK_DIR "scan interrupted inside a large txn" check_sync_diff $WORK_DIR $CUR/conf/diff_config.toml 200 3 + $CUR/../_utils/check_logs_contains $WORK_DIR "scan interrupted inside a large txn" cleanup_process $CDC_BINARY echo "[$(date)] <<<<<< run test case $TEST_NAME success! >>>>>>" From 205a7c14995cbddd03fe79dc733f5c00fe480983 Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Mon, 29 Jun 2026 10:53:15 +0800 Subject: [PATCH 10/35] tests: fix large txn split diff config --- tests/integration_tests/large_txn_split/conf/diff_config.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration_tests/large_txn_split/conf/diff_config.toml b/tests/integration_tests/large_txn_split/conf/diff_config.toml index ddf7f7d633..ef585e95f6 100644 --- a/tests/integration_tests/large_txn_split/conf/diff_config.toml +++ b/tests/integration_tests/large_txn_split/conf/diff_config.toml @@ -1,4 +1,3 @@ -tz = "Asia/Shanghai" check-thread-count = 4 export-fix-sql = true check-struct-only = false From 7652d402addd8796bae2f152937b6b392a6f8e2c Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Mon, 29 Jun 2026 15:19:30 +0800 Subject: [PATCH 11/35] eventcollector,eventservice: add split txn reset coverage --- .../eventcollector/dispatcher_stat.go | 16 +++++++- .../eventcollector/dispatcher_stat_test.go | 39 +++++++++++++++++++ .../integration_tests/large_txn_split/run.sh | 18 ++++++++- 3 files changed, 70 insertions(+), 3 deletions(-) diff --git a/downstreamadapter/eventcollector/dispatcher_stat.go b/downstreamadapter/eventcollector/dispatcher_stat.go index 350cb0dae7..694c67b2af 100644 --- a/downstreamadapter/eventcollector/dispatcher_stat.go +++ b/downstreamadapter/eventcollector/dispatcher_stat.go @@ -16,6 +16,7 @@ package eventcollector import ( "sync/atomic" + "github.com/pingcap/failpoint" "github.com/pingcap/log" "github.com/pingcap/ticdc/downstreamadapter/dispatcher" "github.com/pingcap/ticdc/pkg/common" @@ -333,6 +334,7 @@ func (d *dispatcherStat) isFromCurrentEpoch(event dispatcher.DispatcherEvent, st // 3. Finally: Forward valid events to target with wake callback func (d *dispatcherStat) handleBatchDataEvents(events []dispatcher.DispatcherEvent) bool { var validEvents []dispatcher.DispatcherEvent + hasDML := false state := d.loadCurrentEpochState() for _, event := range events { if !d.isFromCurrentEpoch(event, state) { @@ -354,6 +356,7 @@ func (d *dispatcherStat) handleBatchDataEvents(events []dispatcher.DispatcherEve validEvents = append(validEvents, event) case commonEvent.TypeDMLEvent: if d.shouldForwardEventByCommitTs(event) { + hasDML = true validEvents = append(validEvents, event) } case commonEvent.TypeBatchDMLEvent: @@ -374,6 +377,7 @@ func (d *dispatcherStat) handleBatchDataEvents(events []dispatcher.DispatcherEve dml.TableInfoVersion = tableInfoVersion dmlEvent := dispatcher.NewDispatcherEvent(event.From, dml) if d.shouldForwardEventByCommitTs(dmlEvent) { + hasDML = true validEvents = append(validEvents, dmlEvent) } } @@ -388,7 +392,17 @@ func (d *dispatcherStat) handleBatchDataEvents(events []dispatcher.DispatcherEve return false } d.updateCommitTsStateByEvents(state, validEvents) - return d.target.HandleEvents(validEvents, func() { d.wake() }) + handled := d.target.HandleEvents(validEvents, func() { d.wake() }) + if hasDML { + failpoint.Inject("InjectResetDispatcherAfterBatchDataEvents", func() { + log.Info("inject dispatcher reset after batch data events", + zap.Stringer("changefeedID", d.target.GetChangefeedID()), + zap.Stringer("dispatcherID", d.getDispatcherID()), + zap.Uint64("checkpointTs", d.target.GetCheckpointTs())) + d.session.resetCurrentEventService() + }) + } + return handled } // handleSingleDataEvents processes a single DDL or SyncPoint event with the following algorithm: diff --git a/downstreamadapter/eventcollector/dispatcher_stat_test.go b/downstreamadapter/eventcollector/dispatcher_stat_test.go index cc3f272c71..ee16fe39cb 100644 --- a/downstreamadapter/eventcollector/dispatcher_stat_test.go +++ b/downstreamadapter/eventcollector/dispatcher_stat_test.go @@ -19,6 +19,7 @@ import ( "testing" "time" + "github.com/pingcap/failpoint" "github.com/pingcap/ticdc/downstreamadapter/dispatcher" "github.com/pingcap/ticdc/eventpb" "github.com/pingcap/ticdc/heartbeatpb" @@ -1277,6 +1278,44 @@ func TestHandleBatchDataEvents(t *testing.T) { } } +func TestInjectResetDispatcherAfterBatchDataEvents(t *testing.T) { + failpointName := "github.com/pingcap/ticdc/downstreamadapter/eventcollector/InjectResetDispatcherAfterBatchDataEvents" + require.NoError(t, failpoint.Enable(failpointName, `1*return(true)`)) + defer func() { + require.NoError(t, failpoint.Disable(failpointName)) + }() + + localServerID := node.ID("local-server") + dispatcherID := common.NewDispatcherID() + mockDisp := newMockDispatcher(dispatcherID, 100) + mockDisp.handleEvents = func(events []dispatcher.DispatcherEvent, wakeCallback func()) (block bool) { + return len(events) > 0 + } + collector := newTestEventCollector(localServerID) + stat := newDispatcherStat(mockDisp, collector, nil) + stat.currentEpoch.Store(newDispatcherEpochState(1, 1, stat.target.GetStartTs())) + stat.lastEventCommitTs.Store(100) + markSessionReceiving(stat.session, localServerID) + + require.True(t, stat.handleBatchDataEvents([]dispatcher.DispatcherEvent{ + { + From: &localServerID, + Event: &commonEvent.DMLEvent{ + Seq: 2, + Epoch: 1, + CommitTs: 101, + }, + }, + })) + requireDispatcherRequests( + t, + readDispatcherRequests(t, collector, 1), + dispatcherRequestRecord{to: localServerID, action: eventpb.ActionType_ACTION_TYPE_RESET}, + ) + require.Equal(t, uint64(2), stat.loadCurrentEpochState().epoch) + require.Equal(t, uint64(101), stat.loadCurrentEpochState().maxEventTs.Load()) +} + func TestHandleSingleDataEvents(t *testing.T) { t.Parallel() diff --git a/tests/integration_tests/large_txn_split/run.sh b/tests/integration_tests/large_txn_split/run.sh index 67e70a616a..d4b9d6fdc6 100755 --- a/tests/integration_tests/large_txn_split/run.sh +++ b/tests/integration_tests/large_txn_split/run.sh @@ -8,6 +8,7 @@ WORK_DIR=$OUT_DIR/$TEST_NAME CDC_BINARY=cdc.test SINK_TYPE=$1 CDC_ADDR="127.0.0.1:18300" +RESET_AFTER_BATCH_FAILPOINT="github.com/pingcap/ticdc/downstreamadapter/eventcollector/InjectResetDispatcherAfterBatchDataEvents=3*return(true)" function prepare() { rm -rf $WORK_DIR && mkdir -p $WORK_DIR @@ -22,7 +23,8 @@ function prepare() { --binary $CDC_BINARY \ --addr "$CDC_ADDR" \ --pd $pd_addr \ - --config "$CUR/conf/server.toml" + --config "$CUR/conf/server.toml" \ + --failpoint "$RESET_AFTER_BATCH_FAILPOINT" cdc_cli_changefeed create \ --server "$CDC_ADDR" \ @@ -70,6 +72,15 @@ function run_workload() { fi } +function render_diff_config() { + local output=$1 + + sed \ + -e "s/port = 3306/port = ${DOWN_TIDB_PORT}/" \ + -e "s/port = 4000/port = ${UP_TIDB_PORT}/" \ + "$CUR/conf/diff_config.toml" >"$output" +} + trap 'stop_test $WORK_DIR' EXIT if [ "$SINK_TYPE" == "mysql" ]; then @@ -84,11 +95,14 @@ if [ "$SINK_TYPE" == "mysql" ]; then generate_workload "$workload_sql" set -x run_workload "$workload_sql" + diff_config=$WORK_DIR/diff_config.toml + render_diff_config "$diff_config" echo "[$(date)] Workload completed, verifying split path and data consistency..." - check_sync_diff $WORK_DIR $CUR/conf/diff_config.toml 200 3 + check_sync_diff $WORK_DIR "$diff_config" 200 3 $CUR/../_utils/check_logs_contains $WORK_DIR "scan interrupted inside a large txn" + $CUR/../_utils/check_logs_contains $WORK_DIR "inject dispatcher reset after batch data events" cleanup_process $CDC_BINARY echo "[$(date)] <<<<<< run test case $TEST_NAME success! >>>>>>" From 8d6bf1f20cd2a1d6ac1352c624c49d99b456a85f Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Wed, 1 Jul 2026 10:21:45 +0800 Subject: [PATCH 12/35] eventstore: test row scan position across sub switch --- logservice/eventstore/event_store_test.go | 117 ++++++++++++++++++++++ 1 file changed, 117 insertions(+) diff --git a/logservice/eventstore/event_store_test.go b/logservice/eventstore/event_store_test.go index f07e7ff86e..99e6312a7a 100644 --- a/logservice/eventstore/event_store_test.go +++ b/logservice/eventstore/event_store_test.go @@ -1135,6 +1135,123 @@ func TestEventStoreSwitchSubStat(t *testing.T) { } } +func TestEventStoreRowLevelScanPositionSurvivesSubStatSwitch(t *testing.T) { + restoreCfg := setDataSharingForTest(t, true) + defer restoreCfg() + + ctx := context.Background() + _, storeInt := newEventStoreForTest(t.TempDir()) + store := storeInt.(*eventStore) + defer store.Close(ctx) + + const ( + tableID int64 = 1 + txnStartTs uint64 = 120 + txnCommitTs uint64 = 200 + nextStartTs uint64 = 130 + nextCommitTs uint64 = 201 + ) + + dispatcherID1 := common.NewDispatcherID() + dispatcherID2 := common.NewDispatcherID() + cfID := common.NewChangefeedID4Test("default", "test-cf") + fullSpan := &heartbeatpb.TableSpan{TableID: tableID, StartKey: []byte("a"), EndKey: []byte("z")} + dispatcherSpan := &heartbeatpb.TableSpan{TableID: tableID, StartKey: []byte("b"), EndKey: []byte("h")} + + require.True(t, store.RegisterDispatcher(cfID, dispatcherID1, fullSpan, 100, func(uint64, uint64) {}, false, false)) + require.True(t, store.RegisterDispatcher(cfID, dispatcherID2, dispatcherSpan, 100, func(uint64, uint64) {}, false, false)) + + dispatcherStat := store.dispatcherMeta.dispatcherStats[dispatcherID2] + require.NotNil(t, dispatcherStat) + oldSubStat := dispatcherStat.subStat + newSubStat := dispatcherStat.pendingSubStat + require.NotNil(t, oldSubStat) + require.NotNil(t, newSubStat) + require.NotEqual(t, oldSubStat.subID, newSubStat.subID) + + rows := []common.RawKVEntry{ + {OpType: common.OpTypePut, StartTs: txnStartTs, CRTs: txnCommitTs, Key: []byte("c-row-1"), Value: []byte("value-1")}, + {OpType: common.OpTypePut, StartTs: txnStartTs, CRTs: txnCommitTs, Key: []byte("c-row-2"), Value: []byte("value-2")}, + {OpType: common.OpTypePut, StartTs: nextStartTs, CRTs: nextCommitTs, Key: []byte("c-next-row"), Value: []byte("value-3")}, + } + encoder, err := zstd.NewWriter(nil) + require.NoError(t, err) + defer encoder.Close() + var compressionBuf []byte + var rawValueBuf []byte + writeRows := func(subStat *subscriptionStat) { + err := store.writeEvents(store.dbs[subStat.dbIndex], []eventWithCallback{{ + subID: subStat.subID, + tableID: tableID, + kvs: rows, + callback: func() {}, + }}, encoder, &compressionBuf, &rawValueBuf) + require.NoError(t, err) + } + writeRows(oldSubStat) + writeRows(newSubStat) + + type scannedEvent struct { + key string + position common.ScanPosition + } + collectEvents := func(dataRange common.DataRange) []scannedEvent { + iter, err := store.GetIterator(dispatcherID2, dataRange) + require.NoError(t, err) + require.NotNil(t, iter) + positionIter, ok := iter.(EventIteratorWithScanPosition) + require.True(t, ok) + + events := make([]scannedEvent, 0) + for { + rawKV, position, _ := positionIter.NextWithScanPosition() + if rawKV == nil { + break + } + require.NotEmpty(t, position) + events = append(events, scannedEvent{ + key: string(rawKV.Key), + position: position, + }) + } + rowCount, err := iter.Close() + require.NoError(t, err) + require.Equal(t, int64(len(events)), rowCount) + return events + } + + oldSubStat.resolvedTs.Store(nextCommitTs) + firstScanEvents := collectEvents(common.DataRange{ + Span: dispatcherSpan, + CommitTsStart: txnCommitTs - 1, + CommitTsEnd: nextCommitTs, + }) + require.Len(t, firstScanEvents, 3) + require.Equal(t, []string{"c-row-1", "c-row-2", "c-next-row"}, []string{ + firstScanEvents[0].key, + firstScanEvents[1].key, + firstScanEvents[2].key, + }) + require.Equal(t, oldSubStat.subID, dispatcherStat.subStat.subID) + require.Equal(t, newSubStat.subID, dispatcherStat.pendingSubStat.subID) + + newSubStat.resolvedTs.Store(nextCommitTs) + resumedEvents := collectEvents(common.DataRange{ + Span: dispatcherSpan, + CommitTsStart: txnCommitTs, + CommitTsEnd: nextCommitTs, + RowLevelScanPosition: firstScanEvents[0].position, + }) + require.Len(t, resumedEvents, 2) + require.Equal(t, []string{"c-row-2", "c-next-row"}, []string{ + resumedEvents[0].key, + resumedEvents[1].key, + }) + require.Equal(t, newSubStat.subID, dispatcherStat.subStat.subID) + require.Nil(t, dispatcherStat.pendingSubStat) + require.Equal(t, oldSubStat.subID, dispatcherStat.removingSubStat.subID) +} + func TestWriteToEventStore(t *testing.T) { dir := t.TempDir() _, storeInt := newEventStoreForTest(dir) From b523967ff7f00b951af72ba1be23090e294ab01c Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Fri, 3 Jul 2026 15:17:51 +0800 Subject: [PATCH 13/35] eventservice: lower large txn threshold default Signed-off-by: dongmen <414110582@qq.com> --- pkg/config/debug.go | 4 ++-- pkg/config/server_config_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/config/debug.go b/pkg/config/debug.go index 5afc77eff4..23863d443d 100644 --- a/pkg/config/debug.go +++ b/pkg/config/debug.go @@ -142,8 +142,8 @@ func NewDefaultEventServiceConfig() *EventServiceConfig { ScanTaskQueueSize: 1024 * 8, ScanLimitInBytes: 1024 * 1024 * 256, // 256MB DMLEventMaxRows: 256, - DMLEventMaxBytes: 1024 * 1024 * 1, // 1MB - LargeTxnThresholdInBytes: 1024 * 1024 * 32, // 32MB + DMLEventMaxBytes: 1024 * 1024 * 1, // 1MB + LargeTxnThresholdInBytes: 1024 * 1024 * 1, // 1MB EnableScanWindow: true, EnableRemoteEventService: true, } diff --git a/pkg/config/server_config_test.go b/pkg/config/server_config_test.go index 0199960121..8cb56e5158 100644 --- a/pkg/config/server_config_test.go +++ b/pkg/config/server_config_test.go @@ -58,7 +58,7 @@ enable-legacy-safepoint = true func TestServerConfigClone(t *testing.T) { t.Parallel() conf := GetDefaultServerConfig() - require.Equal(t, int64(32*1024*1024), conf.Debug.EventService.LargeTxnThresholdInBytes) + require.Equal(t, int64(1024*1024), conf.Debug.EventService.LargeTxnThresholdInBytes) conf.Addr = "192.155.22.33:8887" conf.Sorter.SortDir = "/tmp" conf2 := conf.Clone() From c4acc550c8ea89fb4f6aac60d4bb58b7612485d5 Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Mon, 6 Jul 2026 15:47:27 +0800 Subject: [PATCH 14/35] tests: cover large txn UK update split Signed-off-by: dongmen <414110582@qq.com> --- .../large_txn_split/conf/diff_config.toml | 2 +- tests/integration_tests/large_txn_split/run.sh | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/integration_tests/large_txn_split/conf/diff_config.toml b/tests/integration_tests/large_txn_split/conf/diff_config.toml index ef585e95f6..0d4a3e57c0 100644 --- a/tests/integration_tests/large_txn_split/conf/diff_config.toml +++ b/tests/integration_tests/large_txn_split/conf/diff_config.toml @@ -19,4 +19,4 @@ check-struct-only = false output-dir = "./output" source-instances = ["mysql1"] target-instance = "tidb0" - target-check-tables = ["large_txn_split.large_txn_table"] + target-check-tables = ["large_txn_split.large_txn_table", "large_txn_split.large_txn_uk_update_table"] diff --git a/tests/integration_tests/large_txn_split/run.sh b/tests/integration_tests/large_txn_split/run.sh index d4b9d6fdc6..49d6e67a68 100755 --- a/tests/integration_tests/large_txn_split/run.sh +++ b/tests/integration_tests/large_txn_split/run.sh @@ -38,6 +38,7 @@ function generate_workload() { { echo "USE large_txn_split;" echo "CREATE TABLE IF NOT EXISTS large_txn_table (id INT AUTO_INCREMENT PRIMARY KEY, batch_id INT, data LONGTEXT);" + echo "CREATE TABLE IF NOT EXISTS large_txn_uk_update_table (id INT PRIMARY KEY, uk INT NOT NULL, data LONGTEXT, UNIQUE KEY uk_idx (uk));" echo "BEGIN;" for i in $(seq 1 "$rows"); do @@ -59,6 +60,16 @@ function generate_workload() { echo "INSERT INTO large_txn_table (batch_id, data) VALUES (1, REPEAT('c', 1024));" done echo "COMMIT;" + + echo "BEGIN;" + for i in $(seq 1 "$rows"); do + echo "INSERT INTO large_txn_uk_update_table (id, uk, data) VALUES ($i, $i, REPEAT('u', 1024));" + done + echo "COMMIT;" + + echo "BEGIN;" + echo "UPDATE large_txn_uk_update_table SET uk = uk + 1000000, data = REPEAT('v', 1024);" + echo "COMMIT;" } >"$sql_file" } @@ -102,6 +113,7 @@ if [ "$SINK_TYPE" == "mysql" ]; then check_sync_diff $WORK_DIR "$diff_config" 200 3 $CUR/../_utils/check_logs_contains $WORK_DIR "scan interrupted inside a large txn" + $CUR/../_utils/check_logs_contains $WORK_DIR "split update event" $CUR/../_utils/check_logs_contains $WORK_DIR "inject dispatcher reset after batch data events" cleanup_process $CDC_BINARY From c2b6983de7f16bb83e6075ee591b50c4f7284182 Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Mon, 6 Jul 2026 16:24:26 +0800 Subject: [PATCH 15/35] eventservice: fix large txn spill test lint Signed-off-by: dongmen <414110582@qq.com> --- pkg/eventservice/large_txn_spill_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/eventservice/large_txn_spill_test.go b/pkg/eventservice/large_txn_spill_test.go index 17b7c59236..5bbf5d80e8 100644 --- a/pkg/eventservice/large_txn_spill_test.go +++ b/pkg/eventservice/large_txn_spill_test.go @@ -110,7 +110,7 @@ func newTestSpillRawKVEntry(index int) *common.RawKVEntry { CRTs: 100, StartTs: 90, RegionID: uint64(index), - Key: []byte(fmt.Sprintf("key-%02d", index)), - Value: []byte(fmt.Sprintf("value-%02d", index)), + Key: fmt.Appendf(nil, "key-%02d", index), + Value: fmt.Appendf(nil, "value-%02d", index), } } From 7d6bdfa91bd730841b9c9f8571f97a0a394ec3ee Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Tue, 7 Jul 2026 10:33:35 +0800 Subject: [PATCH 16/35] eventservice: avoid redundant scan position copy Signed-off-by: dongmen <414110582@qq.com> --- pkg/eventservice/dispatcher_stat.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/pkg/eventservice/dispatcher_stat.go b/pkg/eventservice/dispatcher_stat.go index 16c26a1941..4b6f3804b3 100644 --- a/pkg/eventservice/dispatcher_stat.go +++ b/pkg/eventservice/dispatcher_stat.go @@ -233,9 +233,8 @@ func (a *dispatcherStat) storeLastScannedPosition(position common.ScanPosition) a.lastScannedPosition.Store(common.ScanPosition{}) return } - positionCopy := make(common.ScanPosition, len(position)) - copy(positionCopy, position) - a.lastScannedPosition.Store(positionCopy) + // ScanPosition is produced as an immutable snapshot by the scanner path. + a.lastScannedPosition.Store(position) } func (a *dispatcherStat) getLastScannedPosition() common.ScanPosition { @@ -247,9 +246,7 @@ func (a *dispatcherStat) getLastScannedPosition() common.ScanPosition { if len(position) == 0 { return nil } - positionCopy := make(common.ScanPosition, len(position)) - copy(positionCopy, position) - return positionCopy + return position } // onResolvedTs try to update the resolved ts of the dispatcher. From 284c6559cc2ed568e59fbc9823e62dbcae75b15d Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Tue, 7 Jul 2026 10:59:06 +0800 Subject: [PATCH 17/35] pkg: add reusable spill record files Introduce pkg/spill as a shared temp-file record store with chunked append, handle reads, sequential reads, and cleanup. Refactor eventservice large transaction insert spill to reuse it while keeping RawKVEntry encoding local. --- pkg/errors/error.go | 4 + pkg/eventservice/large_txn_spill.go | 123 ++----------- pkg/spill/record_file.go | 259 ++++++++++++++++++++++++++++ pkg/spill/record_file_test.go | 87 ++++++++++ 4 files changed, 369 insertions(+), 104 deletions(-) create mode 100644 pkg/spill/record_file.go create mode 100644 pkg/spill/record_file_test.go diff --git a/pkg/errors/error.go b/pkg/errors/error.go index b5d50e4e0a..e7925bfa3e 100644 --- a/pkg/errors/error.go +++ b/pkg/errors/error.go @@ -725,6 +725,10 @@ var ( "fail to open storage for redo log", errors.RFCCodeText("CDC:ErrStorageInitialize"), ) + ErrSpillFileOp = errors.Normalize( + "spill file operation failed: %s", + errors.RFCCodeText("CDC:ErrSpillFileOp"), + ) ErrRedoConfigInvalid = errors.Normalize( "redo log config invalid", diff --git a/pkg/eventservice/large_txn_spill.go b/pkg/eventservice/large_txn_spill.go index 7cb2da25f1..6256a2aa16 100644 --- a/pkg/eventservice/large_txn_spill.go +++ b/pkg/eventservice/large_txn_spill.go @@ -14,144 +14,80 @@ package eventservice import ( - "encoding/binary" "io" - "os" "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/errors" + recordspill "github.com/pingcap/ticdc/pkg/spill" ) -const largeTxnSpillRecordLenSize = 8 - // largeTxnInsertSpill stores deferred insert rows for a single large transaction. type largeTxnInsertSpill struct { - path string - file *os.File - closed bool - cleaned bool + path string + file *recordspill.RecordFile } func newLargeTxnInsertSpill(dir string) (*largeTxnInsertSpill, error) { if dir == "" { return nil, errors.New("large txn spill dir is empty") } - if err := os.MkdirAll(dir, 0o700); err != nil { - return nil, errors.Trace(err) - } - - file, err := os.CreateTemp(dir, "eventservice-large-txn-insert-*.spill") + file, err := recordspill.NewRecordFile(dir, "eventservice-large-txn-insert-*.spill") if err != nil { - return nil, errors.Trace(err) + return nil, err } return &largeTxnInsertSpill{ - path: file.Name(), + path: file.Path(), file: file, }, nil } func (s *largeTxnInsertSpill) Append(entry *common.RawKVEntry) error { - if s.cleaned { - return errors.New("large txn spill has been cleaned up") - } - if s.closed { - return errors.New("large txn spill is closed") - } if entry == nil { return errors.New("cannot append nil RawKVEntry to large txn spill") } - payload := entry.Encode() - var lenBuf [largeTxnSpillRecordLenSize]byte - binary.LittleEndian.PutUint64(lenBuf[:], uint64(len(payload))) - if err := writeLargeTxnSpillRecord(s.file, lenBuf[:]); err != nil { - return err - } - return writeLargeTxnSpillRecord(s.file, payload) + _, err := s.file.Append(entry.Encode()) + return err } func (s *largeTxnInsertSpill) NewReader() (*largeTxnInsertSpillReader, error) { - if s.cleaned { - return nil, errors.New("large txn spill has been cleaned up") - } if err := s.Close(); err != nil { return nil, err } - file, err := os.Open(s.path) + reader, err := s.file.NewReader() if err != nil { - return nil, errors.Trace(err) + return nil, err } - return &largeTxnInsertSpillReader{file: file}, nil + return &largeTxnInsertSpillReader{reader: reader}, nil } func (s *largeTxnInsertSpill) Close() error { - if s == nil || s.closed { - return nil - } - s.closed = true - if s.file == nil { + if s == nil { return nil } - - err := s.file.Close() - s.file = nil - return errors.Trace(err) + return s.file.Close() } func (s *largeTxnInsertSpill) Cleanup() error { if s == nil { return nil } - if err := s.Close(); err != nil { - return err - } - if s.cleaned { - return nil - } - s.cleaned = true - if s.path == "" { - return nil - } - - err := os.Remove(s.path) - if err != nil && !os.IsNotExist(err) { - return errors.Trace(err) - } - return nil + return s.file.Cleanup() } type largeTxnInsertSpillReader struct { - file *os.File - closed bool + reader *recordspill.Reader } func (r *largeTxnInsertSpillReader) Next() (*common.RawKVEntry, error) { - if r.closed { - return nil, errors.New("large txn spill reader is closed") - } - - var lenBuf [largeTxnSpillRecordLenSize]byte - _, err := io.ReadFull(r.file, lenBuf[:]) + data, err := r.reader.Next() if err != nil { if errors.Is(err, io.EOF) { return nil, io.EOF } - return nil, errors.Trace(err) - } - - recordLen := binary.LittleEndian.Uint64(lenBuf[:]) - if recordLen == 0 { - return nil, errors.New("large txn spill contains empty record") - } - if recordLen > uint64(int(^uint(0)>>1)) { - return nil, errors.Errorf("large txn spill record is too large: %d", recordLen) - } - - data := make([]byte, int(recordLen)) - if _, err := io.ReadFull(r.file, data); err != nil { - return nil, errors.Trace(err) + return nil, err } entry := &common.RawKVEntry{} @@ -162,29 +98,8 @@ func (r *largeTxnInsertSpillReader) Next() (*common.RawKVEntry, error) { } func (r *largeTxnInsertSpillReader) Close() error { - if r == nil || r.closed { + if r == nil { return nil } - r.closed = true - if r.file == nil { - return nil - } - - err := r.file.Close() - r.file = nil - return errors.Trace(err) -} - -func writeLargeTxnSpillRecord(writer io.Writer, data []byte) error { - for len(data) > 0 { - n, err := writer.Write(data) - if err != nil { - return errors.Trace(err) - } - if n == 0 { - return errors.New("failed to write large txn spill record") - } - data = data[n:] - } - return nil + return r.reader.Close() } diff --git a/pkg/spill/record_file.go b/pkg/spill/record_file.go new file mode 100644 index 0000000000..df4e3356eb --- /dev/null +++ b/pkg/spill/record_file.go @@ -0,0 +1,259 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package spill + +import ( + "encoding/binary" + "io" + "os" + + cerror "github.com/pingcap/ticdc/pkg/errors" +) + +const recordLenSize = 8 + +// Handle points to one framed record in a RecordFile. +type Handle struct { + Offset int64 + Length uint64 +} + +// Valid returns whether the handle points to a non-empty record. +func (h Handle) Valid() bool { + return h.Length > 0 +} + +// RecordFile stores temporary length-prefixed records in a local file. +type RecordFile struct { + path string + file *os.File + closed bool + cleaned bool +} + +// NewRecordFile creates a temporary spill file under dir. +func NewRecordFile(dir string, pattern string) (*RecordFile, error) { + if dir == "" { + return nil, cerror.ErrSpillFileOp.GenWithStackByArgs("empty spill directory") + } + if pattern == "" { + return nil, cerror.ErrSpillFileOp.GenWithStackByArgs("empty spill file pattern") + } + if err := os.MkdirAll(dir, 0o700); err != nil { + return nil, cerror.WrapError(cerror.ErrSpillFileOp, err, "create spill directory") + } + + file, err := os.CreateTemp(dir, pattern) + if err != nil { + return nil, cerror.WrapError(cerror.ErrSpillFileOp, err, "create spill file") + } + return &RecordFile{ + path: file.Name(), + file: file, + }, nil +} + +// Path returns the underlying temporary file path. +func (s *RecordFile) Path() string { + if s == nil { + return "" + } + return s.path +} + +// Append writes one record and returns its handle. +func (s *RecordFile) Append(data []byte) (Handle, error) { + return s.AppendChunks(data) +} + +// AppendChunks writes one record assembled from chunks without first joining +// them into a single byte slice. +func (s *RecordFile) AppendChunks(chunks ...[]byte) (Handle, error) { + if s == nil || s.cleaned { + return Handle{}, cerror.ErrSpillFileOp.GenWithStackByArgs("spill file has been cleaned up") + } + if s.closed || s.file == nil { + return Handle{}, cerror.ErrSpillFileOp.GenWithStackByArgs("spill file is closed") + } + + recordLen := uint64(0) + for _, chunk := range chunks { + recordLen += uint64(len(chunk)) + } + if recordLen == 0 { + return Handle{}, cerror.ErrSpillFileOp.GenWithStackByArgs("empty spill record") + } + + offset, err := s.file.Seek(0, io.SeekEnd) + if err != nil { + return Handle{}, cerror.WrapError(cerror.ErrSpillFileOp, err, "seek spill file") + } + + var lenBuf [recordLenSize]byte + binary.LittleEndian.PutUint64(lenBuf[:], recordLen) + if err := writeFull(s.file, lenBuf[:]); err != nil { + return Handle{}, err + } + for _, chunk := range chunks { + if err := writeFull(s.file, chunk); err != nil { + return Handle{}, err + } + } + + return Handle{Offset: offset + recordLenSize, Length: recordLen}, nil +} + +// Read reads the record at handle. +func (s *RecordFile) Read(handle Handle) ([]byte, error) { + if s == nil || s.cleaned { + return nil, cerror.ErrSpillFileOp.GenWithStackByArgs("spill file has been cleaned up") + } + if !handle.Valid() { + return nil, cerror.ErrSpillFileOp.GenWithStackByArgs("invalid spill record handle") + } + if handle.Length > uint64(int(^uint(0)>>1)) { + return nil, cerror.ErrSpillFileOp.GenWithStackByArgs("spill record is too large") + } + + file := s.file + if file == nil { + var err error + file, err = os.Open(s.path) + if err != nil { + return nil, cerror.WrapError(cerror.ErrSpillFileOp, err, "open spill file") + } + defer file.Close() + } + + data := make([]byte, int(handle.Length)) + if _, err := file.ReadAt(data, handle.Offset); err != nil { + return nil, cerror.WrapError(cerror.ErrSpillFileOp, err, "read spill record") + } + return data, nil +} + +// NewReader returns a sequential reader over the spill records. +func (s *RecordFile) NewReader() (*Reader, error) { + if s == nil || s.cleaned { + return nil, cerror.ErrSpillFileOp.GenWithStackByArgs("spill file has been cleaned up") + } + file, err := os.Open(s.path) + if err != nil { + return nil, cerror.WrapError(cerror.ErrSpillFileOp, err, "open spill file") + } + return &Reader{file: file}, nil +} + +// Close closes the writer side of the spill file. +func (s *RecordFile) Close() error { + if s == nil || s.closed { + return nil + } + s.closed = true + if s.file == nil { + return nil + } + + err := s.file.Close() + s.file = nil + return cerror.WrapError(cerror.ErrSpillFileOp, err, "close spill file") +} + +// Cleanup closes and removes the spill file. +func (s *RecordFile) Cleanup() error { + if s == nil { + return nil + } + if err := s.Close(); err != nil { + return err + } + if s.cleaned { + return nil + } + s.cleaned = true + if s.path == "" { + return nil + } + + err := os.Remove(s.path) + if err != nil && !os.IsNotExist(err) { + return cerror.WrapError(cerror.ErrSpillFileOp, err, "remove spill file") + } + return nil +} + +// Reader reads records sequentially from a RecordFile. +type Reader struct { + file *os.File + closed bool +} + +// Next returns the next record payload. +func (r *Reader) Next() ([]byte, error) { + if r == nil || r.closed { + return nil, cerror.ErrSpillFileOp.GenWithStackByArgs("spill reader is closed") + } + + var lenBuf [recordLenSize]byte + _, err := io.ReadFull(r.file, lenBuf[:]) + if err != nil { + if cerror.Is(err, io.EOF) { + return nil, io.EOF + } + return nil, cerror.WrapError(cerror.ErrSpillFileOp, err, "read spill record length") + } + + recordLen := binary.LittleEndian.Uint64(lenBuf[:]) + if recordLen == 0 { + return nil, cerror.ErrSpillFileOp.GenWithStackByArgs("empty spill record") + } + if recordLen > uint64(int(^uint(0)>>1)) { + return nil, cerror.ErrSpillFileOp.GenWithStackByArgs("spill record is too large") + } + + data := make([]byte, int(recordLen)) + if _, err := io.ReadFull(r.file, data); err != nil { + return nil, cerror.WrapError(cerror.ErrSpillFileOp, err, "read spill record") + } + return data, nil +} + +// Close closes the sequential reader. +func (r *Reader) Close() error { + if r == nil || r.closed { + return nil + } + r.closed = true + if r.file == nil { + return nil + } + + err := r.file.Close() + r.file = nil + return cerror.WrapError(cerror.ErrSpillFileOp, err, "close spill reader") +} + +func writeFull(writer io.Writer, data []byte) error { + for len(data) > 0 { + n, err := writer.Write(data) + if err != nil { + return cerror.WrapError(cerror.ErrSpillFileOp, err, "write spill record") + } + if n == 0 { + return cerror.ErrSpillFileOp.GenWithStackByArgs("zero bytes written") + } + data = data[n:] + } + return nil +} diff --git a/pkg/spill/record_file_test.go b/pkg/spill/record_file_test.go new file mode 100644 index 0000000000..cabec15713 --- /dev/null +++ b/pkg/spill/record_file_test.go @@ -0,0 +1,87 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package spill + +import ( + "io" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestRecordFileReadByHandle(t *testing.T) { + store, err := NewRecordFile(t.TempDir(), "test-*.spill") + require.NoError(t, err) + defer func() { + require.NoError(t, store.Cleanup()) + }() + + first, err := store.Append([]byte("first")) + require.NoError(t, err) + second, err := store.AppendChunks([]byte("sec"), []byte("ond")) + require.NoError(t, err) + + data, err := store.Read(second) + require.NoError(t, err) + require.Equal(t, []byte("second"), data) + data, err = store.Read(first) + require.NoError(t, err) + require.Equal(t, []byte("first"), data) +} + +func TestRecordFileSequentialReader(t *testing.T) { + store, err := NewRecordFile(t.TempDir(), "test-*.spill") + require.NoError(t, err) + defer func() { + require.NoError(t, store.Cleanup()) + }() + + _, err = store.Append([]byte("first")) + require.NoError(t, err) + _, err = store.Append([]byte("second")) + require.NoError(t, err) + require.NoError(t, store.Close()) + + reader, err := store.NewReader() + require.NoError(t, err) + defer func() { + require.NoError(t, reader.Close()) + }() + + data, err := reader.Next() + require.NoError(t, err) + require.Equal(t, []byte("first"), data) + data, err = reader.Next() + require.NoError(t, err) + require.Equal(t, []byte("second"), data) + data, err = reader.Next() + require.ErrorIs(t, err, io.EOF) + require.Nil(t, data) +} + +func TestRecordFileCreatesDirAndCleansUp(t *testing.T) { + dir := filepath.Join(t.TempDir(), "nested", "spill") + store, err := NewRecordFile(dir, "test-*.spill") + require.NoError(t, err) + + require.DirExists(t, dir) + path := store.Path() + require.NoError(t, store.Cleanup()) + require.NoError(t, store.Cleanup()) + + _, err = os.Stat(path) + require.True(t, os.IsNotExist(err)) +} From a2eef5b5d554300d8cc97a6d44e15e4ca88151fc Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Tue, 7 Jul 2026 14:30:31 +0800 Subject: [PATCH 18/35] eventservice: add big txn scan metrics --- pkg/eventservice/dispatcher_stat.go | 70 ++++++++++++++++++++++++ pkg/eventservice/dispatcher_stat_test.go | 42 ++++++++++++++ pkg/eventservice/event_scanner.go | 50 +++++++++++++++++ pkg/eventservice/metrics_collector.go | 8 +++ pkg/metrics/event_service.go | 15 +++++ 5 files changed, 185 insertions(+) diff --git a/pkg/eventservice/dispatcher_stat.go b/pkg/eventservice/dispatcher_stat.go index 4b6f3804b3..6ae2c5a7d8 100644 --- a/pkg/eventservice/dispatcher_stat.go +++ b/pkg/eventservice/dispatcher_stat.go @@ -113,6 +113,9 @@ type dispatcherStat struct { largeTxnStateMu sync.Mutex largeTxnState *largeTxnScanState + // bigTxnMetricState is updated only by the serialized scan task of this dispatcher. + bigTxnMetricState *bigTxnMetricState + // isRemoved is used to indicate whether the dispatcher is removed. // it is set to true in the following two cases: // 1) the dispatcher is removed @@ -249,6 +252,73 @@ func (a *dispatcherStat) getLastScannedPosition() common.ScanPosition { return position } +type bigTxnMetricState struct { + startTs uint64 + commitTs uint64 + rawKVBytes int64 + largeTxnThresholdInBytes int64 +} + +func (a *dispatcherStat) addBigTxnMetricFragment( + startTs uint64, + commitTs uint64, + rawKVBytes int64, + largeTxnThresholdInBytes int64, +) { + if rawKVBytes <= largeTxnThresholdInBytes { + return + } + if a.bigTxnMetricState == nil || + a.bigTxnMetricState.startTs != startTs || + a.bigTxnMetricState.commitTs != commitTs { + a.bigTxnMetricState = &bigTxnMetricState{ + startTs: startTs, + commitTs: commitTs, + largeTxnThresholdInBytes: largeTxnThresholdInBytes, + } + } + a.bigTxnMetricState.rawKVBytes += rawKVBytes +} + +func (a *dispatcherStat) finishBigTxnMetric( + startTs uint64, + commitTs uint64, + rawKVBytes int64, + largeTxnThresholdInBytes int64, +) { + totalRawKVBytes := rawKVBytes + if a.bigTxnMetricState != nil { + if a.bigTxnMetricState.startTs == startTs && a.bigTxnMetricState.commitTs == commitTs { + totalRawKVBytes += a.bigTxnMetricState.rawKVBytes + } else if a.bigTxnMetricState.rawKVBytes > a.bigTxnMetricState.largeTxnThresholdInBytes { + updateMetricEventServiceBigTxn(a.bigTxnMetricState.rawKVBytes) + } + a.bigTxnMetricState = nil + } + if totalRawKVBytes > largeTxnThresholdInBytes { + updateMetricEventServiceBigTxn(totalRawKVBytes) + } +} + +func (a *dispatcherStat) finishPendingBigTxnMetric() { + if a.bigTxnMetricState == nil { + return + } + state := a.bigTxnMetricState + a.bigTxnMetricState = nil + if state.rawKVBytes > state.largeTxnThresholdInBytes { + updateMetricEventServiceBigTxn(state.rawKVBytes) + } +} + +func (a *dispatcherStat) finishPendingBigTxnMetricBefore(startTs uint64, commitTs uint64) { + if a.bigTxnMetricState == nil || + (a.bigTxnMetricState.startTs == startTs && a.bigTxnMetricState.commitTs == commitTs) { + return + } + a.finishPendingBigTxnMetric() +} + // onResolvedTs try to update the resolved ts of the dispatcher. func (a *dispatcherStat) onResolvedTs(resolvedTs uint64) bool { if resolvedTs <= a.receivedResolvedTs.Load() { diff --git a/pkg/eventservice/dispatcher_stat_test.go b/pkg/eventservice/dispatcher_stat_test.go index 08ccb7dd91..a4fb6cdaad 100644 --- a/pkg/eventservice/dispatcher_stat_test.go +++ b/pkg/eventservice/dispatcher_stat_test.go @@ -21,6 +21,8 @@ import ( "github.com/pingcap/ticdc/eventpb" "github.com/pingcap/ticdc/pkg/common" pevent "github.com/pingcap/ticdc/pkg/common/event" + "github.com/pingcap/ticdc/pkg/metrics" + dto "github.com/prometheus/client_model/go" "github.com/stretchr/testify/require" "github.com/tikv/client-go/v2/oracle" ) @@ -77,6 +79,46 @@ func TestDispatcherStatResolvedTs(t *testing.T) { require.False(t, updated) } +func TestDispatcherStatBigTxnMetricsCountSplitTxnOnce(t *testing.T) { + beforeHistogramCount, beforeHistogramSum := readBigTxnSizeMetric(t) + beforeCounter := readBigTxnCountMetric(t) + + stat := &dispatcherStat{} + stat.addBigTxnMetricFragment(100, 200, 70, 50) + + histogramCount, histogramSum := readBigTxnSizeMetric(t) + require.Equal(t, beforeHistogramCount, histogramCount) + require.Equal(t, beforeHistogramSum, histogramSum) + require.Equal(t, beforeCounter, readBigTxnCountMetric(t)) + + stat.finishBigTxnMetric(100, 200, 30, 50) + + histogramCount, histogramSum = readBigTxnSizeMetric(t) + require.Equal(t, beforeHistogramCount+1, histogramCount) + require.Equal(t, beforeHistogramSum+100, histogramSum) + require.Equal(t, beforeCounter+1, readBigTxnCountMetric(t)) +} + +func readBigTxnSizeMetric(t *testing.T) (uint64, float64) { + t.Helper() + + metric := &dto.Metric{} + require.NoError(t, metrics.EventServiceBigTxnSize.Write(metric)) + histogram := metric.GetHistogram() + require.NotNil(t, histogram) + return histogram.GetSampleCount(), histogram.GetSampleSum() +} + +func readBigTxnCountMetric(t *testing.T) float64 { + t.Helper() + + metric := &dto.Metric{} + require.NoError(t, metrics.EventServiceBigTxnCount.Write(metric)) + counter := metric.GetCounter() + require.NotNil(t, counter) + return counter.GetValue() +} + func TestDispatcherStatGetDataRange(t *testing.T) { t.Parallel() diff --git a/pkg/eventservice/event_scanner.go b/pkg/eventservice/event_scanner.go index 09b77aabef..072b7e210a 100644 --- a/pkg/eventservice/event_scanner.go +++ b/pkg/eventservice/event_scanner.go @@ -278,6 +278,7 @@ func (s *eventScanner) scanAndMergeEvents( if state := dispatcher.getLargeTxnState(); processor.currentTxn == nil && state != nil && state.getPhase() == largeTxnScanPhaseOriginal { + dispatcher.finishPendingBigTxnMetric() dispatcher.markLargeTxnDrainInserts(state.startTs, state.commitTs, false, 0) session.progress = newTxnScanProgress(state.commitTs, state.startTs) return true, nil @@ -296,11 +297,13 @@ func (s *eventScanner) scanAndMergeEvents( err = finalizeScan(merger, processor, session, session.dataRange.CommitTsEnd) return false, err } + dispatcher.finishPendingBigTxnMetricBefore(rawEvent.StartTs, rawEvent.CRTs) if state := dispatcher.getLargeTxnState(); processor.currentTxn == nil && state != nil && state.getPhase() == largeTxnScanPhaseOriginal && (rawEvent.StartTs != state.startTs || rawEvent.CRTs != state.commitTs) { + dispatcher.finishPendingBigTxnMetric() dispatcher.markLargeTxnDrainInserts(state.startTs, state.commitTs, true, rawEvent.CRTs) session.progress = newTxnScanProgress(state.commitTs, state.startTs) return true, nil @@ -438,9 +441,27 @@ func (s *eventScanner) commitTxn( processor *dmlProcessor, eventCommitTs, tableInfoUpdateTs uint64, ) error { + var ( + startTs uint64 + commitTs uint64 + rawKVBytes int64 + largeTxnThresholdInBytes int64 + hasCurrentTxn bool + ) + if processor.currentTxn != nil { + currentTxn := processor.currentTxn + startTs = currentTxn.CurrentDMLEvent.GetStartTs() + commitTs = currentTxn.CurrentDMLEvent.GetCommitTs() + rawKVBytes = currentTxn.rawKVBytes + largeTxnThresholdInBytes = currentTxn.largeTxnThresholdInBytes + hasCurrentTxn = true + } if err := processor.commitTxn(); err != nil { return err } + if hasCurrentTxn { + session.dispatcherStat.finishBigTxnMetric(startTs, commitTs, rawKVBytes, largeTxnThresholdInBytes) + } currentBatchDML := processor.getCurrentBatchDML() // Use DMLCount() instead of Len() to check if the batchDML is empty @@ -540,9 +561,29 @@ func finalizeScan( sess *session, endTs uint64, ) error { + var ( + startTs uint64 + commitTs uint64 + rawKVBytes int64 + largeTxnThresholdInBytes int64 + hasCurrentTxn bool + ) + if processor.currentTxn != nil { + currentTxn := processor.currentTxn + startTs = currentTxn.CurrentDMLEvent.GetStartTs() + commitTs = currentTxn.CurrentDMLEvent.GetCommitTs() + rawKVBytes = currentTxn.rawKVBytes + largeTxnThresholdInBytes = currentTxn.largeTxnThresholdInBytes + hasCurrentTxn = true + } if err := processor.commitTxn(); err != nil { return err } + if hasCurrentTxn { + sess.dispatcherStat.finishBigTxnMetric(startTs, commitTs, rawKVBytes, largeTxnThresholdInBytes) + } else { + sess.dispatcherStat.finishPendingBigTxnMetric() + } resolvedBatch := processor.getCurrentBatchDML() events := merger.mergeWithPrecedingDDLs(resolvedBatch) @@ -606,6 +647,15 @@ func interruptCurrentTxn( startTs uint64, position common.ScanPosition, ) { + if processor.currentTxn != nil { + currentTxn := processor.currentTxn + currentDML := currentTxn.CurrentDMLEvent + session.dispatcherStat.addBigTxnMetricFragment( + currentDML.GetStartTs(), + currentDML.GetCommitTs(), + currentTxn.rawKVBytes, + currentTxn.largeTxnThresholdInBytes) + } events := merger.mergeWithPrecedingDDLs(processor.getCurrentBatchDML()) session.appendEvents(events) session.progress = newRowLevelScanProgress(commitTs, startTs, position) diff --git a/pkg/eventservice/metrics_collector.go b/pkg/eventservice/metrics_collector.go index 0952ed9260..61cb4d2fff 100644 --- a/pkg/eventservice/metrics_collector.go +++ b/pkg/eventservice/metrics_collector.go @@ -106,6 +106,14 @@ func updateMetricEventServiceSendDMLTypeCount(mode int64, rawType string, update metrics.EventServiceSendDMLTypeCount.WithLabelValues(common.StringMode(mode), rawType).Inc() } +func updateMetricEventServiceBigTxn(rawKVBytes int64) { + if rawKVBytes <= 0 { + return + } + metrics.EventServiceBigTxnSize.Observe(float64(rawKVBytes)) + metrics.EventServiceBigTxnCount.Inc() +} + // dispatcherHeapItem wraps dispatcherStat to implement heap.Item interface. // The heap maintains the slowest dispatchers by checkpointTs. // The heap top is the fastest (largest checkpointTs) among the slowest ones. diff --git a/pkg/metrics/event_service.go b/pkg/metrics/event_service.go index a8d96a2bbd..283eb1fdfd 100644 --- a/pkg/metrics/event_service.go +++ b/pkg/metrics/event_service.go @@ -199,6 +199,19 @@ var ( Help: "The number of transactions scanned from eventStore", Buckets: prometheus.ExponentialBuckets(1, 2.0, 8), // 1 ~ 256 }) + EventServiceBigTxnSize = prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: "ticdc", + Subsystem: "event_service", + Name: "big_txn_size", + Help: "The raw KV size of big transactions scanned from eventStore", + Buckets: prometheus.ExponentialBuckets(1024*1024, 2.0, 16), // 1MB to 32GB + }) + EventServiceBigTxnCount = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "ticdc", + Subsystem: "event_service", + Name: "big_txn_count", + Help: "The number of big transactions scanned from eventStore", + }) EventServiceSkipScanCount = prometheus.NewCounterVec( prometheus.CounterOpts{ @@ -268,6 +281,8 @@ func initEventServiceMetrics(registry *prometheus.Registry) { registry.MustRegister(EventServiceAvailableMemoryQuotaGaugeVec) registry.MustRegister(EventServiceScannedDMLSize) registry.MustRegister(EventServiceScannedTxnCount) + registry.MustRegister(EventServiceBigTxnSize) + registry.MustRegister(EventServiceBigTxnCount) registry.MustRegister(EventServiceSkipScanCount) registry.MustRegister(EventServiceInterruptScanCount) registry.MustRegister(EventServiceGetDDLEventDuration) From dbb2f6e3c3d67dd1ed7982261d4e953047837d5b Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Tue, 7 Jul 2026 15:21:11 +0800 Subject: [PATCH 19/35] *: fix go mod tidy Signed-off-by: dongmen <414110582@qq.com> --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 372f022fa3..bc77aba970 100644 --- a/go.mod +++ b/go.mod @@ -60,6 +60,7 @@ require ( github.com/pingcap/tidb/pkg/parser v0.0.0-20260604031706-f9faeaf4828f github.com/pingcap/tiflow v0.0.0-20260610095716-97d622547231 github.com/prometheus/client_golang v1.23.0 + github.com/prometheus/client_model v0.6.2 github.com/r3labs/diff v1.1.0 github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 github.com/robfig/cron v1.2.0 @@ -298,7 +299,6 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect - github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.65.0 // indirect github.com/prometheus/procfs v0.19.2 // indirect github.com/qri-io/jsonpointer v0.1.1 // indirect From cb1b3a242240e0c0ae2db819893f3d34171363d1 Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Wed, 8 Jul 2026 10:00:45 +0800 Subject: [PATCH 20/35] spill: fix errors import alias Signed-off-by: dongmen <414110582@qq.com> --- pkg/spill/record_file.go | 58 +++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 28 deletions(-) diff --git a/pkg/spill/record_file.go b/pkg/spill/record_file.go index df4e3356eb..42ce066488 100644 --- a/pkg/spill/record_file.go +++ b/pkg/spill/record_file.go @@ -18,7 +18,7 @@ import ( "io" "os" - cerror "github.com/pingcap/ticdc/pkg/errors" + "github.com/pingcap/ticdc/pkg/errors" ) const recordLenSize = 8 @@ -45,18 +45,18 @@ type RecordFile struct { // NewRecordFile creates a temporary spill file under dir. func NewRecordFile(dir string, pattern string) (*RecordFile, error) { if dir == "" { - return nil, cerror.ErrSpillFileOp.GenWithStackByArgs("empty spill directory") + return nil, errors.ErrSpillFileOp.GenWithStackByArgs("empty spill directory") } if pattern == "" { - return nil, cerror.ErrSpillFileOp.GenWithStackByArgs("empty spill file pattern") + return nil, errors.ErrSpillFileOp.GenWithStackByArgs("empty spill file pattern") } if err := os.MkdirAll(dir, 0o700); err != nil { - return nil, cerror.WrapError(cerror.ErrSpillFileOp, err, "create spill directory") + return nil, errors.WrapError(errors.ErrSpillFileOp, err, "create spill directory") } file, err := os.CreateTemp(dir, pattern) if err != nil { - return nil, cerror.WrapError(cerror.ErrSpillFileOp, err, "create spill file") + return nil, errors.WrapError(errors.ErrSpillFileOp, err, "create spill file") } return &RecordFile{ path: file.Name(), @@ -81,10 +81,10 @@ func (s *RecordFile) Append(data []byte) (Handle, error) { // them into a single byte slice. func (s *RecordFile) AppendChunks(chunks ...[]byte) (Handle, error) { if s == nil || s.cleaned { - return Handle{}, cerror.ErrSpillFileOp.GenWithStackByArgs("spill file has been cleaned up") + return Handle{}, errors.ErrSpillFileOp.GenWithStackByArgs("spill file has been cleaned up") } if s.closed || s.file == nil { - return Handle{}, cerror.ErrSpillFileOp.GenWithStackByArgs("spill file is closed") + return Handle{}, errors.ErrSpillFileOp.GenWithStackByArgs("spill file is closed") } recordLen := uint64(0) @@ -92,12 +92,12 @@ func (s *RecordFile) AppendChunks(chunks ...[]byte) (Handle, error) { recordLen += uint64(len(chunk)) } if recordLen == 0 { - return Handle{}, cerror.ErrSpillFileOp.GenWithStackByArgs("empty spill record") + return Handle{}, errors.ErrSpillFileOp.GenWithStackByArgs("empty spill record") } offset, err := s.file.Seek(0, io.SeekEnd) if err != nil { - return Handle{}, cerror.WrapError(cerror.ErrSpillFileOp, err, "seek spill file") + return Handle{}, errors.WrapError(errors.ErrSpillFileOp, err, "seek spill file") } var lenBuf [recordLenSize]byte @@ -117,13 +117,13 @@ func (s *RecordFile) AppendChunks(chunks ...[]byte) (Handle, error) { // Read reads the record at handle. func (s *RecordFile) Read(handle Handle) ([]byte, error) { if s == nil || s.cleaned { - return nil, cerror.ErrSpillFileOp.GenWithStackByArgs("spill file has been cleaned up") + return nil, errors.ErrSpillFileOp.GenWithStackByArgs("spill file has been cleaned up") } if !handle.Valid() { - return nil, cerror.ErrSpillFileOp.GenWithStackByArgs("invalid spill record handle") + return nil, errors.ErrSpillFileOp.GenWithStackByArgs("invalid spill record handle") } if handle.Length > uint64(int(^uint(0)>>1)) { - return nil, cerror.ErrSpillFileOp.GenWithStackByArgs("spill record is too large") + return nil, errors.ErrSpillFileOp.GenWithStackByArgs("spill record is too large") } file := s.file @@ -131,14 +131,16 @@ func (s *RecordFile) Read(handle Handle) ([]byte, error) { var err error file, err = os.Open(s.path) if err != nil { - return nil, cerror.WrapError(cerror.ErrSpillFileOp, err, "open spill file") + return nil, errors.WrapError(errors.ErrSpillFileOp, err, "open spill file") } - defer file.Close() + defer func() { + _ = file.Close() + }() } data := make([]byte, int(handle.Length)) if _, err := file.ReadAt(data, handle.Offset); err != nil { - return nil, cerror.WrapError(cerror.ErrSpillFileOp, err, "read spill record") + return nil, errors.WrapError(errors.ErrSpillFileOp, err, "read spill record") } return data, nil } @@ -146,11 +148,11 @@ func (s *RecordFile) Read(handle Handle) ([]byte, error) { // NewReader returns a sequential reader over the spill records. func (s *RecordFile) NewReader() (*Reader, error) { if s == nil || s.cleaned { - return nil, cerror.ErrSpillFileOp.GenWithStackByArgs("spill file has been cleaned up") + return nil, errors.ErrSpillFileOp.GenWithStackByArgs("spill file has been cleaned up") } file, err := os.Open(s.path) if err != nil { - return nil, cerror.WrapError(cerror.ErrSpillFileOp, err, "open spill file") + return nil, errors.WrapError(errors.ErrSpillFileOp, err, "open spill file") } return &Reader{file: file}, nil } @@ -167,7 +169,7 @@ func (s *RecordFile) Close() error { err := s.file.Close() s.file = nil - return cerror.WrapError(cerror.ErrSpillFileOp, err, "close spill file") + return errors.WrapError(errors.ErrSpillFileOp, err, "close spill file") } // Cleanup closes and removes the spill file. @@ -188,7 +190,7 @@ func (s *RecordFile) Cleanup() error { err := os.Remove(s.path) if err != nil && !os.IsNotExist(err) { - return cerror.WrapError(cerror.ErrSpillFileOp, err, "remove spill file") + return errors.WrapError(errors.ErrSpillFileOp, err, "remove spill file") } return nil } @@ -202,29 +204,29 @@ type Reader struct { // Next returns the next record payload. func (r *Reader) Next() ([]byte, error) { if r == nil || r.closed { - return nil, cerror.ErrSpillFileOp.GenWithStackByArgs("spill reader is closed") + return nil, errors.ErrSpillFileOp.GenWithStackByArgs("spill reader is closed") } var lenBuf [recordLenSize]byte _, err := io.ReadFull(r.file, lenBuf[:]) if err != nil { - if cerror.Is(err, io.EOF) { + if errors.Is(err, io.EOF) { return nil, io.EOF } - return nil, cerror.WrapError(cerror.ErrSpillFileOp, err, "read spill record length") + return nil, errors.WrapError(errors.ErrSpillFileOp, err, "read spill record length") } recordLen := binary.LittleEndian.Uint64(lenBuf[:]) if recordLen == 0 { - return nil, cerror.ErrSpillFileOp.GenWithStackByArgs("empty spill record") + return nil, errors.ErrSpillFileOp.GenWithStackByArgs("empty spill record") } if recordLen > uint64(int(^uint(0)>>1)) { - return nil, cerror.ErrSpillFileOp.GenWithStackByArgs("spill record is too large") + return nil, errors.ErrSpillFileOp.GenWithStackByArgs("spill record is too large") } data := make([]byte, int(recordLen)) if _, err := io.ReadFull(r.file, data); err != nil { - return nil, cerror.WrapError(cerror.ErrSpillFileOp, err, "read spill record") + return nil, errors.WrapError(errors.ErrSpillFileOp, err, "read spill record") } return data, nil } @@ -241,17 +243,17 @@ func (r *Reader) Close() error { err := r.file.Close() r.file = nil - return cerror.WrapError(cerror.ErrSpillFileOp, err, "close spill reader") + return errors.WrapError(errors.ErrSpillFileOp, err, "close spill reader") } func writeFull(writer io.Writer, data []byte) error { for len(data) > 0 { n, err := writer.Write(data) if err != nil { - return cerror.WrapError(cerror.ErrSpillFileOp, err, "write spill record") + return errors.WrapError(errors.ErrSpillFileOp, err, "write spill record") } if n == 0 { - return cerror.ErrSpillFileOp.GenWithStackByArgs("zero bytes written") + return errors.ErrSpillFileOp.GenWithStackByArgs("zero bytes written") } data = data[n:] } From b758366011e3b7e7201c3f5abc7620357b399c7f Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Fri, 10 Jul 2026 10:14:44 +0800 Subject: [PATCH 21/35] eventservice: harden large transaction spill handling --- pkg/eventservice/event_broker.go | 1 - pkg/eventservice/event_broker_test.go | 2 +- pkg/eventservice/event_scanner.go | 41 ++--- pkg/eventservice/event_scanner_test.go | 198 +++++++++++++++++++++++ pkg/eventservice/event_service.go | 16 +- pkg/eventservice/event_service_test.go | 23 +++ pkg/eventservice/large_txn_spill.go | 46 +++++- pkg/eventservice/large_txn_spill_test.go | 25 ++- pkg/eventservice/large_txn_state.go | 34 +++- 9 files changed, 354 insertions(+), 32 deletions(-) diff --git a/pkg/eventservice/event_broker.go b/pkg/eventservice/event_broker.go index b1c00e5c84..04e1bb0173 100644 --- a/pkg/eventservice/event_broker.go +++ b/pkg/eventservice/event_broker.go @@ -787,7 +787,6 @@ func (c *eventBroker) doScan(ctx context.Context, task scanTask) { progress.rowLevelScanPosition, ) } - task.info.GetMode() // Update metrics metricEventBrokerScanTaskCount.Inc() } diff --git a/pkg/eventservice/event_broker_test.go b/pkg/eventservice/event_broker_test.go index 38c2314470..e931a2604c 100644 --- a/pkg/eventservice/event_broker_test.go +++ b/pkg/eventservice/event_broker_test.go @@ -674,7 +674,7 @@ func mustCreateLargeTxnState(t *testing.T, stat *dispatcherStat, tableID int64) state, err := stat.getOrCreateLargeTxnState(t.TempDir(), tableID, nil, 90, 100) require.NoError(t, err) require.NoError(t, state.appendInsert(newTestSpillRawKVEntry(1))) - return state.spill.path + return state.spill.file.Path() } func TestResetDispatcherSendsHandshakeWithoutNextNotify(t *testing.T) { diff --git a/pkg/eventservice/event_scanner.go b/pkg/eventservice/event_scanner.go index 072b7e210a..037b0d1a1a 100644 --- a/pkg/eventservice/event_scanner.go +++ b/pkg/eventservice/event_scanner.go @@ -16,7 +16,6 @@ package eventservice import ( "context" "io" - "path/filepath" "time" "github.com/pingcap/log" @@ -155,9 +154,6 @@ func (s *eventScanner) scan( if state := dispatcherStat.getLargeTxnState(); state != nil && state.getPhase() == largeTxnScanPhaseDrainInserts { interrupted, err := s.drainLargeTxnInserts(sess, state) - if err != nil { - _ = dispatcherStat.cleanupLargeTxnState() - } return sess.eventBytes, sess.events, sess.progress, interrupted, err } @@ -174,6 +170,7 @@ func (s *eventScanner) scan( return 0, nil, scanProgress{}, false, err } if iter == nil { + dispatcherStat.finishPendingBigTxnMetric() if state := dispatcherStat.getLargeTxnState(); state != nil && state.getPhase() == largeTxnScanPhaseOriginal { dispatcherStat.markLargeTxnDrainInserts(state.startTs, state.commitTs, false, 0) @@ -671,6 +668,16 @@ func (s *eventScanner) drainLargeTxnInserts( session *session, state *largeTxnScanState, ) (bool, error) { + drainedInsertCount := 0 + returnWithError := func(scanErr error) (bool, error) { + if rollbackErr := state.rollbackDrain(); rollbackErr != nil { + log.Warn("reset large transaction spill reader failed", + zap.Stringer("dispatcherID", session.dispatcherStat.id), + zap.Error(rollbackErr)) + } + return false, scanErr + } + processor := newDMLProcessor( s.mounter, s.schemaGetter, @@ -683,7 +690,7 @@ func (s *eventScanner) drainLargeTxnInserts( for { shouldStop, err := s.checkScanConditions(session) if err != nil { - return false, err + return returnWithError(err) } if shouldStop { return false, nil @@ -697,18 +704,21 @@ func (s *eventScanner) drainLargeTxnInserts( if errors.Is(err, io.EOF) { if processor.currentTxn != nil { if err := processor.commitTxn(); err != nil { - return false, err + return returnWithError(err) } if processor.getCurrentBatchDML().DMLCount() != 0 { session.appendEvents([]event.Event{processor.getCurrentBatchDML()}) } } + state.commitDrainedInserts(drainedInsertCount) session.progress = newTxnScanProgress(state.commitTs, state.startTs) hasFollowingTxn, followingCommitTs := state.snapshotDrainInfo() shouldResolveCommitTs := (!hasFollowingTxn || followingCommitTs > state.commitTs) && session.dataRange.CommitTsEnd == state.commitTs if err := session.dispatcherStat.cleanupLargeTxnState(); err != nil { - return false, err + log.Warn("cleanup drained large transaction spill failed", + zap.Stringer("dispatcherID", session.dispatcherStat.id), + zap.Error(err)) } if shouldResolveCommitTs { resolved := event.NewResolvedEvent(state.commitTs, session.dispatcherStat.id, session.dispatcherStat.epoch) @@ -718,8 +728,9 @@ func (s *eventScanner) drainLargeTxnInserts( } return true, nil } - return false, err + return returnWithError(err) } + drainedInsertCount++ if processor.currentTxn == nil { if err := processor.startTxn( @@ -730,18 +741,19 @@ func (s *eventScanner) drainLargeTxnInserts( state.commitTs, true, ); err != nil { - return false, err + return returnWithError(err) } } session.observeRawEntry(entry, nil) if err := processor.appendInsertRow(entry); err != nil { - return false, err + return returnWithError(err) } if session.exceedLimit(processor.batchDML.GetSize(), processor.batchDML) { if err := processor.commitTxn(); err != nil { - return false, err + return returnWithError(err) } session.appendEvents([]event.Event{processor.getCurrentBatchDML()}) + state.commitDrainedInserts(drainedInsertCount) session.progress = newTxnScanProgress(state.commitTs, state.startTs) return true, nil } @@ -975,12 +987,6 @@ type TxnEvent struct { shouldSplitTxn bool } -const largeTxnInsertSpillDirName = "eventservice" - -func getLargeTxnInsertSpillDir() string { - return filepath.Join(config.GetGlobalServerConfig().DataDir, largeTxnInsertSpillDirName) -} - func newTxnEvent( batchDML *event.BatchDMLEvent, dispatcherID common.DispatcherID, @@ -1216,7 +1222,6 @@ func (p *dmlProcessor) appendInsertRow(rawEvent *common.RawKVEntry) error { log.Panic("no current DML event to append to") } rawEvent.Key = event.RemoveKeyspacePrefix(rawEvent.Key) - updateMetricEventServiceSendDMLTypeCount(p.mode, rawEvent.GetType(), false) return p.currentTxn.AppendRow(rawEvent, p.mounter.DecodeToChunk, p.filter, p.filterContext) } diff --git a/pkg/eventservice/event_scanner_test.go b/pkg/eventservice/event_scanner_test.go index 7c72104297..cfbad54bcd 100644 --- a/pkg/eventservice/event_scanner_test.go +++ b/pkg/eventservice/event_scanner_test.go @@ -16,6 +16,7 @@ package eventservice import ( "context" "errors" + "os" "path/filepath" "testing" "time" @@ -30,7 +31,9 @@ import ( "github.com/pingcap/ticdc/pkg/config" "github.com/pingcap/ticdc/pkg/filter" "github.com/pingcap/ticdc/pkg/integrity" + "github.com/pingcap/ticdc/pkg/metrics" "github.com/pingcap/tidb/pkg/util/chunk" + "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/require" "go.uber.org/atomic" ) @@ -40,6 +43,24 @@ type mockMounter struct { decodeCount atomic.Int64 } +type failOnceMounter struct { + event.Mounter + err error +} + +func (m *failOnceMounter) DecodeToChunk( + _ *common.RawKVEntry, + _ *common.TableInfo, + _ *chunk.Chunk, +) (int, *integrity.Checksum, error) { + if m.err != nil { + err := m.err + m.err = nil + return 0, nil, err + } + return 1, nil, nil +} + type countingDMLTypeFilter struct { filter.Filter dmlTypeCallCount atomic.Int64 @@ -73,6 +94,7 @@ func setLargeTxnThresholdForTest(t *testing.T, threshold int64) { original := config.GetGlobalServerConfig().Clone() cfg := original.Clone() cfg.Debug.EventService.LargeTxnThresholdInBytes = threshold + cfg.DataDir = t.TempDir() config.StoreGlobalServerConfig(cfg) t.Cleanup(func() { config.StoreGlobalServerConfig(original) @@ -514,6 +536,7 @@ func TestEventScannerSplitsLargeTxnWithRowLevelProgress(t *testing.T) { resolved, ok := events[0].(event.ResolvedEvent) require.True(t, ok) require.Equal(t, resolvedTs, resolved.ResolvedTs) + require.Nil(t, disp.bigTxnMetricState) } func TestEventScannerDoesNotSplitCurrentTxnBelowLargeTxnThreshold(t *testing.T) { @@ -570,6 +593,8 @@ func TestEventScannerDoesNotSplitCurrentTxnBelowLargeTxnThreshold(t *testing.T) func TestEventScannerSpillsSplitUKUpdateInLargeTxn(t *testing.T) { setLargeTxnThresholdForTest(t, 0) + insertCount := testutil.ToFloat64( + metrics.EventServiceSendDMLTypeCount.WithLabelValues(common.StringMode(0), "insert")) helper := event.NewEventTestHelper(t) defer helper.Close() @@ -650,6 +675,78 @@ func TestEventScannerSpillsSplitUKUpdateInLargeTxn(t *testing.T) { require.True(t, ok) require.Equal(t, resolvedTs, resolved.ResolvedTs) require.Nil(t, disp.getLargeTxnState()) + require.Equal(t, insertCount, testutil.ToFloat64( + metrics.EventServiceSendDMLTypeCount.WithLabelValues(common.StringMode(0), "insert"))) +} + +func TestEventScannerFlushesSpilledUKUpdateBeforeSameCommitDDL(t *testing.T) { + setLargeTxnThresholdForTest(t, 0) + + helper := event.NewEventTestHelper(t) + defer helper.Close() + helper.Tk().MustExec("use test") + ddlEvent := helper.DDL2Event("create table t_uk_same_ddl (id int primary key, a int, b char(50), unique key uk_a(a))") + _, updateEvent := helper.DML2UpdateEvent("test", "t_uk_same_ddl", + "insert into test.t_uk_same_ddl(id,a,b) values (1, 10, 'old_b')", + "update test.t_uk_same_ddl set a = 20 where id = 1") + resolvedTs := updateEvent.CRTs + + broker, _, _, _ := newEventBrokerForTest() + broker.close() + mockStore := broker.eventStore.(*mockEventStore) + mockSchemaStore := broker.schemaStore.(*mockSchemaStore) + + disInfo := newMockDispatcherInfoForTest(t) + disInfo.startTs = updateEvent.StartTs + changefeedStatus := broker.getOrSetChangefeedStatus(disInfo) + disp := newDispatcherStat(disInfo, 1, 1, nil, changefeedStatus) + makeDispatcherReady(disp) + require.NoError(t, broker.addDispatcher(disp.info)) + + createDDL := *ddlEvent + createDDL.FinishedTs = updateEvent.StartTs + sameCommitDDL := event.DDLEvent{ + FinishedTs: updateEvent.CRTs, + TableInfo: ddlEvent.TableInfo, + } + mockSchemaStore.AppendDDLEvent(disInfo.GetTableSpan().TableID, createDDL, sameCommitDDL) + require.NoError(t, mockStore.AppendEvents(disInfo.GetID(), resolvedTs, updateEvent)) + disp.receivedResolvedTs.Store(resolvedTs) + disp.eventStoreCommitTs.Store(resolvedTs) + + scanner := newEventScanner(broker.eventStore, broker.schemaStore, event.NewMounter(time.UTC, &integrity.Config{}), 0) + sl := scanLimit{maxDMLBytes: 1, isInUnitTest: true} + + dataRange, ok := disp.getDataRange() + require.True(t, ok) + ddlEvents, err := scanner.fetchDDLEvents(disp, dataRange) + require.NoError(t, err) + require.Len(t, ddlEvents, 1) + require.Equal(t, sameCommitDDL.FinishedTs, ddlEvents[0].GetCommitTs()) + + _, events, progress, interrupted, err := scanner.scan(context.Background(), disp, dataRange, sl) + require.NoError(t, err) + require.False(t, interrupted) + require.True(t, progress.valid) + require.Empty(t, progress.rowLevelScanPosition) + require.Len(t, events, 3) + + batch := events[0].(*event.BatchDMLEvent) + require.Equal(t, resolvedTs, batch.GetCommitTs()) + require.Equal(t, int32(2), batch.Len()) + deleteRow, ok := batch.DMLEvents[0].GetNextRow() + require.True(t, ok) + require.Equal(t, common.RowTypeDelete, deleteRow.RowType) + insertRow, ok := batch.DMLEvents[0].GetNextRow() + require.True(t, ok) + require.Equal(t, common.RowTypeInsert, insertRow.RowType) + + require.Equal(t, event.TypeDDLEvent, events[1].GetType()) + require.Equal(t, sameCommitDDL.FinishedTs, events[1].GetCommitTs()) + resolved, ok := events[2].(event.ResolvedEvent) + require.True(t, ok) + require.Equal(t, resolvedTs, resolved.ResolvedTs) + require.Nil(t, disp.getLargeTxnState()) } func TestEventScannerDrainsSpillBeforeFollowingSameCommitTxn(t *testing.T) { @@ -772,6 +869,107 @@ func TestDrainLargeTxnInsertsStopsWhenDispatcherRemoved(t *testing.T) { require.NoError(t, disp.cleanupLargeTxnState()) } +func TestEventScannerRetriesLargeTxnDrainAfterOpenError(t *testing.T) { + helper := event.NewEventTestHelper(t) + defer helper.Close() + ddlEvent := helper.DDL2Event("create table test.t_drain_retry (id int primary key, value varchar(32))") + + info := newMockDispatcherInfoForTest(t) + info.span.TableID = ddlEvent.GetTableID() + status := newChangefeedStatusForTest(t, info) + disp := newDispatcherStat(info, 1, 1, nil, status) + state, err := disp.getOrCreateLargeTxnState( + t.TempDir(), info.GetTableSpan().TableID, ddlEvent.TableInfo, 90, 100) + require.NoError(t, err) + require.NoError(t, state.appendInsert(newTestSpillRawKVEntry(1))) + disp.markLargeTxnDrainInserts(90, 100, false, 0) + + spillPath := state.spill.file.Path() + backupPath := spillPath + ".backup" + require.NoError(t, os.Rename(spillPath, backupPath)) + t.Cleanup(func() { + _ = os.Remove(backupPath) + }) + + dataRange := common.DataRange{ + Span: info.GetTableSpan(), + CommitTsStart: 100, + CommitTsEnd: 100, + } + scanner := newEventScanner( + &stubEventGetter{}, NewMockSchemaStore(), &mockMounter{}, 0) + + _, events, _, interrupted, err := scanner.scan( + context.Background(), disp, dataRange, + scanLimit{maxDMLBytes: 100, isInUnitTest: true}) + require.Error(t, err) + require.False(t, interrupted) + require.Empty(t, events) + require.Same(t, state, disp.getLargeTxnState()) + + require.NoError(t, os.Rename(backupPath, spillPath)) + _, events, progress, interrupted, err := scanner.scan( + context.Background(), disp, dataRange, + scanLimit{maxDMLBytes: 100, isInUnitTest: true}) + require.NoError(t, err) + require.False(t, interrupted) + require.True(t, progress.valid) + require.Len(t, events, 2) + + batch := events[0].(*event.BatchDMLEvent) + require.Equal(t, int32(1), batch.Len()) + resolved := events[1].(event.ResolvedEvent) + require.Equal(t, uint64(100), resolved.ResolvedTs) + require.Nil(t, disp.getLargeTxnState()) +} + +func TestEventScannerRetriesLargeTxnDrainAfterDecodeError(t *testing.T) { + helper := event.NewEventTestHelper(t) + defer helper.Close() + ddlEvent := helper.DDL2Event("create table test.t_drain_decode_retry (id int primary key, value varchar(32))") + + info := newMockDispatcherInfoForTest(t) + info.span.TableID = ddlEvent.GetTableID() + status := newChangefeedStatusForTest(t, info) + disp := newDispatcherStat(info, 1, 1, nil, status) + state, err := disp.getOrCreateLargeTxnState( + t.TempDir(), info.GetTableSpan().TableID, ddlEvent.TableInfo, 90, 100) + require.NoError(t, err) + require.NoError(t, state.appendInsert(newTestSpillRawKVEntry(1))) + require.NoError(t, state.appendInsert(newTestSpillRawKVEntry(2))) + disp.markLargeTxnDrainInserts(90, 100, false, 0) + + dataRange := common.DataRange{ + Span: info.GetTableSpan(), + CommitTsStart: 100, + CommitTsEnd: 100, + } + mounter := &failOnceMounter{err: errors.New("injected decode error")} + scanner := newEventScanner( + &stubEventGetter{}, NewMockSchemaStore(), mounter, 0) + + _, events, _, interrupted, err := scanner.scan( + context.Background(), disp, dataRange, + scanLimit{maxDMLBytes: 100, isInUnitTest: true}) + require.Error(t, err) + require.False(t, interrupted) + require.Empty(t, events) + require.Same(t, state, disp.getLargeTxnState()) + + _, events, progress, interrupted, err := scanner.scan( + context.Background(), disp, dataRange, + scanLimit{maxDMLBytes: 100, isInUnitTest: true}) + require.NoError(t, err) + require.False(t, interrupted) + require.True(t, progress.valid) + require.Len(t, events, 2) + + batch := events[0].(*event.BatchDMLEvent) + require.Equal(t, int32(2), batch.Len()) + require.Equal(t, uint64(100), events[1].(event.ResolvedEvent).ResolvedTs) + require.Nil(t, disp.getLargeTxnState()) +} + // Test the case where some DMLs have commit timestamps newer than the table's delete version func TestEventScannerWithDeleteTable(t *testing.T) { broker, _, _, _ := newEventBrokerForTest() diff --git a/pkg/eventservice/event_service.go b/pkg/eventservice/event_service.go index 8839ff9d87..ad81189e03 100644 --- a/pkg/eventservice/event_service.go +++ b/pkg/eventservice/event_service.go @@ -85,13 +85,27 @@ type eventService struct { func New(eventStore eventstore.EventStore, schemaStore schemastore.SchemaStore) common.SubModule { mc := appcontext.GetService[messaging.MessageCenter](appcontext.MessageCenter) - tzName := config.GetGlobalServerConfig().TZ + serverConfig := config.GetGlobalServerConfig() + tzName := serverConfig.TZ tz, err := util.GetTimezone(tzName) if err != nil { log.Panic("load timezone from server config failed", zap.String("timezone", tzName), zap.Error(err)) } + if serverConfig.DataDir != "" { + spillDir := getLargeTxnInsertSpillDir() + removed, err := cleanupLargeTxnInsertSpillFiles(spillDir) + if err != nil { + log.Warn("cleanup orphaned large transaction spill files failed", + zap.String("spillDir", spillDir), + zap.Error(err)) + } else if removed != 0 { + log.Info("removed orphaned large transaction spill files", + zap.String("spillDir", spillDir), + zap.Int("removed", removed)) + } + } es := &eventService{ mc: mc, eventStore: eventStore, diff --git a/pkg/eventservice/event_service_test.go b/pkg/eventservice/event_service_test.go index 8f39f7c5e4..eac53c1579 100644 --- a/pkg/eventservice/event_service_test.go +++ b/pkg/eventservice/event_service_test.go @@ -17,6 +17,8 @@ import ( "context" "encoding/binary" "fmt" + "os" + "path/filepath" "sync" "sync/atomic" "testing" @@ -61,6 +63,27 @@ func startEventService( return esImpl } +func TestNewEventServiceRemovesOrphanedLargeTxnSpillFiles(t *testing.T) { + original := config.GetGlobalServerConfig().Clone() + cfg := original.Clone() + cfg.DataDir = t.TempDir() + config.StoreGlobalServerConfig(cfg) + t.Cleanup(func() { + config.StoreGlobalServerConfig(original) + }) + + spillDir := getLargeTxnInsertSpillDir() + require.NoError(t, os.MkdirAll(spillDir, 0o700)) + orphanPath := filepath.Join(spillDir, "eventservice-large-txn-insert-orphan.spill") + require.NoError(t, os.WriteFile(orphanPath, []byte("orphan"), 0o600)) + + mc := messaging.NewMockMessageCenter() + appcontext.SetService(appcontext.MessageCenter, mc) + _ = New(newMockEventStore(100), NewMockSchemaStore()) + + require.NoFileExists(t, orphanPath) +} + func TestEventServiceBasic(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() diff --git a/pkg/eventservice/large_txn_spill.go b/pkg/eventservice/large_txn_spill.go index 6256a2aa16..a0497ad24a 100644 --- a/pkg/eventservice/large_txn_spill.go +++ b/pkg/eventservice/large_txn_spill.go @@ -15,31 +15,65 @@ package eventservice import ( "io" + "os" + "path/filepath" "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/config" "github.com/pingcap/ticdc/pkg/errors" recordspill "github.com/pingcap/ticdc/pkg/spill" ) +const ( + largeTxnInsertSpillDirName = "eventservice" + largeTxnInsertSpillPattern = "eventservice-large-txn-insert-*.spill" +) + // largeTxnInsertSpill stores deferred insert rows for a single large transaction. type largeTxnInsertSpill struct { - path string file *recordspill.RecordFile } +func getLargeTxnInsertSpillDir() string { + return filepath.Join(config.GetGlobalServerConfig().DataDir, largeTxnInsertSpillDirName) +} + +func cleanupLargeTxnInsertSpillFiles(dir string) (int, error) { + paths, err := filepath.Glob(filepath.Join(dir, largeTxnInsertSpillPattern)) + if err != nil { + return 0, errors.WrapError(errors.ErrSpillFileOp, err, "list large transaction spill files") + } + + removed := 0 + for _, path := range paths { + info, err := os.Lstat(path) + if err != nil { + if os.IsNotExist(err) { + continue + } + return removed, errors.WrapError(errors.ErrSpillFileOp, err, "stat large transaction spill file") + } + if !info.Mode().IsRegular() { + continue + } + if err := os.Remove(path); err != nil { + return removed, errors.WrapError(errors.ErrSpillFileOp, err, "remove large transaction spill file") + } + removed++ + } + return removed, nil +} + func newLargeTxnInsertSpill(dir string) (*largeTxnInsertSpill, error) { if dir == "" { return nil, errors.New("large txn spill dir is empty") } - file, err := recordspill.NewRecordFile(dir, "eventservice-large-txn-insert-*.spill") + file, err := recordspill.NewRecordFile(dir, largeTxnInsertSpillPattern) if err != nil { return nil, err } - return &largeTxnInsertSpill{ - path: file.Path(), - file: file, - }, nil + return &largeTxnInsertSpill{file: file}, nil } func (s *largeTxnInsertSpill) Append(entry *common.RawKVEntry) error { diff --git a/pkg/eventservice/large_txn_spill_test.go b/pkg/eventservice/large_txn_spill_test.go index 5bbf5d80e8..0fb139fcf4 100644 --- a/pkg/eventservice/large_txn_spill_test.go +++ b/pkg/eventservice/large_txn_spill_test.go @@ -66,7 +66,7 @@ func TestLargeTxnInsertSpillCreatesDir(t *testing.T) { }() require.DirExists(t, dir) - require.Equal(t, dir, filepath.Dir(spill.path)) + require.Equal(t, dir, filepath.Dir(spill.file.Path())) } func TestLargeTxnInsertSpillCleanup(t *testing.T) { @@ -74,7 +74,7 @@ func TestLargeTxnInsertSpillCleanup(t *testing.T) { require.NoError(t, err) require.NoError(t, spill.Append(newTestSpillRawKVEntry(1))) - path := spill.path + path := spill.file.Path() require.NoError(t, spill.Cleanup()) require.NoError(t, spill.Cleanup()) @@ -104,6 +104,27 @@ func TestLargeTxnInsertSpillEmpty(t *testing.T) { require.Nil(t, entry) } +func TestCleanupLargeTxnInsertSpillFiles(t *testing.T) { + dir := t.TempDir() + orphanPaths := []string{ + filepath.Join(dir, "eventservice-large-txn-insert-1.spill"), + filepath.Join(dir, "eventservice-large-txn-insert-2.spill"), + } + for _, path := range orphanPaths { + require.NoError(t, os.WriteFile(path, []byte("orphan"), 0o600)) + } + keepPath := filepath.Join(dir, "unrelated.spill") + require.NoError(t, os.WriteFile(keepPath, []byte("keep"), 0o600)) + + removed, err := cleanupLargeTxnInsertSpillFiles(dir) + require.NoError(t, err) + require.Equal(t, len(orphanPaths), removed) + for _, path := range orphanPaths { + require.NoFileExists(t, path) + } + require.FileExists(t, keepPath) +} + func newTestSpillRawKVEntry(index int) *common.RawKVEntry { return &common.RawKVEntry{ OpType: common.OpTypePut, diff --git a/pkg/eventservice/large_txn_state.go b/pkg/eventservice/large_txn_state.go index 780c7d0b1a..4c8757d343 100644 --- a/pkg/eventservice/large_txn_state.go +++ b/pkg/eventservice/large_txn_state.go @@ -41,9 +41,13 @@ type largeTxnScanState struct { hasFollowingTxn bool followingCommitTs uint64 - spill *largeTxnInsertSpill - reader *largeTxnInsertSpillReader - cleaned bool + spill *largeTxnInsertSpill + reader *largeTxnInsertSpillReader + // drainedInsertCount is the number of insert rows returned by completed + // drain scans. It lets an errored drain reopen the reader and retry only the + // rows from the current scan attempt. + drainedInsertCount int + cleaned bool } func (a *dispatcherStat) getOrCreateLargeTxnState( @@ -144,6 +148,13 @@ func (s *largeTxnScanState) nextInsert() (*common.RawKVEntry, error) { if err != nil { return nil, err } + for range s.drainedInsertCount { + if _, err := reader.Next(); err != nil { + _ = reader.Close() + return nil, errors.WrapError( + errors.ErrSpillFileOp, err, "seek committed spill rows") + } + } s.reader = reader } @@ -157,6 +168,23 @@ func (s *largeTxnScanState) nextInsert() (*common.RawKVEntry, error) { return entry, nil } +func (s *largeTxnScanState) commitDrainedInserts(count int) { + s.mu.Lock() + defer s.mu.Unlock() + s.drainedInsertCount += count +} + +func (s *largeTxnScanState) rollbackDrain() error { + s.mu.Lock() + defer s.mu.Unlock() + if s.reader == nil { + return nil + } + err := s.reader.Close() + s.reader = nil + return err +} + func (s *largeTxnScanState) markDrainInserts(hasFollowingTxn bool, followingCommitTs uint64) { s.mu.Lock() defer s.mu.Unlock() From a67388a855fc73a50525010b57d94f7f28086539 Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Fri, 10 Jul 2026 10:45:48 +0800 Subject: [PATCH 22/35] eventservice: address large transaction review feedback --- pkg/common/table_span.go | 1 + pkg/common/table_span_test.go | 3 +++ pkg/eventservice/dispatcher_stat.go | 1 + pkg/eventservice/dispatcher_stat_test.go | 20 ++++++++++++++++++++ pkg/eventservice/large_txn_spill.go | 4 ++-- pkg/eventservice/large_txn_spill_test.go | 14 ++++++++++++++ 6 files changed, 41 insertions(+), 2 deletions(-) diff --git a/pkg/common/table_span.go b/pkg/common/table_span.go index 73661e7f5b..3cb4d107ac 100644 --- a/pkg/common/table_span.go +++ b/pkg/common/table_span.go @@ -62,6 +62,7 @@ func (d *DataRange) Equal(other *DataRange) bool { return d.Span.Equal(other.Span) && d.CommitTsStart == other.CommitTsStart && d.CommitTsEnd == other.CommitTsEnd && + d.LastScannedTxnStartTs == other.LastScannedTxnStartTs && bytes.Equal(d.RowLevelScanPosition, other.RowLevelScanPosition) } diff --git a/pkg/common/table_span_test.go b/pkg/common/table_span_test.go index 3ecc64bbf5..0d07d613c0 100644 --- a/pkg/common/table_span_test.go +++ b/pkg/common/table_span_test.go @@ -74,10 +74,13 @@ func TestDataRangeEqual(t *testing.T) { dataRange2 := NewDataRange(1, span2, 10, 20) dataRange3 := NewDataRange(1, span1, 15, 25) dataRange4 := NewDataRange(2, span3, 10, 20) + dataRange5 := NewDataRange(1, span2, 10, 20) + dataRange5.LastScannedTxnStartTs = 9 require.True(t, dataRange1.Equal(dataRange2)) require.False(t, dataRange1.Equal(dataRange3)) require.False(t, dataRange1.Equal(dataRange4)) + require.False(t, dataRange1.Equal(dataRange5)) } func TestTableSpanLess(t *testing.T) { diff --git a/pkg/eventservice/dispatcher_stat.go b/pkg/eventservice/dispatcher_stat.go index 6ae2c5a7d8..6455544eaf 100644 --- a/pkg/eventservice/dispatcher_stat.go +++ b/pkg/eventservice/dispatcher_stat.go @@ -271,6 +271,7 @@ func (a *dispatcherStat) addBigTxnMetricFragment( if a.bigTxnMetricState == nil || a.bigTxnMetricState.startTs != startTs || a.bigTxnMetricState.commitTs != commitTs { + a.finishPendingBigTxnMetricBefore(startTs, commitTs) a.bigTxnMetricState = &bigTxnMetricState{ startTs: startTs, commitTs: commitTs, diff --git a/pkg/eventservice/dispatcher_stat_test.go b/pkg/eventservice/dispatcher_stat_test.go index a4fb6cdaad..ade40ca8e7 100644 --- a/pkg/eventservice/dispatcher_stat_test.go +++ b/pkg/eventservice/dispatcher_stat_test.go @@ -99,6 +99,26 @@ func TestDispatcherStatBigTxnMetricsCountSplitTxnOnce(t *testing.T) { require.Equal(t, beforeCounter+1, readBigTxnCountMetric(t)) } +func TestDispatcherStatBigTxnMetricsFlushPreviousTxn(t *testing.T) { + beforeHistogramCount, beforeHistogramSum := readBigTxnSizeMetric(t) + beforeCounter := readBigTxnCountMetric(t) + + stat := &dispatcherStat{} + stat.addBigTxnMetricFragment(100, 200, 70, 50) + stat.addBigTxnMetricFragment(101, 201, 80, 50) + + histogramCount, histogramSum := readBigTxnSizeMetric(t) + require.Equal(t, beforeHistogramCount+1, histogramCount) + require.Equal(t, beforeHistogramSum+70, histogramSum) + require.Equal(t, beforeCounter+1, readBigTxnCountMetric(t)) + require.Equal(t, &bigTxnMetricState{ + startTs: 101, + commitTs: 201, + rawKVBytes: 80, + largeTxnThresholdInBytes: 50, + }, stat.bigTxnMetricState) +} + func readBigTxnSizeMetric(t *testing.T) (uint64, float64) { t.Helper() diff --git a/pkg/eventservice/large_txn_spill.go b/pkg/eventservice/large_txn_spill.go index a0497ad24a..30bed1ca34 100644 --- a/pkg/eventservice/large_txn_spill.go +++ b/pkg/eventservice/large_txn_spill.go @@ -66,7 +66,7 @@ func cleanupLargeTxnInsertSpillFiles(dir string) (int, error) { func newLargeTxnInsertSpill(dir string) (*largeTxnInsertSpill, error) { if dir == "" { - return nil, errors.New("large txn spill dir is empty") + return nil, errors.ErrSpillFileOp.GenWithStackByArgs("empty large transaction spill directory") } file, err := recordspill.NewRecordFile(dir, largeTxnInsertSpillPattern) if err != nil { @@ -78,7 +78,7 @@ func newLargeTxnInsertSpill(dir string) (*largeTxnInsertSpill, error) { func (s *largeTxnInsertSpill) Append(entry *common.RawKVEntry) error { if entry == nil { - return errors.New("cannot append nil RawKVEntry to large txn spill") + return errors.ErrSpillFileOp.GenWithStackByArgs("cannot append nil RawKVEntry") } _, err := s.file.Append(entry.Encode()) diff --git a/pkg/eventservice/large_txn_spill_test.go b/pkg/eventservice/large_txn_spill_test.go index 0fb139fcf4..1de4ebcd6f 100644 --- a/pkg/eventservice/large_txn_spill_test.go +++ b/pkg/eventservice/large_txn_spill_test.go @@ -21,6 +21,7 @@ import ( "testing" "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/errors" "github.com/stretchr/testify/require" ) @@ -104,6 +105,19 @@ func TestLargeTxnInsertSpillEmpty(t *testing.T) { require.Nil(t, entry) } +func TestLargeTxnInsertSpillValidationErrors(t *testing.T) { + spill, err := newLargeTxnInsertSpill("") + require.True(t, errors.ErrSpillFileOp.Equal(err)) + require.Nil(t, spill) + + spill, err = newLargeTxnInsertSpill(t.TempDir()) + require.NoError(t, err) + defer func() { + require.NoError(t, spill.Cleanup()) + }() + require.True(t, errors.ErrSpillFileOp.Equal(spill.Append(nil))) +} + func TestCleanupLargeTxnInsertSpillFiles(t *testing.T) { dir := t.TempDir() orphanPaths := []string{ From c9c063829578c1967c91f3828cd8c811016e6b0c Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Tue, 21 Jul 2026 13:15:21 +0800 Subject: [PATCH 23/35] eventservice,eventstore: separate scan state and txn strategy --- logservice/eventstore/event_store.go | 41 +- logservice/eventstore/event_store_test.go | 142 +++--- logservice/eventstore/format.go | 6 +- logservice/eventstore/pebble_test.go | 2 +- logservice/eventstore/scan_request.go | 34 ++ pkg/common/table_span.go | 14 +- pkg/common/table_span_test.go | 3 - pkg/eventservice/dispatcher_stat.go | 79 ++- pkg/eventservice/dispatcher_stat_test.go | 70 ++- pkg/eventservice/event_broker.go | 43 +- pkg/eventservice/event_broker_test.go | 65 ++- pkg/eventservice/event_scanner.go | 439 +++-------------- .../event_scanner_benchmark_test.go | 12 +- pkg/eventservice/event_scanner_test.go | 102 ++-- pkg/eventservice/event_service_test.go | 25 +- pkg/eventservice/scan_progress.go | 48 ++ pkg/eventservice/txn_scan_strategy.go | 465 ++++++++++++++++++ 17 files changed, 936 insertions(+), 654 deletions(-) create mode 100644 logservice/eventstore/scan_request.go create mode 100644 pkg/eventservice/scan_progress.go create mode 100644 pkg/eventservice/txn_scan_strategy.go diff --git a/logservice/eventstore/event_store.go b/logservice/eventstore/event_store.go index 77c2e92817..07ec16e5e2 100644 --- a/logservice/eventstore/event_store.go +++ b/logservice/eventstore/event_store.go @@ -93,8 +93,8 @@ type EventStore interface { UpdateDispatcherCheckpointTs(dispatcherID common.DispatcherID, checkpointTs uint64) - // GetIterator returns an iterator which scans data in ts range (dataRange.CommitTsStart, dataRange.CommitTsEnd]. - GetIterator(dispatcherID common.DispatcherID, dataRange common.DataRange) (EventIterator, error) + // GetIterator returns an iterator for the requested range and resume cursor. + GetIterator(dispatcherID common.DispatcherID, request ScanRequest) (EventIterator, error) GetLogCoordinatorNodeID() node.ID } @@ -120,7 +120,7 @@ type EventIteratorWithScanPosition interface { // NextWithScanPosition returns the next event, the opaque position of the // returned event, and whether this event is from a new txn. - NextWithScanPosition() (*common.RawKVEntry, common.ScanPosition, bool) + NextWithScanPosition() (*common.RawKVEntry, ScanPosition, bool) } type dispatcherStat struct { @@ -817,10 +817,11 @@ func (e *eventStore) UpdateDispatcherCheckpointTs( updateSubStatCheckpoint(dispatcherStat.removingSubStat) } -func (e *eventStore) GetIterator(dispatcherID common.DispatcherID, dataRange common.DataRange) (EventIterator, error) { +func (e *eventStore) GetIterator(dispatcherID common.DispatcherID, request ScanRequest) (EventIterator, error) { if e.closed.Load() { return nil, nil } + dataRange := request.Range e.dispatcherMeta.RLock() stat, ok := e.dispatcherMeta.dispatcherStats[dispatcherID] @@ -838,7 +839,7 @@ func (e *eventStore) GetIterator(dispatcherID common.DispatcherID, dataRange com zap.Int64("tableID", dataRange.Span.GetTableID()), zap.Uint64("commitTsStart", dataRange.CommitTsStart), zap.Uint64("commitTsEnd", dataRange.CommitTsEnd), - zap.Uint64("lastScannedTxnStartTs", dataRange.LastScannedTxnStartTs)) + zap.Uint64("lastScannedTxnStartTs", request.Cursor.TxnStartTs)) } return nil } @@ -849,7 +850,7 @@ func (e *eventStore) GetIterator(dispatcherID common.DispatcherID, dataRange com zap.Int64("tableID", dataRange.Span.GetTableID()), zap.Uint64("commitTsStart", dataRange.CommitTsStart), zap.Uint64("commitTsEnd", dataRange.CommitTsEnd), - zap.Uint64("lastScannedTxnStartTs", dataRange.LastScannedTxnStartTs), + zap.Uint64("lastScannedTxnStartTs", request.Cursor.TxnStartTs), zap.Uint64("subStatCheckpointTs", checkpointTs), zap.Uint64("subStatResolvedTs", subStat.resolvedTs.Load())) } @@ -860,7 +861,7 @@ func (e *eventStore) GetIterator(dispatcherID common.DispatcherID, dataRange com zap.Int64("tableID", dataRange.Span.GetTableID()), zap.Uint64("commitTsStart", dataRange.CommitTsStart), zap.Uint64("commitTsEnd", dataRange.CommitTsEnd), - zap.Uint64("lastScannedTxnStartTs", dataRange.LastScannedTxnStartTs), + zap.Uint64("lastScannedTxnStartTs", request.Cursor.TxnStartTs), zap.Uint64("subStatCheckpointTs", checkpointTs), zap.Uint64("subStatResolvedTs", subStat.resolvedTs.Load())) } @@ -917,38 +918,38 @@ func (e *eventStore) GetIterator(dispatcherID common.DispatcherID, dataRange com e.dispatcherMeta.Unlock() } - // dataRange fields: - // CommitTsStart and CommitTsEnd define the commit-ts scan window. - // LastScannedTxnStartTs records how far the previous scan progressed inside + // request fields: + // Range defines the commit-ts scan window. Cursor.TxnStartTs records how far + // the previous scan progressed inside // CommitTsStart. It is zero if there is no unfinished scan at CommitTsStart. // // Iterator key bounds: // Pebble uses [LowerBound, UpperBound), so end is always encoded as // CommitTsEnd+1. // - // If RowLevelScanPosition is present, continue scanning from the next + // If Cursor.Position is present, continue scanning from the next // eventstore key after that opaque position. // - // If LastScannedTxnStartTs is zero, scan commit ts in + // If Cursor.TxnStartTs is zero, scan commit ts in // (CommitTsStart, CommitTsEnd], and use CommitTsStart+1 as LowerBound. // - // If LastScannedTxnStartTs is non-zero, continue scanning commit ts - // CommitTsStart with start ts greater than LastScannedTxnStartTs, then scan + // If Cursor.TxnStartTs is non-zero, continue scanning commit ts + // CommitTsStart with start ts greater than Cursor.TxnStartTs, then scan // later commit ts up to CommitTsEnd. // var start []byte - if len(dataRange.RowLevelScanPosition) != 0 { + if len(request.Cursor.Position) != 0 { start = encodeRowLevelScanPositionLowerBound( uint64(subStat.subID), stat.tableSpan.TableID, - dataRange.RowLevelScanPosition, + request.Cursor.Position, ) - } else if dataRange.LastScannedTxnStartTs != 0 { + } else if request.Cursor.TxnStartTs != 0 { start = encodeScanLowerBound( uint64(subStat.subID), stat.tableSpan.TableID, dataRange.CommitTsStart, - dataRange.LastScannedTxnStartTs+1, + request.Cursor.TxnStartTs+1, ) } else { start = encodeTxnCommitTsBoundaryKey(uint64(subStat.subID), stat.tableSpan.TableID, dataRange.CommitTsStart+1) @@ -1584,9 +1585,9 @@ func (iter *eventStoreIter) Next() (*common.RawKVEntry, bool) { return rawKV, isNewTxn } -func (iter *eventStoreIter) NextWithScanPosition() (*common.RawKVEntry, common.ScanPosition, bool) { +func (iter *eventStoreIter) NextWithScanPosition() (*common.RawKVEntry, ScanPosition, bool) { rawKV := &common.RawKVEntry{} - var scanPosition common.ScanPosition + var scanPosition ScanPosition for { if !iter.innerIter.Valid() { return nil, nil, false diff --git a/logservice/eventstore/event_store_test.go b/logservice/eventstore/event_store_test.go index 99e6312a7a..6b58725b18 100644 --- a/logservice/eventstore/event_store_test.go +++ b/logservice/eventstore/event_store_test.go @@ -156,7 +156,7 @@ func requireEventIterator( t testing.TB, store EventStore, dispatcherID common.DispatcherID, dataRange common.DataRange, ) EventIterator { t.Helper() - iter, err := store.GetIterator(dispatcherID, dataRange) + iter, err := store.GetIterator(dispatcherID, ScanRequest{Range: dataRange}) require.NoError(t, err) return iter } @@ -307,7 +307,7 @@ func TestEventStoreUsesKeyspaceIDForEncryption(t *testing.T) { CommitTsStart: 0, CommitTsEnd: largeKV.CRTs, } - iter, err := es.GetIterator(dispatcherID, dataRange) + iter, err := es.GetIterator(dispatcherID, ScanRequest{Range: dataRange}) require.NoError(t, err) require.NotNil(t, iter) @@ -394,10 +394,12 @@ func TestEventStoreHandlesUnencryptedValuesFromEncryptionLayer(t *testing.T) { require.NoError(t, err) subStat.resolvedTs.Store(largeKV.CRTs) - iter, err := es.GetIterator(dispatcherID, common.DataRange{ - Span: span, - CommitTsStart: 0, - CommitTsEnd: largeKV.CRTs, + iter, err := es.GetIterator(dispatcherID, ScanRequest{ + Range: common.DataRange{ + Span: span, + CommitTsStart: 0, + CommitTsEnd: largeKV.CRTs, + }, }) require.NoError(t, err) require.NotNil(t, iter) @@ -730,10 +732,12 @@ func TestGetIteratorPanicWhenStartLessThanCheckpoint(t *testing.T) { store.UpdateDispatcherCheckpointTs(dispatcherID, 120) require.Panics(t, func() { - _, _ = store.GetIterator(dispatcherID, common.DataRange{ - Span: span, - CommitTsStart: 110, - CommitTsEnd: 150, + _, _ = store.GetIterator(dispatcherID, ScanRequest{ + Range: common.DataRange{ + Span: span, + CommitTsStart: 110, + CommitTsEnd: 150, + }, }) }) } @@ -984,14 +988,16 @@ func TestEventStoreSwitchSubStat(t *testing.T) { subStat.resolvedTs.Store(ts) } getIterator := func() { - iter, err := store.GetIterator(dispatcherID2, common.DataRange{ - Span: &heartbeatpb.TableSpan{ - TableID: tableID, - StartKey: []byte("b"), - EndKey: []byte("h"), + iter, err := store.GetIterator(dispatcherID2, ScanRequest{ + Range: common.DataRange{ + Span: &heartbeatpb.TableSpan{ + TableID: tableID, + StartKey: []byte("b"), + EndKey: []byte("h"), + }, + CommitTsStart: 100, + CommitTsEnd: 150, }, - CommitTsStart: 100, - CommitTsEnd: 150, }) require.NoError(t, err) if iter != nil { @@ -1065,14 +1071,16 @@ func TestEventStoreSwitchSubStat(t *testing.T) { // case 3: subStat 1 advance quicker than subStat 2, dispatcher 2 can still read data from subStat 1 updateSubStatResolvedTs(1, 220) { - iter, err := store.GetIterator(dispatcherID2, common.DataRange{ - Span: &heartbeatpb.TableSpan{ - TableID: tableID, - StartKey: []byte("b"), - EndKey: []byte("h"), + iter, err := store.GetIterator(dispatcherID2, ScanRequest{ + Range: common.DataRange{ + Span: &heartbeatpb.TableSpan{ + TableID: tableID, + StartKey: []byte("b"), + EndKey: []byte("h"), + }, + CommitTsStart: 100, + CommitTsEnd: 220, }, - CommitTsStart: 100, - CommitTsEnd: 220, }) require.NoError(t, err) if iter != nil { @@ -1102,14 +1110,16 @@ func TestEventStoreSwitchSubStat(t *testing.T) { // dispatcher 2 read data from subStat 2 and totally remove itself from the subsriber list of subStat 1 updateSubStatResolvedTs(2, 220) { - iter, err := store.GetIterator(dispatcherID2, common.DataRange{ - Span: &heartbeatpb.TableSpan{ - TableID: tableID, - StartKey: []byte("b"), - EndKey: []byte("h"), + iter, err := store.GetIterator(dispatcherID2, ScanRequest{ + Range: common.DataRange{ + Span: &heartbeatpb.TableSpan{ + TableID: tableID, + StartKey: []byte("b"), + EndKey: []byte("h"), + }, + CommitTsStart: 100, + CommitTsEnd: 220, }, - CommitTsStart: 100, - CommitTsEnd: 220, }) require.NoError(t, err) if iter != nil { @@ -1193,10 +1203,10 @@ func TestEventStoreRowLevelScanPositionSurvivesSubStatSwitch(t *testing.T) { type scannedEvent struct { key string - position common.ScanPosition + position ScanPosition } - collectEvents := func(dataRange common.DataRange) []scannedEvent { - iter, err := store.GetIterator(dispatcherID2, dataRange) + collectEvents := func(request ScanRequest) []scannedEvent { + iter, err := store.GetIterator(dispatcherID2, request) require.NoError(t, err) require.NotNil(t, iter) positionIter, ok := iter.(EventIteratorWithScanPosition) @@ -1221,10 +1231,12 @@ func TestEventStoreRowLevelScanPositionSurvivesSubStatSwitch(t *testing.T) { } oldSubStat.resolvedTs.Store(nextCommitTs) - firstScanEvents := collectEvents(common.DataRange{ - Span: dispatcherSpan, - CommitTsStart: txnCommitTs - 1, - CommitTsEnd: nextCommitTs, + firstScanEvents := collectEvents(ScanRequest{ + Range: common.DataRange{ + Span: dispatcherSpan, + CommitTsStart: txnCommitTs - 1, + CommitTsEnd: nextCommitTs, + }, }) require.Len(t, firstScanEvents, 3) require.Equal(t, []string{"c-row-1", "c-row-2", "c-next-row"}, []string{ @@ -1236,11 +1248,13 @@ func TestEventStoreRowLevelScanPositionSurvivesSubStatSwitch(t *testing.T) { require.Equal(t, newSubStat.subID, dispatcherStat.pendingSubStat.subID) newSubStat.resolvedTs.Store(nextCommitTs) - resumedEvents := collectEvents(common.DataRange{ - Span: dispatcherSpan, - CommitTsStart: txnCommitTs, - CommitTsEnd: nextCommitTs, - RowLevelScanPosition: firstScanEvents[0].position, + resumedEvents := collectEvents(ScanRequest{ + Range: common.DataRange{ + Span: dispatcherSpan, + CommitTsStart: txnCommitTs, + CommitTsEnd: nextCommitTs, + }, + Cursor: ScanCursor{Position: firstScanEvents[0].position}, }) require.Len(t, resumedEvents, 2) require.Equal(t, []string{"c-row-2", "c-next-row"}, []string{ @@ -1774,10 +1788,10 @@ func TestEventStoreResumeTokenSupportsRowLevelResume(t *testing.T) { type scannedEvent struct { key string - position common.ScanPosition + position ScanPosition } - collectEvents := func(dataRange common.DataRange) []scannedEvent { - iter, err := store.GetIterator(dispatcherID, dataRange) + collectEvents := func(request ScanRequest) []scannedEvent { + iter, err := store.GetIterator(dispatcherID, request) require.NoError(t, err) if iter == nil { return nil @@ -1803,10 +1817,12 @@ func TestEventStoreResumeTokenSupportsRowLevelResume(t *testing.T) { return events } - fullRange := common.DataRange{ - Span: span, - CommitTsStart: txnCommitTs - 1, - CommitTsEnd: nextCommitTs, + fullRange := ScanRequest{ + Range: common.DataRange{ + Span: span, + CommitTsStart: txnCommitTs - 1, + CommitTsEnd: nextCommitTs, + }, } fullEvents := collectEvents(fullRange) require.Len(t, fullEvents, 3) @@ -1816,24 +1832,28 @@ func TestEventStoreResumeTokenSupportsRowLevelResume(t *testing.T) { fullEvents[2].key, }) - resumeAfterTxnStart := common.DataRange{ - Span: span, - CommitTsStart: txnCommitTs, - CommitTsEnd: nextCommitTs, - LastScannedTxnStartTs: txnStartTs, + resumeAfterTxnStart := ScanRequest{ + Range: common.DataRange{ + Span: span, + CommitTsStart: txnCommitTs, + CommitTsEnd: nextCommitTs, + }, + Cursor: ScanCursor{TxnStartTs: txnStartTs}, } - // LastScannedTxnStartTs can resume after a txn start-ts, but it cannot + // Cursor.TxnStartTs can resume after a txn start-ts, but it cannot // identify a specific row inside the same txn. Once set to txnStartTs, all // rows in that txn are skipped, including row-2. txnLevelEvents := collectEvents(resumeAfterTxnStart) require.Len(t, txnLevelEvents, 1) require.Equal(t, []string{"next-row"}, []string{txnLevelEvents[0].key}) - resumeAfterRow1 := common.DataRange{ - Span: span, - CommitTsStart: txnCommitTs, - CommitTsEnd: nextCommitTs, - RowLevelScanPosition: fullEvents[0].position, + resumeAfterRow1 := ScanRequest{ + Range: common.DataRange{ + Span: span, + CommitTsStart: txnCommitTs, + CommitTsEnd: nextCommitTs, + }, + Cursor: ScanCursor{Position: fullEvents[0].position}, } rowLevelEvents := collectEvents(resumeAfterRow1) require.Len(t, rowLevelEvents, 2) diff --git a/logservice/eventstore/format.go b/logservice/eventstore/format.go index c75dbc3be8..407145f5ff 100644 --- a/logservice/eventstore/format.go +++ b/logservice/eventstore/format.go @@ -93,13 +93,13 @@ func encodeScanLowerBound(uniqueID uint64, tableID int64, txnCommitTs uint64, tx return buf } -func encodeRowLevelScanPosition(key []byte) common.ScanPosition { - position := make(common.ScanPosition, len(key)-encodedKeyTxnCommitTsOffset) +func encodeRowLevelScanPosition(key []byte) ScanPosition { + position := make(ScanPosition, len(key)-encodedKeyTxnCommitTsOffset) copy(position, key[encodedKeyTxnCommitTsOffset:]) return position } -func encodeRowLevelScanPositionLowerBound(uniqueID uint64, tableID int64, position common.ScanPosition) []byte { +func encodeRowLevelScanPositionLowerBound(uniqueID uint64, tableID int64, position ScanPosition) []byte { buf := make([]byte, encodedKeyTxnCommitTsOffset, encodedKeyTxnCommitTsOffset+len(position)+1) binary.BigEndian.PutUint64(buf[encodedKeyUniqueIDOffset:encodedKeyUniqueIDOffset+encodedKeyUniqueIDLen], uniqueID) binary.BigEndian.PutUint64(buf[encodedKeyTableIDOffset:encodedKeyTableIDOffset+encodedKeyTableIDLen], uint64(tableID)) diff --git a/logservice/eventstore/pebble_test.go b/logservice/eventstore/pebble_test.go index ad8d64722f..712b711ada 100644 --- a/logservice/eventstore/pebble_test.go +++ b/logservice/eventstore/pebble_test.go @@ -171,7 +171,7 @@ func TestEventStoreKeyBounds(t *testing.T) { require.True(t, bytes.HasPrefix(key, lowerBound)) rowLevelPosition := encodeRowLevelScanPosition(key) - require.Equal(t, common.ScanPosition(key[encodedKeyTxnCommitTsOffset:]), rowLevelPosition) + require.Equal(t, ScanPosition(key[encodedKeyTxnCommitTsOffset:]), rowLevelPosition) rowLevelLowerBound := encodeRowLevelScanPositionLowerBound(1, 1, rowLevelPosition) require.Less(t, bytes.Compare(key, rowLevelLowerBound), 0) diff --git a/logservice/eventstore/scan_request.go b/logservice/eventstore/scan_request.go new file mode 100644 index 0000000000..db2179fc7e --- /dev/null +++ b/logservice/eventstore/scan_request.go @@ -0,0 +1,34 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package eventstore + +import "github.com/pingcap/ticdc/pkg/common" + +// ScanPosition is an opaque eventstore-owned token used to resume after a +// specific row inside a scan window. +type ScanPosition []byte + +// ScanCursor identifies where scanning should resume inside a DataRange. +// Position is an opaque eventstore-owned token and takes precedence over +// TxnStartTs when both are present. +type ScanCursor struct { + TxnStartTs uint64 + Position ScanPosition +} + +// ScanRequest combines a pure data range with its resume cursor. +type ScanRequest struct { + Range common.DataRange + Cursor ScanCursor +} diff --git a/pkg/common/table_span.go b/pkg/common/table_span.go index 3cb4d107ac..6c55257bc2 100644 --- a/pkg/common/table_span.go +++ b/pkg/common/table_span.go @@ -25,18 +25,8 @@ type DataRange struct { Span *heartbeatpb.TableSpan CommitTsStart uint64 CommitTsEnd uint64 - - // RowLevelScanPosition is an opaque eventstore-owned token used to resume - // scanning after a specific row inside the scan window. - RowLevelScanPosition ScanPosition - - // LastScannedTxnStartTs is the start-ts of the last scanned DML event. - // it should less than the CommitTsStart - LastScannedTxnStartTs uint64 } -type ScanPosition []byte - func NewDataRange(clusterID uint64, span *heartbeatpb.TableSpan, startTs, endTs uint64) *DataRange { return &DataRange{ ClusterID: clusterID, @@ -61,9 +51,7 @@ func (d *DataRange) String() string { func (d *DataRange) Equal(other *DataRange) bool { return d.Span.Equal(other.Span) && d.CommitTsStart == other.CommitTsStart && - d.CommitTsEnd == other.CommitTsEnd && - d.LastScannedTxnStartTs == other.LastScannedTxnStartTs && - bytes.Equal(d.RowLevelScanPosition, other.RowLevelScanPosition) + d.CommitTsEnd == other.CommitTsEnd } // Merge merges two DataRange, if the two DataRange have different Span, return nil. diff --git a/pkg/common/table_span_test.go b/pkg/common/table_span_test.go index 0d07d613c0..3ecc64bbf5 100644 --- a/pkg/common/table_span_test.go +++ b/pkg/common/table_span_test.go @@ -74,13 +74,10 @@ func TestDataRangeEqual(t *testing.T) { dataRange2 := NewDataRange(1, span2, 10, 20) dataRange3 := NewDataRange(1, span1, 15, 25) dataRange4 := NewDataRange(2, span3, 10, 20) - dataRange5 := NewDataRange(1, span2, 10, 20) - dataRange5.LastScannedTxnStartTs = 9 require.True(t, dataRange1.Equal(dataRange2)) require.False(t, dataRange1.Equal(dataRange3)) require.False(t, dataRange1.Equal(dataRange4)) - require.False(t, dataRange1.Equal(dataRange5)) } func TestTableSpanLess(t *testing.T) { diff --git a/pkg/eventservice/dispatcher_stat.go b/pkg/eventservice/dispatcher_stat.go index 6455544eaf..9ca209f88d 100644 --- a/pkg/eventservice/dispatcher_stat.go +++ b/pkg/eventservice/dispatcher_stat.go @@ -18,6 +18,7 @@ import ( "time" "github.com/pingcap/log" + "github.com/pingcap/ticdc/logservice/eventstore" "github.com/pingcap/ticdc/pkg/common" pevent "github.com/pingcap/ticdc/pkg/common/event" "github.com/pingcap/ticdc/pkg/config" @@ -104,11 +105,9 @@ type dispatcherStat struct { // Note: Please don't changed this value directly, use updateSentResolvedTs instead. sentResolvedTs atomic.Uint64 - // The last scanned DML event start-ts. - // These two values are used to construct the scan range for the next scan task. - lastScannedCommitTs atomic.Uint64 - lastScannedStartTs atomic.Uint64 - lastScannedPosition atomic.Value + // lastScanProgress is published as one immutable value so the next scan + // cannot observe a cursor assembled from different scan results. + lastScanProgress atomic.Pointer[scanProgress] largeTxnStateMu sync.Mutex largeTxnState *largeTxnScanState @@ -174,8 +173,7 @@ func newDispatcherStat( dispStat.sentResolvedTs.Store(startTs) - dispStat.lastScannedCommitTs.Store(startTs) - dispStat.lastScannedStartTs.Store(0) + dispStat.storeScanProgress(newTxnScanProgress(startTs, 0)) dispStat.lastReadySendTime.Store(0) dispStat.readyInterval.Store(1) dispStat.resetScanLimit() @@ -224,32 +222,24 @@ func (a *dispatcherStat) updateScanRange(txnCommitTs, txnStartTs uint64) { func (a *dispatcherStat) updateScanRangeWithPosition( txnCommitTs uint64, txnStartTs uint64, - position common.ScanPosition, + position eventstore.ScanPosition, ) { - a.lastScannedCommitTs.Store(txnCommitTs) - a.lastScannedStartTs.Store(txnStartTs) - a.storeLastScannedPosition(position) + a.storeScanProgress(newRowLevelScanProgress(txnCommitTs, txnStartTs, position)) } -func (a *dispatcherStat) storeLastScannedPosition(position common.ScanPosition) { - if len(position) == 0 { - a.lastScannedPosition.Store(common.ScanPosition{}) - return - } - // ScanPosition is produced as an immutable snapshot by the scanner path. - a.lastScannedPosition.Store(position) +func (a *dispatcherStat) storeScanProgress(progress scanProgress) { + progress.rowLevelScanPosition = cloneScanPosition(progress.rowLevelScanPosition) + a.lastScanProgress.Store(&progress) } -func (a *dispatcherStat) getLastScannedPosition() common.ScanPosition { - value := a.lastScannedPosition.Load() - if value == nil { - return nil - } - position := value.(common.ScanPosition) - if len(position) == 0 { - return nil +func (a *dispatcherStat) loadScanProgress() scanProgress { + progress := a.lastScanProgress.Load() + if progress == nil { + return scanProgress{} } - return position + result := *progress + result.rowLevelScanPosition = cloneScanPosition(result.rowLevelScanPosition) + return result } type bigTxnMetricState struct { @@ -340,30 +330,35 @@ func (a *dispatcherStat) onLatestCommitTs(latestCommitTs uint64) bool { return util.CompareAndMonotonicIncrease(&a.eventStoreCommitTs, latestCommitTs) } -// getDataRange returns the data range that the dispatcher needs to scan. -func (a *dispatcherStat) getDataRange() (common.DataRange, bool) { - lastTxnCommitTs := a.lastScannedCommitTs.Load() - lastTxnStartTs := a.lastScannedStartTs.Load() - lastPosition := a.getLastScannedPosition() +// getScanRequest returns the range and cursor that the dispatcher needs to scan. +func (a *dispatcherStat) getScanRequest() (eventstore.ScanRequest, bool) { + progress := a.loadScanProgress() + lastTxnCommitTs := progress.txnCommitTs + lastTxnStartTs := progress.txnStartTs + lastPosition := progress.rowLevelScanPosition hasPendingLargeTxn := a.hasPendingLargeTxnState() // the data not received by the event store yet, so just skip it. resolvedTs := a.receivedResolvedTs.Load() if lastTxnCommitTs > resolvedTs { - return common.DataRange{}, false + return eventstore.ScanRequest{}, false } if lastTxnCommitTs == resolvedTs && lastTxnStartTs == 0 && len(lastPosition) == 0 && !hasPendingLargeTxn { - return common.DataRange{}, false + return eventstore.ScanRequest{}, false } - // Range: (CommitTsStart-lastScannedStartTs, CommitTsEnd], - // since the CommitTsStart(and the data before startTs) is already sent to the dispatcher. - r := common.DataRange{ - Span: a.info.GetTableSpan(), - CommitTsStart: lastTxnCommitTs, - CommitTsEnd: resolvedTs, - RowLevelScanPosition: lastPosition, - LastScannedTxnStartTs: lastTxnStartTs, + // Range is (CommitTsStart, CommitTsEnd], with Cursor identifying any + // unfinished transaction or row at CommitTsStart. + r := eventstore.ScanRequest{ + Range: common.DataRange{ + Span: a.info.GetTableSpan(), + CommitTsStart: lastTxnCommitTs, + CommitTsEnd: resolvedTs, + }, + Cursor: eventstore.ScanCursor{ + TxnStartTs: lastTxnStartTs, + Position: lastPosition, + }, } return r, true } diff --git a/pkg/eventservice/dispatcher_stat_test.go b/pkg/eventservice/dispatcher_stat_test.go index ade40ca8e7..00755d4ee6 100644 --- a/pkg/eventservice/dispatcher_stat_test.go +++ b/pkg/eventservice/dispatcher_stat_test.go @@ -19,6 +19,7 @@ import ( "github.com/pingcap/ticdc/downstreamadapter/syncpoint" "github.com/pingcap/ticdc/eventpb" + "github.com/pingcap/ticdc/logservice/eventstore" "github.com/pingcap/ticdc/pkg/common" pevent "github.com/pingcap/ticdc/pkg/common/event" "github.com/pingcap/ticdc/pkg/metrics" @@ -139,7 +140,7 @@ func readBigTxnCountMetric(t *testing.T) float64 { return counter.GetValue() } -func TestDispatcherStatGetDataRange(t *testing.T) { +func TestDispatcherStatGetScanRequest(t *testing.T) { t.Parallel() info := newMockDispatcherInfo(t, 100, common.NewDispatcherID(), 1, eventpb.ActionType_ACTION_TYPE_REGISTER) @@ -149,50 +150,75 @@ func TestDispatcherStatGetDataRange(t *testing.T) { // case 1: get range after resolved ts update stat.onResolvedTs(200) - r, ok := stat.getDataRange() + r, ok := stat.getScanRequest() require.True(t, ok) - require.Equal(t, uint64(100), r.CommitTsStart) - require.Equal(t, uint64(0), r.LastScannedTxnStartTs) - require.Equal(t, uint64(200), r.CommitTsEnd) - require.Equal(t, info.GetTableSpan(), r.Span) + require.Equal(t, uint64(100), r.Range.CommitTsStart) + require.Equal(t, uint64(0), r.Cursor.TxnStartTs) + require.Equal(t, uint64(200), r.Range.CommitTsEnd) + require.Equal(t, info.GetTableSpan(), r.Range.Span) // case 2: get range after scan range update stat.updateScanRange(130, 120) - r, ok = stat.getDataRange() + r, ok = stat.getScanRequest() require.True(t, ok) - require.Equal(t, uint64(130), r.CommitTsStart) - require.Equal(t, uint64(120), r.LastScannedTxnStartTs) - require.Equal(t, uint64(200), r.CommitTsEnd) - require.Equal(t, info.GetTableSpan(), r.Span) + require.Equal(t, uint64(130), r.Range.CommitTsStart) + require.Equal(t, uint64(120), r.Cursor.TxnStartTs) + require.Equal(t, uint64(200), r.Range.CommitTsEnd) + require.Equal(t, info.GetTableSpan(), r.Range.Span) // case 3: get range after sent resolved ts update stat.updateSentResolvedTs(200) - r, ok = stat.getDataRange() + r, ok = stat.getScanRequest() require.False(t, ok) // case 4: same commit-ts may still have later transactions when the // transaction-level resume start-ts is set. stat.updateScanRange(200, 150) - r, ok = stat.getDataRange() + r, ok = stat.getScanRequest() require.True(t, ok) - require.Equal(t, uint64(200), r.CommitTsStart) - require.Equal(t, uint64(150), r.LastScannedTxnStartTs) - require.Equal(t, uint64(200), r.CommitTsEnd) + require.Equal(t, uint64(200), r.Range.CommitTsStart) + require.Equal(t, uint64(150), r.Cursor.TxnStartTs) + require.Equal(t, uint64(200), r.Range.CommitTsEnd) // case 5: get range after resolved ts update again stat.onResolvedTs(300) - r, ok = stat.getDataRange() + r, ok = stat.getScanRequest() require.True(t, ok) - require.Equal(t, uint64(200), r.CommitTsStart) - require.Equal(t, uint64(150), r.LastScannedTxnStartTs) - require.Equal(t, uint64(300), r.CommitTsEnd) - require.Equal(t, info.GetTableSpan(), r.Span) + require.Equal(t, uint64(200), r.Range.CommitTsStart) + require.Equal(t, uint64(150), r.Cursor.TxnStartTs) + require.Equal(t, uint64(300), r.Range.CommitTsEnd) + require.Equal(t, info.GetTableSpan(), r.Range.Span) stat.updateSentResolvedTs(300) - r, ok = stat.getDataRange() + r, ok = stat.getScanRequest() require.False(t, ok) } +func TestDispatcherStatScanProgressSnapshotIsImmutable(t *testing.T) { + t.Parallel() + + info := newMockDispatcherInfo(t, 100, common.NewDispatcherID(), 1, eventpb.ActionType_ACTION_TYPE_REGISTER) + status := newChangefeedStatusForTest(t, info) + stat := newDispatcherStat(info, 1, 1, nil, status) + stat.onResolvedTs(200) + + position := eventstore.ScanPosition("position") + stat.updateScanRangeWithPosition(150, 120, position) + position[0] = 'X' + + progress := stat.loadScanProgress() + require.Equal(t, eventstore.ScanPosition("position"), progress.rowLevelScanPosition) + progress.rowLevelScanPosition[0] = 'Y' + + progress = stat.loadScanProgress() + require.Equal(t, eventstore.ScanPosition("position"), progress.rowLevelScanPosition) + request, ok := stat.getScanRequest() + require.True(t, ok) + require.Equal(t, uint64(150), request.Range.CommitTsStart) + require.Equal(t, uint64(120), request.Cursor.TxnStartTs) + require.Equal(t, eventstore.ScanPosition("position"), request.Cursor.Position) +} + func TestDispatcherStatUpdateWatermark(t *testing.T) { startTs := uint64(100) info := newMockDispatcherInfo(t, startTs, common.NewDispatcherID(), 1, eventpb.ActionType_ACTION_TYPE_REGISTER) diff --git a/pkg/eventservice/event_broker.go b/pkg/eventservice/event_broker.go index 04e1bb0173..39e94cba32 100644 --- a/pkg/eventservice/event_broker.go +++ b/pkg/eventservice/event_broker.go @@ -279,8 +279,9 @@ func (c *eventBroker) refreshMinSentResolvedTs(ctx context.Context) error { func (c *eventBroker) sendSignalResolvedTs(d *dispatcherStat) { // Can't send resolvedTs if there was a interrupted scan task happened before. - // d.lastScannedStartTs.Load() != 0 indicates that there was a interrupted scan task happened before. - if time.Since(d.lastSentResolvedTsTime.Load()) < defaultSendResolvedTsInterval || d.lastScannedStartTs.Load() != 0 { + // A non-zero scan-progress start-ts indicates that there was an interrupted scan task before. + if time.Since(d.lastSentResolvedTsTime.Load()) < defaultSendResolvedTsInterval || + d.loadScanProgress().txnStartTs != 0 { return } watermark := d.sentResolvedTs.Load() @@ -403,17 +404,18 @@ func (c *eventBroker) logUninitializedDispatchers(ctx context.Context) error { } } -// getScanTaskDataRange determines the valid data range for scanning a given task. +// getScanTaskRequest determines the valid range and resume cursor for a scan task. // It checks various conditions (dispatcher status, DDL state, max commit ts of dml event) // to decide whether scanning is needed and returns the appropriate time range. -// If no valid range is found, it returns an empty DataRange. -func (c *eventBroker) getScanTaskDataRange(task scanTask) (bool, common.DataRange) { - // 1. Get the data range of the dispatcher. - dataRange, needScan := task.getDataRange() +// If no valid range is found, it returns an empty ScanRequest. +func (c *eventBroker) getScanTaskRequest(task scanTask) (bool, eventstore.ScanRequest) { + // 1. Get the range and resume cursor of the dispatcher. + request, needScan := task.getScanRequest() if !needScan { updateMetricEventServiceSkipResolvedTsCount(task.info.GetMode()) - return false, common.DataRange{} + return false, eventstore.ScanRequest{} } + dataRange := &request.Range keyspaceMeta := common.KeyspaceMeta{ ID: task.info.GetTableSpan().KeyspaceID, @@ -424,7 +426,7 @@ func (c *eventBroker) getScanTaskDataRange(task scanTask) (bool, common.DataRang ddlState, err := c.schemaStore.GetTableDDLEventState(keyspaceMeta, task.info.GetTableSpan().TableID) if err != nil { log.Error("GetTableDDLEventState failed", zap.Uint32("keyspaceID", task.info.GetTableSpan().KeyspaceID), zap.Int64("tableID", task.info.GetTableSpan().TableID), zap.Error(err)) - return false, common.DataRange{} + return false, eventstore.ScanRequest{} } dataRange.CommitTsEnd = min(dataRange.CommitTsEnd, ddlState.ResolvedTs) commitTsEndBeforeWindow := dataRange.CommitTsEnd @@ -481,23 +483,24 @@ func (c *eventBroker) getScanTaskDataRange(task scanTask) (bool, common.DataRang } if dataRange.CommitTsEnd <= dataRange.CommitTsStart { - hasSameCommitTxnResume := dataRange.LastScannedTxnStartTs != 0 && + hasSameCommitTxnResume := request.Cursor.TxnStartTs != 0 && dataRange.CommitTsEnd == dataRange.CommitTsStart - if len(dataRange.RowLevelScanPosition) != 0 || + if len(request.Cursor.Position) != 0 || hasSameCommitTxnResume || task.hasPendingLargeTxnState() { - return true, dataRange + return true, request } updateMetricEventServiceSkipResolvedTsCount(task.info.GetMode()) // Scan range can become empty after applying capping (for example, scan window). // Send a signal resolved-ts event (rate limited) to keep downstream responsive, // but do not advance the watermark here. c.sendSignalResolvedTs(task) - return false, common.DataRange{} + return false, eventstore.ScanRequest{} } // 3. Check whether there is any events in the data range - // Note: target range is (dataRange.CommitTsStart-dataRange.LastScannedTxnStartTs, dataRange.CommitTsEnd] + // Note: target range resumes after request.Cursor inside + // (dataRange.CommitTsStart, dataRange.CommitTsEnd]. // when `dataRange.CommitTsStart` equals `task.eventStoreCommitTs.Load()`, // it is difficult to determine whether any txn events with a commitTs of `dataRange.CommitTsStart` remain unscanned. // because multiple transactions may have the same commit ts. @@ -508,9 +511,9 @@ func (c *eventBroker) getScanTaskDataRange(task scanTask) (bool, common.DataRang // The dispatcher has no new events. In such case, we don't need to scan the event store. // We just send the watermark to the dispatcher. c.sendResolvedTs(task, dataRange.CommitTsEnd) - return false, common.DataRange{} + return false, eventstore.ScanRequest{} } - return true, dataRange + return true, request } // scanReady checks if the dispatcher needs to scan the event store/schema store. @@ -540,7 +543,7 @@ func (c *eventBroker) scanReady(task scanTask) bool { c.sendHandshakeIfNeed(task) - ok, _ := c.getScanTaskDataRange(task) + ok, _ := c.getScanTaskRequest(task) return ok } @@ -659,7 +662,7 @@ func (c *eventBroker) doScan(ctx context.Context, task scanTask) { return } - needScan, dataRange := c.getScanTaskDataRange(task) + needScan, request := c.getScanTaskRequest(task) if !needScan { return } @@ -720,7 +723,7 @@ func (c *eventBroker) doScan(ctx context.Context, task scanTask) { } scanner := newEventScanner(c.eventStore, c.schemaStore, c.mounter, task.info.GetMode()) - scannedBytes, events, progress, interrupted, err := scanner.scan(ctx, task, dataRange, sl) + scannedBytes, events, progress, interrupted, err := scanner.scan(ctx, task, request, sl) if scannedBytes < 0 { releaseQuota(available, uint64(sl.maxDMLBytes)) } else if scannedBytes >= 0 && scannedBytes < sl.maxDMLBytes { @@ -735,7 +738,7 @@ func (c *eventBroker) doScan(ctx context.Context, task scanTask) { log.Error("scan events failed", zap.Stringer("changefeedID", task.changefeedStat.changefeedID), zap.Stringer("dispatcherID", task.id), zap.Int64("tableID", task.info.GetTableSpan().GetTableID()), - zap.Any("dataRange", dataRange), zap.Uint64("receivedResolvedTs", task.receivedResolvedTs.Load()), + zap.Any("scanRequest", request), zap.Uint64("receivedResolvedTs", task.receivedResolvedTs.Load()), zap.Uint64("sentResolvedTs", task.sentResolvedTs.Load()), zap.Error(err)) return } diff --git a/pkg/eventservice/event_broker_test.go b/pkg/eventservice/event_broker_test.go index e931a2604c..40fdea72fa 100644 --- a/pkg/eventservice/event_broker_test.go +++ b/pkg/eventservice/event_broker_test.go @@ -138,8 +138,8 @@ func TestOnNotify(t *testing.T) { err = broker.resetDispatcher(disInfo) require.Nil(t, err) - require.Equal(t, disp.lastScannedCommitTs.Load(), uint64(100)) - require.Equal(t, disp.lastScannedStartTs.Load(), uint64(0)) + require.Equal(t, disp.loadScanProgress().txnCommitTs, uint64(100)) + require.Equal(t, disp.loadScanProgress().txnStartTs, uint64(0)) disp.setHandshaked() @@ -228,9 +228,9 @@ func TestScanRangeCappedByScanWindow(t *testing.T) { disp.eventStoreCommitTs.Store(oracle.GoTimeToTS(baseTime.Add(15 * time.Second))) changefeedStatus.refreshMinSentResolvedTs() - needScan, dataRange := broker.getScanTaskDataRange(disp) + needScan, dataRange := broker.getScanTaskRequest(disp) require.True(t, needScan) - require.Equal(t, oracle.GoTimeToTS(baseTime.Add(defaultScanInterval)), dataRange.CommitTsEnd) + require.Equal(t, oracle.GoTimeToTS(baseTime.Add(defaultScanInterval)), dataRange.Range.CommitTsEnd) } func TestGetScanTaskDataRangeEmptyAfterCappingDoesNotResetScanRange(t *testing.T) { @@ -253,16 +253,15 @@ func TestGetScanTaskDataRangeEmptyAfterCappingDoesNotResetScanRange(t *testing.T disp.sentResolvedTs.Store(baseTs) disp.receivedResolvedTs.Store(oracle.GoTimeToTS(baseTime.Add(40 * time.Second))) disp.eventStoreCommitTs.Store(commitStart) - disp.lastScannedCommitTs.Store(commitStart) - disp.lastScannedStartTs.Store(lastStartTs) + disp.updateScanRange(commitStart, lastStartTs) changefeedStatus.minSentTs.Store(baseTs) changefeedStatus.scanInterval.Store(int64(defaultScanInterval)) - needScan, _ := broker.getScanTaskDataRange(disp) + needScan, _ := broker.getScanTaskRequest(disp) require.False(t, needScan) - require.Equal(t, commitStart, disp.lastScannedCommitTs.Load()) - require.Equal(t, lastStartTs, disp.lastScannedStartTs.Load()) + require.Equal(t, commitStart, disp.loadScanProgress().txnCommitTs) + require.Equal(t, lastStartTs, disp.loadScanProgress().txnStartTs) } func TestGetScanTaskDataRangeEmptyAfterCappingWithPendingDDLEventUsesLocalWindow(t *testing.T) { @@ -286,8 +285,7 @@ func TestGetScanTaskDataRangeEmptyAfterCappingWithPendingDDLEventUsesLocalWindow disp.sentResolvedTs.Store(baseTs) disp.receivedResolvedTs.Store(resolvedTs) disp.eventStoreCommitTs.Store(commitStart) - disp.lastScannedCommitTs.Store(commitStart) - disp.lastScannedStartTs.Store(commitStart - 1) + disp.updateScanRange(commitStart, commitStart-1) changefeedStatus.minSentTs.Store(baseTs) changefeedStatus.scanInterval.Store(int64(defaultScanInterval)) @@ -295,10 +293,10 @@ func TestGetScanTaskDataRangeEmptyAfterCappingWithPendingDDLEventUsesLocalWindow ss.resolvedTs = resolvedTs ss.maxDDLCommitTs = ddlCommitTs - needScan, dataRange := broker.getScanTaskDataRange(disp) + needScan, dataRange := broker.getScanTaskRequest(disp) require.True(t, needScan) - require.Equal(t, commitStart, dataRange.CommitTsStart) - require.Equal(t, oracle.GoTimeToTS(oracle.GetTimeFromTS(commitStart).Add(defaultScanInterval)), dataRange.CommitTsEnd) + require.Equal(t, commitStart, dataRange.Range.CommitTsStart) + require.Equal(t, oracle.GoTimeToTS(oracle.GetTimeFromTS(commitStart).Add(defaultScanInterval)), dataRange.Range.CommitTsEnd) } func TestGetScanTaskDataRangeEmptyAfterCappingWithPendingSyncPointCrossesSyncPoint(t *testing.T) { @@ -325,8 +323,7 @@ func TestGetScanTaskDataRangeEmptyAfterCappingWithPendingSyncPointCrossesSyncPoi disp.sentResolvedTs.Store(baseTs) disp.receivedResolvedTs.Store(resolvedTs) disp.eventStoreCommitTs.Store(commitStart) - disp.lastScannedCommitTs.Store(commitStart) - disp.lastScannedStartTs.Store(commitStart - 1) + disp.updateScanRange(commitStart, commitStart-1) changefeedStatus.minSentTs.Store(baseTs) changefeedStatus.scanInterval.Store(int64(time.Second)) @@ -334,10 +331,10 @@ func TestGetScanTaskDataRangeEmptyAfterCappingWithPendingSyncPointCrossesSyncPoi ss.resolvedTs = resolvedTs ss.maxDDLCommitTs = 0 - needScan, dataRange := broker.getScanTaskDataRange(disp) + needScan, dataRange := broker.getScanTaskRequest(disp) require.True(t, needScan) - require.Equal(t, commitStart, dataRange.CommitTsStart) - require.Equal(t, nextSyncPointTs+1, dataRange.CommitTsEnd) + require.Equal(t, commitStart, dataRange.Range.CommitTsStart) + require.Equal(t, nextSyncPointTs+1, dataRange.Range.CommitTsEnd) } func TestGetScanTaskDataRangeRingWaitWithThreeDispatchersCanAdvancePendingDDL(t *testing.T) { @@ -383,26 +380,24 @@ func TestGetScanTaskDataRangeRingWaitWithThreeDispatchersCanAdvancePendingDDL(t d1.receivedResolvedTs.Store(ts110) d1.eventStoreCommitTs.Store(ts103) - d1.lastScannedCommitTs.Store(ts101) - d1.lastScannedStartTs.Store(ts101 - 1) + d1.updateScanRange(ts101, ts101-1) ss.resolvedTs = ts110 ss.maxDDLCommitTs = ts103 // Round 1: global cap makes range empty (end=ts101), fallback should locally move it to ts102. - needScan, dataRange := broker.getScanTaskDataRange(d1) + needScan, dataRange := broker.getScanTaskRequest(d1) require.True(t, needScan) - require.Equal(t, ts101, dataRange.CommitTsStart) - require.Equal(t, ts102, dataRange.CommitTsEnd) + require.Equal(t, ts101, dataRange.Range.CommitTsStart) + require.Equal(t, ts102, dataRange.Range.CommitTsEnd) // Round 2: still globally capped by ts100, but fallback should continue moving to ts103, // which allows this dispatcher to eventually reach the pending truncate ddl barrier. - d1.lastScannedCommitTs.Store(ts102) - d1.lastScannedStartTs.Store(0) - needScan, dataRange = broker.getScanTaskDataRange(d1) + d1.updateScanRange(ts102, 0) + needScan, dataRange = broker.getScanTaskRequest(d1) require.True(t, needScan) - require.Equal(t, ts102, dataRange.CommitTsStart) - require.Equal(t, ts103, dataRange.CommitTsEnd) + require.Equal(t, ts102, dataRange.Range.CommitTsStart) + require.Equal(t, ts103, dataRange.Range.CommitTsEnd) } func TestHandleCongestionControlV2DoesNotResetScanIntervalOnMemoryRelease(t *testing.T) { @@ -499,9 +494,9 @@ func TestDoScanKeepsRowLevelProgressAfterSendingFragment(t *testing.T) { broker.doScan(context.Background(), disp) - require.Equal(t, resolvedTs, disp.lastScannedCommitTs.Load()) - require.Equal(t, kvEvents[0].StartTs, disp.lastScannedStartTs.Load()) - require.NotEmpty(t, disp.getLastScannedPosition()) + require.Equal(t, resolvedTs, disp.loadScanProgress().txnCommitTs) + require.Equal(t, kvEvents[0].StartTs, disp.loadScanProgress().txnStartTs) + require.NotEmpty(t, disp.loadScanProgress().rowLevelScanPosition) require.True(t, disp.isTaskScanning.Load()) } @@ -734,7 +729,7 @@ func TestResetTableTriggerDispatcherDoesNotUseNormalScan(t *testing.T) { require.NotSame(t, oldStat, newStat) require.Equal(t, uint64(0), newStat.seq.Load()) require.Equal(t, uint64(100), newStat.sentResolvedTs.Load()) - require.Equal(t, uint64(100), newStat.lastScannedCommitTs.Load()) + require.Equal(t, uint64(100), newStat.loadScanProgress().txnCommitTs) require.False(t, newStat.isTaskScanning.Load()) require.Empty(t, broker.messageCh[newStat.messageWorkerIndex]) } @@ -1128,8 +1123,8 @@ func TestSendHandshakeUsesStartTs(t *testing.T) { } require.Equal(t, uint64(100), disp.sentResolvedTs.Load()) - require.Equal(t, uint64(100), disp.lastScannedCommitTs.Load()) - require.Equal(t, uint64(0), disp.lastScannedStartTs.Load()) + require.Equal(t, uint64(100), disp.loadScanProgress().txnCommitTs) + require.Equal(t, uint64(0), disp.loadScanProgress().txnStartTs) } func TestAddDispatcherFailure(t *testing.T) { diff --git a/pkg/eventservice/event_scanner.go b/pkg/eventservice/event_scanner.go index 037b0d1a1a..6656e67975 100644 --- a/pkg/eventservice/event_scanner.go +++ b/pkg/eventservice/event_scanner.go @@ -15,7 +15,6 @@ package eventservice import ( "context" - "io" "time" "github.com/pingcap/log" @@ -35,7 +34,7 @@ import ( // eventGetter is the interface for getting iterator of events // The implementation of eventGetter is eventstore.EventStore type eventGetter interface { - GetIterator(dispatcherID common.DispatcherID, dataRange common.DataRange) (eventstore.EventIterator, error) + GetIterator(dispatcherID common.DispatcherID, request eventstore.ScanRequest) (eventstore.EventIterator, error) } // schemaGetter is the interface for getting schema info and ddl events @@ -64,30 +63,6 @@ type eventScanner struct { mode int64 } -type scanProgress struct { - valid bool - txnCommitTs uint64 - txnStartTs uint64 - rowLevelScanPosition common.ScanPosition -} - -func newTxnScanProgress(commitTs uint64, startTs uint64) scanProgress { - return scanProgress{ - valid: true, - txnCommitTs: commitTs, - txnStartTs: startTs, - } -} - -func newRowLevelScanProgress(commitTs uint64, startTs uint64, position common.ScanPosition) scanProgress { - progress := newTxnScanProgress(commitTs, startTs) - if len(position) > 0 { - progress.rowLevelScanPosition = make(common.ScanPosition, len(position)) - copy(progress.rowLevelScanPosition, position) - } - return progress -} - // newEventScanner creates a new EventScanner func newEventScanner( eventStore eventGetter, @@ -144,16 +119,21 @@ func newEventScanner( func (s *eventScanner) scan( ctx context.Context, dispatcherStat *dispatcherStat, - dataRange common.DataRange, + request eventstore.ScanRequest, limit scanLimit, ) (int64, []event.Event, scanProgress, bool, error) { + dataRange := request.Range // Initialize scan session sess := newSession(ctx, dispatcherStat, dataRange, limit) defer sess.recordMetrics() + strategy := newTxnScanStrategy(dispatcherStat.txnAtomicity.ShouldSplitTxn()) + scanCtx := &txnScanContext{ + scanner: s, + session: sess, + } - if state := dispatcherStat.getLargeTxnState(); state != nil && - state.getPhase() == largeTxnScanPhaseDrainInserts { - interrupted, err := s.drainLargeTxnInserts(sess, state) + handled, interrupted, err := strategy.resumePending(scanCtx) + if handled { return sess.eventBytes, sess.events, sess.progress, interrupted, err } @@ -165,28 +145,31 @@ func (s *eventScanner) scan( } metrics.EventServiceGetDDLEventDuration.Observe(time.Since(start).Seconds()) - iter, err := s.eventGetter.GetIterator(dispatcherStat.info.GetID(), dataRange) + scanCtx.merger = newEventMerger(events) + scanCtx.processor = newDMLProcessor( + s.mounter, + s.schemaGetter, + dispatcherStat.filter, + dispatcherStat.info.IsOutputRawChangeEvent(), + s.mode, + dispatcherStat.info.EnableIgnoreUpdateOnlyColumns()) + scanCtx.processor.dispatcherStat = dispatcherStat + + iter, err := s.eventGetter.GetIterator(dispatcherStat.info.GetID(), request) if err != nil { return 0, nil, scanProgress{}, false, err } if iter == nil { - dispatcherStat.finishPendingBigTxnMetric() - if state := dispatcherStat.getLargeTxnState(); state != nil && - state.getPhase() == largeTxnScanPhaseOriginal { - dispatcherStat.markLargeTxnDrainInserts(state.startTs, state.commitTs, false, 0) - sess.progress = newTxnScanProgress(state.commitTs, state.startTs) - return 0, sess.events, sess.progress, true, nil + interrupted, err := strategy.finishTxn(scanCtx, nextTxnMeta{}) + if err != nil || interrupted { + return 0, sess.events, sess.progress, interrupted, err } - resolved := event.NewResolvedEvent(dataRange.CommitTsEnd, dispatcherStat.id, dispatcherStat.epoch) - events = append(events, resolved) - sess.appendEvents(events) - sess.progress = newTxnScanProgress(dataRange.CommitTsEnd, 0) - return 0, sess.events, sess.progress, false, nil + err = finalizeScan(scanCtx.merger, scanCtx.processor, sess, dataRange.CommitTsEnd) + return 0, sess.events, sess.progress, false, err } // Execute event scanning and merging - merger := newEventMerger(events) - interrupted, scanErr := s.scanAndMergeEvents(sess, merger, iter) + interrupted, scanErr := s.scanAndMergeEvents(scanCtx, strategy, iter) closeErr := s.closeIterator(iter) if scanErr != nil { if closeErr != nil { @@ -246,20 +229,15 @@ func (s *eventScanner) closeIterator(iter eventstore.EventIterator) error { // scanAndMergeEvents performs the main scanning and merging logic func (s *eventScanner) scanAndMergeEvents( - session *session, - merger *eventMerger, + scanCtx *txnScanContext, + strategy txnScanStrategy, iter eventstore.EventIterator, ) (bool, error) { + session := scanCtx.session + merger := scanCtx.merger + processor := scanCtx.processor tableID := session.dataRange.Span.TableID dispatcher := session.dispatcherStat - processor := newDMLProcessor( - s.mounter, - s.schemaGetter, - dispatcher.filter, - dispatcher.info.IsOutputRawChangeEvent(), - s.mode, - dispatcher.info.EnableIgnoreUpdateOnlyColumns()) - processor.dispatcherStat = dispatcher for { shouldStop, err := s.checkScanConditions(session) @@ -272,22 +250,7 @@ func (s *eventScanner) scanAndMergeEvents( rawEvent, position, isNewTxn := nextEventWithScanPosition(iter) if rawEvent == nil { - if state := dispatcher.getLargeTxnState(); processor.currentTxn == nil && - state != nil && - state.getPhase() == largeTxnScanPhaseOriginal { - dispatcher.finishPendingBigTxnMetric() - dispatcher.markLargeTxnDrainInserts(state.startTs, state.commitTs, false, 0) - session.progress = newTxnScanProgress(state.commitTs, state.startTs) - return true, nil - } - interrupted, err := s.finishCurrentTxn( - session, - merger, - processor, - 0, - 0, - false, - ) + interrupted, err := strategy.finishTxn(scanCtx, nextTxnMeta{}) if err != nil || interrupted { return interrupted, err } @@ -296,14 +259,14 @@ func (s *eventScanner) scanAndMergeEvents( } dispatcher.finishPendingBigTxnMetricBefore(rawEvent.StartTs, rawEvent.CRTs) - if state := dispatcher.getLargeTxnState(); processor.currentTxn == nil && - state != nil && - state.getPhase() == largeTxnScanPhaseOriginal && - (rawEvent.StartTs != state.startTs || rawEvent.CRTs != state.commitTs) { - dispatcher.finishPendingBigTxnMetric() - dispatcher.markLargeTxnDrainInserts(state.startTs, state.commitTs, true, rawEvent.CRTs) - session.progress = newTxnScanProgress(state.commitTs, state.startTs) - return true, nil + if processor.currentTxn == nil { + interrupted, err := strategy.finishTxn(scanCtx, nextTxnMeta{ + startTs: rawEvent.StartTs, + commitTs: rawEvent.CRTs, + }) + if err != nil || interrupted { + return interrupted, err + } } if isNewTxn { @@ -311,14 +274,12 @@ func (s *eventScanner) scanAndMergeEvents( if err != nil { return false, err } - interrupted, err := s.finishCurrentTxn( - session, - merger, - processor, - rawEvent.CRTs, - getTableInfoUpdateTs(tableInfo), - tableInfo == nil, - ) + interrupted, err := strategy.finishTxn(scanCtx, nextTxnMeta{ + startTs: rawEvent.StartTs, + commitTs: rawEvent.CRTs, + tableInfoUpdateTs: getTableInfoUpdateTs(tableInfo), + tableDeleted: tableInfo == nil, + }) if err != nil || interrupted { return interrupted, err } @@ -336,7 +297,7 @@ func (s *eventScanner) scanAndMergeEvents( return true, nil } - err = s.startTxn(session, processor, rawEvent.StartTs, rawEvent.CRTs, tableInfo, tableID) + err = strategy.startTxn(scanCtx, rawEvent.StartTs, rawEvent.CRTs, tableInfo, tableID) if err != nil { return false, err } @@ -352,16 +313,16 @@ func (s *eventScanner) scanAndMergeEvents( zap.Int64("mode", s.mode)) return false, err } - if s.canInterruptCurrentTxn(session, merger, processor, rawEvent, position) { - interruptCurrentTxn(session, merger, processor, rawEvent.CRTs, rawEvent.StartTs, position) - return true, nil + interrupted, err := strategy.afterAppend(scanCtx, rawEvent, position) + if err != nil || interrupted { + return interrupted, err } } } func nextEventWithScanPosition( iter eventstore.EventIterator, -) (*common.RawKVEntry, common.ScanPosition, bool) { +) (*common.RawKVEntry, eventstore.ScanPosition, bool) { if positionIter, ok := iter.(eventstore.EventIteratorWithScanPosition); ok { return positionIter.NextWithScanPosition() } @@ -416,22 +377,6 @@ func (s *eventScanner) getTableInfo4Txn(dispatcher *dispatcherStat, tableID int6 return nil, err } -func (s *eventScanner) startTxn( - session *session, - processor *dmlProcessor, - startTs, commitTs uint64, - tableInfo *common.TableInfo, - tableID int64, -) error { - shouldSplitTxn := session.dispatcherStat.txnAtomicity.ShouldSplitTxn() - err := processor.startTxn(session.dispatcherStat.id, tableID, tableInfo, startTs, commitTs, shouldSplitTxn) - if err != nil { - return err - } - session.dmlCount++ - return nil -} - func (s *eventScanner) commitTxn( session *session, merger *eventMerger, @@ -478,77 +423,6 @@ func (s *eventScanner) commitTxn( return nil } -func (s *eventScanner) finishCurrentTxn( - session *session, - merger *eventMerger, - processor *dmlProcessor, - nextCommitTs uint64, - nextTableInfoUpdateTs uint64, - nextTableDeleted bool, -) (bool, error) { - if processor.currentTxn == nil { - return false, nil - } - currentStartTs := processor.currentTxn.CurrentDMLEvent.GetStartTs() - currentCommitTs := processor.currentTxn.CurrentDMLEvent.GetCommitTs() - - if processor.hasSpilledInsertsForCurrentTxn() && merger.hasDDLAtCommitTs(currentCommitTs) { - if err := processor.flushCachedInsertRows(); err != nil { - return false, err - } - if err := processor.flushSpilledInsertsIntoCurrentTxn(); err != nil { - return false, err - } - } - - if err := s.commitTxn(session, merger, processor, nextCommitTs, nextTableInfoUpdateTs); err != nil { - return false, err - } - if !processor.hasLargeTxnState(currentStartTs, currentCommitTs) { - return false, nil - } - - events := merger.mergeWithPrecedingDDLs(processor.getCurrentBatchDML()) - session.appendEvents(events) - processor.resetBatchDML() - - hasFollowingTxn := nextCommitTs != 0 && !nextTableDeleted - session.dispatcherStat.markLargeTxnDrainInserts( - currentStartTs, - currentCommitTs, - hasFollowingTxn, - nextCommitTs, - ) - if len(session.lastRowPosition) > 0 { - session.progress = newRowLevelScanProgress(currentCommitTs, currentStartTs, session.lastRowPosition) - } else { - session.progress = newTxnScanProgress(currentCommitTs, currentStartTs) - } - return true, nil -} - -func (s *eventScanner) canInterruptCurrentTxn( - session *session, - merger *eventMerger, - processor *dmlProcessor, - rawEvent *common.RawKVEntry, - position common.ScanPosition, -) bool { - if len(position) == 0 || !session.dispatcherStat.txnAtomicity.ShouldSplitTxn() { - return false - } - if len(processor.insertRowCache) > 0 { - return false - } - if processor.currentTxn == nil || !processor.currentTxn.exceedsLargeTxnThreshold() { - return false - } - if !merger.canInterrupt(rawEvent.CRTs, processor.batchDML) { - return false - } - return true -} - // finalizeScan finalizes the scan when all events have been processed // it's called when the iterator is nil, always indicates that all entries // with the same commit-ts is processed, so it's ok to append resolved-ts event @@ -636,130 +510,6 @@ func interruptScan( session.appendEvents(events) } -func interruptCurrentTxn( - session *session, - merger *eventMerger, - processor *dmlProcessor, - commitTs uint64, - startTs uint64, - position common.ScanPosition, -) { - if processor.currentTxn != nil { - currentTxn := processor.currentTxn - currentDML := currentTxn.CurrentDMLEvent - session.dispatcherStat.addBigTxnMetricFragment( - currentDML.GetStartTs(), - currentDML.GetCommitTs(), - currentTxn.rawKVBytes, - currentTxn.largeTxnThresholdInBytes) - } - events := merger.mergeWithPrecedingDDLs(processor.getCurrentBatchDML()) - session.appendEvents(events) - session.progress = newRowLevelScanProgress(commitTs, startTs, position) - log.Debug("scan interrupted inside a large txn", - zap.Stringer("dispatcherID", session.dispatcherStat.id), - zap.Uint64("startTs", startTs), - zap.Uint64("commitTs", commitTs), - zap.Int("scannedEntryCount", session.scannedEntryCount), - zap.Duration("duration", time.Since(session.startTime))) -} - -func (s *eventScanner) drainLargeTxnInserts( - session *session, - state *largeTxnScanState, -) (bool, error) { - drainedInsertCount := 0 - returnWithError := func(scanErr error) (bool, error) { - if rollbackErr := state.rollbackDrain(); rollbackErr != nil { - log.Warn("reset large transaction spill reader failed", - zap.Stringer("dispatcherID", session.dispatcherStat.id), - zap.Error(rollbackErr)) - } - return false, scanErr - } - - processor := newDMLProcessor( - s.mounter, - s.schemaGetter, - session.dispatcherStat.filter, - session.dispatcherStat.info.IsOutputRawChangeEvent(), - s.mode, - session.dispatcherStat.info.EnableIgnoreUpdateOnlyColumns()) - processor.dispatcherStat = session.dispatcherStat - - for { - shouldStop, err := s.checkScanConditions(session) - if err != nil { - return returnWithError(err) - } - if shouldStop { - return false, nil - } - - entry, err := state.nextInsert() - if err != nil { - if session.dispatcherStat.isRemoved.Load() { - return false, nil - } - if errors.Is(err, io.EOF) { - if processor.currentTxn != nil { - if err := processor.commitTxn(); err != nil { - return returnWithError(err) - } - if processor.getCurrentBatchDML().DMLCount() != 0 { - session.appendEvents([]event.Event{processor.getCurrentBatchDML()}) - } - } - state.commitDrainedInserts(drainedInsertCount) - session.progress = newTxnScanProgress(state.commitTs, state.startTs) - hasFollowingTxn, followingCommitTs := state.snapshotDrainInfo() - shouldResolveCommitTs := (!hasFollowingTxn || followingCommitTs > state.commitTs) && - session.dataRange.CommitTsEnd == state.commitTs - if err := session.dispatcherStat.cleanupLargeTxnState(); err != nil { - log.Warn("cleanup drained large transaction spill failed", - zap.Stringer("dispatcherID", session.dispatcherStat.id), - zap.Error(err)) - } - if shouldResolveCommitTs { - resolved := event.NewResolvedEvent(state.commitTs, session.dispatcherStat.id, session.dispatcherStat.epoch) - session.appendEvents([]event.Event{resolved}) - session.progress = newTxnScanProgress(state.commitTs, 0) - return false, nil - } - return true, nil - } - return returnWithError(err) - } - drainedInsertCount++ - - if processor.currentTxn == nil { - if err := processor.startTxn( - session.dispatcherStat.id, - state.tableID, - state.tableInfo, - state.startTs, - state.commitTs, - true, - ); err != nil { - return returnWithError(err) - } - } - session.observeRawEntry(entry, nil) - if err := processor.appendInsertRow(entry); err != nil { - return returnWithError(err) - } - if session.exceedLimit(processor.batchDML.GetSize(), processor.batchDML) { - if err := processor.commitTxn(); err != nil { - return returnWithError(err) - } - session.appendEvents([]event.Event{processor.getCurrentBatchDML()}) - state.commitDrainedInserts(drainedInsertCount) - session.progress = newTxnScanProgress(state.commitTs, state.startTs) - return true, nil - } - } -} - // session manages the state and context of a scan operation type session struct { ctx context.Context @@ -772,7 +522,7 @@ type session struct { scannedBytes int64 scannedEntryCount int - lastRowPosition common.ScanPosition + lastRowPosition eventstore.ScanPosition // dmlCount is the count of transactions. dmlCount int @@ -800,14 +550,14 @@ func newSession( } // observeRawEntry adds to the total bytes scanned -func (s *session) observeRawEntry(entry *common.RawKVEntry, position common.ScanPosition) { +func (s *session) observeRawEntry(entry *common.RawKVEntry, position eventstore.ScanPosition) { s.scannedBytes += entry.GetSize() s.scannedEntryCount++ if len(position) == 0 { s.lastRowPosition = nil return } - s.lastRowPosition = make(common.ScanPosition, len(position)) + s.lastRowPosition = make(eventstore.ScanPosition, len(position)) copy(s.lastRowPosition, position) } @@ -1138,85 +888,6 @@ func (p *dmlProcessor) flushCachedInsertRows() error { return nil } -func (p *dmlProcessor) spillCachedInsertRows() error { - if len(p.insertRowCache) == 0 { - return nil - } - state, err := p.getOrCreateLargeTxnState() - if err != nil { - return err - } - for _, insertRow := range p.insertRowCache { - if err := state.appendInsert(insertRow); err != nil { - return err - } - } - p.insertRowCache = make([]*common.RawKVEntry, 0) - return nil -} - -func (p *dmlProcessor) shouldSpillSplitUpdateInsert() bool { - if p.currentTxn == nil { - return false - } - return p.currentTxn.exceedsLargeTxnThreshold() || p.hasSpilledInsertsForCurrentTxn() -} - -func (p *dmlProcessor) hasSpilledInsertsForCurrentTxn() bool { - if p.currentTxn == nil || p.dispatcherStat == nil { - return false - } - current := p.currentTxn.CurrentDMLEvent - return p.hasLargeTxnState(current.GetStartTs(), current.GetCommitTs()) -} - -func (p *dmlProcessor) hasLargeTxnState(startTs uint64, commitTs uint64) bool { - if p.dispatcherStat == nil { - return false - } - state := p.dispatcherStat.getLargeTxnState() - return state != nil && - state.startTs == startTs && - state.commitTs == commitTs && - state.getPhase() == largeTxnScanPhaseOriginal -} - -func (p *dmlProcessor) flushSpilledInsertsIntoCurrentTxn() error { - if p.currentTxn == nil || p.dispatcherStat == nil { - return nil - } - state := p.dispatcherStat.getLargeTxnState() - if state == nil || state.getPhase() != largeTxnScanPhaseOriginal { - return nil - } - for { - insertRow, err := state.nextInsert() - if err != nil { - if errors.Is(err, io.EOF) { - return p.dispatcherStat.cleanupLargeTxnState() - } - return err - } - if err := p.appendInsertRow(insertRow); err != nil { - return err - } - } -} - -func (p *dmlProcessor) getOrCreateLargeTxnState() (*largeTxnScanState, error) { - if p.dispatcherStat == nil { - return nil, errors.New("dispatcher stat is required for large txn update spill") - } - currentDML := p.currentTxn.CurrentDMLEvent - return p.dispatcherStat.getOrCreateLargeTxnState( - p.spillDir, - currentDML.GetTableID(), - currentDML.TableInfo, - currentDML.GetStartTs(), - currentDML.GetCommitTs(), - ) -} - func (p *dmlProcessor) appendInsertRow(rawEvent *common.RawKVEntry) error { if p.currentTxn == nil { log.Panic("no current DML event to append to") diff --git a/pkg/eventservice/event_scanner_benchmark_test.go b/pkg/eventservice/event_scanner_benchmark_test.go index 9a2a0b8ff7..2927dd24bd 100644 --- a/pkg/eventservice/event_scanner_benchmark_test.go +++ b/pkg/eventservice/event_scanner_benchmark_test.go @@ -48,7 +48,7 @@ type benchmarkEventGetter struct { func (g *benchmarkEventGetter) GetIterator( common.DispatcherID, - common.DataRange, + eventstore.ScanRequest, ) (eventstore.EventIterator, error) { return &singleTxnIterator{ raw: g.raw, @@ -204,10 +204,12 @@ func BenchmarkEventScannerIgnoreDelete(b *testing.B) { event.NewMounter(time.UTC, &integrity.Config{}), common.DefaultMode, ) - dataRange := common.DataRange{ - Span: disInfo.GetTableSpan(), - CommitTsStart: ddlEvent.FinishedTs, - CommitTsEnd: deleteRow.CRTs + 1, + dataRange := eventstore.ScanRequest{ + Range: common.DataRange{ + Span: disInfo.GetTableSpan(), + CommitTsStart: ddlEvent.FinishedTs, + CommitTsEnd: deleteRow.CRTs + 1, + }, } limit := scanLimit{maxDMLBytes: 1 << 60} diff --git a/pkg/eventservice/event_scanner_test.go b/pkg/eventservice/event_scanner_test.go index cfbad54bcd..0256c28e08 100644 --- a/pkg/eventservice/event_scanner_test.go +++ b/pkg/eventservice/event_scanner_test.go @@ -81,7 +81,7 @@ type stubEventGetter struct { } func (g *stubEventGetter) GetIterator( - dispatcherID common.DispatcherID, dataRange common.DataRange, + dispatcherID common.DispatcherID, request eventstore.ScanRequest, ) (eventstore.EventIterator, error) { return g.iter, g.err } @@ -129,7 +129,8 @@ func TestEventScannerReturnsIteratorErrors(t *testing.T) { &mockMounter{}, 0, ) - _, events, _, interrupted, err := scanner.scan(context.Background(), disp, dataRange, scanLimit{}) + request := eventstore.ScanRequest{Range: dataRange} + _, events, _, interrupted, err := scanner.scan(context.Background(), disp, request, scanLimit{}) require.ErrorIs(t, err, getIterErr) require.Nil(t, events) require.False(t, interrupted) @@ -141,7 +142,7 @@ func TestEventScannerReturnsIteratorErrors(t *testing.T) { &mockMounter{}, 0, ) - _, events, _, interrupted, err = scanner.scan(context.Background(), disp, dataRange, scanLimit{}) + _, events, _, interrupted, err = scanner.scan(context.Background(), disp, request, scanLimit{}) require.ErrorIs(t, err, closeErr) require.Nil(t, events) require.False(t, interrupted) @@ -183,7 +184,7 @@ func TestEventScanner(t *testing.T) { sl := scanLimit{ maxDMLBytes: 1000, } - ok, dataRange := broker.getScanTaskDataRange(disp) + ok, dataRange := broker.getScanTaskRequest(disp) require.True(t, ok) _, events, _, isInterrupted, err := scanner.scan(ctx, disp, dataRange, sl) @@ -199,7 +200,7 @@ func TestEventScanner(t *testing.T) { resolvedTs := kvEvents[len(kvEvents)-1].CRTs + 1 disp.receivedResolvedTs.Store(resolvedTs) - ok, dataRange = broker.getScanTaskDataRange(disp) + ok, dataRange = broker.getScanTaskRequest(disp) require.True(t, ok) sl = scanLimit{ @@ -230,7 +231,7 @@ func TestEventScanner(t *testing.T) { disp.receivedResolvedTs.Store(resolvedTs) require.True(t, ok) - dataRange.CommitTsStart = ddlEvent.GetCommitTs() + dataRange.Range.CommitTsStart = ddlEvent.GetCommitTs() sl = scanLimit{ maxDMLBytes: 1000, @@ -269,7 +270,7 @@ func TestEventScanner(t *testing.T) { // Expected result: // [DDL(x), BatchDML_1[DML(x+1)], BatchDML_2[DML(x+2), DML(x+3), DML(x+4)], Resolved(x+5)] disp.receivedResolvedTs.Store(resolvedTs) - ok, dataRange = broker.getScanTaskDataRange(disp) + ok, dataRange = broker.getScanTaskRequest(disp) require.True(t, ok) sl = scanLimit{ @@ -499,7 +500,7 @@ func TestEventScannerSplitsLargeTxnWithRowLevelProgress(t *testing.T) { scanner := newEventScanner(broker.eventStore, broker.schemaStore, &mockMounter{}, 0) sl := scanLimit{maxDMLBytes: 1, isInUnitTest: true} - dataRange, ok := disp.getDataRange() + dataRange, ok := disp.getScanRequest() require.True(t, ok) _, events, progress, interrupted, err := scanner.scan(context.Background(), disp, dataRange, sl) require.NoError(t, err) @@ -512,9 +513,9 @@ func TestEventScannerSplitsLargeTxnWithRowLevelProgress(t *testing.T) { require.Equal(t, kvEvents[0].CRTs, firstBatch.GetCommitTs()) disp.updateScanRangeWithPosition(progress.txnCommitTs, progress.txnStartTs, progress.rowLevelScanPosition) - dataRange, ok = disp.getDataRange() + dataRange, ok = disp.getScanRequest() require.True(t, ok) - require.Equal(t, progress.rowLevelScanPosition, dataRange.RowLevelScanPosition) + require.Equal(t, progress.rowLevelScanPosition, dataRange.Cursor.Position) _, events, progress, interrupted, err = scanner.scan(context.Background(), disp, dataRange, sl) require.NoError(t, err) require.True(t, interrupted) @@ -526,7 +527,7 @@ func TestEventScannerSplitsLargeTxnWithRowLevelProgress(t *testing.T) { require.Equal(t, kvEvents[1].CRTs, secondBatch.GetCommitTs()) disp.updateScanRangeWithPosition(progress.txnCommitTs, progress.txnStartTs, progress.rowLevelScanPosition) - dataRange, ok = disp.getDataRange() + dataRange, ok = disp.getScanRequest() require.True(t, ok) _, events, progress, interrupted, err = scanner.scan(context.Background(), disp, dataRange, sl) require.NoError(t, err) @@ -574,7 +575,7 @@ func TestEventScannerDoesNotSplitCurrentTxnBelowLargeTxnThreshold(t *testing.T) scanner := newEventScanner(broker.eventStore, broker.schemaStore, &mockMounter{}, 0) sl := scanLimit{maxDMLBytes: 1, isInUnitTest: true} - dataRange, ok := disp.getDataRange() + dataRange, ok := disp.getScanRequest() require.True(t, ok) _, events, progress, interrupted, err := scanner.scan(context.Background(), disp, dataRange, sl) require.NoError(t, err) @@ -625,7 +626,7 @@ func TestEventScannerSpillsSplitUKUpdateInLargeTxn(t *testing.T) { scanner := newEventScanner(broker.eventStore, broker.schemaStore, event.NewMounter(time.UTC, &integrity.Config{}), 0) sl := scanLimit{maxDMLBytes: 1, isInUnitTest: true} - dataRange, ok := disp.getDataRange() + dataRange, ok := disp.getScanRequest() require.True(t, ok) _, events, progress, interrupted, err := scanner.scan(context.Background(), disp, dataRange, sl) require.NoError(t, err) @@ -641,7 +642,7 @@ func TestEventScannerSpillsSplitUKUpdateInLargeTxn(t *testing.T) { require.NotNil(t, disp.getLargeTxnState()) disp.updateScanRangeWithPosition(progress.txnCommitTs, progress.txnStartTs, progress.rowLevelScanPosition) - dataRange, ok = disp.getDataRange() + dataRange, ok = disp.getScanRequest() require.True(t, ok) _, events, progress, interrupted, err = scanner.scan(context.Background(), disp, dataRange, sl) require.NoError(t, err) @@ -650,7 +651,7 @@ func TestEventScannerSpillsSplitUKUpdateInLargeTxn(t *testing.T) { require.True(t, progress.valid) disp.updateScanRangeWithPosition(progress.txnCommitTs, progress.txnStartTs, progress.rowLevelScanPosition) - dataRange, ok = disp.getDataRange() + dataRange, ok = disp.getScanRequest() require.True(t, ok) _, events, progress, interrupted, err = scanner.scan(context.Background(), disp, dataRange, sl) require.NoError(t, err) @@ -664,7 +665,7 @@ func TestEventScannerSpillsSplitUKUpdateInLargeTxn(t *testing.T) { require.Equal(t, common.RowTypeInsert, insertRow.RowType) disp.updateScanRangeWithPosition(progress.txnCommitTs, progress.txnStartTs, progress.rowLevelScanPosition) - dataRange, ok = disp.getDataRange() + dataRange, ok = disp.getScanRequest() require.True(t, ok) _, events, progress, interrupted, err = scanner.scan(context.Background(), disp, dataRange, sl) require.NoError(t, err) @@ -717,9 +718,9 @@ func TestEventScannerFlushesSpilledUKUpdateBeforeSameCommitDDL(t *testing.T) { scanner := newEventScanner(broker.eventStore, broker.schemaStore, event.NewMounter(time.UTC, &integrity.Config{}), 0) sl := scanLimit{maxDMLBytes: 1, isInUnitTest: true} - dataRange, ok := disp.getDataRange() + dataRange, ok := disp.getScanRequest() require.True(t, ok) - ddlEvents, err := scanner.fetchDDLEvents(disp, dataRange) + ddlEvents, err := scanner.fetchDDLEvents(disp, dataRange.Range) require.NoError(t, err) require.Len(t, ddlEvents, 1) require.Equal(t, sameCommitDDL.FinishedTs, ddlEvents[0].GetCommitTs()) @@ -792,7 +793,7 @@ func TestEventScannerDrainsSpillBeforeFollowingSameCommitTxn(t *testing.T) { smallLimit := scanLimit{maxDMLBytes: 1, isInUnitTest: true} scanAndAdvance := func(limit scanLimit) ([]event.Event, scanProgress, bool) { - ok, dataRange := broker.getScanTaskDataRange(disp) + ok, dataRange := broker.getScanTaskRequest(disp) require.True(t, ok) _, events, progress, interrupted, err := scanner.scan(context.Background(), disp, dataRange, limit) require.NoError(t, err) @@ -862,7 +863,10 @@ func TestDrainLargeTxnInsertsStopsWhenDispatcherRemoved(t *testing.T) { CommitTsEnd: 100, }, scanLimit{maxDMLBytes: 1, isInUnitTest: true}) - interrupted, err := scanner.drainLargeTxnInserts(sess, state) + interrupted, err := drainLargeTxnInserts(&txnScanContext{ + scanner: scanner, + session: sess, + }, state) require.NoError(t, err) require.False(t, interrupted) require.Empty(t, sess.events) @@ -891,10 +895,12 @@ func TestEventScannerRetriesLargeTxnDrainAfterOpenError(t *testing.T) { _ = os.Remove(backupPath) }) - dataRange := common.DataRange{ - Span: info.GetTableSpan(), - CommitTsStart: 100, - CommitTsEnd: 100, + dataRange := eventstore.ScanRequest{ + Range: common.DataRange{ + Span: info.GetTableSpan(), + CommitTsStart: 100, + CommitTsEnd: 100, + }, } scanner := newEventScanner( &stubEventGetter{}, NewMockSchemaStore(), &mockMounter{}, 0) @@ -939,10 +945,12 @@ func TestEventScannerRetriesLargeTxnDrainAfterDecodeError(t *testing.T) { require.NoError(t, state.appendInsert(newTestSpillRawKVEntry(2))) disp.markLargeTxnDrainInserts(90, 100, false, 0) - dataRange := common.DataRange{ - Span: info.GetTableSpan(), - CommitTsStart: 100, - CommitTsEnd: 100, + dataRange := eventstore.ScanRequest{ + Range: common.DataRange{ + Span: info.GetTableSpan(), + CommitTsStart: 100, + CommitTsEnd: 100, + }, } mounter := &failOnceMounter{err: errors.New("injected decode error")} scanner := newEventScanner( @@ -1013,7 +1021,7 @@ func TestEventScannerWithDeleteTable(t *testing.T) { dml3 := kvEvents[3] mockSchemaStore.DeleteTable(tableID, dml2.CRTs) disp.receivedResolvedTs.Store(resolvedTs) - ok, dataRange := broker.getScanTaskDataRange(disp) + ok, dataRange := broker.getScanTaskRequest(disp) require.True(t, ok) sl := scanLimit{ @@ -1106,7 +1114,7 @@ func TestEventScannerWithDDL(t *testing.T) { mockSchemaStore.AppendDDLEvent(tableID, fakeDDL) disp.receivedResolvedTs.Store(resolvedTs) - ok, dataRange := broker.getScanTaskDataRange(disp) + ok, dataRange := broker.getScanTaskRequest(disp) require.True(t, ok) // case 1: Scanning interrupted at dml1 @@ -1260,7 +1268,7 @@ func TestEventScannerWithDDL(t *testing.T) { resolvedTs = resolvedTs + 3 disp.receivedResolvedTs.Store(resolvedTs) - ok, dataRange = broker.getScanTaskDataRange(disp) + ok, dataRange = broker.getScanTaskRequest(disp) require.True(t, ok) _, events, _, isInterrupted, err = scanner.scan(ctx, disp, dataRange, sl) @@ -2370,9 +2378,23 @@ func TestScanAndMergeEventsSingleUKUpdate(t *testing.T) { events: make([]event.Event, 0), } merger := newEventMerger([]event.Event{}) + processor := newDMLProcessor( + scanner.mounter, + scanner.schemaGetter, + stat.filter, + stat.info.IsOutputRawChangeEvent(), + scanner.mode, + stat.info.EnableIgnoreUpdateOnlyColumns()) + processor.dispatcherStat = stat + scanCtx := &txnScanContext{ + scanner: scanner, + session: sess, + merger: merger, + processor: processor, + } // Execute scanAndMergeEvents - isInterrupted, err := scanner.scanAndMergeEvents(sess, merger, mockIter) + isInterrupted, err := scanner.scanAndMergeEvents(scanCtx, newTxnScanStrategy(true), mockIter) events := sess.events // Verify results @@ -2462,8 +2484,22 @@ func TestScanAndMergeEventsSkipsDeletedTableTxn(t *testing.T) { events: make([]event.Event, 0), } merger := newEventMerger(nil) + processor := newDMLProcessor( + scanner.mounter, + scanner.schemaGetter, + disp.filter, + disp.info.IsOutputRawChangeEvent(), + scanner.mode, + disp.info.EnableIgnoreUpdateOnlyColumns()) + processor.dispatcherStat = disp + scanCtx := &txnScanContext{ + scanner: scanner, + session: sess, + merger: merger, + processor: processor, + } - isInterrupted, err := scanner.scanAndMergeEvents(sess, merger, &mockEventIterator{ + isInterrupted, err := scanner.scanAndMergeEvents(scanCtx, newTxnScanStrategy(false), &mockEventIterator{ events: []*common.RawKVEntry{rawEvent}, }) require.NoError(t, err) diff --git a/pkg/eventservice/event_service_test.go b/pkg/eventservice/event_service_test.go index eac53c1579..f6f900db99 100644 --- a/pkg/eventservice/event_service_test.go +++ b/pkg/eventservice/event_service_test.go @@ -277,8 +277,9 @@ func (m *mockEventStore) UnregisterDispatcher(changefeedID common.ChangeFeedID, } func (m *mockEventStore) GetIterator( - dispatcherID common.DispatcherID, dataRange common.DataRange, + dispatcherID common.DispatcherID, request eventstore.ScanRequest, ) (eventstore.EventIterator, error) { + dataRange := request.Range span, ok := m.dispatcherMap.Load(dispatcherID) if !ok { log.Panic("dispatcher not found", zap.Stringer("dispatcherID", dispatcherID)) @@ -293,21 +294,21 @@ func (m *mockEventStore) GetIterator( events := spanStats.getAllEvents() entries := make([]*common.RawKVEntry, 0) - positions := make([]common.ScanPosition, 0) - rowLevelStart := decodeMockScanPosition(dataRange.RowLevelScanPosition) + positions := make([]eventstore.ScanPosition, 0) + rowLevelStart := decodeMockScanPosition(request.Cursor.Position) for i, e := range events { if rowLevelStart >= 0 && i <= rowLevelStart { continue } - if len(dataRange.RowLevelScanPosition) != 0 { + if len(request.Cursor.Position) != 0 { if e.CRTs >= dataRange.CommitTsStart && e.CRTs <= dataRange.CommitTsEnd { entries = append(entries, e) positions = append(positions, encodeMockScanPosition(i)) } continue } - if dataRange.LastScannedTxnStartTs != 0 { - if e.CRTs == dataRange.CommitTsStart && e.StartTs <= dataRange.LastScannedTxnStartTs { + if request.Cursor.TxnStartTs != 0 { + if e.CRTs == dataRange.CommitTsStart && e.StartTs <= request.Cursor.TxnStartTs { continue } if e.CRTs >= dataRange.CommitTsStart && e.CRTs <= dataRange.CommitTsEnd { @@ -358,7 +359,7 @@ func (m *mockEventStore) RegisterDispatcher( type mockEventIterator struct { events []*common.RawKVEntry - positions []common.ScanPosition + positions []eventstore.ScanPosition prevStartTS uint64 prevCommitTS uint64 rowCount int @@ -370,14 +371,14 @@ func (iter *mockEventIterator) Next() (*common.RawKVEntry, bool) { return row, isNewTxn } -func (iter *mockEventIterator) NextWithScanPosition() (*common.RawKVEntry, common.ScanPosition, bool) { +func (iter *mockEventIterator) NextWithScanPosition() (*common.RawKVEntry, eventstore.ScanPosition, bool) { if len(iter.events) == 0 { return nil, nil, false } row := iter.events[0] iter.events = iter.events[1:] - var position common.ScanPosition + var position eventstore.ScanPosition if len(iter.positions) > 0 { position = iter.positions[0] iter.positions = iter.positions[1:] @@ -396,15 +397,15 @@ func (m *mockEventIterator) Close() (int64, error) { return int64(m.rowCount), m.closeErr } -func encodeMockScanPosition(index int) common.ScanPosition { +func encodeMockScanPosition(index int) eventstore.ScanPosition { var buf [8]byte binary.BigEndian.PutUint64(buf[:], uint64(index)) - position := make(common.ScanPosition, len(buf)) + position := make(eventstore.ScanPosition, len(buf)) copy(position, buf[:]) return position } -func decodeMockScanPosition(position common.ScanPosition) int { +func decodeMockScanPosition(position eventstore.ScanPosition) int { if len(position) == 0 { return -1 } diff --git a/pkg/eventservice/scan_progress.go b/pkg/eventservice/scan_progress.go new file mode 100644 index 0000000000..d5f1cc725c --- /dev/null +++ b/pkg/eventservice/scan_progress.go @@ -0,0 +1,48 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package eventservice + +import "github.com/pingcap/ticdc/logservice/eventstore" + +// scanProgress is an immutable snapshot published after a scan result has +// been sent. A non-empty rowLevelScanPosition resumes inside the transaction. +type scanProgress struct { + valid bool + txnCommitTs uint64 + txnStartTs uint64 + rowLevelScanPosition eventstore.ScanPosition +} + +func newTxnScanProgress(commitTs uint64, startTs uint64) scanProgress { + return scanProgress{ + valid: true, + txnCommitTs: commitTs, + txnStartTs: startTs, + } +} + +func newRowLevelScanProgress(commitTs uint64, startTs uint64, position eventstore.ScanPosition) scanProgress { + progress := newTxnScanProgress(commitTs, startTs) + progress.rowLevelScanPosition = cloneScanPosition(position) + return progress +} + +func cloneScanPosition(position eventstore.ScanPosition) eventstore.ScanPosition { + if len(position) == 0 { + return nil + } + cloned := make(eventstore.ScanPosition, len(position)) + copy(cloned, position) + return cloned +} diff --git a/pkg/eventservice/txn_scan_strategy.go b/pkg/eventservice/txn_scan_strategy.go new file mode 100644 index 0000000000..47945ffff0 --- /dev/null +++ b/pkg/eventservice/txn_scan_strategy.go @@ -0,0 +1,465 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package eventservice + +import ( + "io" + "time" + + "github.com/pingcap/log" + "github.com/pingcap/ticdc/logservice/eventstore" + "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/common/event" + "github.com/pingcap/ticdc/pkg/errors" + "go.uber.org/zap" +) + +type txnScanContext struct { + scanner *eventScanner + session *session + merger *eventMerger + processor *dmlProcessor +} + +type nextTxnMeta struct { + startTs uint64 + commitTs uint64 + tableInfoUpdateTs uint64 + tableDeleted bool +} + +// txnScanStrategy owns behavior that differs between atomic and split +// transaction scanning while the event scanner retains the common iterator and +// DDL/DML merge loop. +type txnScanStrategy interface { + resumePending(ctx *txnScanContext) (handled bool, interrupted bool, err error) + startTxn( + ctx *txnScanContext, + startTs uint64, + commitTs uint64, + tableInfo *common.TableInfo, + tableID int64, + ) error + finishTxn(ctx *txnScanContext, next nextTxnMeta) (interrupted bool, err error) + afterAppend( + ctx *txnScanContext, + rawEvent *common.RawKVEntry, + position eventstore.ScanPosition, + ) (interrupted bool, err error) +} + +func newTxnScanStrategy(shouldSplitTxn bool) txnScanStrategy { + if shouldSplitTxn { + return splitTxnScanStrategy{} + } + return atomicTxnScanStrategy{} +} + +type atomicTxnScanStrategy struct{} + +func (atomicTxnScanStrategy) resumePending( + ctx *txnScanContext, +) (bool, bool, error) { + return false, false, nil +} + +func (atomicTxnScanStrategy) startTxn( + ctx *txnScanContext, + startTs uint64, + commitTs uint64, + tableInfo *common.TableInfo, + tableID int64, +) error { + return startTxn(ctx, startTs, commitTs, tableInfo, tableID, false) +} + +func (atomicTxnScanStrategy) finishTxn( + ctx *txnScanContext, + next nextTxnMeta, +) (bool, error) { + if ctx.processor == nil || ctx.processor.currentTxn == nil { + return false, nil + } + return false, ctx.scanner.commitTxn( + ctx.session, + ctx.merger, + ctx.processor, + next.commitTs, + next.tableInfoUpdateTs, + ) +} + +func (atomicTxnScanStrategy) afterAppend( + ctx *txnScanContext, + rawEvent *common.RawKVEntry, + position eventstore.ScanPosition, +) (bool, error) { + return false, nil +} + +type splitTxnScanStrategy struct{} + +func (splitTxnScanStrategy) resumePending( + ctx *txnScanContext, +) (bool, bool, error) { + state := ctx.session.dispatcherStat.getLargeTxnState() + if state == nil || state.getPhase() != largeTxnScanPhaseDrainInserts { + return false, false, nil + } + interrupted, err := drainLargeTxnInserts(ctx, state) + return true, interrupted, err +} + +func (splitTxnScanStrategy) startTxn( + ctx *txnScanContext, + startTs uint64, + commitTs uint64, + tableInfo *common.TableInfo, + tableID int64, +) error { + return startTxn(ctx, startTs, commitTs, tableInfo, tableID, true) +} + +func (splitTxnScanStrategy) finishTxn( + ctx *txnScanContext, + next nextTxnMeta, +) (bool, error) { + if ctx.processor == nil || ctx.processor.currentTxn == nil { + return finishPendingSplitTxn(ctx.session, next), nil + } + return finishCurrentSplitTxn(ctx, next) +} + +func finishPendingSplitTxn(session *session, next nextTxnMeta) bool { + dispatcher := session.dispatcherStat + state := dispatcher.getLargeTxnState() + if state == nil || state.getPhase() != largeTxnScanPhaseOriginal { + return false + } + if next.commitTs != 0 && next.startTs == state.startTs && next.commitTs == state.commitTs { + return false + } + + dispatcher.finishPendingBigTxnMetric() + dispatcher.markLargeTxnDrainInserts(state.startTs, state.commitTs, next.commitTs != 0, next.commitTs) + session.progress = newTxnScanProgress(state.commitTs, state.startTs) + return true +} + +func (splitTxnScanStrategy) afterAppend( + ctx *txnScanContext, + rawEvent *common.RawKVEntry, + position eventstore.ScanPosition, +) (bool, error) { + if !canInterruptCurrentTxn(ctx, rawEvent, position) { + return false, nil + } + interruptCurrentTxn(ctx, rawEvent.CRTs, rawEvent.StartTs, position) + return true, nil +} + +func startTxn( + ctx *txnScanContext, + startTs uint64, + commitTs uint64, + tableInfo *common.TableInfo, + tableID int64, + shouldSplitTxn bool, +) error { + err := ctx.processor.startTxn( + ctx.session.dispatcherStat.id, + tableID, + tableInfo, + startTs, + commitTs, + shouldSplitTxn, + ) + if err != nil { + return err + } + ctx.session.dmlCount++ + return nil +} + +func finishCurrentSplitTxn(ctx *txnScanContext, next nextTxnMeta) (bool, error) { + processor := ctx.processor + currentStartTs := processor.currentTxn.CurrentDMLEvent.GetStartTs() + currentCommitTs := processor.currentTxn.CurrentDMLEvent.GetCommitTs() + + if processor.hasSpilledInsertsForCurrentTxn() && ctx.merger.hasDDLAtCommitTs(currentCommitTs) { + if err := processor.flushCachedInsertRows(); err != nil { + return false, err + } + if err := processor.flushSpilledInsertsIntoCurrentTxn(); err != nil { + return false, err + } + } + + if err := ctx.scanner.commitTxn( + ctx.session, + ctx.merger, + processor, + next.commitTs, + next.tableInfoUpdateTs, + ); err != nil { + return false, err + } + if !processor.hasLargeTxnState(currentStartTs, currentCommitTs) { + return false, nil + } + + events := ctx.merger.mergeWithPrecedingDDLs(processor.getCurrentBatchDML()) + ctx.session.appendEvents(events) + processor.resetBatchDML() + + hasFollowingTxn := next.commitTs != 0 && !next.tableDeleted + ctx.session.dispatcherStat.markLargeTxnDrainInserts( + currentStartTs, + currentCommitTs, + hasFollowingTxn, + next.commitTs, + ) + if len(ctx.session.lastRowPosition) > 0 { + ctx.session.progress = newRowLevelScanProgress( + currentCommitTs, + currentStartTs, + ctx.session.lastRowPosition, + ) + } else { + ctx.session.progress = newTxnScanProgress(currentCommitTs, currentStartTs) + } + return true, nil +} + +func canInterruptCurrentTxn( + ctx *txnScanContext, + rawEvent *common.RawKVEntry, + position eventstore.ScanPosition, +) bool { + if len(position) == 0 || len(ctx.processor.insertRowCache) > 0 { + return false + } + if ctx.processor.currentTxn == nil || !ctx.processor.currentTxn.exceedsLargeTxnThreshold() { + return false + } + return ctx.merger.canInterrupt(rawEvent.CRTs, ctx.processor.batchDML) +} + +func interruptCurrentTxn( + ctx *txnScanContext, + commitTs uint64, + startTs uint64, + position eventstore.ScanPosition, +) { + processor := ctx.processor + if processor.currentTxn != nil { + currentTxn := processor.currentTxn + currentDML := currentTxn.CurrentDMLEvent + ctx.session.dispatcherStat.addBigTxnMetricFragment( + currentDML.GetStartTs(), + currentDML.GetCommitTs(), + currentTxn.rawKVBytes, + currentTxn.largeTxnThresholdInBytes) + } + events := ctx.merger.mergeWithPrecedingDDLs(processor.getCurrentBatchDML()) + ctx.session.appendEvents(events) + ctx.session.progress = newRowLevelScanProgress(commitTs, startTs, position) + log.Debug("scan interrupted inside a large txn", + zap.Stringer("dispatcherID", ctx.session.dispatcherStat.id), + zap.Uint64("startTs", startTs), + zap.Uint64("commitTs", commitTs), + zap.Int("scannedEntryCount", ctx.session.scannedEntryCount), + zap.Duration("duration", time.Since(ctx.session.startTime))) +} + +func drainLargeTxnInserts( + ctx *txnScanContext, + state *largeTxnScanState, +) (bool, error) { + session := ctx.session + drainedInsertCount := 0 + returnWithError := func(scanErr error) (bool, error) { + if rollbackErr := state.rollbackDrain(); rollbackErr != nil { + log.Warn("reset large transaction spill reader failed", + zap.Stringer("dispatcherID", session.dispatcherStat.id), + zap.Error(rollbackErr)) + } + return false, scanErr + } + + processor := newDMLProcessor( + ctx.scanner.mounter, + ctx.scanner.schemaGetter, + session.dispatcherStat.filter, + session.dispatcherStat.info.IsOutputRawChangeEvent(), + ctx.scanner.mode, + session.dispatcherStat.info.EnableIgnoreUpdateOnlyColumns()) + processor.dispatcherStat = session.dispatcherStat + + for { + shouldStop, err := ctx.scanner.checkScanConditions(session) + if err != nil { + return returnWithError(err) + } + if shouldStop { + return false, nil + } + + entry, err := state.nextInsert() + if err != nil { + if session.dispatcherStat.isRemoved.Load() { + return false, nil + } + if errors.Is(err, io.EOF) { + if processor.currentTxn != nil { + if err := processor.commitTxn(); err != nil { + return returnWithError(err) + } + if processor.getCurrentBatchDML().DMLCount() != 0 { + session.appendEvents([]event.Event{processor.getCurrentBatchDML()}) + } + } + state.commitDrainedInserts(drainedInsertCount) + session.progress = newTxnScanProgress(state.commitTs, state.startTs) + hasFollowingTxn, followingCommitTs := state.snapshotDrainInfo() + shouldResolveCommitTs := (!hasFollowingTxn || followingCommitTs > state.commitTs) && + session.dataRange.CommitTsEnd == state.commitTs + if err := session.dispatcherStat.cleanupLargeTxnState(); err != nil { + log.Warn("cleanup drained large transaction spill failed", + zap.Stringer("dispatcherID", session.dispatcherStat.id), + zap.Error(err)) + } + if shouldResolveCommitTs { + resolved := event.NewResolvedEvent( + state.commitTs, + session.dispatcherStat.id, + session.dispatcherStat.epoch, + ) + session.appendEvents([]event.Event{resolved}) + session.progress = newTxnScanProgress(state.commitTs, 0) + return false, nil + } + return true, nil + } + return returnWithError(err) + } + drainedInsertCount++ + + if processor.currentTxn == nil { + if err := processor.startTxn( + session.dispatcherStat.id, + state.tableID, + state.tableInfo, + state.startTs, + state.commitTs, + true, + ); err != nil { + return returnWithError(err) + } + } + session.observeRawEntry(entry, nil) + if err := processor.appendInsertRow(entry); err != nil { + return returnWithError(err) + } + if session.exceedLimit(processor.batchDML.GetSize(), processor.batchDML) { + if err := processor.commitTxn(); err != nil { + return returnWithError(err) + } + session.appendEvents([]event.Event{processor.getCurrentBatchDML()}) + state.commitDrainedInserts(drainedInsertCount) + session.progress = newTxnScanProgress(state.commitTs, state.startTs) + return true, nil + } + } +} + +func (p *dmlProcessor) spillCachedInsertRows() error { + if len(p.insertRowCache) == 0 { + return nil + } + state, err := p.getOrCreateLargeTxnState() + if err != nil { + return err + } + for _, insertRow := range p.insertRowCache { + if err := state.appendInsert(insertRow); err != nil { + return err + } + } + p.insertRowCache = make([]*common.RawKVEntry, 0) + return nil +} + +func (p *dmlProcessor) shouldSpillSplitUpdateInsert() bool { + if p.currentTxn == nil { + return false + } + return p.currentTxn.exceedsLargeTxnThreshold() || p.hasSpilledInsertsForCurrentTxn() +} + +func (p *dmlProcessor) hasSpilledInsertsForCurrentTxn() bool { + if p.currentTxn == nil || p.dispatcherStat == nil { + return false + } + current := p.currentTxn.CurrentDMLEvent + return p.hasLargeTxnState(current.GetStartTs(), current.GetCommitTs()) +} + +func (p *dmlProcessor) hasLargeTxnState(startTs uint64, commitTs uint64) bool { + if p.dispatcherStat == nil { + return false + } + state := p.dispatcherStat.getLargeTxnState() + return state != nil && + state.startTs == startTs && + state.commitTs == commitTs && + state.getPhase() == largeTxnScanPhaseOriginal +} + +func (p *dmlProcessor) flushSpilledInsertsIntoCurrentTxn() error { + if p.currentTxn == nil || p.dispatcherStat == nil { + return nil + } + state := p.dispatcherStat.getLargeTxnState() + if state == nil || state.getPhase() != largeTxnScanPhaseOriginal { + return nil + } + for { + insertRow, err := state.nextInsert() + if err != nil { + if errors.Is(err, io.EOF) { + return p.dispatcherStat.cleanupLargeTxnState() + } + return err + } + if err := p.appendInsertRow(insertRow); err != nil { + return err + } + } +} + +func (p *dmlProcessor) getOrCreateLargeTxnState() (*largeTxnScanState, error) { + if p.dispatcherStat == nil { + return nil, errors.New("dispatcher stat is required for large txn update spill") + } + currentDML := p.currentTxn.CurrentDMLEvent + return p.dispatcherStat.getOrCreateLargeTxnState( + p.spillDir, + currentDML.GetTableID(), + currentDML.TableInfo, + currentDML.GetStartTs(), + currentDML.GetCommitTs(), + ) +} From 8a9646d581bdf6e479d71c770ca036476e6f6733 Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Tue, 21 Jul 2026 13:16:19 +0800 Subject: [PATCH 24/35] eventservice: encrypt large transaction spill data --- pkg/eventservice/large_txn_spill.go | 50 ++++++++++++++--- pkg/eventservice/large_txn_spill_test.go | 68 +++++++++++++++++++++--- pkg/eventservice/large_txn_state.go | 6 ++- 3 files changed, 111 insertions(+), 13 deletions(-) diff --git a/pkg/eventservice/large_txn_spill.go b/pkg/eventservice/large_txn_spill.go index 30bed1ca34..41c9b30579 100644 --- a/pkg/eventservice/large_txn_spill.go +++ b/pkg/eventservice/large_txn_spill.go @@ -14,12 +14,15 @@ package eventservice import ( + "context" "io" "os" "path/filepath" "github.com/pingcap/ticdc/pkg/common" + appcontext "github.com/pingcap/ticdc/pkg/common/context" "github.com/pingcap/ticdc/pkg/config" + "github.com/pingcap/ticdc/pkg/encryption" "github.com/pingcap/ticdc/pkg/errors" recordspill "github.com/pingcap/ticdc/pkg/spill" ) @@ -31,7 +34,9 @@ const ( // largeTxnInsertSpill stores deferred insert rows for a single large transaction. type largeTxnInsertSpill struct { - file *recordspill.RecordFile + file *recordspill.RecordFile + keyspaceID uint32 + encryptionManager encryption.EncryptionManager } func getLargeTxnInsertSpillDir() string { @@ -64,7 +69,16 @@ func cleanupLargeTxnInsertSpillFiles(dir string) (int, error) { return removed, nil } -func newLargeTxnInsertSpill(dir string) (*largeTxnInsertSpill, error) { +func newLargeTxnInsertSpill(dir string, keyspaceID uint32) (*largeTxnInsertSpill, error) { + encryptionManager, _ := appcontext.TryGetService[encryption.EncryptionManager](appcontext.EncryptionManager) + return newLargeTxnInsertSpillWithEncryption(dir, keyspaceID, encryptionManager) +} + +func newLargeTxnInsertSpillWithEncryption( + dir string, + keyspaceID uint32, + encryptionManager encryption.EncryptionManager, +) (*largeTxnInsertSpill, error) { if dir == "" { return nil, errors.ErrSpillFileOp.GenWithStackByArgs("empty large transaction spill directory") } @@ -73,7 +87,11 @@ func newLargeTxnInsertSpill(dir string) (*largeTxnInsertSpill, error) { return nil, err } - return &largeTxnInsertSpill{file: file}, nil + return &largeTxnInsertSpill{ + file: file, + keyspaceID: keyspaceID, + encryptionManager: encryptionManager, + }, nil } func (s *largeTxnInsertSpill) Append(entry *common.RawKVEntry) error { @@ -81,7 +99,15 @@ func (s *largeTxnInsertSpill) Append(entry *common.RawKVEntry) error { return errors.ErrSpillFileOp.GenWithStackByArgs("cannot append nil RawKVEntry") } - _, err := s.file.Append(entry.Encode()) + data := entry.Encode() + var err error + if s.encryptionManager != nil { + data, err = s.encryptionManager.EncryptData(context.Background(), s.keyspaceID, data) + if err != nil { + return err + } + } + _, err = s.file.Append(data) return err } @@ -94,7 +120,11 @@ func (s *largeTxnInsertSpill) NewReader() (*largeTxnInsertSpillReader, error) { if err != nil { return nil, err } - return &largeTxnInsertSpillReader{reader: reader}, nil + return &largeTxnInsertSpillReader{ + reader: reader, + keyspaceID: s.keyspaceID, + encryptionManager: s.encryptionManager, + }, nil } func (s *largeTxnInsertSpill) Close() error { @@ -112,7 +142,9 @@ func (s *largeTxnInsertSpill) Cleanup() error { } type largeTxnInsertSpillReader struct { - reader *recordspill.Reader + reader *recordspill.Reader + keyspaceID uint32 + encryptionManager encryption.EncryptionManager } func (r *largeTxnInsertSpillReader) Next() (*common.RawKVEntry, error) { @@ -123,6 +155,12 @@ func (r *largeTxnInsertSpillReader) Next() (*common.RawKVEntry, error) { } return nil, err } + if r.encryptionManager != nil { + data, err = r.encryptionManager.DecryptData(context.Background(), r.keyspaceID, data) + if err != nil { + return nil, err + } + } entry := &common.RawKVEntry{} if err := entry.Decode(data); err != nil { diff --git a/pkg/eventservice/large_txn_spill_test.go b/pkg/eventservice/large_txn_spill_test.go index 1de4ebcd6f..ef1b325cf9 100644 --- a/pkg/eventservice/large_txn_spill_test.go +++ b/pkg/eventservice/large_txn_spill_test.go @@ -14,6 +14,8 @@ package eventservice import ( + "bytes" + "context" "fmt" "io" "os" @@ -26,7 +28,7 @@ import ( ) func TestLargeTxnInsertSpillReadOrder(t *testing.T) { - spill, err := newLargeTxnInsertSpill(t.TempDir()) + spill, err := newLargeTxnInsertSpill(t.TempDir(), 0) require.NoError(t, err) defer func() { require.NoError(t, spill.Cleanup()) @@ -60,7 +62,7 @@ func TestLargeTxnInsertSpillReadOrder(t *testing.T) { func TestLargeTxnInsertSpillCreatesDir(t *testing.T) { dir := filepath.Join(t.TempDir(), "data-dir", largeTxnInsertSpillDirName) - spill, err := newLargeTxnInsertSpill(dir) + spill, err := newLargeTxnInsertSpill(dir, 0) require.NoError(t, err) defer func() { require.NoError(t, spill.Cleanup()) @@ -71,7 +73,7 @@ func TestLargeTxnInsertSpillCreatesDir(t *testing.T) { } func TestLargeTxnInsertSpillCleanup(t *testing.T) { - spill, err := newLargeTxnInsertSpill(t.TempDir()) + spill, err := newLargeTxnInsertSpill(t.TempDir(), 0) require.NoError(t, err) require.NoError(t, spill.Append(newTestSpillRawKVEntry(1))) @@ -88,7 +90,7 @@ func TestLargeTxnInsertSpillCleanup(t *testing.T) { } func TestLargeTxnInsertSpillEmpty(t *testing.T) { - spill, err := newLargeTxnInsertSpill(t.TempDir()) + spill, err := newLargeTxnInsertSpill(t.TempDir(), 0) require.NoError(t, err) defer func() { require.NoError(t, spill.Cleanup()) @@ -106,11 +108,11 @@ func TestLargeTxnInsertSpillEmpty(t *testing.T) { } func TestLargeTxnInsertSpillValidationErrors(t *testing.T) { - spill, err := newLargeTxnInsertSpill("") + spill, err := newLargeTxnInsertSpill("", 0) require.True(t, errors.ErrSpillFileOp.Equal(err)) require.Nil(t, spill) - spill, err = newLargeTxnInsertSpill(t.TempDir()) + spill, err = newLargeTxnInsertSpill(t.TempDir(), 0) require.NoError(t, err) defer func() { require.NoError(t, spill.Cleanup()) @@ -139,6 +141,60 @@ func TestCleanupLargeTxnInsertSpillFiles(t *testing.T) { require.FileExists(t, keepPath) } +type xorEncryptionManager struct { + encryptKeyspaceID uint32 + decryptKeyspaceID uint32 +} + +func (m *xorEncryptionManager) EncryptData( + ctx context.Context, keyspaceID uint32, data []byte, +) ([]byte, error) { + m.encryptKeyspaceID = keyspaceID + return xorBytes(data), nil +} + +func (m *xorEncryptionManager) DecryptData( + ctx context.Context, keyspaceID uint32, data []byte, +) ([]byte, error) { + m.decryptKeyspaceID = keyspaceID + return xorBytes(data), nil +} + +func xorBytes(data []byte) []byte { + result := make([]byte, len(data)) + for i := range data { + result[i] = data[i] ^ 0xff + } + return result +} + +func TestLargeTxnInsertSpillUsesEncryptionManager(t *testing.T) { + const keyspaceID uint32 = 42 + manager := &xorEncryptionManager{} + spill, err := newLargeTxnInsertSpillWithEncryption(t.TempDir(), keyspaceID, manager) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, spill.Cleanup()) + }) + + entry := newTestSpillRawKVEntry(1) + encoded := entry.Encode() + require.NoError(t, spill.Append(entry)) + + onDisk, err := os.ReadFile(spill.file.Path()) + require.NoError(t, err) + require.False(t, bytes.Contains(onDisk, encoded)) + require.Equal(t, keyspaceID, manager.encryptKeyspaceID) + + reader, err := spill.NewReader() + require.NoError(t, err) + decoded, err := reader.Next() + require.NoError(t, err) + require.Equal(t, entry, decoded) + require.Equal(t, keyspaceID, manager.decryptKeyspaceID) + require.NoError(t, reader.Close()) +} + func newTestSpillRawKVEntry(index int) *common.RawKVEntry { return &common.RawKVEntry{ OpType: common.OpTypePut, diff --git a/pkg/eventservice/large_txn_state.go b/pkg/eventservice/large_txn_state.go index 4c8757d343..78cbb4c923 100644 --- a/pkg/eventservice/large_txn_state.go +++ b/pkg/eventservice/large_txn_state.go @@ -70,7 +70,11 @@ func (a *dispatcherStat) getOrCreateLargeTxnState( return state, nil } - spill, err := newLargeTxnInsertSpill(spillDir) + var keyspaceID uint32 + if a.info != nil { + keyspaceID = a.info.GetTableSpan().KeyspaceID + } + spill, err := newLargeTxnInsertSpill(spillDir, keyspaceID) if err != nil { return nil, err } From ee603b230ec9103c6a5d03f159f48ea3c3b95dcc Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Tue, 21 Jul 2026 16:39:56 +0800 Subject: [PATCH 25/35] eventservice: fix unused parameter lint errors Signed-off-by: dongmen <414110582@qq.com> --- pkg/eventservice/txn_scan_strategy.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/eventservice/txn_scan_strategy.go b/pkg/eventservice/txn_scan_strategy.go index 47945ffff0..799b779dab 100644 --- a/pkg/eventservice/txn_scan_strategy.go +++ b/pkg/eventservice/txn_scan_strategy.go @@ -70,7 +70,7 @@ func newTxnScanStrategy(shouldSplitTxn bool) txnScanStrategy { type atomicTxnScanStrategy struct{} func (atomicTxnScanStrategy) resumePending( - ctx *txnScanContext, + _ *txnScanContext, ) (bool, bool, error) { return false, false, nil } @@ -102,9 +102,9 @@ func (atomicTxnScanStrategy) finishTxn( } func (atomicTxnScanStrategy) afterAppend( - ctx *txnScanContext, - rawEvent *common.RawKVEntry, - position eventstore.ScanPosition, + _ *txnScanContext, + _ *common.RawKVEntry, + _ eventstore.ScanPosition, ) (bool, error) { return false, nil } From 453a512892df307e3c57fa7948ce3d4cff0ab560 Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Tue, 21 Jul 2026 18:00:40 +0800 Subject: [PATCH 26/35] eventservice: keep row cursor within scan window Signed-off-by: dongmen <414110582@qq.com> --- pkg/eventservice/event_broker.go | 9 ++++++++ pkg/eventservice/event_broker_test.go | 33 +++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/pkg/eventservice/event_broker.go b/pkg/eventservice/event_broker.go index de846479d6..517738ba9e 100644 --- a/pkg/eventservice/event_broker.go +++ b/pkg/eventservice/event_broker.go @@ -482,6 +482,15 @@ func (c *eventBroker) getScanTaskRequest(task scanTask) (bool, eventstore.ScanRe } } + // A row cursor resumes inside the transaction at CommitTsStart. The adaptive + // window may shrink after that cursor was published, but it must not move the + // effective upper bound behind the cursor's commit ts. + if len(request.Cursor.Position) != 0 && + dataRange.CommitTsEnd < dataRange.CommitTsStart && + commitTsEndBeforeWindow >= dataRange.CommitTsStart { + dataRange.CommitTsEnd = dataRange.CommitTsStart + } + if dataRange.CommitTsEnd <= dataRange.CommitTsStart { hasSameCommitTxnResume := request.Cursor.TxnStartTs != 0 && dataRange.CommitTsEnd == dataRange.CommitTsStart diff --git a/pkg/eventservice/event_broker_test.go b/pkg/eventservice/event_broker_test.go index 31be19f35b..a49c122a41 100644 --- a/pkg/eventservice/event_broker_test.go +++ b/pkg/eventservice/event_broker_test.go @@ -23,6 +23,7 @@ import ( "github.com/pingcap/log" "github.com/pingcap/ticdc/eventpb" + "github.com/pingcap/ticdc/logservice/eventstore" "github.com/pingcap/ticdc/pkg/common" appcontext "github.com/pingcap/ticdc/pkg/common/context" "github.com/pingcap/ticdc/pkg/common/event" @@ -335,6 +336,38 @@ func TestGetScanTaskDataRangeEmptyAfterCappingDoesNotResetScanRange(t *testing.T require.Equal(t, lastStartTs, disp.loadScanProgress().txnStartTs) } +func TestGetScanTaskRequestKeepsRowCursorInsideShrunkWindow(t *testing.T) { + broker, _, schemaStore, _ := newEventBrokerForTest() + // Close the broker, so we can catch all messages in the test. + broker.close() + + info := newMockDispatcherInfoForTest(t) + info.epoch = 1 + changefeedStatus := broker.getOrSetChangefeedStatus(info) + disp := newDispatcherStat(info, 1, 1, nil, changefeedStatus) + disp.seq.Store(1) + + baseTime := time.Now() + baseTs := oracle.GoTimeToTS(baseTime) + cursorCommitTs := oracle.GoTimeToTS(baseTime.Add(20 * time.Second)) + resolvedTs := oracle.GoTimeToTS(baseTime.Add(40 * time.Second)) + position := eventstore.ScanPosition("row-cursor") + + disp.sentResolvedTs.Store(baseTs) + disp.receivedResolvedTs.Store(resolvedTs) + disp.eventStoreCommitTs.Store(cursorCommitTs) + disp.updateScanRangeWithPosition(cursorCommitTs, cursorCommitTs-1, position) + changefeedStatus.minSentTs.Store(baseTs) + changefeedStatus.scanInterval.Store(int64(defaultScanInterval)) + schemaStore.resolvedTs = resolvedTs + + needScan, request := broker.getScanTaskRequest(disp) + require.True(t, needScan) + require.Equal(t, cursorCommitTs, request.Range.CommitTsStart) + require.Equal(t, cursorCommitTs, request.Range.CommitTsEnd) + require.Equal(t, position, request.Cursor.Position) +} + func TestGetScanTaskDataRangeEmptyAfterCappingWithPendingDDLEventUsesLocalWindow(t *testing.T) { broker, _, ss, _ := newEventBrokerForTest() // Close the broker, so we can catch all message in the test. From a721f145a01980a46c51d7cc0f3659d3be8fbeb0 Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Tue, 21 Jul 2026 18:01:00 +0800 Subject: [PATCH 27/35] eventservice,spill: harden large transaction spill lifecycle Signed-off-by: dongmen <414110582@qq.com> --- pkg/eventservice/event_broker_test.go | 2 +- pkg/eventservice/event_scanner.go | 7 ++- pkg/eventservice/event_scanner_test.go | 38 ++++++++++++++--- pkg/eventservice/large_txn_spill.go | 8 ++-- pkg/eventservice/large_txn_spill_test.go | 54 ++++++++++++++++++++---- pkg/eventservice/large_txn_state.go | 31 +++++++++----- pkg/eventservice/txn_scan_strategy.go | 7 +-- pkg/spill/record_file.go | 13 +++--- pkg/spill/record_file_test.go | 31 ++++++++++++++ 9 files changed, 149 insertions(+), 42 deletions(-) diff --git a/pkg/eventservice/event_broker_test.go b/pkg/eventservice/event_broker_test.go index a49c122a41..932967d35d 100644 --- a/pkg/eventservice/event_broker_test.go +++ b/pkg/eventservice/event_broker_test.go @@ -772,7 +772,7 @@ func mustCreateLargeTxnState(t *testing.T, stat *dispatcherStat, tableID int64) state, err := stat.getOrCreateLargeTxnState(t.TempDir(), tableID, nil, 90, 100) require.NoError(t, err) - require.NoError(t, state.appendInsert(newTestSpillRawKVEntry(1))) + require.NoError(t, state.appendInsert(context.Background(), newTestSpillRawKVEntry(1))) return state.spill.file.Path() } diff --git a/pkg/eventservice/event_scanner.go b/pkg/eventservice/event_scanner.go index 6656e67975..f6a235e256 100644 --- a/pkg/eventservice/event_scanner.go +++ b/pkg/eventservice/event_scanner.go @@ -153,6 +153,7 @@ func (s *eventScanner) scan( dispatcherStat.info.IsOutputRawChangeEvent(), s.mode, dispatcherStat.info.EnableIgnoreUpdateOnlyColumns()) + scanCtx.processor.ctx = ctx scanCtx.processor.dispatcherStat = dispatcherStat iter, err := s.eventGetter.GetIterator(dispatcherStat.info.GetID(), request) @@ -797,6 +798,9 @@ const dmlTypeFilterCacheSize = int(common.RowTypeUpdate) + 1 // dmlProcessor handles DML event processing and batching type dmlProcessor struct { + // ctx belongs to the current scan attempt. Large-transaction spill I/O uses + // it so external encryption key lookups stop when the scan is canceled. + ctx context.Context mounter event.Mounter schemaGetter schemaGetter @@ -837,6 +841,7 @@ func newDMLProcessor( filterContext.EnableIgnoreUpdateOnlyColumns = true } return &dmlProcessor{ + ctx: context.Background(), mounter: mounter, schemaGetter: schemaGetter, filter: dmlFilter, @@ -990,7 +995,7 @@ func (p *dmlProcessor) appendRow(rawEvent *common.RawKVEntry) error { if err != nil { return err } - if err := state.appendInsert(insertRow); err != nil { + if err := state.appendInsert(p.ctx, insertRow); err != nil { return err } } else { diff --git a/pkg/eventservice/event_scanner_test.go b/pkg/eventservice/event_scanner_test.go index 0256c28e08..f13e423fbb 100644 --- a/pkg/eventservice/event_scanner_test.go +++ b/pkg/eventservice/event_scanner_test.go @@ -852,7 +852,7 @@ func TestDrainLargeTxnInsertsStopsWhenDispatcherRemoved(t *testing.T) { disp := newDispatcherStat(info, 1, 1, nil, status) state, err := disp.getOrCreateLargeTxnState(t.TempDir(), info.GetTableSpan().TableID, nil, 90, 100) require.NoError(t, err) - require.NoError(t, state.appendInsert(newTestSpillRawKVEntry(1))) + require.NoError(t, state.appendInsert(context.Background(), newTestSpillRawKVEntry(1))) disp.markLargeTxnDrainInserts(90, 100, false, 0) disp.isRemoved.Store(true) @@ -873,6 +873,30 @@ func TestDrainLargeTxnInsertsStopsWhenDispatcherRemoved(t *testing.T) { require.NoError(t, disp.cleanupLargeTxnState()) } +func TestDispatcherLargeTxnCleanupRetriesRemoveFailure(t *testing.T) { + info := newMockDispatcherInfoForTest(t) + status := newChangefeedStatusForTest(t, info) + disp := newDispatcherStat(info, 1, 1, nil, status) + state, err := disp.getOrCreateLargeTxnState( + t.TempDir(), info.GetTableSpan().TableID, nil, 90, 100) + require.NoError(t, err) + require.NoError(t, state.appendInsert(context.Background(), newTestSpillRawKVEntry(1))) + + spillPath := state.spill.file.Path() + require.NoError(t, os.Remove(spillPath)) + require.NoError(t, os.Mkdir(spillPath, 0o700)) + childPath := filepath.Join(spillPath, "child") + require.NoError(t, os.WriteFile(childPath, []byte("keep directory non-empty"), 0o600)) + + require.Error(t, disp.cleanupLargeTxnState()) + require.Same(t, state, disp.getLargeTxnState()) + require.False(t, state.cleaned) + require.NoError(t, os.Remove(childPath)) + require.NoError(t, disp.cleanupLargeTxnState()) + require.Nil(t, disp.getLargeTxnState()) + require.NoFileExists(t, spillPath) +} + func TestEventScannerRetriesLargeTxnDrainAfterOpenError(t *testing.T) { helper := event.NewEventTestHelper(t) defer helper.Close() @@ -885,7 +909,7 @@ func TestEventScannerRetriesLargeTxnDrainAfterOpenError(t *testing.T) { state, err := disp.getOrCreateLargeTxnState( t.TempDir(), info.GetTableSpan().TableID, ddlEvent.TableInfo, 90, 100) require.NoError(t, err) - require.NoError(t, state.appendInsert(newTestSpillRawKVEntry(1))) + require.NoError(t, state.appendInsert(context.Background(), newTestSpillRawKVEntry(1))) disp.markLargeTxnDrainInserts(90, 100, false, 0) spillPath := state.spill.file.Path() @@ -941,8 +965,8 @@ func TestEventScannerRetriesLargeTxnDrainAfterDecodeError(t *testing.T) { state, err := disp.getOrCreateLargeTxnState( t.TempDir(), info.GetTableSpan().TableID, ddlEvent.TableInfo, 90, 100) require.NoError(t, err) - require.NoError(t, state.appendInsert(newTestSpillRawKVEntry(1))) - require.NoError(t, state.appendInsert(newTestSpillRawKVEntry(2))) + require.NoError(t, state.appendInsert(context.Background(), newTestSpillRawKVEntry(1))) + require.NoError(t, state.appendInsert(context.Background(), newTestSpillRawKVEntry(2))) disp.markLargeTxnDrainInserts(90, 100, false, 0) dataRange := eventstore.ScanRequest{ @@ -1577,11 +1601,11 @@ func TestDMLProcessor(t *testing.T) { require.Empty(t, processor.insertRowCache) state := disp.getLargeTxnState() require.NotNil(t, state) - insertRow, err := state.nextInsert() + insertRow, err := state.nextInsert(context.Background()) require.NoError(t, err) require.Equal(t, common.OpTypePut, insertRow.OpType) require.False(t, insertRow.IsUpdate()) - insertRow, err = state.nextInsert() + insertRow, err = state.nextInsert(context.Background()) require.NoError(t, err) require.Equal(t, common.OpTypePut, insertRow.OpType) require.False(t, insertRow.IsUpdate()) @@ -1615,7 +1639,7 @@ func TestDMLProcessor(t *testing.T) { require.NoError(t, processor.appendRow(updateEvent)) require.Empty(t, processor.insertRowCache) - insertRow, err := state.nextInsert() + insertRow, err := state.nextInsert(context.Background()) require.NoError(t, err) require.Equal(t, common.OpTypePut, insertRow.OpType) require.False(t, insertRow.IsUpdate()) diff --git a/pkg/eventservice/large_txn_spill.go b/pkg/eventservice/large_txn_spill.go index 41c9b30579..d0468f45b4 100644 --- a/pkg/eventservice/large_txn_spill.go +++ b/pkg/eventservice/large_txn_spill.go @@ -94,7 +94,7 @@ func newLargeTxnInsertSpillWithEncryption( }, nil } -func (s *largeTxnInsertSpill) Append(entry *common.RawKVEntry) error { +func (s *largeTxnInsertSpill) Append(ctx context.Context, entry *common.RawKVEntry) error { if entry == nil { return errors.ErrSpillFileOp.GenWithStackByArgs("cannot append nil RawKVEntry") } @@ -102,7 +102,7 @@ func (s *largeTxnInsertSpill) Append(entry *common.RawKVEntry) error { data := entry.Encode() var err error if s.encryptionManager != nil { - data, err = s.encryptionManager.EncryptData(context.Background(), s.keyspaceID, data) + data, err = s.encryptionManager.EncryptData(ctx, s.keyspaceID, data) if err != nil { return err } @@ -147,7 +147,7 @@ type largeTxnInsertSpillReader struct { encryptionManager encryption.EncryptionManager } -func (r *largeTxnInsertSpillReader) Next() (*common.RawKVEntry, error) { +func (r *largeTxnInsertSpillReader) Next(ctx context.Context) (*common.RawKVEntry, error) { data, err := r.reader.Next() if err != nil { if errors.Is(err, io.EOF) { @@ -156,7 +156,7 @@ func (r *largeTxnInsertSpillReader) Next() (*common.RawKVEntry, error) { return nil, err } if r.encryptionManager != nil { - data, err = r.encryptionManager.DecryptData(context.Background(), r.keyspaceID, data) + data, err = r.encryptionManager.DecryptData(ctx, r.keyspaceID, data) if err != nil { return nil, err } diff --git a/pkg/eventservice/large_txn_spill_test.go b/pkg/eventservice/large_txn_spill_test.go index ef1b325cf9..0ea0919a3a 100644 --- a/pkg/eventservice/large_txn_spill_test.go +++ b/pkg/eventservice/large_txn_spill_test.go @@ -40,7 +40,7 @@ func TestLargeTxnInsertSpillReadOrder(t *testing.T) { newTestSpillRawKVEntry(3), } for _, entry := range entries { - require.NoError(t, spill.Append(entry)) + require.NoError(t, spill.Append(context.Background(), entry)) } reader, err := spill.NewReader() @@ -50,11 +50,11 @@ func TestLargeTxnInsertSpillReadOrder(t *testing.T) { }() for _, expected := range entries { - actual, err := reader.Next() + actual, err := reader.Next(context.Background()) require.NoError(t, err) require.Equal(t, expected, actual) } - actual, err := reader.Next() + actual, err := reader.Next(context.Background()) require.ErrorIs(t, err, io.EOF) require.Nil(t, actual) } @@ -75,7 +75,7 @@ func TestLargeTxnInsertSpillCreatesDir(t *testing.T) { func TestLargeTxnInsertSpillCleanup(t *testing.T) { spill, err := newLargeTxnInsertSpill(t.TempDir(), 0) require.NoError(t, err) - require.NoError(t, spill.Append(newTestSpillRawKVEntry(1))) + require.NoError(t, spill.Append(context.Background(), newTestSpillRawKVEntry(1))) path := spill.file.Path() require.NoError(t, spill.Cleanup()) @@ -102,7 +102,7 @@ func TestLargeTxnInsertSpillEmpty(t *testing.T) { require.NoError(t, reader.Close()) }() - entry, err := reader.Next() + entry, err := reader.Next(context.Background()) require.ErrorIs(t, err, io.EOF) require.Nil(t, entry) } @@ -117,7 +117,7 @@ func TestLargeTxnInsertSpillValidationErrors(t *testing.T) { defer func() { require.NoError(t, spill.Cleanup()) }() - require.True(t, errors.ErrSpillFileOp.Equal(spill.Append(nil))) + require.True(t, errors.ErrSpillFileOp.Equal(spill.Append(context.Background(), nil))) } func TestCleanupLargeTxnInsertSpillFiles(t *testing.T) { @@ -146,6 +146,26 @@ type xorEncryptionManager struct { decryptKeyspaceID uint32 } +type cancelAwareEncryptionManager struct{} + +func (*cancelAwareEncryptionManager) EncryptData( + ctx context.Context, _ uint32, data []byte, +) ([]byte, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + return data, nil +} + +func (*cancelAwareEncryptionManager) DecryptData( + ctx context.Context, _ uint32, data []byte, +) ([]byte, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + return data, nil +} + func (m *xorEncryptionManager) EncryptData( ctx context.Context, keyspaceID uint32, data []byte, ) ([]byte, error) { @@ -179,7 +199,7 @@ func TestLargeTxnInsertSpillUsesEncryptionManager(t *testing.T) { entry := newTestSpillRawKVEntry(1) encoded := entry.Encode() - require.NoError(t, spill.Append(entry)) + require.NoError(t, spill.Append(context.Background(), entry)) onDisk, err := os.ReadFile(spill.file.Path()) require.NoError(t, err) @@ -188,13 +208,31 @@ func TestLargeTxnInsertSpillUsesEncryptionManager(t *testing.T) { reader, err := spill.NewReader() require.NoError(t, err) - decoded, err := reader.Next() + decoded, err := reader.Next(context.Background()) require.NoError(t, err) require.Equal(t, entry, decoded) require.Equal(t, keyspaceID, manager.decryptKeyspaceID) require.NoError(t, reader.Close()) } +func TestLargeTxnStateUsesOperationContext(t *testing.T) { + spill, err := newLargeTxnInsertSpillWithEncryption( + t.TempDir(), 42, &cancelAwareEncryptionManager{}) + require.NoError(t, err) + state := &largeTxnScanState{spill: spill} + t.Cleanup(func() { + require.NoError(t, state.cleanup()) + }) + + canceledCtx, cancel := context.WithCancel(context.Background()) + cancel() + require.ErrorIs(t, state.appendInsert(canceledCtx, newTestSpillRawKVEntry(1)), context.Canceled) + + require.NoError(t, state.appendInsert(context.Background(), newTestSpillRawKVEntry(2))) + _, err = state.nextInsert(canceledCtx) + require.ErrorIs(t, err, context.Canceled) +} + func newTestSpillRawKVEntry(index int) *common.RawKVEntry { return &common.RawKVEntry{ OpType: common.OpTypePut, diff --git a/pkg/eventservice/large_txn_state.go b/pkg/eventservice/large_txn_state.go index 78cbb4c923..a0c834cadb 100644 --- a/pkg/eventservice/large_txn_state.go +++ b/pkg/eventservice/large_txn_state.go @@ -14,6 +14,7 @@ package eventservice import ( + "context" "io" "sync" @@ -120,16 +121,24 @@ func (a *dispatcherStat) markLargeTxnDrainInserts( func (a *dispatcherStat) cleanupLargeTxnState() error { a.largeTxnStateMu.Lock() state := a.largeTxnState - a.largeTxnState = nil a.largeTxnStateMu.Unlock() if state == nil { return nil } - return state.cleanup() + if err := state.cleanup(); err != nil { + return err + } + + a.largeTxnStateMu.Lock() + if a.largeTxnState == state { + a.largeTxnState = nil + } + a.largeTxnStateMu.Unlock() + return nil } -func (s *largeTxnScanState) appendInsert(entry *common.RawKVEntry) error { +func (s *largeTxnScanState) appendInsert(ctx context.Context, entry *common.RawKVEntry) error { s.mu.Lock() defer s.mu.Unlock() if s.cleaned { @@ -138,10 +147,10 @@ func (s *largeTxnScanState) appendInsert(entry *common.RawKVEntry) error { if s.phase != largeTxnScanPhaseOriginal { return errors.New("large txn spill is no longer accepting original txn rows") } - return s.spill.Append(entry) + return s.spill.Append(ctx, entry) } -func (s *largeTxnScanState) nextInsert() (*common.RawKVEntry, error) { +func (s *largeTxnScanState) nextInsert(ctx context.Context) (*common.RawKVEntry, error) { s.mu.Lock() defer s.mu.Unlock() if s.cleaned { @@ -153,7 +162,7 @@ func (s *largeTxnScanState) nextInsert() (*common.RawKVEntry, error) { return nil, err } for range s.drainedInsertCount { - if _, err := reader.Next(); err != nil { + if _, err := reader.Next(ctx); err != nil { _ = reader.Close() return nil, errors.WrapError( errors.ErrSpillFileOp, err, "seek committed spill rows") @@ -162,7 +171,7 @@ func (s *largeTxnScanState) nextInsert() (*common.RawKVEntry, error) { s.reader = reader } - entry, err := s.reader.Next() + entry, err := s.reader.Next(ctx) if err != nil { if errors.Is(err, io.EOF) { return nil, io.EOF @@ -218,15 +227,15 @@ func (s *largeTxnScanState) cleanup() error { if s.cleaned { return nil } - s.cleaned = true var closeErr error if s.reader != nil { closeErr = s.reader.Close() s.reader = nil } cleanupErr := s.spill.Cleanup() - if closeErr != nil { - return closeErr + if cleanupErr != nil { + return cleanupErr } - return cleanupErr + s.cleaned = true + return closeErr } diff --git a/pkg/eventservice/txn_scan_strategy.go b/pkg/eventservice/txn_scan_strategy.go index 799b779dab..fc68fcc0fa 100644 --- a/pkg/eventservice/txn_scan_strategy.go +++ b/pkg/eventservice/txn_scan_strategy.go @@ -306,6 +306,7 @@ func drainLargeTxnInserts( session.dispatcherStat.info.IsOutputRawChangeEvent(), ctx.scanner.mode, session.dispatcherStat.info.EnableIgnoreUpdateOnlyColumns()) + processor.ctx = session.ctx processor.dispatcherStat = session.dispatcherStat for { @@ -317,7 +318,7 @@ func drainLargeTxnInserts( return false, nil } - entry, err := state.nextInsert() + entry, err := state.nextInsert(session.ctx) if err != nil { if session.dispatcherStat.isRemoved.Load() { return false, nil @@ -394,7 +395,7 @@ func (p *dmlProcessor) spillCachedInsertRows() error { return err } for _, insertRow := range p.insertRowCache { - if err := state.appendInsert(insertRow); err != nil { + if err := state.appendInsert(p.ctx, insertRow); err != nil { return err } } @@ -437,7 +438,7 @@ func (p *dmlProcessor) flushSpilledInsertsIntoCurrentTxn() error { return nil } for { - insertRow, err := state.nextInsert() + insertRow, err := state.nextInsert(p.ctx) if err != nil { if errors.Is(err, io.EOF) { return p.dispatcherStat.cleanupLargeTxnState() diff --git a/pkg/spill/record_file.go b/pkg/spill/record_file.go index 42ce066488..730b3b3c01 100644 --- a/pkg/spill/record_file.go +++ b/pkg/spill/record_file.go @@ -177,22 +177,21 @@ func (s *RecordFile) Cleanup() error { if s == nil { return nil } - if err := s.Close(); err != nil { - return err - } + closeErr := s.Close() if s.cleaned { - return nil + return closeErr } - s.cleaned = true if s.path == "" { - return nil + s.cleaned = true + return closeErr } err := os.Remove(s.path) if err != nil && !os.IsNotExist(err) { return errors.WrapError(errors.ErrSpillFileOp, err, "remove spill file") } - return nil + s.cleaned = true + return closeErr } // Reader reads records sequentially from a RecordFile. diff --git a/pkg/spill/record_file_test.go b/pkg/spill/record_file_test.go index cabec15713..414c7b8908 100644 --- a/pkg/spill/record_file_test.go +++ b/pkg/spill/record_file_test.go @@ -85,3 +85,34 @@ func TestRecordFileCreatesDirAndCleansUp(t *testing.T) { _, err = os.Stat(path) require.True(t, os.IsNotExist(err)) } + +func TestRecordFileCleanupRetriesRemoveFailure(t *testing.T) { + store, err := NewRecordFile(t.TempDir(), "test-*.spill") + require.NoError(t, err) + path := store.Path() + + require.NoError(t, store.Close()) + require.NoError(t, os.Remove(path)) + require.NoError(t, os.Mkdir(path, 0o700)) + childPath := filepath.Join(path, "child") + require.NoError(t, os.WriteFile(childPath, []byte("keep directory non-empty"), 0o600)) + + require.Error(t, store.Cleanup()) + require.False(t, store.cleaned) + require.NoError(t, os.Remove(childPath)) + require.NoError(t, store.Cleanup()) + require.True(t, store.cleaned) + require.NoFileExists(t, path) +} + +func TestRecordFileCleanupUnlinksAfterCloseError(t *testing.T) { + store, err := NewRecordFile(t.TempDir(), "test-*.spill") + require.NoError(t, err) + path := store.Path() + + require.NoError(t, store.file.Close()) + require.Error(t, store.Cleanup()) + require.True(t, store.cleaned) + require.NoFileExists(t, path) + require.NoError(t, store.Cleanup()) +} From 089ba2fe0d4238f084f56be278b2a9d06054aacd Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Wed, 22 Jul 2026 10:02:36 +0800 Subject: [PATCH 28/35] eventservice: fix dispatcher cleanup lifecycle Signed-off-by: dongmen <414110582@qq.com> --- pkg/eventservice/dispatcher_stat.go | 40 ++++++++ pkg/eventservice/event_broker.go | 76 ++++++++++++--- pkg/eventservice/event_broker_test.go | 132 ++++++++++++++++++++++++++ 3 files changed, 234 insertions(+), 14 deletions(-) diff --git a/pkg/eventservice/dispatcher_stat.go b/pkg/eventservice/dispatcher_stat.go index 9ca209f88d..59268ee29f 100644 --- a/pkg/eventservice/dispatcher_stat.go +++ b/pkg/eventservice/dispatcher_stat.go @@ -14,6 +14,7 @@ package eventservice import ( + "context" "sync" "time" @@ -135,6 +136,13 @@ type dispatcherStat struct { // If so, we should wait until it is done before we send next resolvedTs event of // this dispatcher. isTaskScanning atomic.Bool + + activeScanMu sync.Mutex + activeScan *activeDispatcherScan +} + +type activeDispatcherScan struct { + cancel context.CancelFunc } func newDispatcherStat( @@ -205,6 +213,38 @@ func (a *dispatcherStat) isHandshaked() bool { return a.seq.Load() > 0 } +func (a *dispatcherStat) beginScan(parent context.Context) (context.Context, func()) { + ctx, cancel := context.WithCancel(parent) + activeScan := &activeDispatcherScan{cancel: cancel} + + a.activeScanMu.Lock() + if a.isRemoved.Load() { + cancel() + } else { + a.activeScan = activeScan + } + a.activeScanMu.Unlock() + + return ctx, func() { + cancel() + a.activeScanMu.Lock() + if a.activeScan == activeScan { + a.activeScan = nil + } + a.activeScanMu.Unlock() + } +} + +func (a *dispatcherStat) markRemoved() { + a.activeScanMu.Lock() + a.isRemoved.Store(true) + activeScan := a.activeScan + a.activeScanMu.Unlock() + if activeScan != nil { + activeScan.cancel() + } +} + func (a *dispatcherStat) setHandshaked() { a.seq.Store(1) } diff --git a/pkg/eventservice/event_broker.go b/pkg/eventservice/event_broker.go index 517738ba9e..712bbc123e 100644 --- a/pkg/eventservice/event_broker.go +++ b/pkg/eventservice/event_broker.go @@ -48,6 +48,7 @@ const ( defaultFlushResolvedTsInterval = 25 * time.Millisecond defaultReportDispatcherStatToStoreInterval = time.Second * 10 + defaultLargeTxnCleanupRetryInterval = time.Second * 10 maxReadyEventIntervalSeconds = 10 // defaultSendResolvedTsInterval use to control whether to send a resolvedTs event to the dispatcher when its scan is skipped. @@ -79,6 +80,10 @@ type eventBroker struct { // dispatcherID -> dispatcherStat map, track all table trigger dispatchers. tableTriggerDispatchers sync.Map + // dispatcherStat -> struct{}, retains removed/replaced dispatchers whose + // large transaction spill cleanup needs another attempt. + pendingLargeTxnCleanup sync.Map + // taskChan is used to send the scan tasks to the scan workers. taskChan []chan scanTask @@ -188,6 +193,10 @@ func newEventBroker( return c.refreshMinSentResolvedTs(ctx) }) + g.Go(func() error { + return c.runLargeTxnCleanupWorker(ctx, defaultLargeTxnCleanupRetryInterval) + }) + log.Info("new event broker created", zap.Uint64("id", id), zap.Uint64("scanLimitInBytes", c.scanLimitInBytes)) return c } @@ -651,6 +660,8 @@ func (c *eventBroker) doScan(ctx context.Context, task scanTask) { c.pushTask(task, false) } }() + scanCtx, finishScan := task.beginScan(ctx) + defer finishScan() var ( remoteID = node.ID(task.info.GetServerID()) @@ -733,13 +744,16 @@ func (c *eventBroker) doScan(ctx context.Context, task scanTask) { } scanner := newEventScanner(c.eventStore, c.schemaStore, c.mounter, task.info.GetMode()) - scannedBytes, events, progress, interrupted, err := scanner.scan(ctx, task, request, sl) + scannedBytes, events, progress, interrupted, err := scanner.scan(scanCtx, task, request, sl) if interrupted { metrics.EventServiceInterruptScanCount.Inc() } if err != nil { releaseQuota(available, uint64(sl.maxDMLBytes)) + if task.isRemoved.Load() { + return + } log.Error("scan events failed", zap.Stringer("changefeedID", task.changefeedStat.changefeedID), zap.Stringer("dispatcherID", task.id), zap.Int64("tableID", task.info.GetTableSpan().GetTableID()), @@ -1099,7 +1113,7 @@ func (c *eventBroker) addDispatcher(info DispatcherInfo) error { zap.Error(err), ) // Mark removed to avoid processing notifications before unregister completes. - dispatcher.isRemoved.Store(true) + dispatcher.markRemoved() c.eventStore.UnregisterDispatcher(changefeedID, id) status.removeDispatcher(id) if status.isEmpty() { @@ -1136,9 +1150,9 @@ func (c *eventBroker) removeDispatcher(dispatcherInfo DispatcherInfo) { } stat := statPtr.(*atomic.Pointer[dispatcherStat]).Load() - stat.isRemoved.Store(true) - if err := stat.cleanupLargeTxnState(); err != nil { - log.Warn("cleanup large txn state failed when removing dispatcher", + stat.markRemoved() + if err := c.cleanupLargeTxnState(stat); err != nil { + log.Warn("cleanup large txn state failed when removing dispatcher, scheduled retry", zap.Stringer("changefeedID", dispatcherInfo.GetChangefeedID()), zap.Stringer("dispatcherID", id), zap.Error(err)) @@ -1212,12 +1226,11 @@ func (c *eventBroker) resetDispatcher(dispatcherInfo DispatcherInfo) error { return nil } - // Mark the old dispatcher as removed. - // No need to worry that the old dispatcher is still scanning, - // because its data will be filtered by event collector because of stale epoch. - oldStat.isRemoved.Store(true) - if err := oldStat.cleanupLargeTxnState(); err != nil { - log.Warn("cleanup large txn state failed when resetting dispatcher", + // Mark the old dispatcher as removed and cancel its scan before cleaning up + // resources shared with that scan. + oldStat.markRemoved() + if err := c.cleanupLargeTxnState(oldStat); err != nil { + log.Warn("cleanup large txn state failed when resetting dispatcher, scheduled retry", zap.Stringer("changefeedID", dispatcherInfo.GetChangefeedID()), zap.Stringer("dispatcherID", dispatcherID), zap.Error(err)) @@ -1271,9 +1284,9 @@ func (c *eventBroker) resetDispatcher(dispatcherInfo DispatcherInfo) error { if oldStat.epoch >= dispatcherInfo.GetEpoch() { return nil } - oldStat.isRemoved.Store(true) - if err := oldStat.cleanupLargeTxnState(); err != nil { - log.Warn("cleanup large txn state failed when retrying dispatcher reset", + oldStat.markRemoved() + if err := c.cleanupLargeTxnState(oldStat); err != nil { + log.Warn("cleanup large txn state failed when retrying dispatcher reset, scheduled retry", zap.Stringer("changefeedID", changefeedID), zap.Stringer("dispatcherID", dispatcherID), zap.Error(err)) @@ -1296,6 +1309,41 @@ func (c *eventBroker) resetDispatcher(dispatcherInfo DispatcherInfo) error { return nil } +func (c *eventBroker) cleanupLargeTxnState(stat *dispatcherStat) error { + err := stat.cleanupLargeTxnState() + if err != nil { + c.pendingLargeTxnCleanup.Store(stat, struct{}{}) + return err + } + c.pendingLargeTxnCleanup.Delete(stat) + return nil +} + +func (c *eventBroker) retryPendingLargeTxnCleanup() { + c.pendingLargeTxnCleanup.Range(func(key, _ any) bool { + stat := key.(*dispatcherStat) + if err := stat.cleanupLargeTxnState(); err == nil { + c.pendingLargeTxnCleanup.Delete(stat) + } + return true + }) +} + +func (c *eventBroker) runLargeTxnCleanupWorker( + ctx context.Context, interval time.Duration, +) error { + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return context.Cause(ctx) + case <-ticker.C: + c.retryPendingLargeTxnCleanup() + } + } +} + func (c *eventBroker) getOrSetChangefeedStatus(info DispatcherInfo) *changefeedStatus { changefeedID := info.GetChangefeedID() if stat, ok := c.changefeedMap.Load(changefeedID); ok { diff --git a/pkg/eventservice/event_broker_test.go b/pkg/eventservice/event_broker_test.go index 932967d35d..33342f506d 100644 --- a/pkg/eventservice/event_broker_test.go +++ b/pkg/eventservice/event_broker_test.go @@ -17,6 +17,7 @@ import ( "context" "errors" "os" + "path/filepath" "sync" "testing" "time" @@ -767,6 +768,137 @@ func TestDispatcherLifecycleCleansLargeTxnState(t *testing.T) { }) } +type blockingEncryptionManager struct { + started chan struct{} + once sync.Once +} + +func (m *blockingEncryptionManager) EncryptData( + ctx context.Context, _ uint32, _ []byte, +) ([]byte, error) { + m.once.Do(func() { + close(m.started) + }) + <-ctx.Done() + return nil, context.Cause(ctx) +} + +func (*blockingEncryptionManager) DecryptData( + _ context.Context, _ uint32, data []byte, +) ([]byte, error) { + return data, nil +} + +func TestDispatcherLifecycleCancelsActiveScanBeforeCleanup(t *testing.T) { + for _, action := range []string{"reset", "remove"} { + t.Run(action, func(t *testing.T) { + broker, _, _, _ := newEventBrokerForTest() + defer broker.close() + + dispInfo := newMockDispatcherInfoForTest(t) + require.NoError(t, broker.addDispatcher(dispInfo)) + stat := broker.getDispatcher(dispInfo.GetID()).Load() + + manager := &blockingEncryptionManager{started: make(chan struct{})} + spill, err := newLargeTxnInsertSpillWithEncryption( + t.TempDir(), dispInfo.GetTableSpan().KeyspaceID, manager) + require.NoError(t, err) + state := &largeTxnScanState{ + startTs: 90, + commitTs: 100, + tableID: dispInfo.GetTableSpan().TableID, + spill: spill, + } + stat.largeTxnStateMu.Lock() + stat.largeTxnState = state + stat.largeTxnStateMu.Unlock() + spillPath := spill.file.Path() + + scanCtx, finishScan := stat.beginScan(context.Background()) + defer finishScan() + appendErrCh := make(chan error, 1) + go func() { + appendErrCh <- state.appendInsert(scanCtx, newTestSpillRawKVEntry(1)) + }() + + select { + case <-manager.started: + case <-time.After(5 * time.Second): + t.Fatal("spill encryption did not start") + } + + lifecycleErrCh := make(chan error, 1) + go func() { + if action == "reset" { + resetInfo := newMockDispatcherInfo( + t, 500, dispInfo.GetID(), dispInfo.GetTableSpan().TableID, + eventpb.ActionType_ACTION_TYPE_RESET) + resetInfo.epoch = stat.epoch + 1 + lifecycleErrCh <- broker.resetDispatcher(resetInfo) + return + } + broker.removeDispatcher(dispInfo) + lifecycleErrCh <- nil + }() + + select { + case err := <-lifecycleErrCh: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("dispatcher lifecycle operation did not cancel active scan") + } + require.ErrorIs(t, <-appendErrCh, context.Canceled) + require.Nil(t, stat.getLargeTxnState()) + require.NoFileExists(t, spillPath) + }) + } +} + +func TestDispatcherLifecycleRetriesFailedLargeTxnCleanup(t *testing.T) { + for _, action := range []string{"reset", "remove"} { + t.Run(action, func(t *testing.T) { + broker, _, _, _ := newEventBrokerForTest() + defer broker.close() + + dispInfo := newMockDispatcherInfoForTest(t) + require.NoError(t, broker.addDispatcher(dispInfo)) + stat := broker.getDispatcher(dispInfo.GetID()).Load() + spillPath := mustCreateLargeTxnState( + t, stat, dispInfo.GetTableSpan().TableID) + require.NoError(t, os.Remove(spillPath)) + require.NoError(t, os.Mkdir(spillPath, 0o700)) + childPath := filepath.Join(spillPath, "child") + require.NoError(t, os.WriteFile( + childPath, []byte("keep directory non-empty"), 0o600)) + t.Cleanup(func() { + _ = os.RemoveAll(spillPath) + }) + + if action == "reset" { + resetInfo := newMockDispatcherInfo( + t, 500, dispInfo.GetID(), dispInfo.GetTableSpan().TableID, + eventpb.ActionType_ACTION_TYPE_RESET) + resetInfo.epoch = stat.epoch + 1 + require.NoError(t, broker.resetDispatcher(resetInfo)) + } else { + broker.removeDispatcher(dispInfo) + } + + require.NotNil(t, stat.getLargeTxnState()) + _, pending := broker.pendingLargeTxnCleanup.Load(stat) + require.True(t, pending) + + require.NoError(t, os.Remove(childPath)) + broker.retryPendingLargeTxnCleanup() + + require.Nil(t, stat.getLargeTxnState()) + _, pending = broker.pendingLargeTxnCleanup.Load(stat) + require.False(t, pending) + require.NoFileExists(t, spillPath) + }) + } +} + func mustCreateLargeTxnState(t *testing.T, stat *dispatcherStat, tableID int64) string { t.Helper() From d1e4204fd1c33632b0c96ec8d010f29d0a9acac9 Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Wed, 22 Jul 2026 15:25:55 +0800 Subject: [PATCH 29/35] eventservice: simplify big transaction metric tracking Signed-off-by: dongmen <414110582@qq.com> --- pkg/eventservice/big_txn_metric.go | 86 ++++++++++++++++++++ pkg/eventservice/big_txn_metric_test.go | 99 ++++++++++++++++++++++++ pkg/eventservice/dispatcher_stat.go | 83 +++----------------- pkg/eventservice/dispatcher_stat_test.go | 62 --------------- pkg/eventservice/event_scanner.go | 10 ++- pkg/eventservice/event_scanner_test.go | 2 +- pkg/eventservice/metrics_collector.go | 8 -- pkg/eventservice/txn_scan_strategy.go | 4 +- 8 files changed, 205 insertions(+), 149 deletions(-) create mode 100644 pkg/eventservice/big_txn_metric.go create mode 100644 pkg/eventservice/big_txn_metric_test.go diff --git a/pkg/eventservice/big_txn_metric.go b/pkg/eventservice/big_txn_metric.go new file mode 100644 index 0000000000..a618de45a2 --- /dev/null +++ b/pkg/eventservice/big_txn_metric.go @@ -0,0 +1,86 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package eventservice + +import "github.com/pingcap/ticdc/pkg/metrics" + +type pendingBigTxnMetric struct { + startTs uint64 + commitTs uint64 + rawKVBytes int64 +} + +// bigTxnMetricTracker combines fragments from a logical transaction that spans +// multiple scan attempts, then reports that transaction exactly once. +type bigTxnMetricTracker struct { + pending *pendingBigTxnMetric +} + +func (t *bigTxnMetricTracker) addFragment( + startTs, commitTs uint64, + rawKVBytes, largeTxnThresholdInBytes int64, +) { + t.flushBefore(startTs, commitTs) + if rawKVBytes <= largeTxnThresholdInBytes { + return + } + if t.pending == nil { + t.pending = &pendingBigTxnMetric{ + startTs: startTs, + commitTs: commitTs, + } + } + t.pending.rawKVBytes += rawKVBytes +} + +func (t *bigTxnMetricTracker) finishTxn( + startTs, commitTs uint64, + rawKVBytes, largeTxnThresholdInBytes int64, +) { + t.flushBefore(startTs, commitTs) + if t.pending != nil { + rawKVBytes += t.pending.rawKVBytes + t.pending = nil + } + if rawKVBytes > largeTxnThresholdInBytes { + observeBigTxnMetric(rawKVBytes) + } +} + +func (t *bigTxnMetricTracker) flushBefore(startTs, commitTs uint64) { + if t.pending == nil || + (t.pending.startTs == startTs && t.pending.commitTs == commitTs) { + return + } + t.flush() +} + +func (t *bigTxnMetricTracker) flush() { + if t.pending == nil { + return + } + // A pending sample is created only after one fragment exceeds the threshold, + // so it remains a big transaction without storing or rechecking the threshold. + rawKVBytes := t.pending.rawKVBytes + t.pending = nil + observeBigTxnMetric(rawKVBytes) +} + +func observeBigTxnMetric(rawKVBytes int64) { + if rawKVBytes <= 0 { + return + } + metrics.EventServiceBigTxnSize.Observe(float64(rawKVBytes)) + metrics.EventServiceBigTxnCount.Inc() +} diff --git a/pkg/eventservice/big_txn_metric_test.go b/pkg/eventservice/big_txn_metric_test.go new file mode 100644 index 0000000000..5beb6531c0 --- /dev/null +++ b/pkg/eventservice/big_txn_metric_test.go @@ -0,0 +1,99 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package eventservice + +import ( + "testing" + + "github.com/pingcap/ticdc/pkg/metrics" + dto "github.com/prometheus/client_model/go" + "github.com/stretchr/testify/require" +) + +func TestBigTxnMetricTracker(t *testing.T) { + t.Run("count split txn once", func(t *testing.T) { + beforeHistogramCount, beforeHistogramSum := readBigTxnSizeMetric(t) + beforeCounter := readBigTxnCountMetric(t) + + tracker := bigTxnMetricTracker{} + tracker.addFragment(100, 200, 70, 50) + + histogramCount, histogramSum := readBigTxnSizeMetric(t) + require.Equal(t, beforeHistogramCount, histogramCount) + require.Equal(t, beforeHistogramSum, histogramSum) + require.Equal(t, beforeCounter, readBigTxnCountMetric(t)) + + tracker.finishTxn(100, 200, 30, 50) + + histogramCount, histogramSum = readBigTxnSizeMetric(t) + require.Equal(t, beforeHistogramCount+1, histogramCount) + require.Equal(t, beforeHistogramSum+100, histogramSum) + require.Equal(t, beforeCounter+1, readBigTxnCountMetric(t)) + require.Nil(t, tracker.pending) + }) + + t.Run("flush previous txn", func(t *testing.T) { + beforeHistogramCount, beforeHistogramSum := readBigTxnSizeMetric(t) + beforeCounter := readBigTxnCountMetric(t) + + tracker := bigTxnMetricTracker{} + tracker.addFragment(100, 200, 70, 50) + tracker.addFragment(101, 201, 80, 50) + + histogramCount, histogramSum := readBigTxnSizeMetric(t) + require.Equal(t, beforeHistogramCount+1, histogramCount) + require.Equal(t, beforeHistogramSum+70, histogramSum) + require.Equal(t, beforeCounter+1, readBigTxnCountMetric(t)) + require.Equal(t, &pendingBigTxnMetric{ + startTs: 101, + commitTs: 201, + rawKVBytes: 80, + }, tracker.pending) + }) + + t.Run("flush at scan end", func(t *testing.T) { + beforeHistogramCount, beforeHistogramSum := readBigTxnSizeMetric(t) + beforeCounter := readBigTxnCountMetric(t) + + tracker := bigTxnMetricTracker{} + tracker.addFragment(100, 200, 70, 50) + tracker.flush() + + histogramCount, histogramSum := readBigTxnSizeMetric(t) + require.Equal(t, beforeHistogramCount+1, histogramCount) + require.Equal(t, beforeHistogramSum+70, histogramSum) + require.Equal(t, beforeCounter+1, readBigTxnCountMetric(t)) + require.Nil(t, tracker.pending) + }) +} + +func readBigTxnSizeMetric(t *testing.T) (uint64, float64) { + t.Helper() + + metric := &dto.Metric{} + require.NoError(t, metrics.EventServiceBigTxnSize.Write(metric)) + histogram := metric.GetHistogram() + require.NotNil(t, histogram) + return histogram.GetSampleCount(), histogram.GetSampleSum() +} + +func readBigTxnCountMetric(t *testing.T) float64 { + t.Helper() + + metric := &dto.Metric{} + require.NoError(t, metrics.EventServiceBigTxnCount.Write(metric)) + counter := metric.GetCounter() + require.NotNil(t, counter) + return counter.GetValue() +} diff --git a/pkg/eventservice/dispatcher_stat.go b/pkg/eventservice/dispatcher_stat.go index 59268ee29f..ffad7257d1 100644 --- a/pkg/eventservice/dispatcher_stat.go +++ b/pkg/eventservice/dispatcher_stat.go @@ -106,15 +106,18 @@ type dispatcherStat struct { // Note: Please don't changed this value directly, use updateSentResolvedTs instead. sentResolvedTs atomic.Uint64 - // lastScanProgress is published as one immutable value so the next scan - // cannot observe a cursor assembled from different scan results. + // lastScanProgress is the resume point produced by the previous scan. It keeps + // the transaction cursor and optional row-level cursor in one immutable snapshot, + // so the next scan cannot combine fields from different scan attempts and resume + // from an invalid position. lastScanProgress atomic.Pointer[scanProgress] largeTxnStateMu sync.Mutex largeTxnState *largeTxnScanState - // bigTxnMetricState is updated only by the serialized scan task of this dispatcher. - bigTxnMetricState *bigTxnMetricState + // bigTxnMetrics aggregates one sample per logical transaction across row-level + // scan interruptions. It is updated only by this dispatcher's serialized scan task. + bigTxnMetrics bigTxnMetricTracker // isRemoved is used to indicate whether the dispatcher is removed. // it is set to true in the following two cases: @@ -137,6 +140,10 @@ type dispatcherStat struct { // this dispatcher. isTaskScanning atomic.Bool + // activeScanMu protects activeScan and serializes scan registration with + // markRemoved. activeScan lets reset/remove cancel the in-flight scan before + // cleaning up its large-transaction state, including interrupting TiKV/KMS + // calls made by spill encryption. activeScanMu sync.Mutex activeScan *activeDispatcherScan } @@ -282,74 +289,6 @@ func (a *dispatcherStat) loadScanProgress() scanProgress { return result } -type bigTxnMetricState struct { - startTs uint64 - commitTs uint64 - rawKVBytes int64 - largeTxnThresholdInBytes int64 -} - -func (a *dispatcherStat) addBigTxnMetricFragment( - startTs uint64, - commitTs uint64, - rawKVBytes int64, - largeTxnThresholdInBytes int64, -) { - if rawKVBytes <= largeTxnThresholdInBytes { - return - } - if a.bigTxnMetricState == nil || - a.bigTxnMetricState.startTs != startTs || - a.bigTxnMetricState.commitTs != commitTs { - a.finishPendingBigTxnMetricBefore(startTs, commitTs) - a.bigTxnMetricState = &bigTxnMetricState{ - startTs: startTs, - commitTs: commitTs, - largeTxnThresholdInBytes: largeTxnThresholdInBytes, - } - } - a.bigTxnMetricState.rawKVBytes += rawKVBytes -} - -func (a *dispatcherStat) finishBigTxnMetric( - startTs uint64, - commitTs uint64, - rawKVBytes int64, - largeTxnThresholdInBytes int64, -) { - totalRawKVBytes := rawKVBytes - if a.bigTxnMetricState != nil { - if a.bigTxnMetricState.startTs == startTs && a.bigTxnMetricState.commitTs == commitTs { - totalRawKVBytes += a.bigTxnMetricState.rawKVBytes - } else if a.bigTxnMetricState.rawKVBytes > a.bigTxnMetricState.largeTxnThresholdInBytes { - updateMetricEventServiceBigTxn(a.bigTxnMetricState.rawKVBytes) - } - a.bigTxnMetricState = nil - } - if totalRawKVBytes > largeTxnThresholdInBytes { - updateMetricEventServiceBigTxn(totalRawKVBytes) - } -} - -func (a *dispatcherStat) finishPendingBigTxnMetric() { - if a.bigTxnMetricState == nil { - return - } - state := a.bigTxnMetricState - a.bigTxnMetricState = nil - if state.rawKVBytes > state.largeTxnThresholdInBytes { - updateMetricEventServiceBigTxn(state.rawKVBytes) - } -} - -func (a *dispatcherStat) finishPendingBigTxnMetricBefore(startTs uint64, commitTs uint64) { - if a.bigTxnMetricState == nil || - (a.bigTxnMetricState.startTs == startTs && a.bigTxnMetricState.commitTs == commitTs) { - return - } - a.finishPendingBigTxnMetric() -} - // onResolvedTs try to update the resolved ts of the dispatcher. func (a *dispatcherStat) onResolvedTs(resolvedTs uint64) bool { if resolvedTs <= a.receivedResolvedTs.Load() { diff --git a/pkg/eventservice/dispatcher_stat_test.go b/pkg/eventservice/dispatcher_stat_test.go index 00755d4ee6..c350dc7cb6 100644 --- a/pkg/eventservice/dispatcher_stat_test.go +++ b/pkg/eventservice/dispatcher_stat_test.go @@ -22,8 +22,6 @@ import ( "github.com/pingcap/ticdc/logservice/eventstore" "github.com/pingcap/ticdc/pkg/common" pevent "github.com/pingcap/ticdc/pkg/common/event" - "github.com/pingcap/ticdc/pkg/metrics" - dto "github.com/prometheus/client_model/go" "github.com/stretchr/testify/require" "github.com/tikv/client-go/v2/oracle" ) @@ -80,66 +78,6 @@ func TestDispatcherStatResolvedTs(t *testing.T) { require.False(t, updated) } -func TestDispatcherStatBigTxnMetricsCountSplitTxnOnce(t *testing.T) { - beforeHistogramCount, beforeHistogramSum := readBigTxnSizeMetric(t) - beforeCounter := readBigTxnCountMetric(t) - - stat := &dispatcherStat{} - stat.addBigTxnMetricFragment(100, 200, 70, 50) - - histogramCount, histogramSum := readBigTxnSizeMetric(t) - require.Equal(t, beforeHistogramCount, histogramCount) - require.Equal(t, beforeHistogramSum, histogramSum) - require.Equal(t, beforeCounter, readBigTxnCountMetric(t)) - - stat.finishBigTxnMetric(100, 200, 30, 50) - - histogramCount, histogramSum = readBigTxnSizeMetric(t) - require.Equal(t, beforeHistogramCount+1, histogramCount) - require.Equal(t, beforeHistogramSum+100, histogramSum) - require.Equal(t, beforeCounter+1, readBigTxnCountMetric(t)) -} - -func TestDispatcherStatBigTxnMetricsFlushPreviousTxn(t *testing.T) { - beforeHistogramCount, beforeHistogramSum := readBigTxnSizeMetric(t) - beforeCounter := readBigTxnCountMetric(t) - - stat := &dispatcherStat{} - stat.addBigTxnMetricFragment(100, 200, 70, 50) - stat.addBigTxnMetricFragment(101, 201, 80, 50) - - histogramCount, histogramSum := readBigTxnSizeMetric(t) - require.Equal(t, beforeHistogramCount+1, histogramCount) - require.Equal(t, beforeHistogramSum+70, histogramSum) - require.Equal(t, beforeCounter+1, readBigTxnCountMetric(t)) - require.Equal(t, &bigTxnMetricState{ - startTs: 101, - commitTs: 201, - rawKVBytes: 80, - largeTxnThresholdInBytes: 50, - }, stat.bigTxnMetricState) -} - -func readBigTxnSizeMetric(t *testing.T) (uint64, float64) { - t.Helper() - - metric := &dto.Metric{} - require.NoError(t, metrics.EventServiceBigTxnSize.Write(metric)) - histogram := metric.GetHistogram() - require.NotNil(t, histogram) - return histogram.GetSampleCount(), histogram.GetSampleSum() -} - -func readBigTxnCountMetric(t *testing.T) float64 { - t.Helper() - - metric := &dto.Metric{} - require.NoError(t, metrics.EventServiceBigTxnCount.Write(metric)) - counter := metric.GetCounter() - require.NotNil(t, counter) - return counter.GetValue() -} - func TestDispatcherStatGetScanRequest(t *testing.T) { t.Parallel() diff --git a/pkg/eventservice/event_scanner.go b/pkg/eventservice/event_scanner.go index f6a235e256..04f37ee5ef 100644 --- a/pkg/eventservice/event_scanner.go +++ b/pkg/eventservice/event_scanner.go @@ -258,7 +258,7 @@ func (s *eventScanner) scanAndMergeEvents( err = finalizeScan(merger, processor, session, session.dataRange.CommitTsEnd) return false, err } - dispatcher.finishPendingBigTxnMetricBefore(rawEvent.StartTs, rawEvent.CRTs) + dispatcher.bigTxnMetrics.flushBefore(rawEvent.StartTs, rawEvent.CRTs) if processor.currentTxn == nil { interrupted, err := strategy.finishTxn(scanCtx, nextTxnMeta{ @@ -403,7 +403,8 @@ func (s *eventScanner) commitTxn( return err } if hasCurrentTxn { - session.dispatcherStat.finishBigTxnMetric(startTs, commitTs, rawKVBytes, largeTxnThresholdInBytes) + session.dispatcherStat.bigTxnMetrics.finishTxn( + startTs, commitTs, rawKVBytes, largeTxnThresholdInBytes) } currentBatchDML := processor.getCurrentBatchDML() @@ -452,9 +453,10 @@ func finalizeScan( return err } if hasCurrentTxn { - sess.dispatcherStat.finishBigTxnMetric(startTs, commitTs, rawKVBytes, largeTxnThresholdInBytes) + sess.dispatcherStat.bigTxnMetrics.finishTxn( + startTs, commitTs, rawKVBytes, largeTxnThresholdInBytes) } else { - sess.dispatcherStat.finishPendingBigTxnMetric() + sess.dispatcherStat.bigTxnMetrics.flush() } resolvedBatch := processor.getCurrentBatchDML() diff --git a/pkg/eventservice/event_scanner_test.go b/pkg/eventservice/event_scanner_test.go index f13e423fbb..7135b210a5 100644 --- a/pkg/eventservice/event_scanner_test.go +++ b/pkg/eventservice/event_scanner_test.go @@ -537,7 +537,7 @@ func TestEventScannerSplitsLargeTxnWithRowLevelProgress(t *testing.T) { resolved, ok := events[0].(event.ResolvedEvent) require.True(t, ok) require.Equal(t, resolvedTs, resolved.ResolvedTs) - require.Nil(t, disp.bigTxnMetricState) + require.Nil(t, disp.bigTxnMetrics.pending) } func TestEventScannerDoesNotSplitCurrentTxnBelowLargeTxnThreshold(t *testing.T) { diff --git a/pkg/eventservice/metrics_collector.go b/pkg/eventservice/metrics_collector.go index 61cb4d2fff..0952ed9260 100644 --- a/pkg/eventservice/metrics_collector.go +++ b/pkg/eventservice/metrics_collector.go @@ -106,14 +106,6 @@ func updateMetricEventServiceSendDMLTypeCount(mode int64, rawType string, update metrics.EventServiceSendDMLTypeCount.WithLabelValues(common.StringMode(mode), rawType).Inc() } -func updateMetricEventServiceBigTxn(rawKVBytes int64) { - if rawKVBytes <= 0 { - return - } - metrics.EventServiceBigTxnSize.Observe(float64(rawKVBytes)) - metrics.EventServiceBigTxnCount.Inc() -} - // dispatcherHeapItem wraps dispatcherStat to implement heap.Item interface. // The heap maintains the slowest dispatchers by checkpointTs. // The heap top is the fastest (largest checkpointTs) among the slowest ones. diff --git a/pkg/eventservice/txn_scan_strategy.go b/pkg/eventservice/txn_scan_strategy.go index fc68fcc0fa..c34269a132 100644 --- a/pkg/eventservice/txn_scan_strategy.go +++ b/pkg/eventservice/txn_scan_strategy.go @@ -152,7 +152,7 @@ func finishPendingSplitTxn(session *session, next nextTxnMeta) bool { return false } - dispatcher.finishPendingBigTxnMetric() + dispatcher.bigTxnMetrics.flush() dispatcher.markLargeTxnDrainInserts(state.startTs, state.commitTs, next.commitTs != 0, next.commitTs) session.progress = newTxnScanProgress(state.commitTs, state.startTs) return true @@ -267,7 +267,7 @@ func interruptCurrentTxn( if processor.currentTxn != nil { currentTxn := processor.currentTxn currentDML := currentTxn.CurrentDMLEvent - ctx.session.dispatcherStat.addBigTxnMetricFragment( + ctx.session.dispatcherStat.bigTxnMetrics.addFragment( currentDML.GetStartTs(), currentDML.GetCommitTs(), currentTxn.rawKVBytes, From 98c9124def1f53461fe031514356c91708a1ed22 Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Wed, 22 Jul 2026 16:04:49 +0800 Subject: [PATCH 30/35] eventservice: clarify row cursor resume range Signed-off-by: dongmen <414110582@qq.com> --- pkg/eventservice/event_broker.go | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/pkg/eventservice/event_broker.go b/pkg/eventservice/event_broker.go index 712bbc123e..b996871fce 100644 --- a/pkg/eventservice/event_broker.go +++ b/pkg/eventservice/event_broker.go @@ -491,21 +491,22 @@ func (c *eventBroker) getScanTaskRequest(task scanTask) (bool, eventstore.ScanRe } } - // A row cursor resumes inside the transaction at CommitTsStart. The adaptive - // window may shrink after that cursor was published, but it must not move the - // effective upper bound behind the cursor's commit ts. - if len(request.Cursor.Position) != 0 && - dataRange.CommitTsEnd < dataRange.CommitTsStart && - commitTsEndBeforeWindow >= dataRange.CommitTsStart { + hasRowResume := len(request.Cursor.Position) != 0 + // A published row cursor at C came from an earlier scan whose DDL and received + // resolved-ts bounds had already reached C. Since those bounds do not regress, + // only the adaptive scan window can move CommitTsEnd behind C. For example, if + // C=100 and the window caps the end at 80, restore the effective range to + // [100, 100] so scanning resumes after Position inside that transaction. + if hasRowResume && dataRange.CommitTsEnd < dataRange.CommitTsStart { dataRange.CommitTsEnd = dataRange.CommitTsStart } if dataRange.CommitTsEnd <= dataRange.CommitTsStart { - hasSameCommitTxnResume := request.Cursor.TxnStartTs != 0 && - dataRange.CommitTsEnd == dataRange.CommitTsStart - if len(request.Cursor.Position) != 0 || - hasSameCommitTxnResume || - task.hasPendingLargeTxnState() { + // A cursor makes [C, C] meaningful: Position resumes rows inside a + // transaction, while TxnStartTs resumes later transactions at the same C. + canResumeAtStart := dataRange.CommitTsEnd == dataRange.CommitTsStart && + (hasRowResume || request.Cursor.TxnStartTs != 0) + if canResumeAtStart || task.hasPendingLargeTxnState() { return true, request } updateMetricEventServiceSkipResolvedTsCount(task.info.GetMode()) From 376646f4198eddd2d92205d6168e385bc3ec314e Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Fri, 24 Jul 2026 17:47:07 +0800 Subject: [PATCH 31/35] eventservice: clarify large transaction scan flow Document scan progress and transaction scan strategies, and simplify the spill insert drain control flow without changing its behavior. Signed-off-by: dongmen <414110582@qq.com> --- ...tservice-scan-progress-and-txn-strategy.md | 428 ++++++++++++++++++ pkg/eventservice/scan_progress.go | 43 +- pkg/eventservice/txn_scan_strategy.go | 218 +++++++-- 3 files changed, 636 insertions(+), 53 deletions(-) create mode 100644 docs/design/2026-07-22-eventservice-scan-progress-and-txn-strategy.md diff --git a/docs/design/2026-07-22-eventservice-scan-progress-and-txn-strategy.md b/docs/design/2026-07-22-eventservice-scan-progress-and-txn-strategy.md new file mode 100644 index 0000000000..ff0ce3acf8 --- /dev/null +++ b/docs/design/2026-07-22-eventservice-scan-progress-and-txn-strategy.md @@ -0,0 +1,428 @@ +# EventService Scan Progress and Transaction Scan Strategies + +This document explains the roles of the following two files in the TiCDC +new-architecture EventService and how they work together: + +- [`pkg/eventservice/scan_progress.go`](../../pkg/eventservice/scan_progress.go) +- [`pkg/eventservice/txn_scan_strategy.go`](../../pkg/eventservice/txn_scan_strategy.go) + +Related code includes: + +- [`pkg/eventservice/event_scanner.go`](../../pkg/eventservice/event_scanner.go): + the common scan loop, DDL/DML merge, and DML decoding. +- [`pkg/eventservice/dispatcher_stat.go`](../../pkg/eventservice/dispatcher_stat.go): + stores the next scan position of a dispatcher. +- [`pkg/eventservice/event_broker.go`](../../pkg/eventservice/event_broker.go): + creates scan requests, sends scan results, and publishes new progress. +- [`logservice/eventstore/scan_request.go`](../../logservice/eventstore/scan_request.go): + defines EventStore scan ranges and resume cursors. +- [`pkg/eventservice/large_txn_state.go`](../../pkg/eventservice/large_txn_state.go): + stores large-transaction state across scan attempts. + +## 1. Two conclusions to remember + +`scanProgress` answers this question: + +> After this scan attempt finishes, where should the next EventStore read +> resume? + +It is neither a downstream checkpoint nor merely a resolved-ts. A scan may stop +at the end of a timestamp range, between two transactions with the same +commit-ts, or even between two rows of the same transaction. These cases need +resume positions with different levels of precision. + +`txnScanStrategy` answers a different question: + +> While scanning a transaction, may the scanner stop inside it, and how should +> the scanner finish that transaction after such an interruption? + +It contains only the differences between atomic and split modes. Common logic, +including EventStore iteration, SchemaStore access, DDL/DML ordering, and DML +decoding, remains in `eventScanner`. + +## 2. Where they sit in the scan pipeline + +```mermaid +sequenceDiagram + participant D as dispatcherStat + participant B as eventBroker + participant S as eventScanner + participant T as txnScanStrategy + participant E as EventStore + + B->>D: getScanRequest() + D-->>B: Range + Cursor + B->>B: Apply the DDL upper bound and scan window + B->>S: scan(request, limit) + S->>T: resumePending() + alt No large transaction is waiting to drain + S->>E: GetIterator(request) + loop For each RawKVEntry + S->>T: finishTxn(next) / startTxn(...) + S->>S: appendRow(rawEvent) + S->>T: afterAppend(rawEvent, position) + end + end + S-->>B: events + scanProgress + interrupted + B->>B: Hand events to the send pipeline + B->>D: Publish valid scanProgress + opt interrupted == true + B->>B: Reschedule this dispatcher immediately + end +``` + +Three objects in this pipeline are easy to confuse: + +| Object | Meaning | +| --- | --- | +| `ScanRequest.Range` | The commit-ts range that this attempt may scan. | +| `ScanRequest.Cursor` | The starting point inside `Range.CommitTsStart`. | +| `scanProgress` | The resume position produced by this attempt for the next attempt. | + +In short, `ScanRequest` is the input and `scanProgress` is the output. Before +the next scan, the dispatcher converts the previous `scanProgress` into a new +`ScanRequest`. + +## 3. What `scanProgress` represents + +The structure has four fields: + +```go +type scanProgress struct { + valid bool + txnCommitTs uint64 + txnStartTs uint64 + rowLevelScanPosition eventstore.ScanPosition +} +``` + +### 3.1 Field semantics + +| Field | Meaning | +| --- | --- | +| `valid` | Whether this scan produced a complete new resume position that is safe to publish. | +| `txnCommitTs` | The commit-ts containing the resume point and the next `Range.CommitTsStart`. | +| `txnStartTs` | The transaction already processed within the same commit-ts. | +| `rowLevelScanPosition` | The EventStore record after which the scan should resume inside that transaction. | + +The name `txnCommitTs` can be slightly misleading. When `txnStartTs == 0` and +there is no row position, it may denote a fully scanned timestamp boundary +rather than an unfinished transaction. + +### 3.2 Three valid progress forms + +Let `C` be a commit-ts, `S` a start-ts, and `P` a row position returned by +EventStore. + +| Progress | Meaning | How the next attempt scans | +| --- | --- | --- | +| `(C, 0, nil)` | Everything through `C` has been fully processed. | Perform a normal scan over `(C, End]`. | +| `(C, S, nil)` | Original EventStore records at commit-ts `C` with start-ts no greater than `S` have been processed; spilled inserts may still need draining. | Continue with transactions at commit-ts `C` whose start-ts is greater than `S`, then scan later commit-ts values. | +| `(C, S, P)` | Transaction `(S, C)` is in progress and records through position `P` have been processed. | Resume after `P` in EventStore. | + +When `Position` is non-empty, EventStore uses it as the exact lower bound. It +takes precedence over `TxnStartTs`. + +That precedence is necessary because `(C, S)` alone cannot resume inside the +same transaction. For example, transaction `(S=10, C=100)` has three records, +`r1`, `r2`, and `r3`: + +```text +Store only (100, 10, nil) -> EventStore treats the transaction as done and skips r2 and r3 +Store (100, 10, position1) -> EventStore resumes after r1 and returns r2 and r3 +``` + +### 3.3 How `scanProgress` becomes the next request + +`dispatcherStat.getScanRequest()` performs this mapping: + +```text +scanProgress.txnCommitTs -> Range.CommitTsStart +dispatcher.receivedResolvedTs -> Range.CommitTsEnd +scanProgress.txnStartTs -> Cursor.TxnStartTs +scanProgress.rowLevelScanPosition -> Cursor.Position +``` + +`eventBroker.getScanTaskRequest()` then restricts `CommitTsEnd` using the +SchemaStore resolved-ts and the adaptive scan window. + +A row cursor has an additional invariant. A published row cursor at `C` proves +that the previous request had a legal upper bound covering `C`. The DDL +resolved-ts and received resolved-ts do not move backward, so only a later +shrink of the adaptive scan window can move `CommitTsEnd` behind `C`. In that +case, the broker restores the range to `[C, C]`. EventStore can then use +`Position` to return the remaining rows from the transaction at `C`. + +### 3.4 Why `valid` is needed + +The zero value of `session.progress` is not a valid resume point. A scan may +return after context cancellation, dispatcher removal, or another interruption +that did not produce a complete cursor. `eventBroker.doScan()` replaces the +dispatcher progress only when `valid == true`. + +Every value created by `newTxnScanProgress` or `newRowLevelScanProgress` has +`valid=true`. Typical sources are: + +- A complete scan through `CommitTsEnd` publishes `(End, 0, nil)`. +- An interruption after a row in a large transaction publishes + `(C, S, Position)`. +- Completion of the original rows before insert draining publishes + `(C, S, nil)`. +- Partial draining of spilled inserts continues to publish `(C, S, nil)`; the + large-transaction state stores the drain-specific position. + +An ordinary interruption at a transaction boundary may have no explicit +`scanProgress`. DML already emitted through the broker send path updates the +dispatcher to `(lastCommitTs, lastStartTs, nil)`. + +Row-level fragments are different. While sending a DML fragment, the send path +can see only transaction-level progress `(C, S)`. Therefore, after processing +the result events, `doScan()` must overwrite that value with the scanner's +`(C, S, Position)`. Otherwise the next attempt would treat an unfinished +transaction as complete. + +### 3.5 Why progress is an immutable snapshot + +The dispatcher publishes one complete progress value through an +`atomic.Pointer[scanProgress]` instead of publishing three fields separately. + +Separate updates could let the next scan observe a mixed state, such as a new +`txnCommitTs` combined with an old `txnStartTs` and old `Position`. Such a +combination describes no real scan result and may duplicate or skip data. + +The underlying type of `ScanPosition` is `[]byte`, so code also clones the +slice when storing and reading a snapshot. This prevents callers from mutating +an already published cursor. + +## 4. Why `txnScanStrategy` exists + +The main `eventScanner` loop is nearly identical in both transaction modes: + +1. Read the next RawKV from EventStore. +2. Detect transaction boundaries from commit-ts and start-ts. +3. Fetch the appropriate version of TableInfo. +4. Decode and merge DML and DDL events. +5. Check scan limits, context cancellation, and dispatcher lifecycle state. + +Only four transaction-lifecycle points differ, so the interface has four +methods: + +| Method | When it is called | Atomic mode | Split mode | +| --- | --- | --- | --- | +| `resumePending` | At the beginning of every scan, before creating an EventStore iterator. | No operation. | If the previous attempt entered the drain phase, emit more inserts from the spill file first. | +| `startTxn` | When a new transaction begins. | Create a `TxnEvent` that cannot be split across scans. | Create a `TxnEvent` that may be split across scans. | +| `finishTxn` | At the next transaction or iterator EOF. | Commit the complete current transaction. | Commit the current fragment and handle the original-to-drain phase transition. | +| `afterAppend` | After appending each RawKV. | No operation; the scanner cannot stop inside the transaction. | After the large-transaction threshold is exceeded and a row position exists, the scanner may stop after the current row. | + +The configuration selects the strategy: + +```go +newTxnScanStrategy(dispatcherStat.txnAtomicity.ShouldSplitTxn()) +``` + +- `transaction-atomicity=table` selects the atomic strategy. +- `transaction-atomicity=none` selects the split strategy. + +Here, atomic means only that the upstream transaction observed by one table +dispatcher is not divided across multiple scan results. The scanner may still +stop between transactions, and this option does not provide whole-transaction +atomicity across multiple tables. + +## 5. Atomic strategy flow + +The atomic strategy is intentionally thin because it enforces only one rule: +the scanner cannot stop inside a transaction. + +```text +startTxn + -> Append all RawKV records with the same (startTs, commitTs) to the current TxnEvent + -> afterAppend never requests an interruption + -> Reach the next transaction or EOF + -> finishTxn commits the whole transaction +``` + +Even after the scan byte limit is reached, the common loop waits for a +transaction boundary before stopping. Therefore, one very large transaction +may make a scan exceed its normal limit. This is the cost of preserving table +transaction atomicity. + +## 6. Split strategy flow + +The split strategy may emit a large transaction over several scan attempts. It +must still resume precisely and correctly handle updates that change a unique +key. + +### 6.1 Row-level resume for an ordinary large transaction + +Assume transaction `(S=10, C=100)` has three EventStore records and the +threshold allows one record per attempt: + +```text +Attempt 1: read r1 -> emit fragment 1 -> progress=(100, 10, P1) +Attempt 2: resume after P1, read r2 -> emit fragment 2 -> progress=(100, 10, P2) +Attempt 3: resume after P2, read r3 -> finish transaction -> progress=(100, 0, nil) +``` + +An interruption inside a transaction requires all of the following: + +1. The EventStore iterator provides a non-empty `Position`. +2. Raw KV bytes in the current fragment exceed the large-transaction threshold. +3. No insert half of a unique-key-changing update remains only in memory. +4. The DML/DDL merger allows an interruption at this commit-ts. + +`DMLEventMaxRows` and `DMLEventMaxBytes` are different from the +large-transaction threshold. The former values create multiple `DMLEvent` +objects inside one `BatchDMLEvent`. Only the latter allows the transaction to +be interrupted across scan attempts and publishes a row cursor. + +### 6.2 Why a unique-key-changing update must spill + +An update that changes a unique key becomes one delete and one insert. If a +large transaction is divided into fragments, emitting the insert too early may +cause a downstream unique-key conflict before other deletes in the same +transaction have executed. + +The split strategy therefore uses two phases: + +```mermaid +stateDiagram-v2 + [*] --> NoState + NoState --> Original: Large transaction contains a UK-changing update + Original --> Original: Emit original rows/delete fragments
write insert halves to spill + Original --> DrainInserts: All original rows are known to be read + DrainInserts --> DrainInserts: Read and emit spilled inserts in batches + DrainInserts --> NoState: Drain completes and spill is cleaned + Original --> NoState: DDL exists at the same commit-ts
merge inserts back and clean spill +``` + +During `largeTxnScanPhaseOriginal`: + +- The scanner reads original EventStore data. +- The delete half of a unique-key-changing update enters the normal DML fragment. +- The corresponding insert half is written to a spill file. +- A row cursor records how far the original EventStore scan has advanced. + +During `largeTxnScanPhaseDrainInserts`: + +- The scanner no longer creates an EventStore iterator. +- `resumePending` reads directly from the spill file at the beginning of a scan. +- Inserts may be emitted in batches according to the scan limit. +- Completion closes the reader, removes the spill file, and clears the + dispatcher's large-transaction state. + +This guarantees that all delete halves of the same large transaction precede +the delayed insert halves. Draining also completes before any following +transaction, including another transaction with the same commit-ts. + +### 6.3 Why `finishTxn` may run when `currentTxn == nil` + +At the beginning of a scan after row-level resume, +`dmlProcessor.currentTxn` is still nil, but the dispatcher may retain a +`largeTxnScanPhaseOriginal` state. + +The strategy must first inspect the next record returned by the iterator: + +- If it still belongs to the same `(S, C)`, original rows remain and scanning + continues. +- If it belongs to another transaction, or the iterator is at EOF, the + original transaction is complete and the strategy enters the drain phase. + +Consequently, `finishTxn` means more than "commit `currentTxn`". It also +confirms whether a large transaction retained across scan attempts has reached +the end of its original data. That is why the common loop calls it even when +`currentTxn == nil`. + +### 6.4 Why the drain phase does not use a row position + +After entering the drain phase, the data source is the spill file rather than +EventStore: + +- `scanProgress` stays at `(C, S, nil)`, preventing EventStore from moving + beyond the large transaction. +- `largeTxnScanState.drainedInsertCount` records how many inserts are included + in completed drain scan results. +- If a drain attempt fails, the reader rolls back to the committed + `drainedInsertCount` and retries from there next time. + +The system therefore has two cursors at different storage layers: + +| Cursor | Layer | Purpose | +| --- | --- | --- | +| `scanProgress` | EventStore scan layer | Prevent the original-data scan from passing transaction `(S, C)`. | +| `drainedInsertCount` | Spill drain layer | Resume insert output inside the spill file. | + +Putting the spill offset into `scanProgress` would mix the cursor semantics of +two storage layers, so the two values remain separate. + +### 6.5 Special handling for a DDL at the same commit-ts + +TiCDC must order DML before DDL at the same commit-ts and must not divide an +unsafe same-commit-ts DML/DDL group. + +If a large transaction with spilled inserts also has a DDL at `C`, the split +strategy does not enter a separate drain phase. Instead, it merges cached and +spilled inserts back into the current transaction batch, emits events in +`DML(C) -> DDL(C)` order, and then cleans the large-transaction state. + +## 7. The difference between `handled`, `interrupted`, and `valid` + +These booleans belong to different layers: + +| Value | Producer | Meaning | +| --- | --- | --- | +| `handled` | `resumePending` | Pending-state logic completely handled this attempt; do not access EventStore. | +| `interrupted` | Strategy/scanner | This attempt stopped intentionally with unfinished work; the broker should reschedule it immediately. | +| `scanProgress.valid` | Scanner/session | The returned resume position is complete and safe to publish to the dispatcher. | + +Typical combinations are: + +- Normal complete scan: `handled=false, interrupted=false, valid=true`. +- Row-level large-transaction fragment: `handled=false, interrupted=true, valid=true`. +- Partial spill drain: `handled=true, interrupted=true, valid=true`. +- Context cancellation or dispatcher removal: usually no new publishable progress. + +## 8. Invariants that must be preserved + +Changes to these two files or adjacent code must preserve at least these +invariants: + +1. A non-empty `Position` takes precedence over `TxnStartTs`. +2. A row cursor `(C, S, P)` requires the next scan range to cover at least `C`. +3. After sending a row fragment, the final published value must remain a row + cursor rather than being overwritten by transaction-level `(C, S)`. +4. The commit-ts, start-ts, and position in one `scanProgress` must originate + from the same scan result. +5. A `Position` must be cloned before being retained across goroutines or scan + attempts. +6. The atomic strategy may stop only at a transaction boundary. +7. The split strategy may stop inside a transaction only when EventStore + provides an exact row position. +8. Delayed inserts from unique-key-changing updates must drain after the + original transaction ends and before a following transaction begins. +9. EventStore progress must not pass the corresponding large transaction while + its spill drain is incomplete. +10. A failed or canceled scan must not publish an incomplete new cursor. + +## 9. Suggested source-reading order + +For further source inspection, use this order: + +1. `scan_progress.go`: understand the three resume forms first. +2. `logservice/eventstore/scan_request.go` and EventStore `GetIterator`: + understand how a cursor becomes an iterator lower bound. +3. `dispatcherStat.getScanRequest()`: see how the previous progress becomes the + next request. +4. `eventBroker.getScanTaskRequest()`: see how the DDL upper bound and scan + window modify the range. +5. `eventScanner.scan()` and `scanAndMergeEvents()`: find the four strategy + hook locations. +6. `txn_scan_strategy.go`: follow the atomic and split implementations + separately. +7. `large_txn_state.go`: understand the original/drain phases and spill cursor. +8. `eventBroker.doScan()`: see why scanner progress is published after result + events are processed. + +In one sentence: + +> `scanProgress` guarantees that the next scan reads from the correct place; +> `txnScanStrategy` guarantees that the current scan stops only at a safe place. diff --git a/pkg/eventservice/scan_progress.go b/pkg/eventservice/scan_progress.go index d5f1cc725c..ec2c231426 100644 --- a/pkg/eventservice/scan_progress.go +++ b/pkg/eventservice/scan_progress.go @@ -15,15 +15,43 @@ package eventservice import "github.com/pingcap/ticdc/logservice/eventstore" -// scanProgress is an immutable snapshot published after a scan result has -// been sent. A non-empty rowLevelScanPosition resumes inside the transaction. +// scanProgress is the immutable EventStore resume point published after a scan +// result has entered the broker's send pipeline. +// +// For example, transaction (startTs=10, commitTs=100) contains rows r1, r2, +// and r3. If a split scan stops after r1 at EventStore position P1, it must +// publish (100, 10, P1). The next scan starts at commitTs 100 and resumes after +// P1, so it can still read r2 and r3. Publishing (100, 10, nil) instead would +// tell EventStore that the whole transaction has been processed and would skip +// those two rows. +// +// The meaningful forms are: +// - (C, 0, nil): everything through commitTs C is complete. +// - (C, S, nil): transaction (S, C) is complete in EventStore, although its +// delayed spill inserts may still need to be drained. +// - (C, S, P): resume transaction (S, C) after row position P. +// +// A non-empty rowLevelScanPosition is the most precise cursor and takes +// precedence over txnStartTs when EventStore constructs its iterator bounds. type scanProgress struct { - valid bool - txnCommitTs uint64 - txnStartTs uint64 + // valid distinguishes a complete, publishable snapshot from the zero value + // left by a canceled or otherwise unfinished scan. + valid bool + // txnCommitTs becomes the next scan range's CommitTsStart. With a zero + // txnStartTs it can represent a resolved boundary rather than a transaction. + txnCommitTs uint64 + // txnStartTs identifies the completed or partially completed transaction at + // txnCommitTs. Zero means there is no pending transaction at that boundary. + txnStartTs uint64 + // rowLevelScanPosition is an opaque EventStore cursor immediately after the + // last processed row. When non-empty, it resumes inside (txnStartTs, + // txnCommitTs) instead of skipping that transaction. rowLevelScanPosition eventstore.ScanPosition } +// newTxnScanProgress creates boundary- or transaction-level progress. A zero +// startTs means the commit-ts boundary is fully resolved; a non-zero startTs +// keeps the next request at that transaction until any pending work is done. func newTxnScanProgress(commitTs uint64, startTs uint64) scanProgress { return scanProgress{ valid: true, @@ -32,12 +60,17 @@ func newTxnScanProgress(commitTs uint64, startTs uint64) scanProgress { } } +// newRowLevelScanProgress creates an exact resume point inside a transaction. +// The position is cloned because ScanPosition is a byte slice owned by the +// EventStore iterator and the progress snapshot outlives that iterator. func newRowLevelScanProgress(commitTs uint64, startTs uint64, position eventstore.ScanPosition) scanProgress { progress := newTxnScanProgress(commitTs, startTs) progress.rowLevelScanPosition = cloneScanPosition(position) return progress } +// cloneScanPosition prevents a published immutable snapshot from sharing a +// mutable byte slice with an iterator or a later scan request. func cloneScanPosition(position eventstore.ScanPosition) eventstore.ScanPosition { if len(position) == 0 { return nil diff --git a/pkg/eventservice/txn_scan_strategy.go b/pkg/eventservice/txn_scan_strategy.go index c34269a132..e933618745 100644 --- a/pkg/eventservice/txn_scan_strategy.go +++ b/pkg/eventservice/txn_scan_strategy.go @@ -26,6 +26,20 @@ import ( "go.uber.org/zap" ) +// txnScanContext groups the per-attempt objects needed by transaction strategy +// hooks. The scanner owns iteration and DDL/DML ordering; a strategy only +// decides where a transaction may stop and how pending transaction work resumes. +// +// For example, transaction (startTs=10, commitTs=100) contains rows r1, r2, +// and r3. The atomic strategy emits all three rows before it may stop. The split +// strategy may emit r1, publish progress (100, 10, P1), and resume r2 after +// EventStore position P1 in the next attempt. If a row is an update that changes +// a unique key, its delete half stays in the original scan while its insert half +// is spilled and drained only after all original rows, avoiding a downstream +// unique-key conflict across fragments. +// +// See docs/design/2026-07-22-eventservice-scan-progress-and-txn-strategy.md for +// the complete cursor and state-transition model. type txnScanContext struct { scanner *eventScanner session *session @@ -33,6 +47,10 @@ type txnScanContext struct { processor *dmlProcessor } +// nextTxnMeta identifies the next iterator record that the scanner is about to +// process. A zero commitTs means iterator EOF. Split mode also uses this value +// when currentTxn is nil to decide whether a transaction retained from a prior +// row-level interruption has reached the end of its original EventStore rows. type nextTxnMeta struct { startTs uint64 commitTs uint64 @@ -44,7 +62,12 @@ type nextTxnMeta struct { // transaction scanning while the event scanner retains the common iterator and // DDL/DML merge loop. type txnScanStrategy interface { + // resumePending runs before DDL lookup and EventStore iterator creation. + // handled means the pending-state path consumed this attempt completely; + // interrupted asks the broker to schedule another attempt immediately. resumePending(ctx *txnScanContext) (handled bool, interrupted bool, err error) + // startTxn creates the mode-specific TxnEvent after common boundary and + // schema-version handling has selected the next transaction. startTxn( ctx *txnScanContext, startTs uint64, @@ -52,7 +75,12 @@ type txnScanStrategy interface { tableInfo *common.TableInfo, tableID int64, ) error + // finishTxn runs at a transaction boundary or iterator EOF. In split mode it + // may also close an original phase retained from a previous scan even when + // this attempt has not created currentTxn yet. finishTxn(ctx *txnScanContext, next nextTxnMeta) (interrupted bool, err error) + // afterAppend is the only hook that may interrupt inside a transaction. A + // non-empty position is the exact cursor after rawEvent. afterAppend( ctx *txnScanContext, rawEvent *common.RawKVEntry, @@ -60,6 +88,8 @@ type txnScanStrategy interface { ) (interrupted bool, err error) } +// newTxnScanStrategy selects split behavior only when the dispatcher's +// transaction atomicity configuration explicitly permits cross-scan fragments. func newTxnScanStrategy(shouldSplitTxn bool) txnScanStrategy { if shouldSplitTxn { return splitTxnScanStrategy{} @@ -67,6 +97,8 @@ func newTxnScanStrategy(shouldSplitTxn bool) txnScanStrategy { return atomicTxnScanStrategy{} } +// atomicTxnScanStrategy preserves table-level transaction atomicity by making +// both pending-resume and per-row interruption no-ops. type atomicTxnScanStrategy struct{} func (atomicTxnScanStrategy) resumePending( @@ -109,6 +141,8 @@ func (atomicTxnScanStrategy) afterAppend( return false, nil } +// splitTxnScanStrategy permits row-level interruption and owns the two-phase +// original-rows/spilled-inserts lifecycle for large transactions. type splitTxnScanStrategy struct{} func (splitTxnScanStrategy) resumePending( @@ -142,6 +176,11 @@ func (splitTxnScanStrategy) finishTxn( return finishCurrentSplitTxn(ctx, next) } +// finishPendingSplitTxn handles the boundary discovered after row-level resume. +// If the next row still belongs to the retained (startTs, commitTs), the +// original phase continues. A different transaction or EOF proves that all +// original rows were read, so progress becomes (C, S, nil) and the next scan +// starts draining delayed inserts before it may pass this transaction. func finishPendingSplitTxn(session *session, next nextTxnMeta) bool { dispatcher := session.dispatcherStat state := dispatcher.getLargeTxnState() @@ -170,6 +209,8 @@ func (splitTxnScanStrategy) afterAppend( return true, nil } +// startTxn contains setup shared by both strategies; shouldSplitTxn controls +// whether the resulting TxnEvent may be emitted as cross-scan fragments. func startTxn( ctx *txnScanContext, startTs uint64, @@ -193,6 +234,11 @@ func startTxn( return nil } +// finishCurrentSplitTxn commits the current fragment at a transaction boundary. +// A transaction with spilled inserts moves from the original phase to the drain +// phase and deliberately keeps progress at (C, S). If a DDL exists at the same +// commit-ts, the inserts are merged back into the current batch instead so the +// merger can preserve the required DML(C) -> DDL(C) order. func finishCurrentSplitTxn(ctx *txnScanContext, next nextTxnMeta) (bool, error) { processor := ctx.processor currentStartTs := processor.currentTxn.CurrentDMLEvent.GetStartTs() @@ -243,6 +289,10 @@ func finishCurrentSplitTxn(ctx *txnScanContext, next nextTxnMeta) (bool, error) return true, nil } +// canInterruptCurrentTxn requires an exact EventStore row position, a fragment +// above the large-transaction threshold, and a DML/DDL merge boundary that is +// safe to split. Cached UK-update inserts must be spilled first so no insert +// half is lost when the in-memory processor is discarded after interruption. func canInterruptCurrentTxn( ctx *txnScanContext, rawEvent *common.RawKVEntry, @@ -257,6 +307,10 @@ func canInterruptCurrentTxn( return ctx.merger.canInterrupt(rawEvent.CRTs, ctx.processor.batchDML) } +// interruptCurrentTxn emits the current fragment and publishes (C, S, P). +// That row-level progress must later overwrite the transaction-level progress +// observed by the broker send path, or EventStore would skip the remainder of +// this transaction on the next scan. func interruptCurrentTxn( ctx *txnScanContext, commitTs uint64, @@ -284,6 +338,12 @@ func interruptCurrentTxn( zap.Duration("duration", time.Since(ctx.session.startTime))) } +// drainLargeTxnInserts serves a pending drain directly from spill storage; it +// does not create an EventStore iterator. EventStore progress remains (C, S, +// nil) while state.drainedInsertCount tracks the independent position inside +// the spill file. On a failed attempt the reader rolls back to the last +// committed drain count, and on EOF the spill state is cleaned before scanning +// can move beyond the transaction. func drainLargeTxnInserts( ctx *txnScanContext, state *largeTxnScanState, @@ -324,68 +384,119 @@ func drainLargeTxnInserts( return false, nil } if errors.Is(err, io.EOF) { - if processor.currentTxn != nil { - if err := processor.commitTxn(); err != nil { - return returnWithError(err) - } - if processor.getCurrentBatchDML().DMLCount() != 0 { - session.appendEvents([]event.Event{processor.getCurrentBatchDML()}) - } - } - state.commitDrainedInserts(drainedInsertCount) - session.progress = newTxnScanProgress(state.commitTs, state.startTs) - hasFollowingTxn, followingCommitTs := state.snapshotDrainInfo() - shouldResolveCommitTs := (!hasFollowingTxn || followingCommitTs > state.commitTs) && - session.dataRange.CommitTsEnd == state.commitTs - if err := session.dispatcherStat.cleanupLargeTxnState(); err != nil { - log.Warn("cleanup drained large transaction spill failed", - zap.Stringer("dispatcherID", session.dispatcherStat.id), - zap.Error(err)) + interrupted, finishErr := completeLargeTxnInsertDrain( + session, state, processor, drainedInsertCount) + if finishErr != nil { + return returnWithError(finishErr) } - if shouldResolveCommitTs { - resolved := event.NewResolvedEvent( - state.commitTs, - session.dispatcherStat.id, - session.dispatcherStat.epoch, - ) - session.appendEvents([]event.Event{resolved}) - session.progress = newTxnScanProgress(state.commitTs, 0) - return false, nil - } - return true, nil + return interrupted, nil } return returnWithError(err) } drainedInsertCount++ - if processor.currentTxn == nil { - if err := processor.startTxn( - session.dispatcherStat.id, - state.tableID, - state.tableInfo, - state.startTs, - state.commitTs, - true, - ); err != nil { - return returnWithError(err) - } + if err := appendDrainedInsert(session, state, processor, entry); err != nil { + return returnWithError(err) + } + if !session.exceedLimit(processor.batchDML.GetSize(), processor.batchDML) { + continue } - session.observeRawEntry(entry, nil) - if err := processor.appendInsertRow(entry); err != nil { + if err := flushLargeTxnInsertBatch( + session, state, processor, drainedInsertCount); err != nil { return returnWithError(err) } - if session.exceedLimit(processor.batchDML.GetSize(), processor.batchDML) { - if err := processor.commitTxn(); err != nil { - return returnWithError(err) - } + return true, nil + } +} + +// appendDrainedInsert lazily recreates the original transaction in the current +// scan attempt, then appends one insert read from spill storage. +func appendDrainedInsert( + session *session, + state *largeTxnScanState, + processor *dmlProcessor, + entry *common.RawKVEntry, +) error { + if processor.currentTxn == nil { + if err := processor.startTxn( + session.dispatcherStat.id, + state.tableID, + state.tableInfo, + state.startTs, + state.commitTs, + true, + ); err != nil { + return err + } + } + session.observeRawEntry(entry, nil) + return processor.appendInsertRow(entry) +} + +// flushLargeTxnInsertBatch publishes a size-limited drain fragment and +// advances the spill retry boundary only after that fragment is complete. +func flushLargeTxnInsertBatch( + session *session, + state *largeTxnScanState, + processor *dmlProcessor, + drainedInsertCount int, +) error { + if err := processor.commitTxn(); err != nil { + return err + } + session.appendEvents([]event.Event{processor.getCurrentBatchDML()}) + state.commitDrainedInserts(drainedInsertCount) + session.progress = newTxnScanProgress(state.commitTs, state.startTs) + return nil +} + +// completeLargeTxnInsertDrain handles spill EOF. It returns interrupted=true +// when normal EventStore/DDL scanning still needs another attempt. +func completeLargeTxnInsertDrain( + session *session, + state *largeTxnScanState, + processor *dmlProcessor, + drainedInsertCount int, +) (bool, error) { + if processor.currentTxn != nil { + if err := processor.commitTxn(); err != nil { + return false, err + } + if processor.getCurrentBatchDML().DMLCount() != 0 { session.appendEvents([]event.Event{processor.getCurrentBatchDML()}) - state.commitDrainedInserts(drainedInsertCount) - session.progress = newTxnScanProgress(state.commitTs, state.startTs) - return true, nil } } + + state.commitDrainedInserts(drainedInsertCount) + session.progress = newTxnScanProgress(state.commitTs, state.startTs) + + hasFollowingTxn, followingCommitTs := state.snapshotDrainInfo() + noFollowingTxnAtCommitTs := !hasFollowingTxn || followingCommitTs > state.commitTs + rangeEndsAtTxnCommitTs := session.dataRange.CommitTsEnd == state.commitTs + shouldResolveCommitTs := noFollowingTxnAtCommitTs && rangeEndsAtTxnCommitTs + + if err := session.dispatcherStat.cleanupLargeTxnState(); err != nil { + log.Warn("cleanup drained large transaction spill failed", + zap.Stringer("dispatcherID", session.dispatcherStat.id), + zap.Error(err)) + } + if !shouldResolveCommitTs { + return true, nil + } + + resolved := event.NewResolvedEvent( + state.commitTs, + session.dispatcherStat.id, + session.dispatcherStat.epoch, + ) + session.appendEvents([]event.Event{resolved}) + session.progress = newTxnScanProgress(state.commitTs, 0) + return false, nil } +// spillCachedInsertRows persists insert halves collected before a split update +// crossed the large-transaction threshold. They must leave the in-memory cache +// before a row-level interruption discards the current processor. func (p *dmlProcessor) spillCachedInsertRows() error { if len(p.insertRowCache) == 0 { return nil @@ -403,6 +514,9 @@ func (p *dmlProcessor) spillCachedInsertRows() error { return nil } +// shouldSpillSplitUpdateInsert keeps spilling once a transaction has entered +// the spill lifecycle, even if a later in-memory fragment is below the size +// threshold. func (p *dmlProcessor) shouldSpillSplitUpdateInsert() bool { if p.currentTxn == nil { return false @@ -410,6 +524,8 @@ func (p *dmlProcessor) shouldSpillSplitUpdateInsert() bool { return p.currentTxn.exceedsLargeTxnThreshold() || p.hasSpilledInsertsForCurrentTxn() } +// hasSpilledInsertsForCurrentTxn prevents state from one transaction from being +// mistaken for pending inserts of another transaction at the same commit-ts. func (p *dmlProcessor) hasSpilledInsertsForCurrentTxn() bool { if p.currentTxn == nil || p.dispatcherStat == nil { return false @@ -429,6 +545,9 @@ func (p *dmlProcessor) hasLargeTxnState(startTs uint64, commitTs uint64) bool { state.getPhase() == largeTxnScanPhaseOriginal } +// flushSpilledInsertsIntoCurrentTxn is the same-commit-ts DDL path. It folds +// delayed inserts back into the current batch instead of entering a separate +// drain attempt, then removes the exhausted spill state. func (p *dmlProcessor) flushSpilledInsertsIntoCurrentTxn() error { if p.currentTxn == nil || p.dispatcherStat == nil { return nil @@ -451,6 +570,9 @@ func (p *dmlProcessor) flushSpilledInsertsIntoCurrentTxn() error { } } +// getOrCreateLargeTxnState returns dispatcher-owned state that survives the +// current processor and scan attempt. The identity fields prevent fragments +// from different transactions from sharing one spill lifecycle. func (p *dmlProcessor) getOrCreateLargeTxnState() (*largeTxnScanState, error) { if p.dispatcherStat == nil { return nil, errors.New("dispatcher stat is required for large txn update spill") From 6be69c5a8d5f9319fa0c8c6cdbaa737f810803c9 Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Sat, 25 Jul 2026 18:22:36 +0800 Subject: [PATCH 32/35] eventservice: use predefined spill errors Signed-off-by: dongmen <414110582@qq.com> --- pkg/eventservice/large_txn_spill_test.go | 16 ++++++++++++++++ pkg/eventservice/large_txn_state.go | 7 ++++--- pkg/eventservice/txn_scan_strategy.go | 3 ++- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/pkg/eventservice/large_txn_spill_test.go b/pkg/eventservice/large_txn_spill_test.go index 0ea0919a3a..365eb5b6c4 100644 --- a/pkg/eventservice/large_txn_spill_test.go +++ b/pkg/eventservice/large_txn_spill_test.go @@ -120,6 +120,22 @@ func TestLargeTxnInsertSpillValidationErrors(t *testing.T) { require.True(t, errors.ErrSpillFileOp.Equal(spill.Append(context.Background(), nil))) } +func TestLargeTxnStateValidationErrors(t *testing.T) { + cleanedState := &largeTxnScanState{cleaned: true} + err := cleanedState.appendInsert(context.Background(), newTestSpillRawKVEntry(1)) + require.True(t, errors.ErrSpillFileOp.Equal(err)) + _, err = cleanedState.nextInsert(context.Background()) + require.True(t, errors.ErrSpillFileOp.Equal(err)) + + drainingState := &largeTxnScanState{phase: largeTxnScanPhaseDrainInserts} + err = drainingState.appendInsert(context.Background(), newTestSpillRawKVEntry(1)) + require.True(t, errors.ErrSpillFileOp.Equal(err)) + + processor := &dmlProcessor{} + _, err = processor.getOrCreateLargeTxnState() + require.True(t, errors.ErrSpillFileOp.Equal(err)) +} + func TestCleanupLargeTxnInsertSpillFiles(t *testing.T) { dir := t.TempDir() orphanPaths := []string{ diff --git a/pkg/eventservice/large_txn_state.go b/pkg/eventservice/large_txn_state.go index a0c834cadb..d558d1d337 100644 --- a/pkg/eventservice/large_txn_state.go +++ b/pkg/eventservice/large_txn_state.go @@ -142,10 +142,11 @@ func (s *largeTxnScanState) appendInsert(ctx context.Context, entry *common.RawK s.mu.Lock() defer s.mu.Unlock() if s.cleaned { - return errors.New("large txn state has been cleaned up") + return errors.ErrSpillFileOp.GenWithStackByArgs("large txn state has been cleaned up") } if s.phase != largeTxnScanPhaseOriginal { - return errors.New("large txn spill is no longer accepting original txn rows") + return errors.ErrSpillFileOp.GenWithStackByArgs( + "large txn spill is no longer accepting original txn rows") } return s.spill.Append(ctx, entry) } @@ -154,7 +155,7 @@ func (s *largeTxnScanState) nextInsert(ctx context.Context) (*common.RawKVEntry, s.mu.Lock() defer s.mu.Unlock() if s.cleaned { - return nil, errors.New("large txn state has been cleaned up") + return nil, errors.ErrSpillFileOp.GenWithStackByArgs("large txn state has been cleaned up") } if s.reader == nil { reader, err := s.spill.NewReader() diff --git a/pkg/eventservice/txn_scan_strategy.go b/pkg/eventservice/txn_scan_strategy.go index e933618745..e69a5b61a2 100644 --- a/pkg/eventservice/txn_scan_strategy.go +++ b/pkg/eventservice/txn_scan_strategy.go @@ -575,7 +575,8 @@ func (p *dmlProcessor) flushSpilledInsertsIntoCurrentTxn() error { // from different transactions from sharing one spill lifecycle. func (p *dmlProcessor) getOrCreateLargeTxnState() (*largeTxnScanState, error) { if p.dispatcherStat == nil { - return nil, errors.New("dispatcher stat is required for large txn update spill") + return nil, errors.ErrSpillFileOp.GenWithStackByArgs( + "dispatcher stat is required for large txn update spill") } currentDML := p.currentTxn.CurrentDMLEvent return p.dispatcherStat.getOrCreateLargeTxnState( From fd4b3a2925d2c63deca1c3fc165680768aec1e68 Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Sat, 25 Jul 2026 18:22:49 +0800 Subject: [PATCH 33/35] eventservice,tests: cover CMEK-encrypted spill files Signed-off-by: dongmen <414110582@qq.com> --- pkg/eventservice/large_txn_spill.go | 10 ++- .../cmek_keyspace/conf/cdc_valid.toml | 3 + .../cmek_keyspace/conf/diff_config.toml | 2 +- tests/integration_tests/cmek_keyspace/run.sh | 76 +++++++++++++++++++ 4 files changed, 88 insertions(+), 3 deletions(-) diff --git a/pkg/eventservice/large_txn_spill.go b/pkg/eventservice/large_txn_spill.go index d0468f45b4..836f4b186d 100644 --- a/pkg/eventservice/large_txn_spill.go +++ b/pkg/eventservice/large_txn_spill.go @@ -19,6 +19,7 @@ import ( "os" "path/filepath" + "github.com/pingcap/failpoint" "github.com/pingcap/ticdc/pkg/common" appcontext "github.com/pingcap/ticdc/pkg/common/context" "github.com/pingcap/ticdc/pkg/config" @@ -107,8 +108,13 @@ func (s *largeTxnInsertSpill) Append(ctx context.Context, entry *common.RawKVEnt return err } } - _, err = s.file.Append(data) - return err + if _, err := s.file.Append(data); err != nil { + return err + } + // Keep the completed record inspectable before drain cleanup in CMEK + // integration tests. + failpoint.Inject("PauseAfterLargeTxnSpillAppend", nil) + return nil } func (s *largeTxnInsertSpill) NewReader() (*largeTxnInsertSpillReader, error) { diff --git a/tests/integration_tests/cmek_keyspace/conf/cdc_valid.toml b/tests/integration_tests/cmek_keyspace/conf/cdc_valid.toml index f50022e52c..0b84bd01f2 100644 --- a/tests/integration_tests/cmek_keyspace/conf/cdc_valid.toml +++ b/tests/integration_tests/cmek_keyspace/conf/cdc_valid.toml @@ -1,5 +1,8 @@ newarch = true +[debug.event-service] +large-txn-threshold-in-bytes = 1048576 + [debug.encryption] enable-encryption = true allow-degrade-on-error = false diff --git a/tests/integration_tests/cmek_keyspace/conf/diff_config.toml b/tests/integration_tests/cmek_keyspace/conf/diff_config.toml index 51f1d8863a..151abff78e 100644 --- a/tests/integration_tests/cmek_keyspace/conf/diff_config.toml +++ b/tests/integration_tests/cmek_keyspace/conf/diff_config.toml @@ -13,7 +13,7 @@ check-struct-only = false target-instance = "tidb0" - target-check-tables = ["cmek_valid.t"] + target-check-tables = ["cmek_valid.t", "cmek_valid.spill_table"] [data-sources] [data-sources.mysql1] diff --git a/tests/integration_tests/cmek_keyspace/run.sh b/tests/integration_tests/cmek_keyspace/run.sh index 6c95849cab..ef6d952647 100755 --- a/tests/integration_tests/cmek_keyspace/run.sh +++ b/tests/integration_tests/cmek_keyspace/run.sh @@ -10,6 +10,7 @@ CDC_BINARY=cdc.test SINK_TYPE=$1 MAX_RETRIES=30 REQUIRE_ENCRYPTED_KEYSPACE_START=${REQUIRE_ENCRYPTED_KEYSPACE_START:-false} +SPILL_APPEND_FAILPOINT=github.com/pingcap/ticdc/pkg/eventservice/PauseAfterLargeTxnSpillAppend LOCAL_KMS_ADDR=127.0.0.1 LOCAL_KMS_PORT=18080 @@ -281,9 +282,75 @@ function assert_table_not_synced_for_duration() { done } +function generate_spill_workload() { + local sql_file=$1 + local rows=2048 + + { + echo "USE cmek_valid;" + echo "CREATE TABLE spill_table (id BIGINT PRIMARY KEY, uk BIGINT NOT NULL, payload LONGTEXT, UNIQUE KEY uk_idx (uk));" + echo "BEGIN;" + for i in $(seq 1 "$rows"); do + echo "INSERT INTO spill_table VALUES ($i, $i, CONCAT('cmek-spill-before-', REPEAT('a', 1024)));" + done + echo "COMMIT;" + echo "BEGIN;" + echo "UPDATE spill_table SET uk = uk + 1000000, payload = CONCAT('cmek-spill-after-', REPEAT('b', 1024));" + echo "COMMIT;" + } >"$sql_file" +} + +function run_spill_workload() { + local sql_file=$1 + local workload_log=$WORK_DIR/spill-workload.log + + if ! mysql -h"127.0.0.1" -P"$UPSTREAM_VALID_TIDB_PORT" -uroot \ + --default-character-set utf8mb4 <"$sql_file" >"$workload_log" 2>&1; then + cat "$workload_log" + return 1 + fi +} + +function assert_spill_file_encrypted() { + local spill_dir=$WORK_DIR/cdc_data/eventservice + local spill_file= + local version= + local key_id_1= + local key_id_2= + local key_id_3= + local i=0 + + while [ "$i" -lt 60 ]; do + spill_file=$(find "$spill_dir" -maxdepth 1 -type f \ + -name 'eventservice-large-txn-insert-*.spill' -size +11c \ + -print -quit 2>/dev/null || true) + if [ -n "$spill_file" ]; then + read -r version key_id_1 key_id_2 key_id_3 < <( + od -An -tu1 -j 8 -N 4 "$spill_file" + ) + if [ -n "$key_id_3" ]; then + if [ "$version" -eq 0 ] || + { [ "$key_id_1" -eq 0 ] && [ "$key_id_2" -eq 0 ] && [ "$key_id_3" -eq 0 ]; }; then + echo "spill record is not CMEK encrypted: $spill_file" + od -An -tx1 -j 8 -N 4 "$spill_file" + return 1 + fi + echo "verified encrypted spill record: $spill_file" + return 0 + fi + fi + i=$((i + 1)) + sleep 1 + done + + echo "encrypted large transaction spill file was not observed under $spill_dir" + return 1 +} + function run_with_valid_kms() { local sink_uri="mysql://root@127.0.0.1:${DOWNSTREAM_TIDB_PORT}/" local changefeed_id=cmek-valid + local spill_workload=$WORK_DIR/cmek-spill-workload.sql run_cdc_server --workdir "$WORK_DIR" --binary "$CDC_BINARY" --config "$CUR/conf/cdc_valid.toml" run_cdc_cli changefeed -k "$UPSTREAM_VALID_KEYSPACE" create --sink-uri="$sink_uri" -c "$changefeed_id" @@ -293,7 +360,16 @@ function run_with_valid_kms() { run_sql "INSERT INTO cmek_valid.t VALUES (1, 'apple'), (2, 'banana');" "127.0.0.1" "$UPSTREAM_VALID_TIDB_PORT" check_table_exists "cmek_valid.t" "127.0.0.1" "$DOWNSTREAM_TIDB_PORT" 90 + + generate_spill_workload "$spill_workload" + enable_failpoint --addr "127.0.0.1:8300" --name "$SPILL_APPEND_FAILPOINT" --expr "pause" + run_spill_workload "$spill_workload" + assert_spill_file_encrypted + disable_failpoint --addr "127.0.0.1:8300" --name "$SPILL_APPEND_FAILPOINT" + check_sync_diff "$WORK_DIR" "$CUR/conf/diff_config.toml" 180 + check_logs_contains "$WORK_DIR" "scan interrupted inside a large txn" + check_logs_contains "$WORK_DIR" "split update event" run_cdc_cli changefeed -k "$UPSTREAM_VALID_KEYSPACE" remove -c "$changefeed_id" cleanup_process "$CDC_BINARY" From 7e204ba90ad390ab3639c25d96bd42ebf8d3ba09 Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Sat, 25 Jul 2026 18:43:50 +0800 Subject: [PATCH 34/35] tests: run CMEK coverage in NextGen heavy CI Signed-off-by: dongmen <414110582@qq.com> --- tests/integration_tests/cmek_keyspace/run.sh | 27 ++++++++++++++++--- tests/integration_tests/run_heavy_it_in_ci.sh | 2 +- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/tests/integration_tests/cmek_keyspace/run.sh b/tests/integration_tests/cmek_keyspace/run.sh index ef6d952647..3508aeebdf 100755 --- a/tests/integration_tests/cmek_keyspace/run.sh +++ b/tests/integration_tests/cmek_keyspace/run.sh @@ -9,12 +9,13 @@ WORK_DIR=$OUT_DIR/$TEST_NAME CDC_BINARY=cdc.test SINK_TYPE=$1 MAX_RETRIES=30 -REQUIRE_ENCRYPTED_KEYSPACE_START=${REQUIRE_ENCRYPTED_KEYSPACE_START:-false} +REQUIRE_ENCRYPTED_KEYSPACE_START=${REQUIRE_ENCRYPTED_KEYSPACE_START:-true} SPILL_APPEND_FAILPOINT=github.com/pingcap/ticdc/pkg/eventservice/PauseAfterLargeTxnSpillAppend LOCAL_KMS_ADDR=127.0.0.1 LOCAL_KMS_PORT=18080 LOCAL_KMS_PID= +LOCAL_KMS_MODULE=github.com/nsmithuk/local-kms@v0.0.0-20230108155039-ce4561e0cb19 UPSTREAM_VALID_KEYSPACE=keyspace-foo UPSTREAM_INVALID_KEYSPACE=keyspace-foo-invalid @@ -50,9 +51,20 @@ function resolve_local_kms_binary() { echo "$gopath/bin/local-kms" return fi + + local install_dir="$WORK_DIR/local-kms-bin" + mkdir -p "$install_dir" + echo "local-kms is not installed; build ${LOCAL_KMS_MODULE}" >&2 + if ! GOBIN="$install_dir" go install "$LOCAL_KMS_MODULE"; then + echo "failed to build ${LOCAL_KMS_MODULE}" >&2 + return 1 + fi + echo "$install_dir/local-kms" + return fi - return 0 + echo "local-kms is not installed and go is unavailable" >&2 + return 1 } function stop_local_kms() { @@ -65,9 +77,11 @@ function stop_local_kms() { function start_local_kms() { local local_kms_binary local kms_data_dir - local_kms_binary=$(resolve_local_kms_binary) + if ! local_kms_binary=$(resolve_local_kms_binary); then + return 1 + fi if [ -z "${local_kms_binary}" ]; then - echo "skip $TEST_NAME: local-kms binary is not found" + echo "failed to resolve local-kms binary" return 1 fi @@ -405,6 +419,11 @@ function run() { fi if ! start_local_kms; then + if [ "${REQUIRE_ENCRYPTED_KEYSPACE_START}" = "true" ]; then + echo "local-kms startup failed and strict mode is enabled" + exit 1 + fi + echo "skip $TEST_NAME: local-kms startup failed in current environment" return fi diff --git a/tests/integration_tests/run_heavy_it_in_ci.sh b/tests/integration_tests/run_heavy_it_in_ci.sh index a44f6fb87a..7f0de118d8 100755 --- a/tests/integration_tests/run_heavy_it_in_ci.sh +++ b/tests/integration_tests/run_heavy_it_in_ci.sh @@ -50,7 +50,7 @@ mysql_groups=( # G09 'availability resolve_lock merge_table drop_many_tables ddl_for_split_tables' # G10 - 'consistent_replicate_nfs consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3' + 'consistent_replicate_nfs consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 cmek_keyspace' # G11 'multi_changefeeds ddl_wait ddl_reentrant force_replicate_table multi_source' # G12 From 4932b7e009bbf481b513912d71d2da86a5fdbea9 Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Sat, 25 Jul 2026 20:15:32 +0800 Subject: [PATCH 35/35] tests: provide AWS credentials for CMEK integration Signed-off-by: dongmen <414110582@qq.com> --- tests/integration_tests/cmek_keyspace/run.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/integration_tests/cmek_keyspace/run.sh b/tests/integration_tests/cmek_keyspace/run.sh index 3508aeebdf..d446710b0d 100755 --- a/tests/integration_tests/cmek_keyspace/run.sh +++ b/tests/integration_tests/cmek_keyspace/run.sh @@ -427,6 +427,13 @@ function run() { return fi + # TiKV uses the AWS SDK to access the local KMS endpoint configured on each + # encrypted keyspace. Dummy credentials let it sign those local requests; + # disabling IMDS prevents a missing credential from stalling on EC2 lookup. + export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-test} + export AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-test} + export AWS_EC2_METADATA_DISABLED=${AWS_EC2_METADATA_DISABLED:-true} + export KEYSPACE_WAIT_REGION_SPLIT=false export KEYSPACE_WAIT_REGION_SPLIT_TIMEOUT=1m export KEYSPACE_CHECK_REGION_SPLIT_INTERVAL=2s