From 0d6bdb9ef43fe7b8875f3750e3cc4f0d3106f3ee Mon Sep 17 00:00:00 2001 From: wk989898 Date: Mon, 6 Jul 2026 06:31:33 +0000 Subject: [PATCH 01/12] init Signed-off-by: wk989898 --- pkg/sink/codec/canal/canal_json_decoder.go | 50 +++++++- pkg/sink/codec/canal/canal_json_test.go | 127 +++++++++++++++++++++ 2 files changed, 172 insertions(+), 5 deletions(-) diff --git a/pkg/sink/codec/canal/canal_json_decoder.go b/pkg/sink/codec/canal/canal_json_decoder.go index f8b3ac9a84..519ec7ee3b 100644 --- a/pkg/sink/codec/canal/canal_json_decoder.go +++ b/pkg/sink/codec/canal/canal_json_decoder.go @@ -22,6 +22,7 @@ import ( "path/filepath" "reflect" "slices" + "sort" "strconv" "strings" @@ -44,6 +45,12 @@ import ( ) type tableKey struct { + schema string + table string + ddlCommitTs uint64 +} + +type tableNameKey struct { schema string table string } @@ -92,6 +99,7 @@ type decoder struct { storage storeapi.Storage upstreamTiDB *sql.DB tableInfoCache map[tableKey]*commonType.TableInfo + ddlCommitTs map[tableNameKey][]uint64 } var tableIDAllocator = common.NewTableIDAllocator() @@ -125,6 +133,7 @@ func NewDecoder( storage: externalStorage, upstreamTiDB: db, tableInfoCache: make(map[tableKey]*commonType.TableInfo), + ddlCommitTs: make(map[tableNameKey][]uint64), }, nil } @@ -378,9 +387,8 @@ func (d *decoder) NextDDLEvent() *commonEvent.DDLEvent { tableIDAllocator.AddBlockTableID(result.SchemaName, result.TableName, tableIDAllocator.Allocate(result.SchemaName, result.TableName)) result.BlockedTables = common.GetBlockedTables(tableIDAllocator, result) - // if receive a table level DDL, just remove the table info to trigger create a new one. - delete(d.tableInfoCache, tableKey{schema: result.SchemaName, table: result.TableName}) - delete(d.tableInfoCache, tableKey{schema: result.SchemaName, table: result.TableName}) + d.addDDLCommitTs(result.SchemaName, result.TableName, result.GetCommitTs()) + d.addDDLCommitTs(result.ExtraSchemaName, result.ExtraTableName, result.GetCommitTs()) return result } @@ -537,8 +545,9 @@ func (d *decoder) queryTableInfo(msg canalJSONMessageInterface) *commonType.Tabl tableName := *msg.getTable() cacheKey := tableKey{ - schema: schemaName, - table: tableName, + schema: schemaName, + table: tableName, + ddlCommitTs: d.getDDLCommitTs(schemaName, tableName, msg.getCommitTs()), } tableInfo, ok := d.tableInfoCache[cacheKey] if !ok { @@ -557,6 +566,37 @@ func (d *decoder) queryTableInfo(msg canalJSONMessageInterface) *commonType.Tabl return tableInfo } +func (d *decoder) addDDLCommitTs(schema, table string, commitTs uint64) { + if schema == "" || table == "" || commitTs == 0 { + return + } + + key := tableNameKey{schema: schema, table: table} + commitTsList := d.ddlCommitTs[key] + i := sort.Search(len(commitTsList), func(i int) bool { + return commitTsList[i] >= commitTs + }) + if i < len(commitTsList) && commitTsList[i] == commitTs { + return + } + d.ddlCommitTs[key] = slices.Insert(commitTsList, i, commitTs) +} + +func (d *decoder) getDDLCommitTs(schema, table string, commitTs uint64) uint64 { + if commitTs == 0 { + return 0 + } + + commitTsList := d.ddlCommitTs[tableNameKey{schema: schema, table: table}] + i := sort.Search(len(commitTsList), func(i int) bool { + return commitTsList[i] > commitTs + }) + if i == 0 { + return 0 + } + return commitTsList[i-1] +} + func newTiColumns(msg canalJSONMessageInterface) []*timodel.ColumnInfo { type columnPair struct { mysqlType string diff --git a/pkg/sink/codec/canal/canal_json_test.go b/pkg/sink/codec/canal/canal_json_test.go index aa6a242144..bd912dcdca 100644 --- a/pkg/sink/codec/canal/canal_json_test.go +++ b/pkg/sink/codec/canal/canal_json_test.go @@ -19,6 +19,7 @@ import ( "testing" "github.com/pingcap/ticdc/downstreamadapter/sink/columnselector" + commonType "github.com/pingcap/ticdc/pkg/common" commonEvent "github.com/pingcap/ticdc/pkg/common/event" "github.com/pingcap/ticdc/pkg/config" "github.com/pingcap/ticdc/pkg/config/kerneltype" @@ -1129,6 +1130,132 @@ func TestDDLSequence(t *testing.T) { require.Equal(t, obtained.GetBlockedTables().InfluenceType, commonEvent.InfluenceTypeNormal) } +func TestDecoderTableInfoCacheUsesDDLCommitTsAcrossColumnChanges(t *testing.T) { + ctx := context.Background() + codecConfig := common.NewConfig(config.ProtocolCanalJSON) + codecConfig.EnableTiDBExtension = true + + decoder, err := NewDecoder(ctx, codecConfig, nil) + require.NoError(t, err) + + buildRowMessage := func(commitTs uint64, mysqlTypes map[string]string) *canalJSONMessageWithTiDBExtension { + data := map[string]interface{}{ + "data": "insert_1", + "id": "525", + } + if _, ok := mysqlTypes["new_col"]; ok { + data["new_col"] = nil + } + return &canalJSONMessageWithTiDBExtension{ + JSONMessage: &JSONMessage{ + Schema: "test", + Table: "table_5", + PKNames: []string{"id"}, + IsDDL: false, + EventType: "INSERT", + SQLType: map[string]int32{ + "data": 12, + "id": 4, + "new_col": 4, + }, + MySQLType: mysqlTypes, + Data: []map[string]interface{}{data}, + }, + Extensions: &tidbExtension{CommitTs: commitTs}, + } + } + + decodeRow := func(message *canalJSONMessageWithTiDBExtension) *commonEvent.DMLEvent { + payload, err := json.Marshal(message) + require.NoError(t, err) + decoder.AddKeyValue(nil, payload) + messageType, hasNext := decoder.HasNext() + require.True(t, hasNext) + require.Equal(t, common.MessageTypeRow, messageType) + return decoder.NextDMLEvent() + } + + columnNames := func(event *commonEvent.DMLEvent) []string { + columns := event.TableInfo.GetColumns() + result := make([]string, 0, len(columns)) + for _, column := range columns { + result = append(result, column.Name.O) + } + return result + } + + oldSchema := map[string]string{ + "data": "varchar(255)", + "id": "int", + } + newSchema := map[string]string{ + "data": "varchar(255)", + "id": "int", + "new_col": "int", + } + + oldEvent := decodeRow(buildRowMessage(100, oldSchema)) + require.Equal(t, []string{"data", "id"}, columnNames(oldEvent)) + + ddlMessage := &canalJSONMessageWithTiDBExtension{ + JSONMessage: &JSONMessage{ + Schema: "test", + Table: "table_5", + IsDDL: true, + EventType: "ALTER", + Query: "ALTER TABLE `test`.`table_5` ADD COLUMN `new_col` INT", + }, + Extensions: &tidbExtension{CommitTs: 200}, + } + payload, err := json.Marshal(ddlMessage) + require.NoError(t, err) + decoder.AddKeyValue(nil, payload) + messageType, hasNext := decoder.HasNext() + require.True(t, hasNext) + require.Equal(t, common.MessageTypeDDL, messageType) + ddl := decoder.NextDDLEvent() + require.Equal(t, oldEvent.GetTableID(), ddl.GetBlockedTables().TableIDs[0]) + + newEvent := decodeRow(buildRowMessage(300, newSchema)) + require.Equal(t, []string{"data", "id", "new_col"}, columnNames(newEvent)) + + lateOldEvent := decodeRow(buildRowMessage(100, oldSchema)) + require.Equal(t, []string{"data", "id"}, columnNames(lateOldEvent)) + require.Equal(t, oldEvent.GetTableID(), lateOldEvent.GetTableID()) + require.Equal(t, oldEvent.GetTableID(), newEvent.GetTableID()) +} + +func TestDecoderTableInfoCacheUsesDDLCommitTsBoundary(t *testing.T) { + tableIDAllocator.Clean() + dec := &decoder{ + tableInfoCache: make(map[tableKey]*commonType.TableInfo), + ddlCommitTs: make(map[tableNameKey][]uint64), + } + buildMessage := func(commitTs uint64) *canalJSONMessageWithTiDBExtension { + return &canalJSONMessageWithTiDBExtension{ + JSONMessage: &JSONMessage{ + Schema: "test", + Table: "table_5", + PKNames: []string{"id"}, + MySQLType: map[string]string{ + "data": "varchar(255)", + "id": "int", + }, + }, + Extensions: &tidbExtension{CommitTs: commitTs}, + } + } + + beforeDDL := dec.queryTableInfo(buildMessage(100)) + dec.addDDLCommitTs("test", "table_5", 200) + afterDDL := dec.queryTableInfo(buildMessage(300)) + lateBeforeDDL := dec.queryTableInfo(buildMessage(100)) + + require.NotSame(t, beforeDDL, afterDDL) + require.Same(t, beforeDDL, lateBeforeDDL) + require.Equal(t, []uint64{200}, dec.ddlCommitTs[tableNameKey{schema: "test", table: "table_5"}]) +} + func TestCreateTableDDL(t *testing.T) { helper := commonEvent.NewEventTestHelper(t) defer helper.Close() From bf53ba7f502f27fdc3df14f46948f0be4800c1b2 Mon Sep 17 00:00:00 2001 From: wk989898 Date: Mon, 6 Jul 2026 10:06:55 +0000 Subject: [PATCH 02/12] update Signed-off-by: wk989898 --- cmd/kafka-consumer/writer.go | 63 ++++--- cmd/kafka-consumer/writer_test.go | 12 +- cmd/pulsar-consumer/writer.go | 79 +++++--- cmd/pulsar-consumer/writer_test.go | 177 ++++++++++++++++-- cmd/storage-consumer/consumer.go | 36 ++-- cmd/util/event_group.go | 103 ++++++---- cmd/util/event_group_test.go | 114 +++++++---- pkg/sink/codec/avro/avro_test.go | 2 +- pkg/sink/codec/avro/decoder.go | 12 +- pkg/sink/codec/avro/encoder_test.go | 4 +- pkg/sink/codec/canal/canal_json_decoder.go | 47 +++-- .../codec/canal/canal_json_encoder_test.go | 24 +-- pkg/sink/codec/canal/canal_json_test.go | 32 ++-- .../codec/canal/canal_json_txn_decoder.go | 10 +- pkg/sink/codec/common/decoder.go | 59 +++++- pkg/sink/codec/csv/csv_decoder.go | 6 +- pkg/sink/codec/csv/csv_decoder_test.go | 2 +- pkg/sink/codec/debezium/avro_decoder.go | 4 +- pkg/sink/codec/debezium/avro_test.go | 4 +- pkg/sink/codec/debezium/debezium_test.go | 2 +- pkg/sink/codec/debezium/decoder.go | 12 +- pkg/sink/codec/open/decoder.go | 12 +- pkg/sink/codec/open/encoder_test.go | 56 +++--- pkg/sink/codec/simple/decoder.go | 18 +- pkg/sink/codec/simple/encoder_test.go | 28 +-- 25 files changed, 650 insertions(+), 268 deletions(-) diff --git a/cmd/kafka-consumer/writer.go b/cmd/kafka-consumer/writer.go index cd8807e4e3..3777f1566b 100644 --- a/cmd/kafka-consumer/writer.go +++ b/cmd/kafka-consumer/writer.go @@ -151,7 +151,6 @@ func (w *writer) flushDDLEvent(ctx context.Context, ddl *event.DDLEvent) error { var ( done = make(chan struct{}, 1) - total int flushed atomic.Int64 ) @@ -164,12 +163,16 @@ func (w *writer) flushDDLEvent(ctx context.Context, ddl *event.DDLEvent) error { if !ok { continue } - before := len(resolvedEvents) - resolvedEvents = g.ResolveInto(commitTs, resolvedEvents) - total += len(resolvedEvents) - before + messages := g.ResolveInto(commitTs, nil) + events := make([]*event.DMLEvent, 0, len(messages)) + for _, message := range messages { + events = util.AppendOrMergeDMLEvent(events, message.ToDMLEvent()) + } + resolvedEvents = append(resolvedEvents, events...) } } + total := len(resolvedEvents) if total == 0 { return w.mysqlSink.WriteBlockEvent(ddl) } @@ -265,7 +268,6 @@ func (w *writer) flushDMLEventsByWatermark(ctx context.Context) error { var ( done = make(chan struct{}, 1) - total int flushed atomic.Int64 ) @@ -273,11 +275,15 @@ func (w *writer) flushDMLEventsByWatermark(ctx context.Context) error { resolvedEvents := make([]*event.DMLEvent, 0) for _, p := range w.progresses { for _, group := range p.eventsGroup { - before := len(resolvedEvents) - resolvedEvents = group.ResolveInto(watermark, resolvedEvents) - total += len(resolvedEvents) - before + messages := group.ResolveInto(watermark, nil) + events := make([]*event.DMLEvent, 0, len(messages)) + for _, message := range messages { + events = util.AppendOrMergeDMLEvent(events, message.ToDMLEvent()) + } + resolvedEvents = append(resolvedEvents, events...) } } + total := len(resolvedEvents) if total == 0 { return nil } @@ -349,7 +355,7 @@ func (w *writer) WriteMessage(ctx context.Context, message *kafka.Message) bool log.Info("simple protocol cached event resolved, append to the group", zap.Int64("tableID", row.GetTableID()), zap.Uint64("commitTs", row.CommitTs), zap.Int32("partition", partition), zap.Any("offset", offset)) - w.appendRow2Group(row, progress, offset) + w.appendMessage2Group(common.NewDMLMessageFromEvent(row), progress, offset) } } @@ -373,7 +379,11 @@ func (w *writer) WriteMessage(ctx context.Context, message *kafka.Message) bool needFlush = true case common.MessageTypeRow: var counter int - row := progress.decoder.NextDMLEvent() + dmlMessage := progress.decoder.NextDMLMessage() + var row *event.DMLEvent + if dmlMessage != nil { + row = dmlMessage.ToDMLEvent() + } if row == nil { if w.protocol != config.ProtocolSimple { log.Panic("DML event is nil, it's not expected", @@ -383,15 +393,15 @@ func (w *writer) WriteMessage(ctx context.Context, message *kafka.Message) bool break } - w.appendRow2Group(row, progress, offset) + w.appendMessage2Group(common.NewDMLMessageFromEvent(row), progress, offset) counter++ for { _, hasNext = progress.decoder.HasNext() if !hasNext { break } - row = progress.decoder.NextDMLEvent() - w.appendRow2Group(row, progress, offset) + row = progress.decoder.NextDMLMessage().ToDMLEvent() + w.appendMessage2Group(common.NewDMLMessageFromEvent(row), progress, offset) counter++ } // If the message containing only one event exceeds the length limit, CDC will allow it and issue a warning. @@ -578,15 +588,16 @@ func (w *writer) checkPartition(row *event.DMLEvent, partition int32, offset kaf } } -func (w *writer) appendRow2Group(dml *event.DMLEvent, progress *partitionProgress, offset kafka.Offset) { +func (w *writer) appendMessage2Group(message *common.DMLMessage, progress *partitionProgress, offset kafka.Offset) { + dml := message.ToDMLEvent() w.checkPartition(dml, progress.partition, offset) // if the kafka cluster is normal, this should not hit. // else if the cluster is abnormal, the consumer may consume old message, then cause the watermark fallback. var ( - tableID = dml.GetTableID() - schema = dml.TableInfo.GetSchemaName() - table = dml.TableInfo.GetTableName() - commitTs = dml.GetCommitTs() + tableID = message.TableID + schema = message.Schema + table = message.Table + commitTs = message.GetCommitTs() ) group := progress.eventsGroup[tableID] if group == nil { @@ -602,12 +613,12 @@ func (w *writer) appendRow2Group(dml *event.DMLEvent, progress *partitionProgres return } if commitTs >= group.HighWatermark { - group.Append(dml, false) + group.AppendMessage(message, false) log.Debug("DML event append to the group", zap.Int32("partition", group.Partition), zap.Any("offset", offset), zap.Uint64("commitTs", commitTs), zap.Uint64("HighWatermark", group.HighWatermark), zap.String("schema", schema), zap.String("table", table), zap.Int64("tableID", tableID), - zap.Stringer("eventType", dml.RowTypes[0])) + zap.Stringer("eventType", message.RowType)) return } if w.enableTableAcrossNodes { @@ -615,8 +626,8 @@ func (w *writer) appendRow2Group(dml *event.DMLEvent, progress *partitionProgres zap.Int32("partition", group.Partition), zap.Any("offset", offset), zap.Uint64("commitTs", commitTs), zap.Uint64("HighWatermark", group.HighWatermark), zap.String("schema", schema), zap.String("table", table), zap.Int64("tableID", tableID), - zap.Stringer("eventType", dml.RowTypes[0])) - group.Append(dml, true) + zap.Stringer("eventType", message.RowType)) + group.AppendMessage(message, true) return } switch w.protocol { @@ -631,7 +642,7 @@ func (w *writer) appendRow2Group(dml *event.DMLEvent, progress *partitionProgres zap.Uint64("commitTs", commitTs), zap.Uint64("highWatermark", group.HighWatermark), zap.Any("partitionWatermark", progress.watermark), zap.Any("watermarkOffset", progress.watermarkOffset), zap.String("schema", schema), zap.String("table", table), zap.Int64("tableID", tableID), - zap.Stringer("eventType", dml.RowTypes[0]), + zap.Stringer("eventType", message.RowType), // zap.Any("columns", row.Columns), zap.Any("preColumns", row.PreColumns), zap.Any("protocol", w.protocol), zap.Bool("IsPartition", dml.TableInfo.TableName.IsPartition)) case config.ProtocolCanalJSON, config.ProtocolOpen, config.ProtocolAvro, @@ -643,8 +654,8 @@ func (w *writer) appendRow2Group(dml *event.DMLEvent, progress *partitionProgres zap.Int32("partition", group.Partition), zap.Any("offset", offset), zap.Uint64("commitTs", commitTs), zap.Uint64("highWatermark", group.HighWatermark), zap.String("schema", schema), zap.String("table", table), zap.Int64("tableID", tableID), - zap.Stringer("eventType", dml.RowTypes[0]), zap.Any("protocol", w.protocol)) - group.Append(dml, true) + zap.Stringer("eventType", message.RowType), zap.Any("protocol", w.protocol)) + group.AppendMessage(message, true) return } log.Warn("DML event fallback row, since less than the group high watermark, ignore it", @@ -652,7 +663,7 @@ func (w *writer) appendRow2Group(dml *event.DMLEvent, progress *partitionProgres zap.Uint64("commitTs", commitTs), zap.Uint64("HighWatermark", group.HighWatermark), zap.Any("partitionWatermark", progress.watermark), zap.Any("watermarkOffset", progress.watermarkOffset), zap.String("schema", schema), zap.String("table", table), zap.Int64("tableID", tableID), - zap.Stringer("eventType", dml.RowTypes[0]), + zap.Stringer("eventType", message.RowType), // zap.Any("columns", row.Columns), zap.Any("preColumns", row.PreColumns), zap.Any("protocol", w.protocol), zap.Bool("IsPartition", dml.TableInfo.TableName.IsPartition)) default: diff --git a/cmd/kafka-consumer/writer_test.go b/cmd/kafka-consumer/writer_test.go index 35841c253e..e848c61239 100644 --- a/cmd/kafka-consumer/writer_test.go +++ b/cmd/kafka-consumer/writer_test.go @@ -311,12 +311,12 @@ func TestOnDDLMarksRoutedCreateTableLikePartitionTableForAvro(t *testing.T) { } progress := w.progresses[0] - w.appendRow2Group(newDMLEvent(200), progress, kafka.Offset(10)) - w.appendRow2Group(newDMLEvent(100), progress, kafka.Offset(11)) + w.appendMessage2Group(codeccommon.NewDMLMessageFromEvent(newDMLEvent(200)), progress, kafka.Offset(10)) + w.appendMessage2Group(codeccommon.NewDMLMessageFromEvent(newDMLEvent(100)), progress, kafka.Offset(11)) resolved := progress.eventsGroup[1].ResolveInto(150, nil) require.Len(t, resolved, 1) - require.Equal(t, uint64(100), resolved[0].CommitTs) + require.Equal(t, uint64(100), resolved[0].GetCommitTs()) } func TestAppendRow2GroupKeepsDebeziumPartitionTableFallback(t *testing.T) { @@ -359,12 +359,12 @@ func TestAppendRow2GroupKeepsDebeziumPartitionTableFallback(t *testing.T) { } progress := w.progresses[0] - w.appendRow2Group(newDMLEvent(200), progress, kafka.Offset(10)) - w.appendRow2Group(newDMLEvent(100), progress, kafka.Offset(11)) + w.appendMessage2Group(codeccommon.NewDMLMessageFromEvent(newDMLEvent(200)), progress, kafka.Offset(10)) + w.appendMessage2Group(codeccommon.NewDMLMessageFromEvent(newDMLEvent(100)), progress, kafka.Offset(11)) resolved := progress.eventsGroup[1].ResolveInto(150, nil) require.Len(t, resolved, 1) - require.Equal(t, uint64(100), resolved[0].CommitTs) + require.Equal(t, uint64(100), resolved[0].GetCommitTs()) }) } } diff --git a/cmd/pulsar-consumer/writer.go b/cmd/pulsar-consumer/writer.go index c2767191a8..3670c503c7 100644 --- a/cmd/pulsar-consumer/writer.go +++ b/cmd/pulsar-consumer/writer.go @@ -143,7 +143,6 @@ func (w *writer) flushDDLEvent(ctx context.Context, ddl *commonEvent.DDLEvent) e var ( done = make(chan struct{}, 1) - total int flushed atomic.Int64 ) @@ -156,12 +155,16 @@ func (w *writer) flushDDLEvent(ctx context.Context, ddl *commonEvent.DDLEvent) e if !ok { continue } - before := len(resolvedEvents) - resolvedEvents = g.ResolveInto(commitTs, resolvedEvents) - total += len(resolvedEvents) - before + messages := g.ResolveInto(commitTs, nil) + events := make([]*commonEvent.DMLEvent, 0, len(messages)) + for _, message := range messages { + events = util.AppendOrMergeDMLEvent(events, w.assembleDMLEvent(progress, message)) + } + resolvedEvents = append(resolvedEvents, events...) } } + total := len(resolvedEvents) if total == 0 { return w.mysqlSink.WriteBlockEvent(ddl) } @@ -257,7 +260,6 @@ func (w *writer) flushDMLEventsByWatermark(ctx context.Context) error { var ( done = make(chan struct{}, 1) - total int flushed atomic.Int64 ) @@ -265,11 +267,15 @@ func (w *writer) flushDMLEventsByWatermark(ctx context.Context) error { resolvedEvents := make([]*commonEvent.DMLEvent, 0) for _, p := range w.progresses { for _, group := range p.eventsGroup { - before := len(resolvedEvents) - resolvedEvents = group.ResolveInto(watermark, resolvedEvents) - total += len(resolvedEvents) - before + messages := group.ResolveInto(watermark, nil) + events := make([]*commonEvent.DMLEvent, 0, len(messages)) + for _, message := range messages { + events = util.AppendOrMergeDMLEvent(events, w.assembleDMLEvent(p, message)) + } + resolvedEvents = append(resolvedEvents, events...) } } + total := len(resolvedEvents) if total == 0 { return nil } @@ -301,6 +307,24 @@ func (w *writer) flushDMLEventsByWatermark(ctx context.Context) error { } } +func (w *writer) assembleDMLEvent(progress *partitionProgress, message *common.DMLMessage) *commonEvent.DMLEvent { + row := message.ToDMLEvent() + if row == nil { + log.Panic("DML event is nil, it's not expected", + zap.Int32("partition", progress.partition), zap.Int64("tableID", message.TableID), + zap.Uint64("commitTs", message.GetCommitTs())) + } + if row.GetTableID() != message.TableID || row.GetCommitTs() != message.GetCommitTs() || + len(row.RowTypes) == 0 || row.RowTypes[0] != message.RowType { + log.Panic("decoded DML event metadata mismatch", + zap.Int32("partition", progress.partition), + zap.Int64("pendingTableID", message.TableID), zap.Int64("decodedTableID", row.GetTableID()), + zap.Uint64("pendingCommitTs", message.GetCommitTs()), zap.Uint64("decodedCommitTs", row.GetCommitTs()), + zap.Stringer("pendingEventType", message.RowType), zap.Any("decodedEventTypes", row.RowTypes)) + } + return row +} + // WriteMessage is to decode pulsar message to event. // return true if the message is flushed to the downstream. // return error if flush messages failed. @@ -341,12 +365,11 @@ func (w *writer) WriteMessage(ctx context.Context, message pulsar.Message) bool zap.Any("blockedTables", ddl.GetBlockedTables())) needFlush = true case common.MessageTypeRow: - row := progress.decoder.NextDMLEvent() - if row == nil { - log.Panic("DML event is nil, it's not expected") + dmlMessage := progress.decoder.NextDMLMessage() + if dmlMessage == nil { + log.Panic("DML message is nil, it's not expected") } - - w.appendRow2Group(row, progress) + w.appendMessage2Group(dmlMessage, progress) default: log.Panic("unknown message type", zap.Any("messageType", messageType)) } @@ -486,12 +509,12 @@ func (w *writer) addPartitionTable(schema, table string) { w.partitionTableAccessor.Add(schema, table) } -func (w *writer) appendRow2Group(dml *commonEvent.DMLEvent, progress *partitionProgress) { +func (w *writer) appendMessage2Group(message *common.DMLMessage, progress *partitionProgress) { var ( - tableID = dml.GetTableID() - schema = dml.TableInfo.GetSchemaName() - table = dml.TableInfo.GetTableName() - commitTs = dml.GetCommitTs() + tableID = message.TableID + schema = message.Schema + table = message.Table + commitTs = message.GetCommitTs() ) group := progress.eventsGroup[tableID] if group == nil { @@ -506,39 +529,41 @@ func (w *writer) appendRow2Group(dml *commonEvent.DMLEvent, progress *partitionP return } if commitTs >= group.HighWatermark { - group.Append(dml, false) + group.AppendMessage(message, false) log.Debug("DML event append to the group", zap.Uint64("commitTs", commitTs), zap.Uint64("highWatermark", group.HighWatermark), zap.String("schema", schema), zap.String("table", table), zap.Int64("tableID", tableID), - zap.Stringer("eventType", dml.RowTypes[0])) + zap.Stringer("eventType", message.RowType)) return } if w.enableTableAcrossNodes { log.Warn("DML events fallback, but enableTableAcrossNodes is true, still append it", zap.Uint64("commitTs", commitTs), zap.Uint64("highWatermark", group.HighWatermark), zap.String("schema", schema), zap.String("table", table), zap.Int64("tableID", tableID), - zap.Stringer("eventType", dml.RowTypes[0])) - group.Append(dml, true) + zap.Stringer("eventType", message.RowType)) + group.AppendMessage(message, true) return } switch w.protocol { case config.ProtocolCanalJSON: // for partition table, the canal-json message cannot assign physical table id to each dml message, // we cannot distinguish whether it's a real fallback event or not, still append it. - if w.partitionTableAccessor.IsPartitionTable(schema, table) { + isPartitionTable := w.partitionTableAccessor != nil && + w.partitionTableAccessor.IsPartitionTable(schema, table) + if isPartitionTable { log.Warn("DML events fallback, but it's canal-json and partition table, still append it", zap.Uint64("commitTs", commitTs), zap.Uint64("highWatermark", group.HighWatermark), zap.String("schema", schema), zap.String("table", table), zap.Int64("tableID", tableID), - zap.Stringer("eventType", dml.RowTypes[0])) - group.Append(dml, true) + zap.Stringer("eventType", message.RowType)) + group.AppendMessage(message, true) return } log.Warn("DML event fallback row, since less than the group high watermark, ignore it", zap.Uint64("commitTs", commitTs), zap.Uint64("highWatermark", group.HighWatermark), zap.Any("partitionWatermark", progress.watermark), zap.Any("watermark", progress.watermark), zap.String("schema", schema), zap.String("table", table), zap.Int64("tableID", tableID), - zap.Stringer("eventType", dml.RowTypes[0]), - zap.Any("protocol", w.protocol), zap.Bool("IsPartition", dml.TableInfo.TableName.IsPartition)) + zap.Stringer("eventType", message.RowType), + zap.Any("protocol", w.protocol), zap.Bool("IsPartition", isPartitionTable)) default: log.Panic("unknown protocol", zap.Any("protocol", w.protocol)) } diff --git a/cmd/pulsar-consumer/writer_test.go b/cmd/pulsar-consumer/writer_test.go index 9e83ae63d9..dadbe02a8f 100644 --- a/cmd/pulsar-consumer/writer_test.go +++ b/cmd/pulsar-consumer/writer_test.go @@ -16,7 +16,9 @@ package main import ( "context" "testing" + "time" + "github.com/apache/pulsar-client-go/pulsar" "github.com/golang/mock/gomock" "github.com/pingcap/ticdc/cmd/util" sinkmock "github.com/pingcap/ticdc/downstreamadapter/sink/mock" @@ -25,7 +27,6 @@ import ( "github.com/pingcap/ticdc/pkg/config" codeccommon "github.com/pingcap/ticdc/pkg/sink/codec/common" timodel "github.com/pingcap/tidb/pkg/meta/model" - "github.com/pingcap/tidb/pkg/util/chunk" "github.com/stretchr/testify/require" ) @@ -293,23 +294,171 @@ func TestOnDDLMarksRoutedCreateTableLikePartitionTable(t *testing.T) { w.onDDL(ddl) require.True(t, w.partitionTableAccessor.IsPartitionTable("target", "dst")) - newDMLEvent := func(commitTs uint64) *commonEvent.DMLEvent { - return &commonEvent.DMLEvent{ - PhysicalTableID: 1, - CommitTs: commitTs, - RowTypes: []common.RowType{common.RowTypeUpdate}, - Rows: chunk.NewChunkWithCapacity(nil, 0), - TableInfo: &common.TableInfo{ - TableName: common.TableName{Schema: "target", Table: "dst"}, - }, - } + newDMLMessage := func(commitTs uint64) *codeccommon.DMLMessage { + return codeccommon.NewDMLMessage(1, "target", "dst", commitTs, common.RowTypeUpdate, nil) } progress := w.progresses[0] - w.appendRow2Group(newDMLEvent(200), progress) - w.appendRow2Group(newDMLEvent(100), progress) + w.appendMessage2Group(newDMLMessage(200), progress) + w.appendMessage2Group(newDMLMessage(100), progress) resolved := progress.eventsGroup[1].ResolveInto(150, nil) require.Len(t, resolved, 1) - require.Equal(t, uint64(100), resolved[0].CommitTs) + require.Equal(t, uint64(100), resolved[0].GetCommitTs()) +} + +func TestWriteMessageDefersDMLAssemblyUntilFlush(t *testing.T) { + ctx := context.Background() + ctrl := gomock.NewController(t) + s := sinkmock.NewMockSink(ctrl) + s.EXPECT().AddDMLEvent(gomock.Any()).Do(func(event *commonEvent.DMLEvent) { + event.PostFlush() + }).Times(1) + + decoder := &deferredDMLDecoder{ + row: &commonEvent.DMLEvent{ + PhysicalTableID: 1, + CommitTs: 100, + RowTypes: []common.RowType{common.RowTypeInsert}, + TableInfo: &common.TableInfo{ + TableName: common.TableName{Schema: "test", Table: "t", TableID: 1}, + }, + }, + } + progress := &partitionProgress{ + partition: 0, + eventsGroup: make(map[int64]*util.EventsGroup), + decoder: decoder, + } + w := &writer{ + progresses: []*partitionProgress{progress}, + mysqlSink: s, + protocol: config.ProtocolCanalJSON, + } + + needCommit := w.WriteMessage(ctx, fakePulsarMessage{key: "k", payload: []byte(`{"fake":"row"}`)}) + require.False(t, needCommit) + require.Equal(t, 1, decoder.addKeyValueCount) + require.Equal(t, 1, decoder.hasNextCount) + require.Equal(t, 1, decoder.nextDMLMessageCount) + require.Zero(t, decoder.toDMLEventCount) + require.Len(t, progress.eventsGroup[1].ResolveInto(99, nil), 0) + + progress.watermark = 100 + require.True(t, w.Write(ctx, codeccommon.MessageTypeResolved)) + require.Equal(t, 1, decoder.addKeyValueCount) + require.Equal(t, 1, decoder.hasNextCount) + require.Equal(t, 1, decoder.nextDMLMessageCount) + require.Equal(t, 1, decoder.toDMLEventCount) + require.Empty(t, progress.eventsGroup[1].ResolveInto(100, nil)) + require.Equal(t, []byte(`{"fake":"row"}`), decoder.lastValue) +} + +type deferredDMLDecoder struct { + row *commonEvent.DMLEvent + + addKeyValueCount int + hasNextCount int + nextDMLMessageCount int + toDMLEventCount int + lastValue []byte +} + +func (d *deferredDMLDecoder) AddKeyValue(_, value []byte) { + d.addKeyValueCount++ + d.lastValue = append(d.lastValue[:0], value...) +} + +func (d *deferredDMLDecoder) HasNext() (codeccommon.MessageType, bool) { + d.hasNextCount++ + return codeccommon.MessageTypeRow, true +} + +func (d *deferredDMLDecoder) NextResolvedEvent() uint64 { + return 0 +} + +func (d *deferredDMLDecoder) NextDMLMessage() *codeccommon.DMLMessage { + d.nextDMLMessageCount++ + return codeccommon.NewDMLMessage(1, "test", "t", 100, common.RowTypeInsert, func() *commonEvent.DMLEvent { + d.toDMLEventCount++ + return d.row + }) +} + +func (d *deferredDMLDecoder) NextDDLEvent() *commonEvent.DDLEvent { + return nil +} + +type fakePulsarMessage struct { + key string + payload []byte +} + +func (m fakePulsarMessage) Topic() string { + return "" +} + +func (m fakePulsarMessage) ProducerName() string { + return "" +} + +func (m fakePulsarMessage) Properties() map[string]string { + return nil +} + +func (m fakePulsarMessage) Payload() []byte { + return m.payload +} + +func (m fakePulsarMessage) ID() pulsar.MessageID { + return nil +} + +func (m fakePulsarMessage) PublishTime() time.Time { + return time.Time{} +} + +func (m fakePulsarMessage) EventTime() time.Time { + return time.Time{} +} + +func (m fakePulsarMessage) Key() string { + return m.key +} + +func (m fakePulsarMessage) OrderingKey() string { + return "" +} + +func (m fakePulsarMessage) RedeliveryCount() uint32 { + return 0 +} + +func (m fakePulsarMessage) IsReplicated() bool { + return false +} + +func (m fakePulsarMessage) GetReplicatedFrom() string { + return "" +} + +func (m fakePulsarMessage) GetSchemaValue(interface{}) error { + return nil +} + +func (m fakePulsarMessage) SchemaVersion() []byte { + return nil +} + +func (m fakePulsarMessage) GetEncryptionContext() *pulsar.EncryptionContext { + return nil +} + +func (m fakePulsarMessage) Index() *uint64 { + return nil +} + +func (m fakePulsarMessage) BrokerPublishTime() *time.Time { + return nil } diff --git a/cmd/storage-consumer/consumer.go b/cmd/storage-consumer/consumer.go index e6e13f6cd4..d4988a0731 100644 --- a/cmd/storage-consumer/consumer.go +++ b/cmd/storage-consumer/consumer.go @@ -270,12 +270,12 @@ func (c *consumer) getNewFiles( return tableDMLMap, err } -func (c *consumer) appendRow2Group(dml *event.DMLEvent, enableTableAcrossNodes bool) { +func (c *consumer) appendMessage2Group(message *common.DMLMessage, enableTableAcrossNodes bool) { var ( - tableID = dml.GetTableID() - schema = dml.TableInfo.GetSchemaName() - table = dml.TableInfo.GetTableName() - commitTs = dml.GetCommitTs() + tableID = message.TableID + schema = message.Schema + table = message.Table + commitTs = message.GetCommitTs() ) group := c.eventsGroup[tableID] if group == nil { @@ -283,25 +283,26 @@ func (c *consumer) appendRow2Group(dml *event.DMLEvent, enableTableAcrossNodes b c.eventsGroup[tableID] = group } if commitTs >= group.HighWatermark { - group.Append(dml, false) + group.AppendMessage(message, false) log.Debug("DML event append to the group", zap.Uint64("commitTs", commitTs), zap.Uint64("highWatermark", group.HighWatermark), zap.String("schema", schema), zap.String("table", table), zap.Int64("tableID", tableID), - zap.Stringer("eventType", dml.RowTypes[0])) + zap.Stringer("eventType", message.RowType)) return } if enableTableAcrossNodes { log.Warn("DML events fallback, but enableTableAcrossNodes is true, still append it", zap.Uint64("commitTs", commitTs), zap.Uint64("highWatermark", group.HighWatermark), zap.String("schema", schema), zap.String("table", table), zap.Int64("tableID", tableID), - zap.Stringer("eventType", dml.RowTypes[0])) - group.Append(dml, true) + zap.Stringer("eventType", message.RowType)) + group.AppendMessage(message, true) return } log.Warn("dml event commit ts fallback, ignore", - zap.Uint64("commitTs", dml.CommitTs), + zap.Uint64("commitTs", commitTs), zap.Any("highWatermark", group.HighWatermark), - zap.Stringer("row", dml), + zap.String("schema", schema), + zap.String("table", table), ) } @@ -350,9 +351,9 @@ func (c *consumer) appendDMLEvents( if tp == common.MessageTypeRow { c.dmlCount.Add(1) - row := decoder.NextDMLEvent() + row := decoder.NextDMLMessage().ToDMLEvent() row.PhysicalTableID = tableID - c.appendRow2Group(row, fileIdx.EnableTableAcrossNodes) + c.appendMessage2Group(common.NewDMLMessageFromEvent(row), fileIdx.EnableTableAcrossNodes) filteredCnt++ } } @@ -370,7 +371,14 @@ func (c *consumer) flushDMLEvents(ctx context.Context, tableID int64) error { if group == nil { return nil } - events := group.GetAllEvents() + messages := group.GetAllMessages() + if len(messages) == 0 { + return nil + } + events := make([]*event.DMLEvent, 0, len(messages)) + for _, message := range messages { + events = util.AppendOrMergeDMLEvent(events, message.ToDMLEvent()) + } total := len(events) if total == 0 { return nil diff --git a/cmd/util/event_group.go b/cmd/util/event_group.go index f81fb54e6f..1c8d287446 100644 --- a/cmd/util/event_group.go +++ b/cmd/util/event_group.go @@ -14,20 +14,20 @@ package util import ( - "slices" "sort" "github.com/pingcap/log" commonEvent "github.com/pingcap/ticdc/pkg/common/event" + codeccommon "github.com/pingcap/ticdc/pkg/sink/codec/common" "go.uber.org/zap" ) -// EventsGroup could store change event message. +// EventsGroup stores change event messages. type EventsGroup struct { Partition int32 tableID int64 - events []*commonEvent.DMLEvent + messages []*codeccommon.DMLMessage HighWatermark uint64 } @@ -36,73 +36,94 @@ func NewEventsGroup(partition int32, tableID int64) *EventsGroup { return &EventsGroup{ Partition: partition, tableID: tableID, - events: make([]*commonEvent.DMLEvent, 0, 1024), + messages: make([]*codeccommon.DMLMessage, 0, 1024), } } -// Append will append an event to event groups. -func (g *EventsGroup) Append(row *commonEvent.DMLEvent, force bool) { - if row.CommitTs > g.HighWatermark { - g.HighWatermark = row.CommitTs +// AppendMessage appends a message to event groups. +func (g *EventsGroup) AppendMessage(message *codeccommon.DMLMessage, force bool) { + commitTs := message.GetCommitTs() + if commitTs > g.HighWatermark { + g.HighWatermark = commitTs } - var lastDMLEvent *commonEvent.DMLEvent - if len(g.events) > 0 { - lastDMLEvent = g.events[len(g.events)-1] + var lastMessage *codeccommon.DMLMessage + if len(g.messages) > 0 { + lastMessage = g.messages[len(g.messages)-1] } - if lastDMLEvent == nil || lastDMLEvent.GetCommitTs() < row.GetCommitTs() { - g.events = append(g.events, row) - return - } - - if lastDMLEvent.GetCommitTs() == row.GetCommitTs() { - lastDMLEvent.Rows.Append(row.Rows, 0, row.Rows.NumRows()) - lastDMLEvent.RowTypes = append(lastDMLEvent.RowTypes, row.RowTypes...) - lastDMLEvent.Length += row.Length - lastDMLEvent.PostTxnFlushed = append(lastDMLEvent.PostTxnFlushed, row.PostTxnFlushed...) + if lastMessage == nil || lastMessage.GetCommitTs() <= commitTs { + g.messages = append(g.messages, message) return } if force { - i := sort.Search(len(g.events), func(i int) bool { - return g.events[i].CommitTs > row.CommitTs + i := sort.Search(len(g.messages), func(i int) bool { + return g.messages[i].GetCommitTs() > commitTs }) - g.events = slices.Insert(g.events, i, row) + g.messages = append(g.messages, nil) + copy(g.messages[i+1:], g.messages[i:]) + g.messages[i] = message return } log.Panic("append event with smaller commit ts", zap.Int32("partition", g.Partition), zap.Int64("tableID", g.tableID), - zap.Uint64("lastCommitTs", lastDMLEvent.GetCommitTs()), zap.Uint64("commitTs", row.GetCommitTs())) + zap.Uint64("lastCommitTs", lastMessage.GetCommitTs()), zap.Uint64("commitTs", commitTs)) } -// ResolveInto appends all events with CommitTs <= resolve into dst and removes them from the group. -// ResolveInto copies pointers into dst first, then clears the -// resolved prefix so Go GC can reclaim resolved events once downstream is done with them. -func (g *EventsGroup) ResolveInto(resolve uint64, dst []*commonEvent.DMLEvent) []*commonEvent.DMLEvent { - i := sort.Search(len(g.events), func(i int) bool { - return g.events[i].CommitTs > resolve +// ResolveInto appends all messages with CommitTs <= resolve into dst and removes them from the group. +// ResolveInto copies pointers into dst first, then clears the resolved prefix so Go GC can reclaim +// resolved messages once downstream is done with them. +func (g *EventsGroup) ResolveInto(resolve uint64, dst []*codeccommon.DMLMessage) []*codeccommon.DMLMessage { + i := sort.Search(len(g.messages), func(i int) bool { + return g.messages[i].GetCommitTs() > resolve }) if i == 0 { return dst } // Copy pointers out first so we can safely clear the group's slice without affecting callers. - dst = append(dst, g.events[:i]...) - clear(g.events[:i]) - g.events = g.events[i:] - if len(g.events) != 0 { + dst = append(dst, g.messages[:i]...) + clear(g.messages[:i]) + g.messages = g.messages[i:] + if len(g.messages) != 0 { log.Debug("not all events resolved", zap.Int32("partition", g.Partition), zap.Int64("tableID", g.tableID), - zap.Int("resolved", i), zap.Int("remained", len(g.events)), - zap.Uint64("resolveTs", resolve), zap.Uint64("firstCommitTs", g.events[0].CommitTs)) + zap.Int("resolved", i), zap.Int("remained", len(g.messages)), + zap.Uint64("resolveTs", resolve), zap.Uint64("firstCommitTs", g.messages[0].GetCommitTs())) } return dst } -// GetAllEvents will get all events. -func (g *EventsGroup) GetAllEvents() []*commonEvent.DMLEvent { - result := g.events - g.events = nil +// GetAllMessages gets all messages. +func (g *EventsGroup) GetAllMessages() []*codeccommon.DMLMessage { + result := g.messages + g.messages = nil return result } + +// AppendOrMergeDMLEvent appends a DML event, or merges it into the previous event +// when both events belong to the same table group and have the same commit-ts. +func AppendOrMergeDMLEvent(events []*commonEvent.DMLEvent, row *commonEvent.DMLEvent) []*commonEvent.DMLEvent { + var lastDMLEvent *commonEvent.DMLEvent + if len(events) > 0 { + lastDMLEvent = events[len(events)-1] + } + + if lastDMLEvent == nil || lastDMLEvent.GetCommitTs() < row.GetCommitTs() { + return append(events, row) + } + + if lastDMLEvent.GetCommitTs() == row.GetCommitTs() { + lastDMLEvent.Rows.Append(row.Rows, 0, row.Rows.NumRows()) + lastDMLEvent.RowTypes = append(lastDMLEvent.RowTypes, row.RowTypes...) + lastDMLEvent.Length += row.Length + lastDMLEvent.PostTxnFlushed = append(lastDMLEvent.PostTxnFlushed, row.PostTxnFlushed...) + return events + } + + log.Panic("append event with smaller commit ts", + zap.Int64("tableID", row.GetTableID()), + zap.Uint64("lastCommitTs", lastDMLEvent.GetCommitTs()), zap.Uint64("commitTs", row.GetCommitTs())) + return events +} diff --git a/cmd/util/event_group_test.go b/cmd/util/event_group_test.go index 8a731bcac6..a8c5fd83ab 100644 --- a/cmd/util/event_group_test.go +++ b/cmd/util/event_group_test.go @@ -16,10 +16,27 @@ package util import ( "testing" + "github.com/pingcap/ticdc/pkg/common" commonEvent "github.com/pingcap/ticdc/pkg/common/event" + codeccommon "github.com/pingcap/ticdc/pkg/sink/codec/common" + "github.com/pingcap/tidb/pkg/util/chunk" "github.com/stretchr/testify/require" ) +func newTestDMLMessage(commitTs uint64) *codeccommon.DMLMessage { + return codeccommon.NewDMLMessage(1, "test", "t", commitTs, common.RowTypeInsert, nil) +} + +func newTestDMLEvent(commitTs uint64, rowTypes ...common.RowType) *commonEvent.DMLEvent { + return &commonEvent.DMLEvent{ + PhysicalTableID: 1, + CommitTs: commitTs, + Length: int32(len(rowTypes)), + RowTypes: rowTypes, + Rows: chunk.NewChunkWithCapacity(nil, 0), + } +} + func TestEventsGroupResolveIntoAppendsAndClearsResolvedPrefix(t *testing.T) { // Scenario: A consumer resolves a prefix of events by watermark/commit-ts and appends them // into a downstream batch slice. We must clear the resolved prefix in the group's backing @@ -31,75 +48,106 @@ func TestEventsGroupResolveIntoAppendsAndClearsResolvedPrefix(t *testing.T) { // 3. Verify (a) returned events are correct, (b) group keeps only the remaining event, // (c) the resolved prefix in the original backing slice is cleared (nil'd). group := NewEventsGroup(0, 1) - e1 := &commonEvent.DMLEvent{CommitTs: 1} - e2 := &commonEvent.DMLEvent{CommitTs: 2} - e3 := &commonEvent.DMLEvent{CommitTs: 3} - group.Append(e1, false) - group.Append(e2, false) - group.Append(e3, false) + m1 := newTestDMLMessage(1) + m2 := newTestDMLMessage(2) + m3 := newTestDMLMessage(3) + group.AppendMessage(m1, false) + group.AppendMessage(m2, false) + group.AppendMessage(m3, false) // Keep a reference to the original slice header so we can validate that ResolveInto clears // the resolved prefix in-place (this is what prevents GC retention of flushed events). - original := group.events + original := group.messages - var dst []*commonEvent.DMLEvent + var dst []*codeccommon.DMLMessage dst = group.ResolveInto(2, dst) require.Len(t, dst, 2) - require.Same(t, e1, dst[0]) - require.Same(t, e2, dst[1]) + require.Same(t, m1, dst[0]) + require.Same(t, m2, dst[1]) - require.Len(t, group.events, 1) - require.Same(t, e3, group.events[0]) + require.Len(t, group.messages, 1) + require.Same(t, m3, group.messages[0]) // The resolved prefix must be nil so the group doesn't keep flushed events alive via its // backing array (classic Go slice memory retention pitfall). require.Nil(t, original[0]) require.Nil(t, original[1]) - require.Same(t, e3, original[2]) + require.Same(t, m3, original[2]) } func TestEventsGroupResolveIntoNoopWhenNothingResolved(t *testing.T) { // Scenario: resolveTs is behind all buffered events. // Expectation: ResolveInto should be a no-op (dst unchanged, group unchanged). group := NewEventsGroup(0, 1) - e1 := &commonEvent.DMLEvent{CommitTs: 10} - e2 := &commonEvent.DMLEvent{CommitTs: 20} - group.Append(e1, false) - group.Append(e2, false) + m1 := newTestDMLMessage(10) + m2 := newTestDMLMessage(20) + group.AppendMessage(m1, false) + group.AppendMessage(m2, false) - original := group.events - dst := make([]*commonEvent.DMLEvent, 0, 1) + original := group.messages + dst := make([]*codeccommon.DMLMessage, 0, 1) dst = group.ResolveInto(5, dst) require.Len(t, dst, 0) - require.Len(t, group.events, 2) - require.Same(t, e1, group.events[0]) - require.Same(t, e2, group.events[1]) + require.Len(t, group.messages, 2) + require.Same(t, m1, group.messages[0]) + require.Same(t, m2, group.messages[1]) // No prefix should be cleared because nothing was resolved. - require.Same(t, e1, original[0]) - require.Same(t, e2, original[1]) + require.Same(t, m1, original[0]) + require.Same(t, m2, original[1]) } func TestEventsGroupResolveIntoClearsAllWhenFullyResolved(t *testing.T) { // Scenario: resolveTs advances beyond all buffered events. // Expectation: group is emptied and all backing-array pointers for resolved events are cleared. group := NewEventsGroup(0, 1) - e1 := &commonEvent.DMLEvent{CommitTs: 1} - e2 := &commonEvent.DMLEvent{CommitTs: 2} - group.Append(e1, false) - group.Append(e2, false) + m1 := newTestDMLMessage(1) + m2 := newTestDMLMessage(2) + group.AppendMessage(m1, false) + group.AppendMessage(m2, false) - original := group.events - var dst []*commonEvent.DMLEvent + original := group.messages + var dst []*codeccommon.DMLMessage dst = group.ResolveInto(100, dst) require.Len(t, dst, 2) - require.Same(t, e1, dst[0]) - require.Same(t, e2, dst[1]) + require.Same(t, m1, dst[0]) + require.Same(t, m2, dst[1]) - require.Len(t, group.events, 0) + require.Len(t, group.messages, 0) require.Nil(t, original[0]) require.Nil(t, original[1]) } + +func TestAppendOrMergeDMLEventMergesSameCommitTs(t *testing.T) { + var flushed []int + e1 := newTestDMLEvent(10, common.RowTypeInsert) + e1.AddPostFlushFunc(func() { flushed = append(flushed, 1) }) + e2 := newTestDMLEvent(10, common.RowTypeDelete) + e2.AddPostFlushFunc(func() { flushed = append(flushed, 2) }) + + events := AppendOrMergeDMLEvent(nil, e1) + events = AppendOrMergeDMLEvent(events, e2) + + require.Len(t, events, 1) + require.Same(t, e1, events[0]) + require.Equal(t, int32(2), events[0].Length) + require.Equal(t, []common.RowType{common.RowTypeInsert, common.RowTypeDelete}, events[0].RowTypes) + + events[0].PostFlush() + require.Equal(t, []int{1, 2}, flushed) +} + +func TestAppendOrMergeDMLEventAppendsDifferentCommitTs(t *testing.T) { + e1 := newTestDMLEvent(10, common.RowTypeInsert) + e2 := newTestDMLEvent(20, common.RowTypeDelete) + + events := AppendOrMergeDMLEvent(nil, e1) + events = AppendOrMergeDMLEvent(events, e2) + + require.Len(t, events, 2) + require.Same(t, e1, events[0]) + require.Same(t, e2, events[1]) +} diff --git a/pkg/sink/codec/avro/avro_test.go b/pkg/sink/codec/avro/avro_test.go index 798bc611d8..c4e5883e7c 100644 --- a/pkg/sink/codec/avro/avro_test.go +++ b/pkg/sink/codec/avro/avro_test.go @@ -187,7 +187,7 @@ func TestAvroEncodeDeleteEventWithWatermarkCarriesCommitTs(t *testing.T) { require.True(t, exists) require.Equal(t, common.MessageTypeRow, messageType) - decoded := decoder.NextDMLEvent() + decoded := decoder.NextDMLMessage().ToDMLEvent() require.NotNil(t, decoded) require.Equal(t, event.CommitTs, decoded.GetCommitTs()) require.Len(t, decoded.RowTypes, 1) diff --git a/pkg/sink/codec/avro/decoder.go b/pkg/sink/codec/avro/decoder.go index 8d6ae80506..211bf40616 100644 --- a/pkg/sink/codec/avro/decoder.go +++ b/pkg/sink/codec/avro/decoder.go @@ -113,8 +113,16 @@ func (d *decoder) NextResolvedEvent() uint64 { return ts } -// NextDMLEvent returns the next row changed event if exists -func (d *decoder) NextDMLEvent() *commonEvent.DMLEvent { +// NextDMLMessage returns the next row changed message if exists +func (d *decoder) NextDMLMessage() *common.DMLMessage { + event := d.nextDMLEvent() + if event == nil { + return nil + } + return common.NewDMLMessageFromEvent(event) +} + +func (d *decoder) nextDMLEvent() *commonEvent.DMLEvent { var ( valueMap map[string]interface{} valueSchema map[string]interface{} diff --git a/pkg/sink/codec/avro/encoder_test.go b/pkg/sink/codec/avro/encoder_test.go index 83797b551e..30a998a04a 100644 --- a/pkg/sink/codec/avro/encoder_test.go +++ b/pkg/sink/codec/avro/encoder_test.go @@ -69,7 +69,7 @@ func TestDMLEventE2E(t *testing.T) { require.True(t, exist) require.Equal(t, common.MessageTypeRow, messageType) - decodedEvent := decoder.NextDMLEvent() + decodedEvent := decoder.NextDMLMessage().ToDMLEvent() require.NotNil(t, decodedEvent) require.NotZero(t, decodedEvent.GetTableID()) @@ -131,7 +131,7 @@ func TestEncodeRoutedDMLEventUsesTargetNames(t *testing.T) { require.True(t, exists) require.Equal(t, common.MessageTypeRow, messageType) - decoded := decoder.NextDMLEvent() + decoded := decoder.NextDMLMessage().ToDMLEvent() require.Equal(t, "target_db", decoded.TableInfo.GetSchemaName()) require.Equal(t, "target_table", decoded.TableInfo.GetTableName()) } diff --git a/pkg/sink/codec/canal/canal_json_decoder.go b/pkg/sink/codec/canal/canal_json_decoder.go index 519ec7ee3b..c854ff87a8 100644 --- a/pkg/sink/codec/canal/canal_json_decoder.go +++ b/pkg/sink/codec/canal/canal_json_decoder.go @@ -203,8 +203,7 @@ func (d *decoder) assembleClaimCheckDMLEvent( log.Panic("unmarshal claim check message failed", zap.Any("value", util.RedactAny(value)), zap.Error(err)) } - d.msg = message - return d.NextDMLEvent() + return d.decodeDMLMessage(message) } func buildData(holder *common.ColumnsHolder) (map[string]interface{}, map[string]string) { @@ -301,19 +300,46 @@ func (d *decoder) assembleHandleKeyOnlyDMLEvent( result.Data = []map[string]interface{}{data} } - d.msg = result - return d.NextDMLEvent() + return d.decodeDMLMessage(result) } -// NextDMLEvent implements the Decoder interface +// NextDMLMessage implements the Decoder interface // `HasNext` should be called before this. -func (d *decoder) NextDMLEvent() *commonEvent.DMLEvent { +func (d *decoder) NextDMLMessage() *common.DMLMessage { if d.msg == nil || d.msg.messageType() != common.MessageTypeRow { + messageType := common.MessageTypeUnknown + if d.msg != nil { + messageType = d.msg.messageType() + } log.Panic("message type is not row changed", - zap.Any("messageType", d.msg.messageType()), zap.Any("msg", d.msg)) + zap.Any("messageType", messageType), zap.Any("msg", d.msg)) + } + + msg := d.msg + schemaName := *msg.getSchema() + tableName := *msg.getTable() + tableID := tableIDAllocator.Allocate(schemaName, tableName) + tableIDAllocator.AddBlockTableID(schemaName, tableName, tableID) + + var rowType commonType.RowType + switch msg.eventType() { + case canal.EventType_DELETE: + rowType = commonType.RowTypeDelete + case canal.EventType_INSERT: + rowType = commonType.RowTypeInsert + case canal.EventType_UPDATE: + rowType = commonType.RowTypeUpdate + default: + log.Panic("unknown event type for the DML event", zap.Any("eventType", msg.eventType())) } - message, withExtension := d.msg.(*canalJSONMessageWithTiDBExtension) + return common.NewDMLMessage(tableID, schemaName, tableName, msg.getCommitTs(), rowType, func() *commonEvent.DMLEvent { + return d.decodeDMLMessage(msg) + }) +} + +func (d *decoder) decodeDMLMessage(msg canalJSONMessageInterface) *commonEvent.DMLEvent { + message, withExtension := msg.(*canalJSONMessageWithTiDBExtension) if withExtension { ctx := context.Background() if message.Extensions.OnlyHandleKey && d.upstreamTiDB != nil { @@ -323,11 +349,10 @@ func (d *decoder) NextDMLEvent() *commonEvent.DMLEvent { return d.assembleClaimCheckDMLEvent(ctx, message.Extensions.ClaimCheckLocation) } } - return d.canalJSONMessage2DMLEvent() + return d.canalJSONMessage2DMLEvent(msg) } -func (d *decoder) canalJSONMessage2DMLEvent() *commonEvent.DMLEvent { - msg := d.msg +func (d *decoder) canalJSONMessage2DMLEvent(msg canalJSONMessageInterface) *commonEvent.DMLEvent { tableInfo := d.queryTableInfo(msg) result := new(commonEvent.DMLEvent) diff --git a/pkg/sink/codec/canal/canal_json_encoder_test.go b/pkg/sink/codec/canal/canal_json_encoder_test.go index 2953642f89..65191626bc 100644 --- a/pkg/sink/codec/canal/canal_json_encoder_test.go +++ b/pkg/sink/codec/canal/canal_json_encoder_test.go @@ -82,7 +82,7 @@ func TestDMLE2E(t *testing.T) { require.True(t, hasNext) require.Equal(t, messageType, common.MessageTypeRow) - decodedEvent := dml2rowEvent(t, decoder.NextDMLEvent()) + decodedEvent := dml2rowEvent(t, decoder.NextDMLMessage().ToDMLEvent()) require.True(t, decodedEvent.IsInsert()) if enableTiDBExtension { require.Equal(t, insertEvent.CommitTs, decodedEvent.CommitTs) @@ -103,7 +103,7 @@ func TestDMLE2E(t *testing.T) { require.True(t, hasNext) require.EqualValues(t, messageType, common.MessageTypeRow) - decodedEvent = dml2rowEvent(t, decoder.NextDMLEvent()) + decodedEvent = dml2rowEvent(t, decoder.NextDMLMessage().ToDMLEvent()) require.True(t, decodedEvent.IsUpdate()) err = encoder.AppendRowChangedEvent(ctx, "", deleteEvent) @@ -117,7 +117,7 @@ func TestDMLE2E(t *testing.T) { require.True(t, hasNext) require.EqualValues(t, messageType, common.MessageTypeRow) - decodedEvent = dml2rowEvent(t, decoder.NextDMLEvent()) + decodedEvent = dml2rowEvent(t, decoder.NextDMLMessage().ToDMLEvent()) require.NoError(t, err) require.True(t, decodedEvent.IsDelete()) } @@ -156,7 +156,7 @@ func TestCanalJSONCompressionE2E(t *testing.T) { require.True(t, hasNext) require.Equal(t, messageType, common.MessageTypeRow) - decodedEvent := decoder.NextDMLEvent() + decodedEvent := decoder.NextDMLMessage().ToDMLEvent() require.Equal(t, decodedEvent.CommitTs, insertEvent.CommitTs) require.Equal(t, decodedEvent.TableInfo.GetSchemaName(), insertEvent.TableInfo.GetSchemaName()) require.Equal(t, decodedEvent.TableInfo.GetTableName(), insertEvent.TableInfo.GetTableName()) @@ -229,7 +229,7 @@ func TestEncodeRoutedDMLEventUsesTargetNames(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, messageType) - decoded := dml2rowEvent(t, decoder.NextDMLEvent()) + decoded := dml2rowEvent(t, decoder.NextDMLMessage().ToDMLEvent()) require.Equal(t, "target_db", decoded.TableInfo.GetSchemaName()) require.Equal(t, "target_table", decoded.TableInfo.GetTableName()) } @@ -296,7 +296,7 @@ func TestCanalJSONClaimCheckE2E(t *testing.T) { require.Equal(t, messageType, common.MessageTypeRow) require.True(t, ok) - decodedLargeEvent := decoder.NextDMLEvent() + decodedLargeEvent := decoder.NextDMLMessage().ToDMLEvent() require.Equal(t, insertEvent.CommitTs, decodedLargeEvent.CommitTs) require.Equal(t, insertEvent.TableInfo.GetSchemaName(), decodedLargeEvent.TableInfo.GetSchemaName()) @@ -711,7 +711,7 @@ func TestCanalJSONContentCompatibleE2E(t *testing.T) { require.True(t, hasNext) require.Equal(t, messageType, common.MessageTypeRow) - decodedEvent := decoder.NextDMLEvent() + decodedEvent := decoder.NextDMLMessage().ToDMLEvent() require.NoError(t, err) require.Equal(t, decodedEvent.CommitTs, event.CommitTs) require.Equal(t, decodedEvent.TableInfo.GetSchemaName(), event.TableInfo.GetSchemaName()) @@ -772,7 +772,7 @@ func TestE2EPartitionTableByHash(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, tp) - decodedEvent := decoder.NextDMLEvent() + decodedEvent := decoder.NextDMLMessage().ToDMLEvent() require.NotZero(t, decodedEvent.GetTableID()) } @@ -829,7 +829,7 @@ func TestE2EPartitionTableByRange(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, tp) - decodedEvent := decoder.NextDMLEvent() + decodedEvent := decoder.NextDMLMessage().ToDMLEvent() require.NotZero(t, decodedEvent.GetTableID()) } @@ -893,7 +893,7 @@ func TestE2EPartitionTable(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, tp) - decodedEvent := decoder.NextDMLEvent() + decodedEvent := decoder.NextDMLMessage().ToDMLEvent() require.NotZero(t, decodedEvent.GetTableID()) rc, ok = insertEvent1.GetNextRow() @@ -913,7 +913,7 @@ func TestE2EPartitionTable(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, tp) - decodedEvent = decoder.NextDMLEvent() + decodedEvent = decoder.NextDMLMessage().ToDMLEvent() require.NotZero(t, decodedEvent.GetTableID()) rc, ok = insertEvent2.GetNextRow() @@ -933,7 +933,7 @@ func TestE2EPartitionTable(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, tp) - decodedEvent = decoder.NextDMLEvent() + decodedEvent = decoder.NextDMLMessage().ToDMLEvent() require.NotZero(t, decodedEvent.GetTableID()) } diff --git a/pkg/sink/codec/canal/canal_json_test.go b/pkg/sink/codec/canal/canal_json_test.go index bd912dcdca..e81a7c245f 100644 --- a/pkg/sink/codec/canal/canal_json_test.go +++ b/pkg/sink/codec/canal/canal_json_test.go @@ -85,7 +85,7 @@ func TestIntegerContentCompatible(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, messageType) - decodedInsert := decoder.NextDMLEvent() + decodedInsert := decoder.NextDMLMessage().ToDMLEvent() require.NotNil(t, decodedInsert) } @@ -170,7 +170,7 @@ func TestIntegerTypes(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, messageType) - decoded := decoder.NextDMLEvent() + decoded := decoder.NextDMLMessage().ToDMLEvent() if enableTiDBExtension { require.Equal(t, event.CommitTs, decoded.GetCommitTs()) } @@ -231,7 +231,7 @@ func TestFloatTypes(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, messageType) - event := decoder.NextDMLEvent() + event := decoder.NextDMLMessage().ToDMLEvent() change, ok := event.GetNextRow() require.True(t, ok) @@ -280,7 +280,7 @@ func TestTimeTypes(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, messageType) - event := decoder.NextDMLEvent() + event := decoder.NextDMLMessage().ToDMLEvent() change, ok := event.GetNextRow() require.True(t, ok) @@ -330,7 +330,7 @@ func TestStringTypes(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, messageType) - event := decoder.NextDMLEvent() + event := decoder.NextDMLMessage().ToDMLEvent() change, ok := event.GetNextRow() require.True(t, ok) @@ -380,7 +380,7 @@ func TestBlobTypes(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, messageType) - event := decoder.NextDMLEvent() + event := decoder.NextDMLMessage().ToDMLEvent() change, ok := event.GetNextRow() require.True(t, ok) @@ -430,7 +430,7 @@ func TestTextTypes(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, messageType) - event := decoder.NextDMLEvent() + event := decoder.NextDMLMessage().ToDMLEvent() change, ok := event.GetNextRow() require.True(t, ok) @@ -489,7 +489,7 @@ func TestOtherTypes(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, messageType) - event := decoder.NextDMLEvent() + event := decoder.NextDMLMessage().ToDMLEvent() change, ok := event.GetNextRow() require.True(t, ok) @@ -547,7 +547,7 @@ func TestDMLEventWithColumnSelector(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, messageType) - event := decoder.NextDMLEvent() + event := decoder.NextDMLMessage().ToDMLEvent() change, ok := event.GetNextRow() require.True(t, ok) @@ -606,7 +606,7 @@ func TestDMLMultiplePK(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, messageType) - event := decoder.NextDMLEvent() + event := decoder.NextDMLMessage().ToDMLEvent() change, ok := event.GetNextRow() require.True(t, ok) @@ -791,7 +791,7 @@ func TestLargeMessageClaimCheck(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, messageType) - decodedInsert := dec.NextDMLEvent() + decodedInsert := dec.NextDMLMessage().ToDMLEvent() require.NotNil(t, decodedInsert) change, ok := decodedInsert.GetNextRow() @@ -882,7 +882,7 @@ func TestMessageLargeHandleKeyOnly(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, messageType) - event := decoder.NextDMLEvent() + event := decoder.NextDMLMessage().ToDMLEvent() change, ok := event.GetNextRow() require.True(t, ok) @@ -973,7 +973,7 @@ func TestDMLTypeEvent(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, messageType) - decoded := decoder.NextDMLEvent() + decoded := decoder.NextDMLMessage().ToDMLEvent() change, ok := decoded.GetNextRow() require.True(t, ok) @@ -999,7 +999,7 @@ func TestDMLTypeEvent(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, messageType) - decoded := decoder.NextDMLEvent() + decoded := decoder.NextDMLMessage().ToDMLEvent() change, ok := decoded.GetNextRow() require.True(t, ok) @@ -1093,7 +1093,7 @@ func TestDDLSequence(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, messageType) - decodedInsert := dec.NextDMLEvent() + decodedInsert := dec.NextDMLMessage().ToDMLEvent() require.NotZero(t, decodedInsert.GetTableID()) addColumn := helper.DDL2Event(`alter table t add column c int`) @@ -1172,7 +1172,7 @@ func TestDecoderTableInfoCacheUsesDDLCommitTsAcrossColumnChanges(t *testing.T) { messageType, hasNext := decoder.HasNext() require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, messageType) - return decoder.NextDMLEvent() + return decoder.NextDMLMessage().ToDMLEvent() } columnNames := func(event *commonEvent.DMLEvent) []string { diff --git a/pkg/sink/codec/canal/canal_json_txn_decoder.go b/pkg/sink/codec/canal/canal_json_txn_decoder.go index 02129a0fb6..9674c027d1 100644 --- a/pkg/sink/codec/canal/canal_json_txn_decoder.go +++ b/pkg/sink/codec/canal/canal_json_txn_decoder.go @@ -96,7 +96,15 @@ func (d *txnDecoder) HasNext() (common.MessageType, bool) { return d.msg.messageType(), true } -func (d *txnDecoder) NextDMLEvent() *commonEvent.DMLEvent { +func (d *txnDecoder) NextDMLMessage() *common.DMLMessage { + event := d.nextDMLEvent() + if event == nil { + return nil + } + return common.NewDMLMessageFromEvent(event) +} + +func (d *txnDecoder) nextDMLEvent() *commonEvent.DMLEvent { if d.msg == nil || d.msg.messageType() != common.MessageTypeRow { log.Panic("message type is not row changed", zap.Any("messageType", d.msg.messageType()), zap.Any("msg", d.msg)) diff --git a/pkg/sink/codec/common/decoder.go b/pkg/sink/codec/common/decoder.go index d83219d7e7..6031b9b751 100644 --- a/pkg/sink/codec/common/decoder.go +++ b/pkg/sink/codec/common/decoder.go @@ -14,9 +14,64 @@ package common import ( + commonType "github.com/pingcap/ticdc/pkg/common" commonEvent "github.com/pingcap/ticdc/pkg/common/event" ) +type DMLMessage struct { + TableID int64 + Schema string + Table string + RowType commonType.RowType + + commitTs uint64 + toDMLEvent func() *commonEvent.DMLEvent +} + +func NewDMLMessage( + tableID int64, + schema string, + table string, + commitTs uint64, + rowType commonType.RowType, + toDMLEvent func() *commonEvent.DMLEvent, +) *DMLMessage { + return &DMLMessage{ + TableID: tableID, + Schema: schema, + Table: table, + RowType: rowType, + commitTs: commitTs, + toDMLEvent: toDMLEvent, + } +} + +func NewDMLMessageFromEvent(event *commonEvent.DMLEvent) *DMLMessage { + var ( + schema string + table string + rowType commonType.RowType + ) + if event.TableInfo != nil { + schema = event.TableInfo.GetSchemaName() + table = event.TableInfo.GetTableName() + } + if len(event.RowTypes) > 0 { + rowType = event.RowTypes[0] + } + return NewDMLMessage(event.GetTableID(), schema, table, event.GetCommitTs(), rowType, func() *commonEvent.DMLEvent { + return event + }) +} + +func (m *DMLMessage) GetCommitTs() uint64 { + return m.commitTs +} + +func (m *DMLMessage) ToDMLEvent() *commonEvent.DMLEvent { + return m.toDMLEvent() +} + // Decoder is an abstraction for events decoder // this interface is only for testing now type Decoder interface { @@ -34,8 +89,8 @@ type Decoder interface { // NextResolvedEvent returns the next resolved event if exists NextResolvedEvent() uint64 - // NextDMLEvent returns the next DML event if exists - NextDMLEvent() *commonEvent.DMLEvent + // NextDMLMessage returns the next DML message if exists + NextDMLMessage() *DMLMessage // NextDDLEvent returns the next DDL event if exists NextDDLEvent() *commonEvent.DDLEvent diff --git a/pkg/sink/codec/csv/csv_decoder.go b/pkg/sink/codec/csv/csv_decoder.go index df29abacdb..0ece4ea9ab 100644 --- a/pkg/sink/codec/csv/csv_decoder.go +++ b/pkg/sink/codec/csv/csv_decoder.go @@ -129,8 +129,8 @@ func (b *decoder) NextResolvedEvent() uint64 { return 0 } -// NextDMLEvent implements the Decoder interface. -func (b *decoder) NextDMLEvent() *commonEvent.DMLEvent { +// NextDMLMessage implements the Decoder interface. +func (b *decoder) NextDMLMessage() *common.DMLMessage { if b.closed { log.Panic("batch decoder is closed, cannot fetch the next DML event") } @@ -139,7 +139,7 @@ func (b *decoder) NextDMLEvent() *commonEvent.DMLEvent { if err != nil { log.Panic("convert message to event failed", zap.Error(err)) } - return e + return common.NewDMLMessageFromEvent(e) } // NextDDLEvent implements the Decoder interface. diff --git a/pkg/sink/codec/csv/csv_decoder_test.go b/pkg/sink/codec/csv/csv_decoder_test.go index 7faeaebdc8..d974f4ecd4 100644 --- a/pkg/sink/codec/csv/csv_decoder_test.go +++ b/pkg/sink/codec/csv/csv_decoder_test.go @@ -49,7 +49,7 @@ func TestCSVBatchDecoder(t *testing.T) { tp, hasNext := decoder.HasNext() require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, tp) - event := decoder.NextDMLEvent() + event := decoder.NextDMLMessage().ToDMLEvent() require.NotNil(t, event) } diff --git a/pkg/sink/codec/debezium/avro_decoder.go b/pkg/sink/codec/debezium/avro_decoder.go index 083fc3481d..3e735745b1 100644 --- a/pkg/sink/codec/debezium/avro_decoder.go +++ b/pkg/sink/codec/debezium/avro_decoder.go @@ -95,8 +95,8 @@ func (d *avroDecoder) NextResolvedEvent() uint64 { return d.inner.NextResolvedEvent() } -func (d *avroDecoder) NextDMLEvent() *commonEvent.DMLEvent { - return d.inner.NextDMLEvent() +func (d *avroDecoder) NextDMLMessage() *common.DMLMessage { + return d.inner.NextDMLMessage() } func (d *avroDecoder) NextDDLEvent() *commonEvent.DDLEvent { diff --git a/pkg/sink/codec/debezium/avro_test.go b/pkg/sink/codec/debezium/avro_test.go index 79b5fc94c2..a61e2094ed 100644 --- a/pkg/sink/codec/debezium/avro_test.go +++ b/pkg/sink/codec/debezium/avro_test.go @@ -232,7 +232,7 @@ func TestDebeziumConfluentAvroDecodeRowEvent(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, messageType) - decoded := decoder.NextDMLEvent() + decoded := decoder.NextDMLMessage().ToDMLEvent() require.Equal(t, commitTs, decoded.CommitTs) require.Equal(t, "test", decoded.TableInfo.GetSchemaName()) require.Equal(t, "foo", decoded.TableInfo.GetTableName()) @@ -303,7 +303,7 @@ func TestDebeziumConfluentAvroDecodeAccountDMLEvents(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, messageType) - decoded := decoder.NextDMLEvent() + decoded := decoder.NextDMLMessage().ToDMLEvent() require.Equal(t, "test", decoded.TableInfo.GetSchemaName()) require.Equal(t, "tp_account", decoded.TableInfo.GetTableName()) diff --git a/pkg/sink/codec/debezium/debezium_test.go b/pkg/sink/codec/debezium/debezium_test.go index 4b716b404a..80380bfebf 100644 --- a/pkg/sink/codec/debezium/debezium_test.go +++ b/pkg/sink/codec/debezium/debezium_test.go @@ -120,7 +120,7 @@ func TestEncodeRoutedDMLEventUsesTargetNames(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, messageType) - decoded := decoder.NextDMLEvent() + decoded := decoder.NextDMLMessage().ToDMLEvent() require.Equal(t, "target_db", decoded.TableInfo.GetSchemaName()) require.Equal(t, "target_table", decoded.TableInfo.GetTableName()) diff --git a/pkg/sink/codec/debezium/decoder.go b/pkg/sink/codec/debezium/decoder.go index 7dc91f377a..62831e5b7d 100644 --- a/pkg/sink/codec/debezium/decoder.go +++ b/pkg/sink/codec/debezium/decoder.go @@ -150,8 +150,16 @@ func (d *decoder) NextDDLEvent() *commonEvent.DDLEvent { return event } -// NextDMLEvent returns the next dml event if exists -func (d *decoder) NextDMLEvent() *commonEvent.DMLEvent { +// NextDMLMessage returns the next dml message if exists +func (d *decoder) NextDMLMessage() *common.DMLMessage { + event := d.nextDMLEvent() + if event == nil { + return nil + } + return common.NewDMLMessageFromEvent(event) +} + +func (d *decoder) nextDMLEvent() *commonEvent.DMLEvent { if len(d.valuePayload) == 0 { log.Panic("next DML event failed, since value payload is empty") } diff --git a/pkg/sink/codec/open/decoder.go b/pkg/sink/codec/open/decoder.go index 1cf9cad13a..36ca3f864d 100644 --- a/pkg/sink/codec/open/decoder.go +++ b/pkg/sink/codec/open/decoder.go @@ -192,8 +192,16 @@ func (b *decoder) NextDDLEvent() *commonEvent.DDLEvent { return result } -// NextDMLEvent implements the Decoder interface -func (b *decoder) NextDMLEvent() *commonEvent.DMLEvent { +// NextDMLMessage implements the Decoder interface +func (b *decoder) NextDMLMessage() *common.DMLMessage { + event := b.nextDMLEvent() + if event == nil { + return nil + } + return common.NewDMLMessageFromEvent(event) +} + +func (b *decoder) nextDMLEvent() *commonEvent.DMLEvent { if b.nextKey.Type != common.MessageTypeRow { log.Panic("message type is not row", zap.Any("messageType", b.nextKey.Type)) } diff --git a/pkg/sink/codec/open/encoder_test.go b/pkg/sink/codec/open/encoder_test.go index 9b02366709..721f9d9faa 100644 --- a/pkg/sink/codec/open/encoder_test.go +++ b/pkg/sink/codec/open/encoder_test.go @@ -83,7 +83,7 @@ func TestEncodeFlag(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, messageType) - decoded := decoder.NextDMLEvent() + decoded := decoder.NextDMLMessage().ToDMLEvent() change, ok := decoded.GetNextRow() require.True(t, ok) @@ -171,7 +171,7 @@ func TestIntegerTypes(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, messageType) - decoded := decoder.NextDMLEvent() + decoded := decoder.NextDMLMessage().ToDMLEvent() require.Equal(t, event.CommitTs, decoded.GetCommitTs()) @@ -226,7 +226,7 @@ func TestFloatTypes(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, messageType) - event := decoder.NextDMLEvent() + event := decoder.NextDMLMessage().ToDMLEvent() change, ok := event.GetNextRow() require.True(t, ok) @@ -275,7 +275,7 @@ func TestTimeTypes(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, messageType) - event := decoder.NextDMLEvent() + event := decoder.NextDMLMessage().ToDMLEvent() change, ok := event.GetNextRow() require.True(t, ok) @@ -324,7 +324,7 @@ func TestStringTypes(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, messageType) - event := decoder.NextDMLEvent() + event := decoder.NextDMLMessage().ToDMLEvent() change, ok := event.GetNextRow() require.True(t, ok) @@ -374,7 +374,7 @@ func TestBlobTypes(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, messageType) - event := decoder.NextDMLEvent() + event := decoder.NextDMLMessage().ToDMLEvent() change, ok := event.GetNextRow() require.True(t, ok) @@ -424,7 +424,7 @@ func TestTextTypes(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, messageType) - event := decoder.NextDMLEvent() + event := decoder.NextDMLMessage().ToDMLEvent() change, ok := event.GetNextRow() require.True(t, ok) @@ -471,7 +471,7 @@ func TestVectorType(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, messageType) - event := dec.NextDMLEvent() + event := dec.NextDMLMessage().ToDMLEvent() change, ok := event.GetNextRow() require.True(t, ok) @@ -520,7 +520,7 @@ func TestCollation(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, messageType) - event := decoder.NextDMLEvent() + event := decoder.NextDMLMessage().ToDMLEvent() change, ok := event.GetNextRow() require.True(t, ok) @@ -578,7 +578,7 @@ func TestOtherTypes(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, messageType) - event := decoder.NextDMLEvent() + event := decoder.NextDMLMessage().ToDMLEvent() change, ok := event.GetNextRow() require.True(t, ok) @@ -673,7 +673,7 @@ func TestEncodeRoutedDMLEventUsesTargetNames(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, messageType) - decoded := decoder.NextDMLEvent() + decoded := decoder.NextDMLMessage().ToDMLEvent() require.Equal(t, "target_db", decoded.TableInfo.GetSchemaName()) require.Equal(t, "target_table", decoded.TableInfo.GetTableName()) @@ -757,7 +757,7 @@ func TestEncoderOneMessage(t *testing.T) { require.True(t, hasNext) require.Equal(t, messageType, common.MessageTypeRow) - decoded := decoder.NextDMLEvent() + decoded := decoder.NextDMLMessage().ToDMLEvent() change, ok := decoded.GetNextRow() require.True(t, ok) @@ -824,7 +824,7 @@ func TestEncoderMultipleMessage(t *testing.T) { require.True(t, hasNext) require.Equal(t, messageType, common.MessageTypeRow) - decoded := decoder.NextDMLEvent() + decoded := decoder.NextDMLMessage().ToDMLEvent() change, ok := decoded.GetNextRow() require.True(t, ok) @@ -834,7 +834,7 @@ func TestEncoderMultipleMessage(t *testing.T) { require.True(t, hasNext) require.Equal(t, messageType, common.MessageTypeRow) - decoded = decoder.NextDMLEvent() + decoded = decoder.NextDMLMessage().ToDMLEvent() change, ok = decoded.GetNextRow() require.True(t, ok) @@ -846,7 +846,7 @@ func TestEncoderMultipleMessage(t *testing.T) { require.True(t, hasNext) require.Equal(t, messageType, common.MessageTypeRow) - decoded = decoder.NextDMLEvent() + decoded = decoder.NextDMLMessage().ToDMLEvent() change, ok = decoded.GetNextRow() require.True(t, ok) @@ -930,7 +930,7 @@ func TestLargeMessageWithHandleEnableHandleKeyOnly(t *testing.T) { require.True(t, hasNext) require.Equal(t, messageType, common.MessageTypeRow) - decoded := decoder.NextDMLEvent() + decoded := decoder.NextDMLMessage().ToDMLEvent() change, ok := decoded.GetNextRow() require.True(t, ok) @@ -1027,7 +1027,7 @@ func TestDMLEventWithColumnSelector(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, messageType) - event := decoder.NextDMLEvent() + event := decoder.NextDMLMessage().ToDMLEvent() change, ok := event.GetNextRow() require.True(t, ok) @@ -1115,7 +1115,7 @@ func TestE2EPartitionTable(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, tp) - decodedEvent := dec.NextDMLEvent() + decodedEvent := dec.NextDMLMessage().ToDMLEvent() // table id should be set to the partition table id, the PhysicalTableID require.Equal(t, decodedEvent.GetTableID(), tableIDAllocator.Allocate(e.TableInfo.GetSchemaName(), e.TableInfo.GetTableName())) @@ -1228,7 +1228,7 @@ func TestGenerateColumn(t *testing.T) { require.True(t, hasNext) require.Equal(t, messageType, common.MessageTypeRow) - decoded := dec.NextDMLEvent() + decoded := dec.NextDMLMessage().ToDMLEvent() require.NoError(t, err) require.NotNil(t, decoded) require.Equal(t, decoded.Rows.NumCols(), 2) @@ -1258,7 +1258,7 @@ func TestGenerateColumn(t *testing.T) { require.True(t, hasNext) require.Equal(t, messageType, common.MessageTypeRow) - decoded = dec.NextDMLEvent() + decoded = dec.NextDMLMessage().ToDMLEvent() require.NoError(t, err) require.NotNil(t, decoded) require.Equal(t, decoded.Rows.NumCols(), 2) @@ -1289,7 +1289,7 @@ func TestGenerateColumn(t *testing.T) { require.True(t, hasNext) require.Equal(t, messageType, common.MessageTypeRow) - decoded = dec.NextDMLEvent() + decoded = dec.NextDMLMessage().ToDMLEvent() } // Including insert / update / delete @@ -1368,7 +1368,7 @@ func TestDMLEvent(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, messageType) - decoded := decoder.NextDMLEvent() + decoded := decoder.NextDMLMessage().ToDMLEvent() change, ok := decoded.GetNextRow() require.True(t, ok) @@ -1420,7 +1420,7 @@ func TestOnlyOutputUpdatedEvent(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, messageType) - decoded := decoder.NextDMLEvent() + decoded := decoder.NextDMLMessage().ToDMLEvent() change, ok := decoded.GetNextRow() require.True(t, ok) @@ -1465,7 +1465,7 @@ func TestPKWithUK(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, messageType) - event := dec.NextDMLEvent() + event := dec.NextDMLMessage().ToDMLEvent() change, ok := event.GetNextRow() require.True(t, ok) require.Len(t, event.TableInfo.GetIndices(), 2) @@ -1514,7 +1514,7 @@ func TestUniqueKeyWithoutPKDMLEvent(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, messageType) - event := dec.NextDMLEvent() + event := dec.NextDMLMessage().ToDMLEvent() change, ok := event.GetNextRow() require.True(t, ok) @@ -1564,7 +1564,7 @@ func TestHandleOnlyEvent(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, messageType) - decoded := decoder.NextDMLEvent() + decoded := decoder.NextDMLMessage().ToDMLEvent() change, ok := decoded.GetNextRow() require.True(t, ok) @@ -1629,7 +1629,7 @@ func TestRenameTable(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, messageType) - decodedInsert := decoder1.NextDMLEvent() + decodedInsert := decoder1.NextDMLMessage().ToDMLEvent() require.NotZero(t, decodedInsert.GetTableID()) require.Contains(t, tableIDAllocator.GetBlockedTables("test", "t"), decodedInsert.GetTableID()) @@ -1735,7 +1735,7 @@ func TestDDLSequence(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, messageType) - decodedInsert := decoder.NextDMLEvent() + decodedInsert := decoder.NextDMLMessage().ToDMLEvent() require.NotZero(t, decodedInsert.GetTableID()) require.Contains(t, tableIDAllocator.GetBlockedTables("test", "t"), decodedInsert.GetTableID()) diff --git a/pkg/sink/codec/simple/decoder.go b/pkg/sink/codec/simple/decoder.go index 101d73f472..c0ac0edf6c 100644 --- a/pkg/sink/codec/simple/decoder.go +++ b/pkg/sink/codec/simple/decoder.go @@ -147,8 +147,16 @@ func (d *Decoder) NextResolvedEvent() uint64 { return ts } -// NextDMLEvent returns the next dml event if exists -func (d *Decoder) NextDMLEvent() *commonEvent.DMLEvent { +// NextDMLMessage returns the next dml message if exists +func (d *Decoder) NextDMLMessage() *common.DMLMessage { + event := d.nextDMLEvent() + if event == nil { + return nil + } + return common.NewDMLMessageFromEvent(event) +} + +func (d *Decoder) nextDMLEvent() *commonEvent.DMLEvent { if d.msg == nil || (d.msg.Data == nil && d.msg.Old == nil) { log.Panic("invalid data for the DML event", zap.String("message", util.RedactAny(d.msg))) } @@ -211,7 +219,7 @@ func (d *Decoder) assembleClaimCheckRowChangedEvent(claimCheckLocation string) * log.Panic("unmarshal claim check message failed", zap.Any("value", util.RedactAny(value)), zap.Error(err)) } d.msg = m - return d.NextDMLEvent() + return d.nextDMLEvent() } func (d *Decoder) assembleHandleKeyOnlyRowChangedEvent(m *message) *commonEvent.DMLEvent { @@ -260,7 +268,7 @@ func (d *Decoder) assembleHandleKeyOnlyRowChangedEvent(m *message) *commonEvent. } d.msg = result - return d.NextDMLEvent() + return d.nextDMLEvent() } func (d *Decoder) buildData( @@ -292,7 +300,7 @@ func (d *Decoder) NextDDLEvent() *commonEvent.DDLEvent { for ele := d.cachedMessages.Front(); ele != nil; { d.msg = ele.Value.(*message) - event := d.NextDMLEvent() + event := d.nextDMLEvent() d.CachedRowChangedEvents = append(d.CachedRowChangedEvents, event) next := ele.Next() diff --git a/pkg/sink/codec/simple/encoder_test.go b/pkg/sink/codec/simple/encoder_test.go index 68170f6f05..e266f638ce 100644 --- a/pkg/sink/codec/simple/encoder_test.go +++ b/pkg/sink/codec/simple/encoder_test.go @@ -146,7 +146,7 @@ func TestEncodeDMLEnableChecksum(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, messageType) - // decodedRow:= decoder.NextDMLEvent() + // decodedRow:= decoder.NextDMLMessage().ToDMLEvent() // require.NoError(t, err) // require.Equal(t, updateEvent.Checksum.Current, decodedRow.Checksum.Current) // require.Equal(t, updateEvent.Checksum.Previous, decodedRow.Checksum.Previous) @@ -189,7 +189,7 @@ func TestEncodeDMLEnableChecksum(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, messageType) - // decodedRow:= decoder.NextDMLEvent() + // decodedRow:= decoder.NextDMLMessage().ToDMLEvent() // require.Error(t, err) // require.Nil(t, decodedRow) } @@ -238,7 +238,7 @@ func TestEncodeRoutedEventsUsesTargetNames(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, messageType) - decodedDML := decoder.NextDMLEvent() + decodedDML := decoder.NextDMLMessage().ToDMLEvent() require.Equal(t, "target_db", decodedDML.TableInfo.GetSchemaName()) require.Equal(t, "target_table", decodedDML.TableInfo.GetTableName()) } @@ -318,7 +318,7 @@ func TestE2EPartitionTable(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, tp) - decodedEvent := dec.NextDMLEvent() + decodedEvent := dec.NextDMLMessage().ToDMLEvent() // table id should be set to the partition table id, the PhysicalTableID require.Equal(t, decodedEvent.GetTableID(), e.GetTableID()) @@ -927,7 +927,7 @@ func TestEncodeDDLEvent(t *testing.T) { require.Equal(t, common.MessageTypeRow, messageType) require.NotEqual(t, 0, dec.msg.BuildTs) - decodedRow := dec.NextDMLEvent() + decodedRow := dec.NextDMLMessage().ToDMLEvent() require.Equal(t, decodedRow.CommitTs, insertEvent.GetCommitTs()) require.Equal(t, decodedRow.TableInfo.GetSchemaName(), insertEvent.TableInfo.GetSchemaName()) require.Equal(t, decodedRow.TableInfo.GetTableName(), insertEvent.TableInfo.GetTableName()) @@ -976,7 +976,7 @@ func TestEncodeDDLEvent(t *testing.T) { require.Equal(t, common.MessageTypeRow, messageType) require.NotEqual(t, 0, dec.msg.BuildTs) - decodedRow = dec.NextDMLEvent() + decodedRow = dec.NextDMLMessage().ToDMLEvent() require.Equal(t, insertEvent2.GetCommitTs(), decodedRow.GetCommitTs()) require.Equal(t, insertEvent2.TableInfo.GetSchemaName(), decodedRow.TableInfo.GetSchemaName()) require.Equal(t, insertEvent2.TableInfo.GetTableName(), decodedRow.TableInfo.GetTableName()) @@ -1130,7 +1130,7 @@ func TestEncodeIntegerTypes(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, messageType) - decodedRow := dec.NextDMLEvent() + decodedRow := dec.NextDMLMessage().ToDMLEvent() require.Equal(t, decodedRow.CommitTs, event.GetCommitTs()) decoded, ok := decodedRow.GetNextRow() @@ -1205,7 +1205,7 @@ func TestEncoderOtherTypes(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, messageType) - decodedRow := dec.NextDMLEvent() + decodedRow := dec.NextDMLMessage().ToDMLEvent() decoded, ok := decodedRow.GetNextRow() require.True(t, ok) @@ -1273,7 +1273,7 @@ func TestE2EPartitionTableDMLBeforeDDL(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, tp) - decodedEvent := dec.NextDMLEvent() + decodedEvent := dec.NextDMLMessage().ToDMLEvent() require.Nil(t, decodedEvent) e.Rewind() @@ -1339,7 +1339,7 @@ func TestEncodeDMLBeforeDDL(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, messageType) - decodedRow := dec.NextDMLEvent() + decodedRow := dec.NextDMLMessage().ToDMLEvent() require.Nil(t, decodedRow) m, err := enc.EncodeDDLEvent(ddlEvent) @@ -1444,7 +1444,7 @@ func TestEncodeBootstrapEvent(t *testing.T) { require.Equal(t, common.MessageTypeRow, messageType) require.NotEqual(t, 0, dec.msg.BuildTs) - decodedRow := dec.NextDMLEvent() + decodedRow := dec.NextDMLMessage().ToDMLEvent() decode, ok := decodedRow.GetNextRow() require.True(t, ok) require.Equal(t, decodedRow.CommitTs, dmlEvent.CommitTs) @@ -1528,7 +1528,7 @@ func TestEncodeLargeEventsNormal(t *testing.T) { require.Equal(t, dec.msg.Type, DMLTypeInsert) } - decodedRow := dec.NextDMLEvent() + decodedRow := dec.NextDMLMessage().ToDMLEvent() require.Equal(t, decodedRow.CommitTs, event.CommitTs) require.Equal(t, decodedRow.TableInfo.GetSchemaName(), event.TableInfo.GetSchemaName()) @@ -1653,7 +1653,7 @@ func TestLargerMessageHandleClaimCheck(t *testing.T) { require.Equal(t, common.MessageTypeRow, messageType) require.NotEqual(t, "", dec.msg.ClaimCheckLocation) - decodedRow := dec.NextDMLEvent() + decodedRow := dec.NextDMLMessage().ToDMLEvent() require.Equal(t, decodedRow.CommitTs, updateEvent.CommitTs) require.Equal(t, decodedRow.TableInfo.GetSchemaName(), updateEvent.TableInfo.GetSchemaName()) @@ -1725,7 +1725,7 @@ func TestLargeMessageHandleKeyOnly(t *testing.T) { require.Equal(t, common.MessageTypeRow, messageType) require.True(t, dec.msg.HandleKeyOnly) - decodedRow := dec.NextDMLEvent() + decodedRow := dec.NextDMLMessage().ToDMLEvent() require.Nil(t, decodedRow) } From 39d3a6a51230fd7756aa2928932266ab15585c34 Mon Sep 17 00:00:00 2001 From: wk989898 Date: Tue, 7 Jul 2026 03:37:14 +0000 Subject: [PATCH 03/12] update Signed-off-by: wk989898 --- cmd/pulsar-consumer/writer_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/pulsar-consumer/writer_test.go b/cmd/pulsar-consumer/writer_test.go index dadbe02a8f..22e827f850 100644 --- a/cmd/pulsar-consumer/writer_test.go +++ b/cmd/pulsar-consumer/writer_test.go @@ -443,7 +443,7 @@ func (m fakePulsarMessage) GetReplicatedFrom() string { return "" } -func (m fakePulsarMessage) GetSchemaValue(interface{}) error { +func (m fakePulsarMessage) GetSchemaValue(any) error { return nil } From bbb914341d6c0f805f29c3baa8c89dcaec9c5419 Mon Sep 17 00:00:00 2001 From: wk989898 Date: Tue, 7 Jul 2026 09:31:58 +0000 Subject: [PATCH 04/12] chore Signed-off-by: wk989898 --- cmd/pulsar-consumer/consumer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/pulsar-consumer/consumer.go b/cmd/pulsar-consumer/consumer.go index 8641d6b663..8c75a8843b 100644 --- a/cmd/pulsar-consumer/consumer.go +++ b/cmd/pulsar-consumer/consumer.go @@ -114,7 +114,7 @@ func (c *consumer) readMessage(ctx context.Context) error { if !needCommit { continue } - err := c.pulsarConsumer.AckID(consumerMsg.ID()) + err := c.pulsarConsumer.AckIDCumulative(consumerMsg.ID()) if err != nil { log.Panic("Error ack message", zap.Error(err)) } From c738f9a54e98bfcf16b900f89994383c889b810b Mon Sep 17 00:00:00 2001 From: wk989898 Date: Wed, 8 Jul 2026 08:58:50 +0000 Subject: [PATCH 05/12] update Signed-off-by: wk989898 --- cmd/kafka-consumer/writer.go | 43 +++++---- cmd/storage-consumer/consumer.go | 13 ++- pkg/sink/codec/avro/decoder.go | 58 +++++++++--- .../codec/canal/canal_json_txn_decoder.go | 34 ++++++- pkg/sink/codec/csv/csv_decoder.go | 25 ++++- pkg/sink/codec/debezium/decoder.go | 49 +++++++++- pkg/sink/codec/open/decoder.go | 94 ++++++++++++++----- pkg/sink/codec/simple/decoder.go | 28 +++++- 8 files changed, 269 insertions(+), 75 deletions(-) diff --git a/cmd/kafka-consumer/writer.go b/cmd/kafka-consumer/writer.go index 3777f1566b..bf3043b9f9 100644 --- a/cmd/kafka-consumer/writer.go +++ b/cmd/kafka-consumer/writer.go @@ -380,28 +380,32 @@ func (w *writer) WriteMessage(ctx context.Context, message *kafka.Message) bool case common.MessageTypeRow: var counter int dmlMessage := progress.decoder.NextDMLMessage() - var row *event.DMLEvent - if dmlMessage != nil { - row = dmlMessage.ToDMLEvent() - } - if row == nil { + if dmlMessage == nil { if w.protocol != config.ProtocolSimple { - log.Panic("DML event is nil, it's not expected", + log.Panic("DML message is nil, it's not expected", zap.Int32("partition", partition), zap.Any("offset", offset)) } - log.Debug("DML event is nil, it's cached", zap.Int32("partition", partition), zap.Any("offset", offset)) + log.Debug("DML message is nil, it's cached", zap.Int32("partition", partition), zap.Any("offset", offset)) break } - w.appendMessage2Group(common.NewDMLMessageFromEvent(row), progress, offset) + w.appendMessage2Group(dmlMessage, progress, offset) counter++ for { _, hasNext = progress.decoder.HasNext() if !hasNext { break } - row = progress.decoder.NextDMLMessage().ToDMLEvent() - w.appendMessage2Group(common.NewDMLMessageFromEvent(row), progress, offset) + dmlMessage = progress.decoder.NextDMLMessage() + if dmlMessage == nil { + if w.protocol != config.ProtocolSimple { + log.Panic("DML message is nil, it's not expected", + zap.Int32("partition", partition), zap.Any("offset", offset)) + } + log.Debug("DML message is nil, it's cached", zap.Int32("partition", partition), zap.Any("offset", offset)) + break + } + w.appendMessage2Group(dmlMessage, progress, offset) counter++ } // If the message containing only one event exceeds the length limit, CDC will allow it and issue a warning. @@ -588,9 +592,15 @@ func (w *writer) checkPartition(row *event.DMLEvent, partition int32, offset kaf } } +func (w *writer) messageWithPartitionCheck(message *common.DMLMessage, partition int32, offset kafka.Offset) *common.DMLMessage { + return common.NewDMLMessage(message.TableID, message.Schema, message.Table, message.GetCommitTs(), message.RowType, func() *event.DMLEvent { + row := message.ToDMLEvent() + w.checkPartition(row, partition, offset) + return row + }) +} + func (w *writer) appendMessage2Group(message *common.DMLMessage, progress *partitionProgress, offset kafka.Offset) { - dml := message.ToDMLEvent() - w.checkPartition(dml, progress.partition, offset) // if the kafka cluster is normal, this should not hit. // else if the cluster is abnormal, the consumer may consume old message, then cause the watermark fallback. var ( @@ -613,6 +623,7 @@ func (w *writer) appendMessage2Group(message *common.DMLMessage, progress *parti return } if commitTs >= group.HighWatermark { + message = w.messageWithPartitionCheck(message, progress.partition, offset) group.AppendMessage(message, false) log.Debug("DML event append to the group", zap.Int32("partition", group.Partition), zap.Any("offset", offset), @@ -627,7 +638,7 @@ func (w *writer) appendMessage2Group(message *common.DMLMessage, progress *parti zap.Uint64("commitTs", commitTs), zap.Uint64("HighWatermark", group.HighWatermark), zap.String("schema", schema), zap.String("table", table), zap.Int64("tableID", tableID), zap.Stringer("eventType", message.RowType)) - group.AppendMessage(message, true) + group.AppendMessage(w.messageWithPartitionCheck(message, progress.partition, offset), true) return } switch w.protocol { @@ -644,7 +655,7 @@ func (w *writer) appendMessage2Group(message *common.DMLMessage, progress *parti zap.String("schema", schema), zap.String("table", table), zap.Int64("tableID", tableID), zap.Stringer("eventType", message.RowType), // zap.Any("columns", row.Columns), zap.Any("preColumns", row.PreColumns), - zap.Any("protocol", w.protocol), zap.Bool("IsPartition", dml.TableInfo.TableName.IsPartition)) + zap.Any("protocol", w.protocol), zap.Bool("IsPartition", false)) case config.ProtocolCanalJSON, config.ProtocolOpen, config.ProtocolAvro, config.ProtocolDebezium, config.ProtocolDebeziumAvro: // for partition table, these protocols cannot assign physical table id to each dml message, @@ -655,7 +666,7 @@ func (w *writer) appendMessage2Group(message *common.DMLMessage, progress *parti zap.Uint64("commitTs", commitTs), zap.Uint64("highWatermark", group.HighWatermark), zap.String("schema", schema), zap.String("table", table), zap.Int64("tableID", tableID), zap.Stringer("eventType", message.RowType), zap.Any("protocol", w.protocol)) - group.AppendMessage(message, true) + group.AppendMessage(w.messageWithPartitionCheck(message, progress.partition, offset), true) return } log.Warn("DML event fallback row, since less than the group high watermark, ignore it", @@ -665,7 +676,7 @@ func (w *writer) appendMessage2Group(message *common.DMLMessage, progress *parti zap.String("schema", schema), zap.String("table", table), zap.Int64("tableID", tableID), zap.Stringer("eventType", message.RowType), // zap.Any("columns", row.Columns), zap.Any("preColumns", row.PreColumns), - zap.Any("protocol", w.protocol), zap.Bool("IsPartition", dml.TableInfo.TableName.IsPartition)) + zap.Any("protocol", w.protocol), zap.Bool("IsPartition", false)) default: log.Panic("unknown protocol", zap.Any("protocol", w.protocol)) } diff --git a/cmd/storage-consumer/consumer.go b/cmd/storage-consumer/consumer.go index d4988a0731..b5fed843b4 100644 --- a/cmd/storage-consumer/consumer.go +++ b/cmd/storage-consumer/consumer.go @@ -351,9 +351,8 @@ func (c *consumer) appendDMLEvents( if tp == common.MessageTypeRow { c.dmlCount.Add(1) - row := decoder.NextDMLMessage().ToDMLEvent() - row.PhysicalTableID = tableID - c.appendMessage2Group(common.NewDMLMessageFromEvent(row), fileIdx.EnableTableAcrossNodes) + message := decoder.NextDMLMessage() + c.appendMessage2Group(messageWithPhysicalTableID(message, tableID), fileIdx.EnableTableAcrossNodes) filteredCnt++ } } @@ -366,6 +365,14 @@ func (c *consumer) appendDMLEvents( return err } +func messageWithPhysicalTableID(message *common.DMLMessage, tableID int64) *common.DMLMessage { + return common.NewDMLMessage(tableID, message.Schema, message.Table, message.GetCommitTs(), message.RowType, func() *event.DMLEvent { + row := message.ToDMLEvent() + row.PhysicalTableID = tableID + return row + }) +} + func (c *consumer) flushDMLEvents(ctx context.Context, tableID int64) error { group := c.eventsGroup[tableID] if group == nil { diff --git a/pkg/sink/codec/avro/decoder.go b/pkg/sink/codec/avro/decoder.go index 211bf40616..83694af1e8 100644 --- a/pkg/sink/codec/avro/decoder.go +++ b/pkg/sink/codec/avro/decoder.go @@ -115,30 +115,48 @@ func (d *decoder) NextResolvedEvent() uint64 { // NextDMLMessage returns the next row changed message if exists func (d *decoder) NextDMLMessage() *common.DMLMessage { - event := d.nextDMLEvent() - if event == nil { - return nil + keyMap, valueMap, valueSchema, isDelete, deleteCommitTs := d.decodeDMLPayload() + schemaName, tableName := schemaAndTableName(valueSchema) + commitTs := deleteCommitTs + if commitTs == 0 && !isDelete { + commitTs = uint64(valueMap[tidbCommitTs].(int64)) } - return common.NewDMLMessageFromEvent(event) + rowType := commonType.RowTypeInsert + if isDelete { + rowType = commonType.RowTypeDelete + } + tableID := tableIDAllocator.Allocate(schemaName, tableName) + return common.NewDMLMessage(tableID, schemaName, tableName, commitTs, rowType, func() *commonEvent.DMLEvent { + return d.assembleDMLEventFromDecoded(keyMap, valueMap, valueSchema, isDelete, deleteCommitTs) + }) } func (d *decoder) nextDMLEvent() *commonEvent.DMLEvent { + keyMap, valueMap, valueSchema, isDelete, deleteCommitTs := d.decodeDMLPayload() + return d.assembleDMLEventFromDecoded(keyMap, valueMap, valueSchema, isDelete, deleteCommitTs) +} + +func (d *decoder) decodeDMLPayload() ( + keyMap map[string]interface{}, + valueMap map[string]interface{}, + valueSchema map[string]interface{}, + isDelete bool, + deleteCommitTs uint64, +) { var ( - valueMap map[string]interface{} - valueSchema map[string]interface{} - err error + keySchema map[string]interface{} + err error ) ctx := context.Background() - keyMap, keySchema, err := d.decodeKey(ctx) + keyMap, keySchema, err = d.decodeKey(ctx) if err != nil { log.Panic("decode key failed", zap.Error(err)) } // for the delete event, only have key part, it holds primary key or the unique key columns. // for the insert / update, extract the value part, it holds all columns. - isDelete := len(d.value) == 0 || d.isDeleteValue() - deleteCommitTs := uint64(0) + isDelete = len(d.value) == 0 || d.isDeleteValue() if isDelete { // delete event only have key part, treat it as the value part also. if d.isDeleteValue() { @@ -153,6 +171,16 @@ func (d *decoder) nextDMLEvent() *commonEvent.DMLEvent { } } + return keyMap, valueMap, valueSchema, isDelete, deleteCommitTs +} + +func (d *decoder) assembleDMLEventFromDecoded( + keyMap map[string]interface{}, + valueMap map[string]interface{}, + valueSchema map[string]interface{}, + isDelete bool, + deleteCommitTs uint64, +) *commonEvent.DMLEvent { event, err := assembleEvent(keyMap, valueMap, valueSchema, isDelete) if err != nil { log.Panic("assemble event failed", zap.Error(err)) @@ -281,10 +309,7 @@ func assembleEvent( columns = append(columns, tiCol) } - // "namespace.schema" - namespace := schema["namespace"].(string) - schemaName := strings.Split(namespace, ".")[1] - tableName := schema["name"].(string) + schemaName, tableName := schemaAndTableName(schema) var commitTs int64 if !isDelete { @@ -315,6 +340,11 @@ func assembleEvent( return event, nil } +func schemaAndTableName(schema map[string]interface{}) (string, string) { + namespace := schema["namespace"].(string) + return strings.Split(namespace, ".")[1], schema["name"].(string) +} + func queryTableInfo(schemaName, tableName string, columns []*timodel.ColumnInfo, keyMap map[string]interface{}) *commonType.TableInfo { tableInfo := newTableInfo(schemaName, tableName, columns, keyMap) return tableInfo diff --git a/pkg/sink/codec/canal/canal_json_txn_decoder.go b/pkg/sink/codec/canal/canal_json_txn_decoder.go index 9674c027d1..2fa78a73dd 100644 --- a/pkg/sink/codec/canal/canal_json_txn_decoder.go +++ b/pkg/sink/codec/canal/canal_json_txn_decoder.go @@ -97,11 +97,37 @@ func (d *txnDecoder) HasNext() (common.MessageType, bool) { } func (d *txnDecoder) NextDMLMessage() *common.DMLMessage { - event := d.nextDMLEvent() - if event == nil { - return nil + if d.msg == nil || d.msg.messageType() != common.MessageTypeRow { + messageType := common.MessageTypeUnknown + if d.msg != nil { + messageType = d.msg.messageType() + } + log.Panic("message type is not row changed", + zap.Any("messageType", messageType), zap.Any("msg", d.msg)) + } + + msg := d.msg + d.msg = nil + schemaName := *msg.getSchema() + tableName := *msg.getTable() + tableID := tableIDAllocator.Allocate(schemaName, tableName) + + var rowType commonType.RowType + switch msg.eventType() { + case canal.EventType_DELETE: + rowType = commonType.RowTypeDelete + case canal.EventType_INSERT: + rowType = commonType.RowTypeInsert + case canal.EventType_UPDATE: + rowType = commonType.RowTypeUpdate + default: + log.Panic("unknown event type for the DML event", zap.Any("eventType", msg.eventType())) } - return common.NewDMLMessageFromEvent(event) + + return common.NewDMLMessage(tableID, schemaName, tableName, msg.getCommitTs(), rowType, func() *commonEvent.DMLEvent { + d.msg = msg + return d.nextDMLEvent() + }) } func (d *txnDecoder) nextDMLEvent() *commonEvent.DMLEvent { diff --git a/pkg/sink/codec/csv/csv_decoder.go b/pkg/sink/codec/csv/csv_decoder.go index 0ece4ea9ab..5e5f6b80be 100644 --- a/pkg/sink/codec/csv/csv_decoder.go +++ b/pkg/sink/codec/csv/csv_decoder.go @@ -135,11 +135,28 @@ func (b *decoder) NextDMLMessage() *common.DMLMessage { log.Panic("batch decoder is closed, cannot fetch the next DML event") } - e, err := csvMsg2RowChangedEvent(b.codecConfig, b.msg, b.tableInfo) - if err != nil { - log.Panic("convert message to event failed", zap.Error(err)) + msg := *b.msg + msg.columns = append([]any(nil), b.msg.columns...) + msg.preColumns = append([]any(nil), b.msg.preColumns...) + + rowType := commonType.RowTypeInsert + if msg.opType == operationDelete { + rowType = commonType.RowTypeDelete } - return common.NewDMLMessageFromEvent(e) + + return common.NewDMLMessage( + b.tableInfo.TableName.TableID, + msg.schemaName, + msg.tableName, + msg.commitTs, + rowType, + func() *commonEvent.DMLEvent { + e, err := csvMsg2RowChangedEvent(b.codecConfig, &msg, b.tableInfo) + if err != nil { + log.Panic("convert message to event failed", zap.Error(err)) + } + return e + }) } // NextDDLEvent implements the Decoder interface. diff --git a/pkg/sink/codec/debezium/decoder.go b/pkg/sink/codec/debezium/decoder.go index 62831e5b7d..da0a1153e4 100644 --- a/pkg/sink/codec/debezium/decoder.go +++ b/pkg/sink/codec/debezium/decoder.go @@ -152,11 +152,52 @@ func (d *decoder) NextDDLEvent() *commonEvent.DDLEvent { // NextDMLMessage returns the next dml message if exists func (d *decoder) NextDMLMessage() *common.DMLMessage { - event := d.nextDMLEvent() - if event == nil { - return nil + if len(d.valuePayload) == 0 { + log.Panic("next DML message failed, since value payload is empty") + } + if d.config.DebeziumDisableSchema { + log.Panic("next DML message failed, since DebeziumDisableSchema is true") + } + if !d.config.EnableTiDBExtension { + log.Panic("next DML message failed, since EnableTiDBExtension is false") + } + + keyPayload := d.keyPayload + keySchema := d.keySchema + valuePayload := d.valuePayload + valueSchema := d.valueSchema + commitTs := d.getCommitTs() + schemaName := d.getSchemaName() + tableName := d.getTableName() + rowType := d.rowType() + tableID := tableIDAllocator.Allocate(schemaName, tableName) + d.clear() + + return common.NewDMLMessage(tableID, schemaName, tableName, commitTs, rowType, func() *commonEvent.DMLEvent { + d.keyPayload = keyPayload + d.keySchema = keySchema + d.valuePayload = valuePayload + d.valueSchema = valueSchema + return d.nextDMLEvent() + }) +} + +func (d *decoder) rowType() commonType.RowType { + op, ok := d.valuePayload["op"] + if !ok { + log.Panic("DML message op not found") + } + switch op { + case "c": + return commonType.RowTypeInsert + case "u": + return commonType.RowTypeUpdate + case "d": + return commonType.RowTypeDelete + default: + log.Panic("unknown op for the DML message", zap.Any("op", op)) } - return common.NewDMLMessageFromEvent(event) + return commonType.RowTypeInsert } func (d *decoder) nextDMLEvent() *commonEvent.DMLEvent { diff --git a/pkg/sink/codec/open/decoder.go b/pkg/sink/codec/open/decoder.go index 36ca3f864d..691ca99858 100644 --- a/pkg/sink/codec/open/decoder.go +++ b/pkg/sink/codec/open/decoder.go @@ -194,11 +194,22 @@ func (b *decoder) NextDDLEvent() *commonEvent.DDLEvent { // NextDMLMessage implements the Decoder interface func (b *decoder) NextDMLMessage() *common.DMLMessage { - event := b.nextDMLEvent() - if event == nil { - return nil + if b.nextKey.Type != common.MessageTypeRow { + log.Panic("message type is not row", zap.Any("messageType", b.nextKey.Type)) } - return common.NewDMLMessageFromEvent(event) + + key := *b.nextKey + value := b.nextDMLValue() + b.nextKey = nil + + rowType := commonType.RowTypeInsert + if key.ClaimCheckLocation == "" { + rowType = b.rowTypeFromDMLValue(value) + } + tableID := tableIDAllocator.Allocate(key.Schema, key.Table) + return common.NewDMLMessage(tableID, key.Schema, key.Table, key.Ts, rowType, func() *commonEvent.DMLEvent { + return b.decodeDMLMessage(&key, value) + }) } func (b *decoder) nextDMLEvent() *commonEvent.DMLEvent { @@ -206,10 +217,47 @@ func (b *decoder) nextDMLEvent() *commonEvent.DMLEvent { log.Panic("message type is not row", zap.Any("messageType", b.nextKey.Type)) } + key := b.nextKey + value := b.nextDMLValue() + b.nextKey = nil + return b.decodeDMLMessage(key, value) +} + +func (b *decoder) nextDMLValue() []byte { valueLen := binary.BigEndian.Uint64(b.valueBytes[:8]) value := b.valueBytes[8 : valueLen+8] b.valueBytes = b.valueBytes[valueLen+8:] + return append([]byte(nil), value...) +} + +func (b *decoder) rowTypeFromDMLValue(value []byte) commonType.RowType { + value, err := common.Decompress(b.config.LargeMessageHandle.LargeMessageHandleCompression, value) + if err != nil { + log.Panic("decompress failed", + zap.String("compression", b.config.LargeMessageHandle.LargeMessageHandleCompression), + zap.Any("value", util.RedactAny(value)), zap.Error(err)) + } + + nextRow := new(messageRow) + nextRow.decode(value) + return rowTypeFromMessageRow(nextRow) +} + +func rowTypeFromMessageRow(row *messageRow) commonType.RowType { + if len(row.Delete) != 0 { + return commonType.RowTypeDelete + } + if len(row.Update) != 0 && len(row.PreColumns) != 0 { + return commonType.RowTypeUpdate + } + if len(row.Update) != 0 { + return commonType.RowTypeInsert + } + log.Panic("unknown event type") + return commonType.RowTypeInsert +} +func (b *decoder) decodeDMLMessage(key *messageKey, value []byte) *commonEvent.DMLEvent { value, err := common.Decompress(b.config.LargeMessageHandle.LargeMessageHandleCompression, value) if err != nil { log.Panic("decompress failed", @@ -222,15 +270,15 @@ func (b *decoder) nextDMLEvent() *commonEvent.DMLEvent { ctx := context.Background() // claim-check message found - if b.nextKey.ClaimCheckLocation != "" { - return b.assembleEventFromClaimCheckStorage(ctx) + if key.ClaimCheckLocation != "" { + return b.assembleEventFromClaimCheckStorage(ctx, key) } - if b.nextKey.OnlyHandleKey && b.upstreamTiDB != nil { - return b.assembleHandleKeyOnlyDMLEvent(ctx, nextRow) + if key.OnlyHandleKey && b.upstreamTiDB != nil { + return b.assembleHandleKeyOnlyDMLEvent(ctx, key, nextRow) } - return b.assembleDMLEvent(nextRow) + return b.assembleDMLEvent(key, nextRow) } func buildColumns( @@ -267,8 +315,7 @@ func buildColumns( return columns } -func (b *decoder) assembleHandleKeyOnlyDMLEvent(ctx context.Context, row *messageRow) *commonEvent.DMLEvent { - key := b.nextKey +func (b *decoder) assembleHandleKeyOnlyDMLEvent(ctx context.Context, key *messageKey, row *messageRow) *commonEvent.DMLEvent { var ( schema = key.Schema table = key.Table @@ -298,13 +345,12 @@ func (b *decoder) assembleHandleKeyOnlyDMLEvent(ctx context.Context, row *messag } else { log.Panic("unknown event type") } - b.nextKey.OnlyHandleKey = false - return b.assembleDMLEvent(row) + key.OnlyHandleKey = false + return b.assembleDMLEvent(key, row) } -func (b *decoder) assembleEventFromClaimCheckStorage(ctx context.Context) *commonEvent.DMLEvent { - _, claimCheckFileName := filepath.Split(b.nextKey.ClaimCheckLocation) - b.nextKey = nil +func (b *decoder) assembleEventFromClaimCheckStorage(ctx context.Context, key *messageKey) *commonEvent.DMLEvent { + _, claimCheckFileName := filepath.Split(key.ClaimCheckLocation) data, err := b.storage.ReadFile(ctx, claimCheckFileName) if err != nil { log.Panic("read claim check file failed", zap.String("fileName", claimCheckFileName), zap.Error(err)) @@ -319,11 +365,11 @@ func (b *decoder) assembleEventFromClaimCheckStorage(ctx context.Context) *commo log.Panic("the batch version is not supported", zap.Uint64("version", version)) } - key := claimCheckM.Key[8:] - keyLen := binary.BigEndian.Uint64(key[:8]) - key = key[8 : keyLen+8] + encodedKey := claimCheckM.Key[8:] + keyLen := binary.BigEndian.Uint64(encodedKey[:8]) + encodedKey = encodedKey[8 : keyLen+8] msgKey := new(messageKey) - msgKey.Decode(key) + msgKey.Decode(encodedKey) valueLen := binary.BigEndian.Uint64(claimCheckM.Value[:8]) value := claimCheckM.Value[8 : valueLen+8] @@ -337,8 +383,7 @@ func (b *decoder) assembleEventFromClaimCheckStorage(ctx context.Context) *commo rowMsg := new(messageRow) rowMsg.decode(value) - b.nextKey = msgKey - return b.assembleDMLEvent(rowMsg) + return b.assembleDMLEvent(msgKey, rowMsg) } func (b *decoder) queryTableInfo(key *messageKey, value *messageRow) *commonType.TableInfo { @@ -498,10 +543,7 @@ func newTiIndices(columns []*timodel.ColumnInfo) []*timodel.IndexInfo { return indices } -func (b *decoder) assembleDMLEvent(value *messageRow) *commonEvent.DMLEvent { - key := b.nextKey - b.nextKey = nil - +func (b *decoder) assembleDMLEvent(key *messageKey, value *messageRow) *commonEvent.DMLEvent { tableInfo := b.queryTableInfo(key, value) result := new(commonEvent.DMLEvent) result.TableInfo = tableInfo diff --git a/pkg/sink/codec/simple/decoder.go b/pkg/sink/codec/simple/decoder.go index c0ac0edf6c..f47e61ead8 100644 --- a/pkg/sink/codec/simple/decoder.go +++ b/pkg/sink/codec/simple/decoder.go @@ -149,11 +149,31 @@ func (d *Decoder) NextResolvedEvent() uint64 { // NextDMLMessage returns the next dml message if exists func (d *Decoder) NextDMLMessage() *common.DMLMessage { - event := d.nextDMLEvent() - if event == nil { - return nil + if d.msg == nil || (d.msg.Data == nil && d.msg.Old == nil) { + log.Panic("invalid data for the DML event", zap.String("message", util.RedactAny(d.msg))) + } + + msg := d.msg + d.msg = nil + + return common.NewDMLMessage(msg.TableID, msg.Schema, msg.Table, msg.CommitTs, rowTypeFromMessageType(msg.Type), func() *commonEvent.DMLEvent { + d.msg = msg + return d.nextDMLEvent() + }) +} + +func rowTypeFromMessageType(tp MessageType) commonType.RowType { + switch tp { + case DMLTypeInsert: + return commonType.RowTypeInsert + case DMLTypeUpdate: + return commonType.RowTypeUpdate + case DMLTypeDelete: + return commonType.RowTypeDelete + default: + log.Panic("unknown row type for the DML message", zap.Any("type", tp)) } - return common.NewDMLMessageFromEvent(event) + return commonType.RowTypeInsert } func (d *Decoder) nextDMLEvent() *commonEvent.DMLEvent { From 2ec9c3a5a8bf7f06c69667387bc907b1f95f35e8 Mon Sep 17 00:00:00 2001 From: wk989898 Date: Mon, 20 Jul 2026 04:13:04 +0000 Subject: [PATCH 06/12] update Signed-off-by: wk989898 --- cmd/kafka-consumer/writer.go | 12 +- pkg/sink/codec/avro/decoder.go | 5 - pkg/sink/codec/canal/canal_json_decoder.go | 17 ++- .../codec/canal/canal_json_txn_decoder.go | 17 +-- pkg/sink/codec/common/decoder.go | 4 +- pkg/sink/codec/common/table_info_cache.go | 14 +++ pkg/sink/codec/debezium/decoder.go | 84 +++++++------- pkg/sink/codec/open/decoder.go | 11 -- pkg/sink/codec/simple/decoder.go | 101 ++++++++--------- pkg/sink/codec/simple/decoder_test.go | 104 ++++++++++++++++++ pkg/sink/codec/simple/encoder_test.go | 27 +++-- 11 files changed, 245 insertions(+), 151 deletions(-) create mode 100644 pkg/sink/codec/simple/decoder_test.go diff --git a/cmd/kafka-consumer/writer.go b/cmd/kafka-consumer/writer.go index bf3043b9f9..89c8528bfc 100644 --- a/cmd/kafka-consumer/writer.go +++ b/cmd/kafka-consumer/writer.go @@ -350,12 +350,12 @@ func (w *writer) WriteMessage(ctx context.Context, message *kafka.Message) bool ddl := progress.decoder.NextDDLEvent() if dec, ok := progress.decoder.(*simple.Decoder); ok { - cachedEvents := dec.GetCachedEvents() - for _, row := range cachedEvents { + cachedMessages := dec.GetCachedMessages() + for _, dmlMessage := range cachedMessages { log.Info("simple protocol cached event resolved, append to the group", - zap.Int64("tableID", row.GetTableID()), zap.Uint64("commitTs", row.CommitTs), + zap.Int64("tableID", dmlMessage.TableID), zap.Uint64("commitTs", dmlMessage.GetCommitTs()), zap.Int32("partition", partition), zap.Any("offset", offset)) - w.appendMessage2Group(common.NewDMLMessageFromEvent(row), progress, offset) + w.appendMessage2Group(dmlMessage, progress, offset) } } @@ -655,7 +655,7 @@ func (w *writer) appendMessage2Group(message *common.DMLMessage, progress *parti zap.String("schema", schema), zap.String("table", table), zap.Int64("tableID", tableID), zap.Stringer("eventType", message.RowType), // zap.Any("columns", row.Columns), zap.Any("preColumns", row.PreColumns), - zap.Any("protocol", w.protocol), zap.Bool("IsPartition", false)) + zap.Any("protocol", w.protocol)) case config.ProtocolCanalJSON, config.ProtocolOpen, config.ProtocolAvro, config.ProtocolDebezium, config.ProtocolDebeziumAvro: // for partition table, these protocols cannot assign physical table id to each dml message, @@ -676,7 +676,7 @@ func (w *writer) appendMessage2Group(message *common.DMLMessage, progress *parti zap.String("schema", schema), zap.String("table", table), zap.Int64("tableID", tableID), zap.Stringer("eventType", message.RowType), // zap.Any("columns", row.Columns), zap.Any("preColumns", row.PreColumns), - zap.Any("protocol", w.protocol), zap.Bool("IsPartition", false)) + zap.Any("protocol", w.protocol)) default: log.Panic("unknown protocol", zap.Any("protocol", w.protocol)) } diff --git a/pkg/sink/codec/avro/decoder.go b/pkg/sink/codec/avro/decoder.go index 83694af1e8..2b66e078f3 100644 --- a/pkg/sink/codec/avro/decoder.go +++ b/pkg/sink/codec/avro/decoder.go @@ -131,11 +131,6 @@ func (d *decoder) NextDMLMessage() *common.DMLMessage { }) } -func (d *decoder) nextDMLEvent() *commonEvent.DMLEvent { - keyMap, valueMap, valueSchema, isDelete, deleteCommitTs := d.decodeDMLPayload() - return d.assembleDMLEventFromDecoded(keyMap, valueMap, valueSchema, isDelete, deleteCommitTs) -} - func (d *decoder) decodeDMLPayload() ( keyMap map[string]interface{}, valueMap map[string]interface{}, diff --git a/pkg/sink/codec/canal/canal_json_decoder.go b/pkg/sink/codec/canal/canal_json_decoder.go index c854ff87a8..87d743234d 100644 --- a/pkg/sink/codec/canal/canal_json_decoder.go +++ b/pkg/sink/codec/canal/canal_json_decoder.go @@ -25,6 +25,7 @@ import ( "sort" "strconv" "strings" + "sync" "github.com/pingcap/log" commonType "github.com/pingcap/ticdc/pkg/common" @@ -98,6 +99,7 @@ type decoder struct { storage storeapi.Storage upstreamTiDB *sql.DB + tableInfoMu sync.RWMutex tableInfoCache map[tableKey]*commonType.TableInfo ddlCommitTs map[tableNameKey][]uint64 } @@ -433,14 +435,15 @@ func (d *decoder) NextResolvedEvent() uint64 { } func formatAllColumnsValue(data map[string]any, columns []*timodel.ColumnInfo) map[string]any { + result := make(map[string]any, len(data)) for _, col := range columns { raw, ok := data[col.Name.O] if !ok { continue } - data[col.Name.O] = formatValue(raw, col.FieldType) + result[col.Name.O] = formatValue(raw, col.FieldType) } - return data + return result } func formatValue(value any, ft types.FieldType) any { @@ -569,10 +572,13 @@ func (d *decoder) queryTableInfo(msg canalJSONMessageInterface) *commonType.Tabl schemaName := *msg.getSchema() tableName := *msg.getTable() + d.tableInfoMu.Lock() + defer d.tableInfoMu.Unlock() + cacheKey := tableKey{ schema: schemaName, table: tableName, - ddlCommitTs: d.getDDLCommitTs(schemaName, tableName, msg.getCommitTs()), + ddlCommitTs: d.getDDLCommitTsLocked(schemaName, tableName, msg.getCommitTs()), } tableInfo, ok := d.tableInfoCache[cacheKey] if !ok { @@ -596,6 +602,9 @@ func (d *decoder) addDDLCommitTs(schema, table string, commitTs uint64) { return } + d.tableInfoMu.Lock() + defer d.tableInfoMu.Unlock() + key := tableNameKey{schema: schema, table: table} commitTsList := d.ddlCommitTs[key] i := sort.Search(len(commitTsList), func(i int) bool { @@ -607,7 +616,7 @@ func (d *decoder) addDDLCommitTs(schema, table string, commitTs uint64) { d.ddlCommitTs[key] = slices.Insert(commitTsList, i, commitTs) } -func (d *decoder) getDDLCommitTs(schema, table string, commitTs uint64) uint64 { +func (d *decoder) getDDLCommitTsLocked(schema, table string, commitTs uint64) uint64 { if commitTs == 0 { return 0 } diff --git a/pkg/sink/codec/canal/canal_json_txn_decoder.go b/pkg/sink/codec/canal/canal_json_txn_decoder.go index 2fa78a73dd..9cdb70f383 100644 --- a/pkg/sink/codec/canal/canal_json_txn_decoder.go +++ b/pkg/sink/codec/canal/canal_json_txn_decoder.go @@ -125,24 +125,11 @@ func (d *txnDecoder) NextDMLMessage() *common.DMLMessage { } return common.NewDMLMessage(tableID, schemaName, tableName, msg.getCommitTs(), rowType, func() *commonEvent.DMLEvent { - d.msg = msg - return d.nextDMLEvent() + return d.canalJSONMessage2RowChange(msg) }) } -func (d *txnDecoder) nextDMLEvent() *commonEvent.DMLEvent { - if d.msg == nil || d.msg.messageType() != common.MessageTypeRow { - log.Panic("message type is not row changed", - zap.Any("messageType", d.msg.messageType()), zap.Any("msg", d.msg)) - } - result := d.canalJSONMessage2RowChange() - d.msg = nil - return result -} - -func (d *txnDecoder) canalJSONMessage2RowChange() *commonEvent.DMLEvent { - msg := d.msg - +func (d *txnDecoder) canalJSONMessage2RowChange(msg canalJSONMessageInterface) *commonEvent.DMLEvent { tableInfo := newTableInfo(msg) result := new(commonEvent.DMLEvent) result.Length++ // todo: set this field correctly diff --git a/pkg/sink/codec/common/decoder.go b/pkg/sink/codec/common/decoder.go index 6031b9b751..86e5a8f5fa 100644 --- a/pkg/sink/codec/common/decoder.go +++ b/pkg/sink/codec/common/decoder.go @@ -24,7 +24,9 @@ type DMLMessage struct { Table string RowType commonType.RowType - commitTs uint64 + commitTs uint64 + // toDMLEvent may be called after the decoder has consumed later messages. + // It must only use data captured by this DMLMessage and must not depend on decoder cursor state. toDMLEvent func() *commonEvent.DMLEvent } diff --git a/pkg/sink/codec/common/table_info_cache.go b/pkg/sink/codec/common/table_info_cache.go index 9cccc22c1f..f1c7853ee6 100644 --- a/pkg/sink/codec/common/table_info_cache.go +++ b/pkg/sink/codec/common/table_info_cache.go @@ -15,6 +15,7 @@ package common import ( "strings" + "sync" "github.com/pingcap/log" "go.uber.org/zap" @@ -39,6 +40,7 @@ func newAccessKey(schema, table string) accessKey { // tableIDAllocator is a fake table id allocator type tableIDAllocator struct { + mu sync.RWMutex tableIDs map[accessKey]int64 currentTableID int64 blockedTableIDs map[accessKey]map[int64]struct{} @@ -63,11 +65,17 @@ func (a *tableIDAllocator) allocateByKey(key accessKey) int64 { // Allocate allocates a table id func (a *tableIDAllocator) Allocate(schema, table string) int64 { + a.mu.Lock() + defer a.mu.Unlock() + key := newAccessKey(schema, table) return a.allocateByKey(key) } func (a *tableIDAllocator) GetBlockedTables(schema, table string) []int64 { + a.mu.RLock() + defer a.mu.RUnlock() + key := newAccessKey(schema, table) blocked := a.blockedTableIDs[key] result := make([]int64, 0, len(blocked)) @@ -78,6 +86,9 @@ func (a *tableIDAllocator) GetBlockedTables(schema, table string) []int64 { } func (a *tableIDAllocator) AddBlockTableID(schema string, table string, physicalTableID int64) { + a.mu.Lock() + defer a.mu.Unlock() + key := newAccessKey(schema, table) if _, ok := a.blockedTableIDs[key]; !ok { a.blockedTableIDs[key] = make(map[int64]struct{}) @@ -91,6 +102,9 @@ func (a *tableIDAllocator) AddBlockTableID(schema string, table string, physical } func (a *tableIDAllocator) Clean() { + a.mu.Lock() + defer a.mu.Unlock() + a.currentTableID = 0 clear(a.tableIDs) clear(a.blockedTableIDs) diff --git a/pkg/sink/codec/debezium/decoder.go b/pkg/sink/codec/debezium/decoder.go index da0a1153e4..9691fb6406 100644 --- a/pkg/sink/codec/debezium/decoder.go +++ b/pkg/sink/codec/debezium/decoder.go @@ -163,27 +163,22 @@ func (d *decoder) NextDMLMessage() *common.DMLMessage { } keyPayload := d.keyPayload - keySchema := d.keySchema valuePayload := d.valuePayload valueSchema := d.valueSchema - commitTs := d.getCommitTs() - schemaName := d.getSchemaName() - tableName := d.getTableName() - rowType := d.rowType() + commitTs := getCommitTsFromPayload(valuePayload) + schemaName := getSchemaNameFromPayload(valuePayload) + tableName := getTableNameFromPayload(valuePayload) + rowType := rowTypeFromPayload(valuePayload) tableID := tableIDAllocator.Allocate(schemaName, tableName) d.clear() return common.NewDMLMessage(tableID, schemaName, tableName, commitTs, rowType, func() *commonEvent.DMLEvent { - d.keyPayload = keyPayload - d.keySchema = keySchema - d.valuePayload = valuePayload - d.valueSchema = valueSchema - return d.nextDMLEvent() + return d.assembleDMLEventFromPayload(keyPayload, valuePayload, valueSchema) }) } -func (d *decoder) rowType() commonType.RowType { - op, ok := d.valuePayload["op"] +func rowTypeFromPayload(valuePayload map[string]interface{}) commonType.RowType { + op, ok := valuePayload["op"] if !ok { log.Panic("DML message op not found") } @@ -200,19 +195,13 @@ func (d *decoder) rowType() commonType.RowType { return commonType.RowTypeInsert } -func (d *decoder) nextDMLEvent() *commonEvent.DMLEvent { - if len(d.valuePayload) == 0 { - log.Panic("next DML event failed, since value payload is empty") - } - if d.config.DebeziumDisableSchema { - log.Panic("next DML event failed, since DebeziumDisableSchema is true") - } - if !d.config.EnableTiDBExtension { - log.Panic("next DML event failed, since EnableTiDBExtension is false") - } - defer d.clear() - tableInfo := d.queryTableInfo() - commitTs := d.getCommitTs() +func (d *decoder) assembleDMLEventFromPayload( + keyPayload map[string]interface{}, + valuePayload map[string]interface{}, + valueSchema map[string]interface{}, +) *commonEvent.DMLEvent { + tableInfo := queryTableInfoFromPayload(keyPayload, valuePayload, valueSchema) + commitTs := getCommitTsFromPayload(valuePayload) event := &commonEvent.DMLEvent{ Rows: chunk.NewChunkFromPoolWithCapacity(tableInfo.GetFieldSlice(), chunk.InitialCapacity), StartTs: commitTs, @@ -225,12 +214,12 @@ func (d *decoder) nextDMLEvent() *commonEvent.DMLEvent { event.Rows.Destroy(chunk.InitialCapacity, tableInfo.GetFieldSlice()) }) columns := tableInfo.GetColumns() - before, ok1 := d.valuePayload["before"].(map[string]interface{}) + before, ok1 := valuePayload["before"].(map[string]interface{}) if ok1 { data := assembleColumnData(before, columns, d.config.TimeZone) common.AppendRow2Chunk(data, columns, event.Rows) } - after, ok2 := d.valuePayload["after"].(map[string]interface{}) + after, ok2 := valuePayload["after"].(map[string]interface{}) if ok2 { data := assembleColumnData(after, columns, d.config.TimeZone) common.AppendRow2Chunk(data, columns, event.Rows) @@ -249,7 +238,11 @@ func (d *decoder) nextDMLEvent() *commonEvent.DMLEvent { } func (d *decoder) getCommitTs() uint64 { - source := d.valuePayload["source"].(map[string]interface{}) + return getCommitTsFromPayload(d.valuePayload) +} + +func getCommitTsFromPayload(valuePayload map[string]interface{}) uint64 { + source := valuePayload["source"].(map[string]interface{}) commitTs, err := source["commit_ts"].(json.Number).Int64() if err != nil { log.Error("decode value failed", zap.Error(err), zap.String("value", util.RedactAny(source))) @@ -258,15 +251,21 @@ func (d *decoder) getCommitTs() uint64 { } func (d *decoder) getSchemaName() string { - source := d.valuePayload["source"].(map[string]interface{}) - schemaName := source["db"].(string) - return schemaName + return getSchemaNameFromPayload(d.valuePayload) +} + +func getSchemaNameFromPayload(valuePayload map[string]interface{}) string { + source := valuePayload["source"].(map[string]interface{}) + return source["db"].(string) } func (d *decoder) getTableName() string { - source := d.valuePayload["source"].(map[string]interface{}) - tableName := source["table"].(string) - return tableName + return getTableNameFromPayload(d.valuePayload) +} + +func getTableNameFromPayload(valuePayload map[string]interface{}) string { + source := valuePayload["source"].(map[string]interface{}) + return source["table"].(string) } func (d *decoder) clear() { @@ -276,19 +275,22 @@ func (d *decoder) clear() { d.valueSchema = nil } -func (d *decoder) queryTableInfo() *commonType.TableInfo { - schemaName := d.getSchemaName() - tableName := d.getTableName() - +func queryTableInfoFromPayload( + keyPayload map[string]interface{}, + valuePayload map[string]interface{}, + valueSchema map[string]interface{}, +) *commonType.TableInfo { + schemaName := getSchemaNameFromPayload(valuePayload) + tableName := getTableNameFromPayload(valuePayload) tidbTableInfo := new(timodel.TableInfo) tidbTableInfo.ID = tableIDAllocator.Allocate(schemaName, tableName) tableIDAllocator.AddBlockTableID(schemaName, tableName, tidbTableInfo.ID) tidbTableInfo.Name = ast.NewCIStr(tableName) - fields := d.valueSchema["fields"].([]interface{}) + fields := valueSchema["fields"].([]interface{}) after := fields[1].(map[string]interface{}) columnsField := after["fields"].([]interface{}) - indexColumns := make([]*timodel.IndexColumn, 0, len(d.keyPayload)) + indexColumns := make([]*timodel.IndexColumn, 0, len(keyPayload)) for idx, column := range columnsField { col := column.(map[string]interface{}) colName := col["field"].(string) @@ -306,7 +308,7 @@ func (d *decoder) queryTableInfo() *commonType.TableInfo { fieldType.SetDecimal(6) } } - if _, ok := d.keyPayload[colName]; ok { + if _, ok := keyPayload[colName]; ok { indexColumns = append(indexColumns, &timodel.IndexColumn{ Name: ast.NewCIStr(colName), Offset: idx, diff --git a/pkg/sink/codec/open/decoder.go b/pkg/sink/codec/open/decoder.go index 691ca99858..19cb2cebb1 100644 --- a/pkg/sink/codec/open/decoder.go +++ b/pkg/sink/codec/open/decoder.go @@ -212,17 +212,6 @@ func (b *decoder) NextDMLMessage() *common.DMLMessage { }) } -func (b *decoder) nextDMLEvent() *commonEvent.DMLEvent { - if b.nextKey.Type != common.MessageTypeRow { - log.Panic("message type is not row", zap.Any("messageType", b.nextKey.Type)) - } - - key := b.nextKey - value := b.nextDMLValue() - b.nextKey = nil - return b.decodeDMLMessage(key, value) -} - func (b *decoder) nextDMLValue() []byte { valueLen := binary.BigEndian.Uint64(b.valueBytes[:8]) value := b.valueBytes[8 : valueLen+8] diff --git a/pkg/sink/codec/simple/decoder.go b/pkg/sink/codec/simple/decoder.go index f47e61ead8..76450349cb 100644 --- a/pkg/sink/codec/simple/decoder.go +++ b/pkg/sink/codec/simple/decoder.go @@ -57,8 +57,8 @@ type Decoder struct { // cachedMessages is used to store the messages which does not have received corresponding table info yet. cachedMessages *list.List - // CachedRowChangedEvents are events just decoded from the cachedMessages - CachedRowChangedEvents []*commonEvent.DMLEvent + // CachedDMLMessages are messages just released from the cachedMessages. + CachedDMLMessages []*common.DMLMessage } // NewDecoder returns a new Decoder @@ -156,9 +156,23 @@ func (d *Decoder) NextDMLMessage() *common.DMLMessage { msg := d.msg d.msg = nil + tableInfo := d.memo.Read(msg.Schema, msg.Table, msg.SchemaVersion) + if tableInfo == nil { + log.Debug("table info not found for the message, "+ + "the consumer should cache this message temporarily, and update the tableInfo after it's received", + zap.String("schema", msg.Schema), + zap.String("table", msg.Table), + zap.Uint64("version", msg.SchemaVersion)) + d.cachedMessages.PushBack(msg) + return nil + } + + return d.newDMLMessage(msg, tableInfo) +} + +func (d *Decoder) newDMLMessage(msg *message, tableInfo *commonType.TableInfo) *common.DMLMessage { return common.NewDMLMessage(msg.TableID, msg.Schema, msg.Table, msg.CommitTs, rowTypeFromMessageType(msg.Type), func() *commonEvent.DMLEvent { - d.msg = msg - return d.nextDMLEvent() + return d.assembleDMLEvent(msg, tableInfo) }) } @@ -176,33 +190,16 @@ func rowTypeFromMessageType(tp MessageType) commonType.RowType { return commonType.RowTypeInsert } -func (d *Decoder) nextDMLEvent() *commonEvent.DMLEvent { - if d.msg == nil || (d.msg.Data == nil && d.msg.Old == nil) { - log.Panic("invalid data for the DML event", zap.String("message", util.RedactAny(d.msg))) +func (d *Decoder) assembleDMLEvent(msg *message, tableInfo *commonType.TableInfo) *commonEvent.DMLEvent { + if msg.ClaimCheckLocation != "" { + return d.assembleClaimCheckRowChangedEvent(msg.ClaimCheckLocation, tableInfo) } - if d.msg.ClaimCheckLocation != "" { - return d.assembleClaimCheckRowChangedEvent(d.msg.ClaimCheckLocation) + if msg.HandleKeyOnly { + return d.assembleHandleKeyOnlyRowChangedEvent(msg, tableInfo) } - if d.msg.HandleKeyOnly { - return d.assembleHandleKeyOnlyRowChangedEvent(d.msg) - } - - tableInfo := d.memo.Read(d.msg.Schema, d.msg.Table, d.msg.SchemaVersion) - if tableInfo == nil { - log.Debug("table info not found for the event, "+ - "the consumer should cache this event temporarily, and update the tableInfo after it's received", - zap.String("schema", d.msg.Schema), - zap.String("table", d.msg.Table), - zap.Uint64("version", d.msg.SchemaVersion)) - d.cachedMessages.PushBack(d.msg) - d.msg = nil - return nil - } - - event := buildDMLEvent(d.msg, tableInfo, d.config.EnableRowChecksum, d.upstreamTiDB) - d.msg = nil + event := buildDMLEvent(msg, tableInfo, d.config.EnableRowChecksum, d.upstreamTiDB) tableIDAllocator.AddBlockTableID(event.TableInfo.GetSchemaName(), event.TableInfo.GetTableName(), event.GetTableID()) @@ -210,7 +207,9 @@ func (d *Decoder) nextDMLEvent() *commonEvent.DMLEvent { return event } -func (d *Decoder) assembleClaimCheckRowChangedEvent(claimCheckLocation string) *commonEvent.DMLEvent { +func (d *Decoder) assembleClaimCheckRowChangedEvent( + claimCheckLocation string, tableInfo *commonType.TableInfo, +) *commonEvent.DMLEvent { _, claimCheckFileName := filepath.Split(claimCheckLocation) data, err := d.storage.ReadFile(context.Background(), claimCheckFileName) if err != nil { @@ -238,23 +237,12 @@ func (d *Decoder) assembleClaimCheckRowChangedEvent(claimCheckLocation string) * if err != nil { log.Panic("unmarshal claim check message failed", zap.Any("value", util.RedactAny(value)), zap.Error(err)) } - d.msg = m - return d.nextDMLEvent() + return d.assembleDMLEvent(m, tableInfo) } -func (d *Decoder) assembleHandleKeyOnlyRowChangedEvent(m *message) *commonEvent.DMLEvent { - tableInfo := d.memo.Read(m.Schema, m.Table, m.SchemaVersion) - if tableInfo == nil { - log.Debug("table info not found for the event, "+ - "the consumer should cache this event temporarily, and update the tableInfo after it's received", - zap.String("schema", d.msg.Schema), - zap.String("table", d.msg.Table), - zap.Uint64("version", d.msg.SchemaVersion)) - d.cachedMessages.PushBack(d.msg) - d.msg = nil - return nil - } - +func (d *Decoder) assembleHandleKeyOnlyRowChangedEvent( + m *message, tableInfo *commonType.TableInfo, +) *commonEvent.DMLEvent { fieldTypeMap := make(map[string]*types.FieldType, len(tableInfo.GetColumns())) for _, col := range tableInfo.GetColumns() { fieldTypeMap[col.Name.O] = &col.FieldType @@ -287,8 +275,7 @@ func (d *Decoder) assembleHandleKeyOnlyRowChangedEvent(m *message) *commonEvent. result.Old = d.buildData(holder, fieldTypeMap, timezone) } - d.msg = result - return d.nextDMLEvent() + return d.assembleDMLEvent(result, tableInfo) } func (d *Decoder) buildData( @@ -319,21 +306,22 @@ func (d *Decoder) NextDDLEvent() *commonEvent.DDLEvent { d.memo.Write(ddl.MultipleTableInfos[1]) for ele := d.cachedMessages.Front(); ele != nil; { - d.msg = ele.Value.(*message) - event := d.nextDMLEvent() - d.CachedRowChangedEvents = append(d.CachedRowChangedEvents, event) - + msg := ele.Value.(*message) next := ele.Next() - d.cachedMessages.Remove(ele) + tableInfo := d.memo.Read(msg.Schema, msg.Table, msg.SchemaVersion) + if tableInfo != nil { + d.CachedDMLMessages = append(d.CachedDMLMessages, d.newDMLMessage(msg, tableInfo)) + d.cachedMessages.Remove(ele) + } ele = next } return ddl } -// GetCachedEvents returns the cached events -func (d *Decoder) GetCachedEvents() []*commonEvent.DMLEvent { - result := d.CachedRowChangedEvents - d.CachedRowChangedEvents = nil +// GetCachedMessages returns the cached messages. +func (d *Decoder) GetCachedMessages() []*common.DMLMessage { + result := d.CachedDMLMessages + d.CachedDMLMessages = nil return result } @@ -667,14 +655,15 @@ func buildDMLEvent(msg *message, tableInfo *commonType.TableInfo, enableRowCheck } func formatAllColumnsValue(data map[string]any, columns []*timodel.ColumnInfo) map[string]any { + result := make(map[string]any, len(data)) for _, col := range columns { raw, ok := data[col.Name.O] if !ok { continue } - data[col.Name.O] = formatValue(raw, col.FieldType) + result[col.Name.O] = formatValue(raw, col.FieldType) } - return data + return result } // formatValue formats the value according to the field type diff --git a/pkg/sink/codec/simple/decoder_test.go b/pkg/sink/codec/simple/decoder_test.go new file mode 100644 index 0000000000..efdfa9206e --- /dev/null +++ b/pkg/sink/codec/simple/decoder_test.go @@ -0,0 +1,104 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package simple + +import ( + "container/list" + "testing" + + commonType "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/config" + "github.com/pingcap/ticdc/pkg/sink/codec/common" + "github.com/stretchr/testify/require" +) + +func TestCachedDMLReturnsMessage(t *testing.T) { + const ( + schema = "test" + table = "t" + tableID = int64(1) + schemaVersion = uint64(100) + commitTs = uint64(90) + ) + + decoder := &Decoder{ + config: common.NewConfig(config.ProtocolSimple), + memo: newMemoryTableInfoProvider(), + cachedMessages: list.New(), + } + decoder.msg = &message{ + Version: defaultVersion, + Schema: schema, + Table: table, + TableID: tableID, + Type: DMLTypeInsert, + CommitTs: commitTs, + SchemaVersion: schemaVersion, + Data: map[string]interface{}{"id": int64(1)}, + } + + require.Nil(t, decoder.NextDMLMessage()) + require.Equal(t, 1, decoder.cachedMessages.Len()) + + decoder.msg = &message{ + Version: defaultVersion, + Type: DDLTypeCreate, + CommitTs: schemaVersion, + TableSchema: &TableSchema{ + Schema: schema, + Table: table, + TableID: tableID, + Version: schemaVersion, + Columns: []*columnSchema{ + { + Name: "id", + DataType: dataType{ + MySQLType: "bigint", + Charset: "binary", + Collate: "binary", + Length: 20, + }, + }, + }, + }, + } + ddl := decoder.NextDDLEvent() + require.NotNil(t, ddl) + + cachedMessages := decoder.GetCachedMessages() + require.Len(t, cachedMessages, 1) + require.Zero(t, decoder.cachedMessages.Len()) + + dmlMessage := cachedMessages[0] + require.Equal(t, tableID, dmlMessage.TableID) + require.Equal(t, schema, dmlMessage.Schema) + require.Equal(t, table, dmlMessage.Table) + require.Equal(t, commitTs, dmlMessage.GetCommitTs()) + require.Equal(t, commonType.RowTypeInsert, dmlMessage.RowType) + + decoder.msg = &message{ + Version: defaultVersion, + Type: MessageTypeWatermark, + CommitTs: commitTs + 1, + } + dmlEvent := dmlMessage.ToDMLEvent() + require.NotNil(t, dmlEvent) + require.Equal(t, tableID, dmlEvent.GetTableID()) + require.Equal(t, commitTs, dmlEvent.GetCommitTs()) + require.Equal(t, schema, dmlEvent.TableInfo.GetSchemaName()) + require.Equal(t, table, dmlEvent.TableInfo.GetTableName()) + require.NotNil(t, decoder.msg) + require.Equal(t, MessageTypeWatermark, decoder.msg.Type) + require.Equal(t, commitTs+1, decoder.msg.CommitTs) +} diff --git a/pkg/sink/codec/simple/encoder_test.go b/pkg/sink/codec/simple/encoder_test.go index e266f638ce..bea43f4102 100644 --- a/pkg/sink/codec/simple/encoder_test.go +++ b/pkg/sink/codec/simple/encoder_test.go @@ -1273,8 +1273,8 @@ func TestE2EPartitionTableDMLBeforeDDL(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, tp) - decodedEvent := dec.NextDMLMessage().ToDMLEvent() - require.Nil(t, decodedEvent) + decodedMessage := dec.NextDMLMessage() + require.Nil(t, decodedMessage) e.Rewind() } @@ -1290,8 +1290,9 @@ func TestE2EPartitionTableDMLBeforeDDL(t *testing.T) { decodedDDL := dec.NextDDLEvent() require.NotNil(t, decodedDDL) - cachedEvents := dec.(*Decoder).GetCachedEvents() - for idx, decodedRow := range cachedEvents { + cachedMessages := dec.(*Decoder).GetCachedMessages() + for idx, message := range cachedMessages { + decodedRow := message.ToDMLEvent() require.NotNil(t, decodedRow) require.NotNil(t, decodedRow.TableInfo) require.Equal(t, decodedRow.GetTableID(), events[idx].GetTableID()) @@ -1339,8 +1340,8 @@ func TestEncodeDMLBeforeDDL(t *testing.T) { require.True(t, hasNext) require.Equal(t, common.MessageTypeRow, messageType) - decodedRow := dec.NextDMLMessage().ToDMLEvent() - require.Nil(t, decodedRow) + decodedMessage := dec.NextDMLMessage() + require.Nil(t, decodedMessage) m, err := enc.EncodeDDLEvent(ddlEvent) require.NoError(t, err) @@ -1354,8 +1355,9 @@ func TestEncodeDMLBeforeDDL(t *testing.T) { ddlEvent = dec.NextDDLEvent() require.NotNil(t, ddlEvent) - cachedEvents := dec.GetCachedEvents() - for _, decodedRow = range cachedEvents { + cachedMessages := dec.GetCachedMessages() + for _, message := range cachedMessages { + decodedRow := message.ToDMLEvent() require.NotNil(t, decodedRow) require.NotNil(t, decodedRow.TableInfo) require.Equal(t, decodedRow.TableInfo.TableName.TableID, ddlEvent.TableInfo.TableName.TableID) @@ -1725,8 +1727,8 @@ func TestLargeMessageHandleKeyOnly(t *testing.T) { require.Equal(t, common.MessageTypeRow, messageType) require.True(t, dec.msg.HandleKeyOnly) - decodedRow := dec.NextDMLMessage().ToDMLEvent() - require.Nil(t, decodedRow) + decodedMessage := dec.NextDMLMessage() + require.Nil(t, decodedMessage) } enc.(*Encoder).config.MaxMessageBytes = config.DefaultMaxMessageBytes @@ -1760,8 +1762,9 @@ func TestLargeMessageHandleKeyOnly(t *testing.T) { } _ = dec.NextDDLEvent() - decodedRows := dec.GetCachedEvents() - for idx, decodedRow := range decodedRows { + decodedMessages := dec.GetCachedMessages() + for idx, message := range decodedMessages { + decodedRow := message.ToDMLEvent() event := events[idx] require.Equal(t, decodedRow.CommitTs, event.CommitTs) From 899f8207356a3e8168a2eec56b28e4283754b9d6 Mon Sep 17 00:00:00 2001 From: wk989898 Date: Mon, 20 Jul 2026 06:16:45 +0000 Subject: [PATCH 07/12] update Signed-off-by: wk989898 --- pkg/sink/codec/canal/canal_json_decoder.go | 3 +- pkg/sink/codec/canal/canal_json_test.go | 44 +++++++++++++++------- 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/pkg/sink/codec/canal/canal_json_decoder.go b/pkg/sink/codec/canal/canal_json_decoder.go index 87d743234d..fe750831b8 100644 --- a/pkg/sink/codec/canal/canal_json_decoder.go +++ b/pkg/sink/codec/canal/canal_json_decoder.go @@ -623,7 +623,8 @@ func (d *decoder) getDDLCommitTsLocked(schema, table string, commitTs uint64) ui commitTsList := d.ddlCommitTs[tableNameKey{schema: schema, table: table}] i := sort.Search(len(commitTsList), func(i int) bool { - return commitTsList[i] > commitTs + // DMLs with the same commit-ts as a DDL are flushed before that DDL. + return commitTsList[i] >= commitTs }) if i == 0 { return 0 diff --git a/pkg/sink/codec/canal/canal_json_test.go b/pkg/sink/codec/canal/canal_json_test.go index e81a7c245f..81182391b9 100644 --- a/pkg/sink/codec/canal/canal_json_test.go +++ b/pkg/sink/codec/canal/canal_json_test.go @@ -1231,28 +1231,44 @@ func TestDecoderTableInfoCacheUsesDDLCommitTsBoundary(t *testing.T) { tableInfoCache: make(map[tableKey]*commonType.TableInfo), ddlCommitTs: make(map[tableNameKey][]uint64), } - buildMessage := func(commitTs uint64) *canalJSONMessageWithTiDBExtension { + buildMessage := func(commitTs uint64, mysqlTypes map[string]string) *canalJSONMessageWithTiDBExtension { return &canalJSONMessageWithTiDBExtension{ JSONMessage: &JSONMessage{ - Schema: "test", - Table: "table_5", - PKNames: []string{"id"}, - MySQLType: map[string]string{ - "data": "varchar(255)", - "id": "int", - }, + Schema: "test", + Table: "table_5", + PKNames: []string{"id"}, + MySQLType: mysqlTypes, }, Extensions: &tidbExtension{CommitTs: commitTs}, } } + columnNames := func(tableInfo *commonType.TableInfo) []string { + columns := tableInfo.GetColumns() + result := make([]string, 0, len(columns)) + for _, column := range columns { + result = append(result, column.Name.O) + } + return result + } + beforeDropSchema := map[string]string{ + "data": "varchar(255)", + "id": "int", + "new_col": "int", + } + afterDropSchema := map[string]string{ + "data": "varchar(255)", + "id": "int", + } - beforeDDL := dec.queryTableInfo(buildMessage(100)) dec.addDDLCommitTs("test", "table_5", 200) - afterDDL := dec.queryTableInfo(buildMessage(300)) - lateBeforeDDL := dec.queryTableInfo(buildMessage(100)) - - require.NotSame(t, beforeDDL, afterDDL) - require.Same(t, beforeDDL, lateBeforeDDL) + sameTsAsDDL := dec.queryTableInfo(buildMessage(200, beforeDropSchema)) + afterDDL := dec.queryTableInfo(buildMessage(300, afterDropSchema)) + lateSameTsAsDDL := dec.queryTableInfo(buildMessage(200, beforeDropSchema)) + + require.Equal(t, []string{"data", "id", "new_col"}, columnNames(sameTsAsDDL)) + require.Equal(t, []string{"data", "id"}, columnNames(afterDDL)) + require.NotSame(t, sameTsAsDDL, afterDDL) + require.Same(t, sameTsAsDDL, lateSameTsAsDDL) require.Equal(t, []uint64{200}, dec.ddlCommitTs[tableNameKey{schema: "test", table: "table_5"}]) } From d43c0373a811d9981d8cdf5b9ff4413cd3dc4804 Mon Sep 17 00:00:00 2001 From: wk989898 Date: Mon, 20 Jul 2026 08:05:55 +0000 Subject: [PATCH 08/12] lint Signed-off-by: wk989898 --- pkg/sink/codec/avro/decoder.go | 64 ++++++++++++++--------------- pkg/sink/codec/open/encoder_test.go | 2 - 2 files changed, 32 insertions(+), 34 deletions(-) diff --git a/pkg/sink/codec/avro/decoder.go b/pkg/sink/codec/avro/decoder.go index 2b66e078f3..ee074954fd 100644 --- a/pkg/sink/codec/avro/decoder.go +++ b/pkg/sink/codec/avro/decoder.go @@ -132,14 +132,14 @@ func (d *decoder) NextDMLMessage() *common.DMLMessage { } func (d *decoder) decodeDMLPayload() ( - keyMap map[string]interface{}, - valueMap map[string]interface{}, - valueSchema map[string]interface{}, + keyMap map[string]any, + valueMap map[string]any, + valueSchema map[string]any, isDelete bool, deleteCommitTs uint64, ) { var ( - keySchema map[string]interface{} + keySchema map[string]any err error ) @@ -170,9 +170,9 @@ func (d *decoder) decodeDMLPayload() ( } func (d *decoder) assembleDMLEventFromDecoded( - keyMap map[string]interface{}, - valueMap map[string]interface{}, - valueSchema map[string]interface{}, + keyMap map[string]any, + valueMap map[string]any, + valueSchema map[string]any, isDelete bool, deleteCommitTs uint64, ) *commonEvent.DMLEvent { @@ -244,18 +244,18 @@ func (d *decoder) decodeDeleteCommitTs() uint64 { // valueMap hold all columns information // schema is corresponding to the valueMap, it can be used to decode the valueMap to construct columns. func assembleEvent( - keyMap, valueMap, schema map[string]interface{}, isDelete bool, + keyMap, valueMap, schema map[string]any, isDelete bool, ) (*commonEvent.DMLEvent, error) { - fields, ok := schema["fields"].([]interface{}) + fields, ok := schema["fields"].([]any) if !ok { return nil, errors.New("schema fields should be a map") } columns := make([]*timodel.ColumnInfo, 0, len(valueMap)) - data := make(map[string]interface{}, 0) + data := make(map[string]any, 0) // fields is ordered by the column id, so iterate over it to build columns // it's also the order to calculate the checksum. for idx, item := range fields { - field, ok := item.(map[string]interface{}) + field, ok := item.(map[string]any) if !ok { return nil, errors.New("schema field should be a map") } @@ -266,18 +266,18 @@ func assembleEvent( break } // query the field to get `tidbType`, and get the mysql type from it. - var holder map[string]interface{} + var holder map[string]any switch ty := field["type"].(type) { - case []interface{}: - if m, ok := ty[0].(map[string]interface{}); ok { - holder = m["connect.parameters"].(map[string]interface{}) - } else if m, ok := ty[1].(map[string]interface{}); ok { - holder = m["connect.parameters"].(map[string]interface{}) + case []any: + if m, ok := ty[0].(map[string]any); ok { + holder = m["connect.parameters"].(map[string]any) + } else if m, ok := ty[1].(map[string]any); ok { + holder = m["connect.parameters"].(map[string]any) } else { log.Panic("type info is anything else", zap.Any("typeInfo", field["type"])) } - case map[string]interface{}: - holder = ty["connect.parameters"].(map[string]interface{}) + case map[string]any: + holder = ty["connect.parameters"].(map[string]any) default: log.Panic("type info is anything else", zap.Any("typeInfo", field["type"])) } @@ -335,17 +335,17 @@ func assembleEvent( return event, nil } -func schemaAndTableName(schema map[string]interface{}) (string, string) { +func schemaAndTableName(schema map[string]any) (string, string) { namespace := schema["namespace"].(string) return strings.Split(namespace, ".")[1], schema["name"].(string) } -func queryTableInfo(schemaName, tableName string, columns []*timodel.ColumnInfo, keyMap map[string]interface{}) *commonType.TableInfo { +func queryTableInfo(schemaName, tableName string, columns []*timodel.ColumnInfo, keyMap map[string]any) *commonType.TableInfo { tableInfo := newTableInfo(schemaName, tableName, columns, keyMap) return tableInfo } -func newTableInfo(schemaName, tableName string, columns []*timodel.ColumnInfo, keyMap map[string]interface{}) *commonType.TableInfo { +func newTableInfo(schemaName, tableName string, columns []*timodel.ColumnInfo, keyMap map[string]any) *commonType.TableInfo { tidbTableInfo := new(timodel.TableInfo) tidbTableInfo.ID = tableIDAllocator.Allocate(schemaName, tableName) tableIDAllocator.AddBlockTableID(schemaName, tableName, tidbTableInfo.ID) @@ -368,7 +368,7 @@ func newTableInfo(schemaName, tableName string, columns []*timodel.ColumnInfo, k return commonType.NewTableInfo4Decoder(schemaName, tidbTableInfo) } -func isCorrupted(valueMap map[string]interface{}) bool { +func isCorrupted(valueMap map[string]any) bool { o, ok := valueMap[tidbCorrupted] if !ok { return false @@ -380,7 +380,7 @@ func isCorrupted(valueMap map[string]interface{}) bool { // extract the checksum from the received value map // return true if the checksum found, and return error if the checksum is not valid -func extractExpectedChecksum(valueMap map[string]interface{}) (uint64, bool) { +func extractExpectedChecksum(valueMap map[string]any) (uint64, bool) { o, ok := valueMap[tidbRowLevelChecksum] if !ok { return 0, false @@ -399,12 +399,12 @@ func extractExpectedChecksum(valueMap map[string]interface{}) (uint64, bool) { // value is an interface, need to convert it to the real value with the help of type info. // holder has the value's column info. func getColumnValue( - value interface{}, holder map[string]interface{}, mysqlType byte, flag uint, -) (interface{}, error) { + value any, holder map[string]any, mysqlType byte, flag uint, +) (any, error) { switch t := value.(type) { // for nullable columns, the value is encoded as a map with one pair. // key is the encoded type, value is the encoded value, only care about the value here. - case map[string]interface{}: + case map[string]any: for _, v := range t { value = v } @@ -569,7 +569,7 @@ func extractGlueSchemaIDAndBinaryData(data []byte) (string, []byte, error) { func decodeRawBytes( ctx context.Context, schemaM SchemaManager, data []byte, topic string, -) (map[string]interface{}, map[string]interface{}, error) { +) (map[string]any, map[string]any, error) { var schemaID schemaID var binary []byte var err error @@ -603,12 +603,12 @@ func decodeRawBytes( return nil, nil, err } - result, ok := native.(map[string]interface{}) + result, ok := native.(map[string]any) if !ok { return nil, nil, errors.ErrCodecDecode.GenWithStack("raw avro message is not a map") } - schema := make(map[string]interface{}) + schema := make(map[string]any) if err := json.Unmarshal([]byte(codec.Schema()), &schema); err != nil { return nil, nil, errors.Trace(err) } @@ -616,13 +616,13 @@ func decodeRawBytes( return result, schema, nil } -func (d *decoder) decodeKey(ctx context.Context) (map[string]interface{}, map[string]interface{}, error) { +func (d *decoder) decodeKey(ctx context.Context) (map[string]any, map[string]any, error) { data := d.key d.key = nil return decodeRawBytes(ctx, d.schemaM, data, d.topic) } -func (d *decoder) decodeValue(ctx context.Context) (map[string]interface{}, map[string]interface{}, error) { +func (d *decoder) decodeValue(ctx context.Context) (map[string]any, map[string]any, error) { data := d.value d.value = nil return decodeRawBytes(ctx, d.schemaM, data, d.topic) diff --git a/pkg/sink/codec/open/encoder_test.go b/pkg/sink/codec/open/encoder_test.go index 721f9d9faa..984e100ccb 100644 --- a/pkg/sink/codec/open/encoder_test.go +++ b/pkg/sink/codec/open/encoder_test.go @@ -1288,8 +1288,6 @@ func TestGenerateColumn(t *testing.T) { messageType, hasNext = dec.HasNext() require.True(t, hasNext) require.Equal(t, messageType, common.MessageTypeRow) - - decoded = dec.NextDMLMessage().ToDMLEvent() } // Including insert / update / delete From b8b72d7991519518e3c65f4881aa57f477d0c19b Mon Sep 17 00:00:00 2001 From: wk989898 Date: Mon, 20 Jul 2026 08:52:33 +0000 Subject: [PATCH 09/12] lint Signed-off-by: wk989898 --- pkg/sink/codec/canal/canal_json_test.go | 4 ++-- pkg/sink/codec/debezium/decoder.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/sink/codec/canal/canal_json_test.go b/pkg/sink/codec/canal/canal_json_test.go index 81182391b9..c92948d198 100644 --- a/pkg/sink/codec/canal/canal_json_test.go +++ b/pkg/sink/codec/canal/canal_json_test.go @@ -1139,7 +1139,7 @@ func TestDecoderTableInfoCacheUsesDDLCommitTsAcrossColumnChanges(t *testing.T) { require.NoError(t, err) buildRowMessage := func(commitTs uint64, mysqlTypes map[string]string) *canalJSONMessageWithTiDBExtension { - data := map[string]interface{}{ + data := map[string]any{ "data": "insert_1", "id": "525", } @@ -1159,7 +1159,7 @@ func TestDecoderTableInfoCacheUsesDDLCommitTsAcrossColumnChanges(t *testing.T) { "new_col": 4, }, MySQLType: mysqlTypes, - Data: []map[string]interface{}{data}, + Data: []map[string]any{data}, }, Extensions: &tidbExtension{CommitTs: commitTs}, } diff --git a/pkg/sink/codec/debezium/decoder.go b/pkg/sink/codec/debezium/decoder.go index 9691fb6406..eac8216ba7 100644 --- a/pkg/sink/codec/debezium/decoder.go +++ b/pkg/sink/codec/debezium/decoder.go @@ -177,7 +177,7 @@ func (d *decoder) NextDMLMessage() *common.DMLMessage { }) } -func rowTypeFromPayload(valuePayload map[string]interface{}) commonType.RowType { +func rowTypeFromPayload(valuePayload map[string]any) commonType.RowType { op, ok := valuePayload["op"] if !ok { log.Panic("DML message op not found") From daf2c7ebe80bf9589d65b328cfbe6615d38ac56e Mon Sep 17 00:00:00 2001 From: wk989898 Date: Mon, 20 Jul 2026 09:04:17 +0000 Subject: [PATCH 10/12] lint Signed-off-by: wk989898 --- pkg/sink/codec/debezium/decoder.go | 60 +++++++++++++++--------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/pkg/sink/codec/debezium/decoder.go b/pkg/sink/codec/debezium/decoder.go index eac8216ba7..5b80cb8047 100644 --- a/pkg/sink/codec/debezium/decoder.go +++ b/pkg/sink/codec/debezium/decoder.go @@ -48,10 +48,10 @@ type decoder struct { upstreamTiDB *sql.DB - keyPayload map[string]interface{} - keySchema map[string]interface{} - valuePayload map[string]interface{} - valueSchema map[string]interface{} + keyPayload map[string]any + keySchema map[string]any + valuePayload map[string]any + valueSchema map[string]any } // NewDecoder return an debezium decoder @@ -196,9 +196,9 @@ func rowTypeFromPayload(valuePayload map[string]any) commonType.RowType { } func (d *decoder) assembleDMLEventFromPayload( - keyPayload map[string]interface{}, - valuePayload map[string]interface{}, - valueSchema map[string]interface{}, + keyPayload map[string]any, + valuePayload map[string]any, + valueSchema map[string]any, ) *commonEvent.DMLEvent { tableInfo := queryTableInfoFromPayload(keyPayload, valuePayload, valueSchema) commitTs := getCommitTsFromPayload(valuePayload) @@ -214,12 +214,12 @@ func (d *decoder) assembleDMLEventFromPayload( event.Rows.Destroy(chunk.InitialCapacity, tableInfo.GetFieldSlice()) }) columns := tableInfo.GetColumns() - before, ok1 := valuePayload["before"].(map[string]interface{}) + before, ok1 := valuePayload["before"].(map[string]any) if ok1 { data := assembleColumnData(before, columns, d.config.TimeZone) common.AppendRow2Chunk(data, columns, event.Rows) } - after, ok2 := valuePayload["after"].(map[string]interface{}) + after, ok2 := valuePayload["after"].(map[string]any) if ok2 { data := assembleColumnData(after, columns, d.config.TimeZone) common.AppendRow2Chunk(data, columns, event.Rows) @@ -241,8 +241,8 @@ func (d *decoder) getCommitTs() uint64 { return getCommitTsFromPayload(d.valuePayload) } -func getCommitTsFromPayload(valuePayload map[string]interface{}) uint64 { - source := valuePayload["source"].(map[string]interface{}) +func getCommitTsFromPayload(valuePayload map[string]any) uint64 { + source := valuePayload["source"].(map[string]any) commitTs, err := source["commit_ts"].(json.Number).Int64() if err != nil { log.Error("decode value failed", zap.Error(err), zap.String("value", util.RedactAny(source))) @@ -254,8 +254,8 @@ func (d *decoder) getSchemaName() string { return getSchemaNameFromPayload(d.valuePayload) } -func getSchemaNameFromPayload(valuePayload map[string]interface{}) string { - source := valuePayload["source"].(map[string]interface{}) +func getSchemaNameFromPayload(valuePayload map[string]any) string { + source := valuePayload["source"].(map[string]any) return source["db"].(string) } @@ -263,8 +263,8 @@ func (d *decoder) getTableName() string { return getTableNameFromPayload(d.valuePayload) } -func getTableNameFromPayload(valuePayload map[string]interface{}) string { - source := valuePayload["source"].(map[string]interface{}) +func getTableNameFromPayload(valuePayload map[string]any) string { + source := valuePayload["source"].(map[string]any) return source["table"].(string) } @@ -276,9 +276,9 @@ func (d *decoder) clear() { } func queryTableInfoFromPayload( - keyPayload map[string]interface{}, - valuePayload map[string]interface{}, - valueSchema map[string]interface{}, + keyPayload map[string]any, + valuePayload map[string]any, + valueSchema map[string]any, ) *commonType.TableInfo { schemaName := getSchemaNameFromPayload(valuePayload) tableName := getTableNameFromPayload(valuePayload) @@ -287,19 +287,19 @@ func queryTableInfoFromPayload( tableIDAllocator.AddBlockTableID(schemaName, tableName, tidbTableInfo.ID) tidbTableInfo.Name = ast.NewCIStr(tableName) - fields := valueSchema["fields"].([]interface{}) - after := fields[1].(map[string]interface{}) - columnsField := after["fields"].([]interface{}) + fields := valueSchema["fields"].([]any) + after := fields[1].(map[string]any) + columnsField := after["fields"].([]any) indexColumns := make([]*timodel.IndexColumn, 0, len(keyPayload)) for idx, column := range columnsField { - col := column.(map[string]interface{}) + col := column.(map[string]any) colName := col["field"].(string) tidbType := col["tidb_type"].(string) optional := col["optional"].(bool) fieldType := parseTiDBType(tidbType, optional) switch fieldType.GetType() { case mysql.TypeEnum, mysql.TypeSet: - parameters := col["parameters"].(map[string]interface{}) + parameters := col["parameters"].(map[string]any) allowed := parameters["allowed"].(string) fieldType.SetElems(strings.Split(allowed, ",")) case mysql.TypeDatetime: @@ -333,8 +333,8 @@ func queryTableInfoFromPayload( return result } -func assembleColumnData(data map[string]interface{}, columns []*timodel.ColumnInfo, timeZone *time.Location) map[string]interface{} { - result := make(map[string]interface{}, 0) +func assembleColumnData(data map[string]any, columns []*timodel.ColumnInfo, timeZone *time.Location) map[string]any { + result := make(map[string]any, 0) for _, col := range columns { val, ok := data[col.Name.O] if !ok { @@ -345,7 +345,7 @@ func assembleColumnData(data map[string]interface{}, columns []*timodel.ColumnIn return result } -func decodeColumn(value interface{}, colInfo *timodel.ColumnInfo, timeZone *time.Location) interface{} { +func decodeColumn(value any, colInfo *timodel.ColumnInfo, timeZone *time.Location) any { if value == nil { return value } @@ -513,18 +513,18 @@ func parseTiDBType(tidbType string, optional bool) *ptypes.FieldType { return ft } -func decodeRawBytes(data []byte) (map[string]interface{}, map[string]interface{}, error) { - var v map[string]interface{} +func decodeRawBytes(data []byte) (map[string]any, map[string]any, error) { + var v map[string]any d := json.NewDecoder(bytes.NewBuffer(data)) d.UseNumber() if err := d.Decode(&v); err != nil { return nil, nil, errors.Trace(err) } - payload, ok := v["payload"].(map[string]interface{}) + payload, ok := v["payload"].(map[string]any) if !ok { return nil, nil, fmt.Errorf("decode payload failed, data: %+v", v) } - schema, ok := v["schema"].(map[string]interface{}) + schema, ok := v["schema"].(map[string]any) if !ok { return nil, nil, fmt.Errorf("decode payload failed, data: %+v", v) } From 748a92395a9477e32d64e47956868954e0f2bde7 Mon Sep 17 00:00:00 2001 From: wk989898 Date: Mon, 20 Jul 2026 09:41:22 +0000 Subject: [PATCH 11/12] . Signed-off-by: wk989898 --- pkg/sink/codec/simple/decoder_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/sink/codec/simple/decoder_test.go b/pkg/sink/codec/simple/decoder_test.go index efdfa9206e..0272380941 100644 --- a/pkg/sink/codec/simple/decoder_test.go +++ b/pkg/sink/codec/simple/decoder_test.go @@ -45,7 +45,7 @@ func TestCachedDMLReturnsMessage(t *testing.T) { Type: DMLTypeInsert, CommitTs: commitTs, SchemaVersion: schemaVersion, - Data: map[string]interface{}{"id": int64(1)}, + Data: map[string]any{"id": int64(1)}, } require.Nil(t, decoder.NextDMLMessage()) From 66fc158bbbc5e3d016c72871989ee2cdc6092c41 Mon Sep 17 00:00:00 2001 From: wk989898 Date: Tue, 21 Jul 2026 03:32:44 +0000 Subject: [PATCH 12/12] update Signed-off-by: wk989898 --- cmd/pulsar-consumer/writer.go | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/cmd/pulsar-consumer/writer.go b/cmd/pulsar-consumer/writer.go index 315b82a92a..507b6c709f 100644 --- a/cmd/pulsar-consumer/writer.go +++ b/cmd/pulsar-consumer/writer.go @@ -158,7 +158,7 @@ func (w *writer) flushDDLEvent(ctx context.Context, ddl *commonEvent.DDLEvent) e messages := g.ResolveInto(commitTs, nil) events := make([]*commonEvent.DMLEvent, 0, len(messages)) for _, message := range messages { - events = util.AppendOrMergeDMLEvent(events, w.assembleDMLEvent(progress, message)) + events = util.AppendOrMergeDMLEvent(events, message.ToDMLEvent()) } resolvedEvents = append(resolvedEvents, events...) } @@ -270,7 +270,7 @@ func (w *writer) flushDMLEventsByWatermark(ctx context.Context) error { messages := group.ResolveInto(watermark, nil) events := make([]*commonEvent.DMLEvent, 0, len(messages)) for _, message := range messages { - events = util.AppendOrMergeDMLEvent(events, w.assembleDMLEvent(p, message)) + events = util.AppendOrMergeDMLEvent(events, message.ToDMLEvent()) } resolvedEvents = append(resolvedEvents, events...) } @@ -307,24 +307,6 @@ func (w *writer) flushDMLEventsByWatermark(ctx context.Context) error { } } -func (w *writer) assembleDMLEvent(progress *partitionProgress, message *common.DMLMessage) *commonEvent.DMLEvent { - row := message.ToDMLEvent() - if row == nil { - log.Panic("DML event is nil, it's not expected", - zap.Int32("partition", progress.partition), zap.Int64("tableID", message.TableID), - zap.Uint64("commitTs", message.GetCommitTs())) - } - if row.GetTableID() != message.TableID || row.GetCommitTs() != message.GetCommitTs() || - len(row.RowTypes) == 0 || row.RowTypes[0] != message.RowType { - log.Panic("decoded DML event metadata mismatch", - zap.Int32("partition", progress.partition), - zap.Int64("pendingTableID", message.TableID), zap.Int64("decodedTableID", row.GetTableID()), - zap.Uint64("pendingCommitTs", message.GetCommitTs()), zap.Uint64("decodedCommitTs", row.GetCommitTs()), - zap.Stringer("pendingEventType", message.RowType), zap.Any("decodedEventTypes", row.RowTypes)) - } - return row -} - // WriteMessage is to decode pulsar message to event. // return true if the message is flushed to the downstream. // return error if flush messages failed.