From 56e5b48a89278b8e609dc349823f0bd04364e91c Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Tue, 16 Jun 2026 18:15:32 +0800 Subject: [PATCH 01/22] first commit --- .../sink/cloudstorage/encoder_group_test.go | 1 + downstreamadapter/sink/cloudstorage/sink.go | 8 +- downstreamadapter/sink/helper/helper.go | 10 +- downstreamadapter/sink/kafka/helper.go | 5 +- downstreamadapter/sink/kafka/sink_test.go | 5 +- downstreamadapter/sink/pulsar/helper.go | 5 +- pkg/sink/codec/avro/encoder.go | 17 +++- pkg/sink/codec/canal/canal_json_encoder.go | 34 ++++--- .../codec/canal/canal_json_encoder_test.go | 16 ++++ .../codec/canal/canal_json_txn_encoder.go | 4 +- pkg/sink/codec/common/config.go | 26 +++++- pkg/sink/codec/open/encoder.go | 39 +++++--- pkg/sink/codec/open/encoder_test.go | 40 ++++++++ pkg/sink/codec/simple/encoder.go | 42 +++++++-- pkg/sink/codec/simple/encoder_test.go | 18 ++++ pkg/sink/kafka/options.go | 91 ++++++++----------- pkg/sink/kafka/options_test.go | 57 +++++++----- pkg/sink/kafka/sarama_config.go | 2 +- 18 files changed, 287 insertions(+), 133 deletions(-) diff --git a/downstreamadapter/sink/cloudstorage/encoder_group_test.go b/downstreamadapter/sink/cloudstorage/encoder_group_test.go index acde673e5a..bd681936ad 100644 --- a/downstreamadapter/sink/cloudstorage/encoder_group_test.go +++ b/downstreamadapter/sink/cloudstorage/encoder_group_test.go @@ -230,6 +230,7 @@ func newTestTxnEncoderConfig(t *testing.T) *common.Config { config.ProtocolCsv, replicaConfig.Sink, config.DefaultMaxMessageBytes, + config.DefaultMaxMessageBytes, ) require.NoError(t, err) return encoderConfig diff --git a/downstreamadapter/sink/cloudstorage/sink.go b/downstreamadapter/sink/cloudstorage/sink.go index 8a64f45396..d93588b97b 100644 --- a/downstreamadapter/sink/cloudstorage/sink.go +++ b/downstreamadapter/sink/cloudstorage/sink.go @@ -87,7 +87,7 @@ func Verify(ctx context.Context, changefeedID common.ChangeFeedID, sinkURI *url. if err != nil { return err } - _, err = helper.GetEncoderConfig(changefeedID, sinkURI, protocol, sinkConfig, math.MaxInt) + _, err = helper.GetEncoderConfig(changefeedID, sinkURI, protocol, sinkConfig, math.MaxInt, math.MaxInt) if err != nil { return err } @@ -117,9 +117,9 @@ func New( } // get cloud storage file extension according to the specific protocol. ext := helper.GetFileExtension(protocol) - // the last param maxMsgBytes is mainly to limit the size of a single message for - // batch protocols in mq scenario. In cloud storage sink, we just set it to max int. - encoderConfig, err := helper.GetEncoderConfig(changefeedID, sinkURI, protocol, sinkConfig, math.MaxInt) + // Message size limits are mainly for MQ batch protocols. Cloud storage uses + // max int for both the final message limit and the batch threshold. + encoderConfig, err := helper.GetEncoderConfig(changefeedID, sinkURI, protocol, sinkConfig, math.MaxInt, math.MaxInt) if err != nil { return nil, err } diff --git a/downstreamadapter/sink/helper/helper.go b/downstreamadapter/sink/helper/helper.go index 47cd949220..23e443de63 100644 --- a/downstreamadapter/sink/helper/helper.go +++ b/downstreamadapter/sink/helper/helper.go @@ -50,17 +50,17 @@ func GetEncoderConfig( sinkURI *url.URL, protocol config.Protocol, sinkConfig *config.SinkConfig, - maxMsgBytes int, + configuredMaxMessageBytes int, + producerMaxMessageBytes int, ) (*common.Config, error) { encoderConfig := common.NewConfig(protocol) if err := encoderConfig.Apply(sinkURI, sinkConfig); err != nil { return nil, errors.WrapError(errors.ErrSinkInvalidConfig, err) } - // Always set encoder's `MaxMessageBytes` equal to producer's `MaxMessageBytes` - // to prevent that the encoder generate batched message too large - // then cause producer meet `message too large`. + batchMaxMessageBytes := min(configuredMaxMessageBytes, producerMaxMessageBytes) encoderConfig = encoderConfig. - WithMaxMessageBytes(maxMsgBytes). + WithMaxMessageBytes(producerMaxMessageBytes). + WithMaxBatchMessageBytes(batchMaxMessageBytes). WithChangefeedID(changefeedID) tz, err := util.GetTimezone(config.GetGlobalServerConfig().TZ) diff --git a/downstreamadapter/sink/kafka/helper.go b/downstreamadapter/sink/kafka/helper.go index de6ce2fb71..23b3692c3c 100644 --- a/downstreamadapter/sink/kafka/helper.go +++ b/downstreamadapter/sink/kafka/helper.go @@ -88,7 +88,10 @@ func newKafkaSinkComponentWithFactory(ctx context.Context, return kafkaComponent, protocol, errors.Trace(err) } - encoderConfig, err := helper.GetEncoderConfig(changefeedID, sinkURI, protocol, sinkConfig, options.MaxMessageBytes) + encoderConfig, err := helper.GetEncoderConfig( + changefeedID, sinkURI, protocol, sinkConfig, + options.MaxMessageBytes, options.ProducerMaxMessageBytes, + ) if err != nil { return kafkaComponent, protocol, errors.Trace(err) } diff --git a/downstreamadapter/sink/kafka/sink_test.go b/downstreamadapter/sink/kafka/sink_test.go index 7d46286e27..b2983f5fa4 100644 --- a/downstreamadapter/sink/kafka/sink_test.go +++ b/downstreamadapter/sink/kafka/sink_test.go @@ -97,7 +97,10 @@ func newKafkaSinkForTestWithProducers(ctx context.Context, if err != nil { return nil, err } - encoderConfig, err := helper.GetEncoderConfig(changefeedID, sinkURI, protocol, sinkConfig, options.MaxMessageBytes) + encoderConfig, err := helper.GetEncoderConfig( + changefeedID, sinkURI, protocol, sinkConfig, + options.MaxMessageBytes, options.ProducerMaxMessageBytes, + ) if err != nil { return nil, err } diff --git a/downstreamadapter/sink/pulsar/helper.go b/downstreamadapter/sink/pulsar/helper.go index d2c33430b5..3ccb299767 100644 --- a/downstreamadapter/sink/pulsar/helper.go +++ b/downstreamadapter/sink/pulsar/helper.go @@ -122,7 +122,10 @@ func newPulsarSinkComponentWithFactory(ctx context.Context, return pulsarComponent, protocol, errors.Trace(err) } - encoderConfig, err := helper.GetEncoderConfig(changefeedID, sinkURI, protocol, sinkConfig, config.DefaultMaxMessageBytes) + encoderConfig, err := helper.GetEncoderConfig( + changefeedID, sinkURI, protocol, sinkConfig, + config.DefaultMaxMessageBytes, config.DefaultMaxMessageBytes, + ) if err != nil { return pulsarComponent, protocol, errors.Trace(err) } diff --git a/pkg/sink/codec/avro/encoder.go b/pkg/sink/codec/avro/encoder.go index fdad6c7dbf..bcb0d1f5f6 100644 --- a/pkg/sink/codec/avro/encoder.go +++ b/pkg/sink/codec/avro/encoder.go @@ -93,7 +93,8 @@ func (a *BatchEncoder) AppendRowChangedEvent( zap.Int("maxMessageBytes", a.config.MaxMessageBytes), zap.Int("length", message.Length()), zap.Any("table", e.TableInfo.TableName)) - return errors.ErrMessageTooLarge.GenWithStackByArgs(e.TableInfo.GetTargetTableName(), message.Length(), a.config.MaxMessageBytes) + return errors.ErrMessageTooLarge.GenWithStackByArgs( + e.TableInfo.GetTargetTableName(), message.Length(), a.config.MaxMessageBytes) } a.result = append(a.result, message) @@ -114,7 +115,12 @@ func (a *BatchEncoder) EncodeCheckpointEvent(ts uint64) (*common.Message, error) } value := buf.Bytes() - return common.NewMsg(nil, value), nil + message := common.NewMsg(nil, value) + if message.Length() > a.config.MaxMessageBytes { + return nil, errors.ErrMessageTooLarge.GenWithStackByArgs( + "checkpoint", message.Length(), a.config.MaxMessageBytes) + } + return message, nil } return nil, nil } @@ -140,7 +146,12 @@ func (a *BatchEncoder) EncodeDDLEvent(e *commonEvent.DDLEvent) (*common.Message, buf.Write(data) value := buf.Bytes() - return common.NewMsg(nil, value), nil + message := common.NewMsg(nil, value) + if message.Length() > a.config.MaxMessageBytes { + return nil, errors.ErrMessageTooLarge.GenWithStackByArgs( + e.GetTargetTableName(), message.Length(), a.config.MaxMessageBytes) + } + return message, nil } return nil, nil diff --git a/pkg/sink/codec/canal/canal_json_encoder.go b/pkg/sink/codec/canal/canal_json_encoder.go index 7425a666ef..d1193a0769 100644 --- a/pkg/sink/codec/canal/canal_json_encoder.go +++ b/pkg/sink/codec/canal/canal_json_encoder.go @@ -441,7 +441,12 @@ func (c *JSONRowEventEncoder) EncodeCheckpointEvent(ts uint64) (*common.Message, return nil, errors.WrapError(errors.ErrCanalEncodeFailed, err) } - return common.NewMsg(nil, value), nil + message := common.NewMsg(nil, value) + if message.Length() > c.config.MaxMessageBytes { + return nil, errors.ErrMessageTooLarge.GenWithStackByArgs( + "checkpoint", message.Length(), c.config.MaxMessageBytes) + } + return message, nil } // AppendRowChangedEvent implements the interface EventJSONBatchEncoder @@ -468,16 +473,7 @@ func (c *JSONRowEventEncoder) AppendRowChangedEvent( targetTable := e.TableInfo.GetTargetTableName() originLength := m.Length() - if m.Length() > c.config.MaxMessageBytes { - // for single message that is longer than max-message-bytes, do not send it. - if c.config.LargeMessageHandle.Disabled() { - log.Error("Single message is too large for canal-json", - zap.Int("maxMessageBytes", c.config.MaxMessageBytes), - zap.Int("length", originLength), - zap.Any("table", e.TableInfo.TableName)) - return errors.ErrMessageTooLarge.GenWithStackByArgs(targetTable, originLength, c.config.MaxMessageBytes) - } - + if m.Length() > c.config.BatchMaxMessageBytes() && !c.config.LargeMessageHandle.Disabled() { if c.config.LargeMessageHandle.HandleKeyOnly() { value, err = newJSONMessageForDML(e, c.config, true, "") if err != nil { @@ -501,6 +497,7 @@ func (c *JSONRowEventEncoder) AppendRowChangedEvent( return errors.ErrMessageTooLarge.GenWithStackByArgs(targetTable, length, c.config.MaxMessageBytes) } log.Warn("Single message is too large for canal-json, only encode handle-key columns", + zap.Int("maxBatchMessageBytes", c.config.BatchMaxMessageBytes()), zap.Int("maxMessageBytes", c.config.MaxMessageBytes), zap.Int("originLength", originLength), zap.Int("length", length), @@ -520,6 +517,14 @@ func (c *JSONRowEventEncoder) AppendRowChangedEvent( } } + if m.Length() > c.config.MaxMessageBytes { + log.Error("Single message is too large for canal-json", + zap.Int("maxMessageBytes", c.config.MaxMessageBytes), + zap.Int("length", m.Length()), + zap.Any("table", e.TableInfo.TableName)) + return errors.ErrMessageTooLarge.GenWithStackByArgs(targetTable, m.Length(), c.config.MaxMessageBytes) + } + c.messages = append(c.messages, m) return nil } @@ -580,7 +585,12 @@ func (c *JSONRowEventEncoder) EncodeDDLEvent(e *commonEvent.DDLEvent) (*common.M return nil, errors.WrapError(errors.ErrCanalEncodeFailed, err) } - return common.NewMsg(nil, value), nil + result := common.NewMsg(nil, value) + if result.Length() > c.config.MaxMessageBytes { + return nil, errors.ErrMessageTooLarge.GenWithStackByArgs( + e.GetTargetTableName(), result.Length(), c.config.MaxMessageBytes) + } + return result, nil } func (c *JSONRowEventEncoder) Clean() { diff --git a/pkg/sink/codec/canal/canal_json_encoder_test.go b/pkg/sink/codec/canal/canal_json_encoder_test.go index 2953642f89..ac68162050 100644 --- a/pkg/sink/codec/canal/canal_json_encoder_test.go +++ b/pkg/sink/codec/canal/canal_json_encoder_test.go @@ -666,6 +666,22 @@ func TestMaxMessageBytes(t *testing.T) { }) require.NoError(t, err) + codecConfig = common.NewConfig(config.ProtocolCanalJSON). + WithMaxMessageBytes(maxMessageBytes). + WithMaxBatchMessageBytes(100) + + encIface, err = NewJSONRowEventEncoder(ctx, codecConfig) + require.NoError(t, err) + + encoder = encIface.(*JSONRowEventEncoder) + err = encoder.AppendRowChangedEvent(ctx, topic, &commonEvent.RowEvent{ + TableInfo: dml.TableInfo, + CommitTs: dml.CommitTs, + Event: rc, + ColumnSelector: columnselector.NewDefaultColumnSelector(), + }) + require.NoError(t, err) + // the test message length is larger than max-message-bytes codecConfig = codecConfig.WithMaxMessageBytes(100) diff --git a/pkg/sink/codec/canal/canal_json_txn_encoder.go b/pkg/sink/codec/canal/canal_json_txn_encoder.go index 0af6f4f3f2..4998378ef7 100644 --- a/pkg/sink/codec/canal/canal_json_txn_encoder.go +++ b/pkg/sink/codec/canal/canal_json_txn_encoder.go @@ -65,13 +65,13 @@ func (j *JSONTxnEventEncoder) AppendTxnEvent(event *commonEvent.DMLEvent) error return err } length := len(value) + common.MaxRecordOverhead - // For single message that is longer than max-message-bytes, do not send it. if length > j.config.MaxMessageBytes { log.Warn("Single message is too large for canal-json", zap.Int("maxMessageBytes", j.config.MaxMessageBytes), zap.Int("length", length), zap.Any("table", event.TableInfo.TableName)) - return errors.ErrMessageTooLarge.GenWithStackByArgs(event.TableInfo.GetTargetTableName(), length, j.config.MaxMessageBytes) + return errors.ErrMessageTooLarge.GenWithStackByArgs( + event.TableInfo.GetTargetTableName(), length, j.config.MaxMessageBytes) } j.valueBuf.Write(value) j.valueBuf.Write(j.terminator) diff --git a/pkg/sink/codec/common/config.go b/pkg/sink/codec/common/config.go index 83486f3da3..2dc181a722 100644 --- a/pkg/sink/codec/common/config.go +++ b/pkg/sink/codec/common/config.go @@ -41,9 +41,12 @@ type Config struct { Protocol config.Protocol - // control batch behavior, only for `open-protocol` and `craft` at the moment. + // MaxMessageBytes is the final encoded message size limit. MaxMessageBytes int - MaxBatchSize int + // MaxBatchMessageBytes controls batch splitting and large-message handling. + // If it is not set, codecs use MaxMessageBytes to keep the old behavior. + MaxBatchMessageBytes int + MaxBatchSize int // DeleteOnlyHandleKeyColumns is true, for the delete event only output the handle key columns. DeleteOnlyHandleKeyColumns bool @@ -344,6 +347,20 @@ func (c *Config) WithMaxMessageBytes(bytes int) *Config { return c } +// WithMaxBatchMessageBytes sets the batch splitting and large-message threshold. +func (c *Config) WithMaxBatchMessageBytes(bytes int) *Config { + c.MaxBatchMessageBytes = bytes + return c +} + +// BatchMaxMessageBytes returns the batch splitting and large-message threshold. +func (c *Config) BatchMaxMessageBytes() int { + if c.MaxBatchMessageBytes > 0 { + return c.MaxBatchMessageBytes + } + return c.MaxMessageBytes +} + // WithChangefeedID set the `changefeedID` func (c *Config) WithChangefeedID(id common.ChangeFeedID) *Config { c.ChangefeedID = id @@ -415,6 +432,11 @@ func (c *Config) Validate() error { errors.Errorf("invalid max-message-bytes %d", c.MaxMessageBytes), ) } + if c.MaxBatchMessageBytes < 0 { + return errors.ErrCodecInvalidConfig.Wrap( + errors.Errorf("invalid max-batch-message-bytes %d", c.MaxBatchMessageBytes), + ) + } if c.MaxBatchSize <= 0 { return errors.ErrCodecInvalidConfig.Wrap( diff --git a/pkg/sink/codec/open/encoder.go b/pkg/sink/codec/open/encoder.go index 75f82e9d03..4f940da41b 100644 --- a/pkg/sink/codec/open/encoder.go +++ b/pkg/sink/codec/open/encoder.go @@ -38,7 +38,7 @@ var ( ) // batchEncoder for open protocol will batch multiple row changed events into a single message. -// One message can contain at most MaxBatchSize events, and the total size of the message cannot exceed MaxMessageBytes. +// One message can contain at most MaxBatchSize events, and the total size of the message cannot exceed BatchMaxMessageBytes. type batchEncoder struct { messages []*common.Message // buff the callback of the latest message @@ -95,17 +95,7 @@ func (d *batchEncoder) AppendRowChangedEvent( return errors.Trace(err) } - if length > d.config.MaxMessageBytes { - // message len is larger than max-message-bytes - if d.config.LargeMessageHandle.Disabled() { - log.Warn("Single message is too large for open-protocol", - zap.Int("maxMessageBytes", d.config.MaxMessageBytes), - zap.Int("length", length), - zap.Any("table", e.TableInfo.TableName), - zap.Any("key", key)) - return errors.ErrMessageTooLarge.GenWithStackByArgs(e.TableInfo.GetTargetTableName(), length, d.config.MaxMessageBytes) - } - + if length > d.config.BatchMaxMessageBytes() && !d.config.LargeMessageHandle.Disabled() { if d.config.LargeMessageHandle.EnableClaimCheck() { // send the large message to the external storage first, then // create a new message contains the reference of the large message. @@ -149,6 +139,15 @@ func (d *batchEncoder) AppendRowChangedEvent( } } + if length > d.config.MaxMessageBytes { + log.Warn("Single message is too large for open-protocol", + zap.Int("maxMessageBytes", d.config.MaxMessageBytes), + zap.Int("length", length), + zap.Any("table", e.TableInfo.TableName), + zap.Any("key", key)) + return errors.ErrMessageTooLarge.GenWithStackByArgs(e.TableInfo.GetTargetTableName(), length, d.config.MaxMessageBytes) + } + d.pushMessage(key, value, e.Callback) return nil } @@ -174,7 +173,7 @@ func (d *batchEncoder) pushMessage(key, value []byte, callback func()) { binary.BigEndian.PutUint64(keyLenByte[:], uint64(len(key))) binary.BigEndian.PutUint64(valueLenByte[:], uint64(len(value))) - if len(d.messages) == 0 || d.messages[len(d.messages)-1].Length()+length > d.config.MaxMessageBytes || d.messages[len(d.messages)-1].GetRowsCount() >= d.config.MaxBatchSize { + if len(d.messages) == 0 || d.messages[len(d.messages)-1].Length()+length > d.config.BatchMaxMessageBytes() || d.messages[len(d.messages)-1].GetRowsCount() >= d.config.MaxBatchSize { d.finalizeCallback() // create a new message versionHead := make([]byte, 8) @@ -244,7 +243,12 @@ func (d *batchEncoder) EncodeDDLEvent(e *commonEvent.DDLEvent) (*common.Message, return nil, errors.Trace(err) } - return common.NewMsg(key, value), nil + message := common.NewMsg(key, value) + if message.Length() > d.config.MaxMessageBytes { + return nil, errors.ErrMessageTooLarge.GenWithStackByArgs( + e.GetTargetTableName(), message.Length(), d.config.MaxMessageBytes) + } + return message, nil } // EncodeCheckpointEvent implements the RowEventEncoder interface @@ -279,5 +283,10 @@ func (d *batchEncoder) EncodeCheckpointEvent(ts uint64) (*common.Message, error) key = keyOutput.Bytes() value := valueOutput.Bytes() - return common.NewMsg(key, value), nil + message := common.NewMsg(key, value) + if message.Length() > d.config.MaxMessageBytes { + return nil, errors.ErrMessageTooLarge.GenWithStackByArgs( + "checkpoint", message.Length(), d.config.MaxMessageBytes) + } + return message, nil } diff --git a/pkg/sink/codec/open/encoder_test.go b/pkg/sink/codec/open/encoder_test.go index 9b02366709..c0aed9ebb2 100644 --- a/pkg/sink/codec/open/encoder_test.go +++ b/pkg/sink/codec/open/encoder_test.go @@ -885,6 +885,46 @@ func TestMessageTooLarge(t *testing.T) { require.Equal(t, count, 0) } +func TestMessageLargerThanBatchLimit(t *testing.T) { + ctx := context.Background() + codecConfig := common.NewConfig(config.ProtocolOpen). + WithMaxMessageBytes(400). + WithMaxBatchMessageBytes(100) + encoder, err := NewBatchEncoder(ctx, codecConfig) + require.NoError(t, err) + + helper := commonEvent.NewEventTestHelper(t) + defer helper.Close() + helper.Tk().MustExec("use test") + + job := helper.DDL2Job(`create table test.t(a tinyint primary key, b int)`) + tableInfo := helper.GetTableInfo(job) + dmlEvent := helper.DML2Event("test", "t", `insert into test.t values (1, 123)`) + require.NotNil(t, dmlEvent) + insertRow, ok := dmlEvent.GetNextRow() + require.True(t, ok) + + count := 0 + insertRowEvent := &commonEvent.RowEvent{ + TableInfo: tableInfo, + CommitTs: dmlEvent.GetCommitTs(), + Event: insertRow, + ColumnSelector: columnselector.NewDefaultColumnSelector(), + Callback: func() { count += 1 }, + } + + err = encoder.AppendRowChangedEvent(ctx, "", insertRowEvent) + require.NoError(t, err) + + messages := encoder.Build() + require.Len(t, messages, 1) + require.Equal(t, 1, messages[0].GetRowsCount()) + require.Equal(t, 0, count) + + messages[0].Callback() + require.Equal(t, 1, count) +} + func TestLargeMessageWithHandleEnableHandleKeyOnly(t *testing.T) { helper := commonEvent.NewEventTestHelper(t) defer helper.Close() diff --git a/pkg/sink/codec/simple/encoder.go b/pkg/sink/codec/simple/encoder.go index b8ef228561..f7a6e4e6c3 100644 --- a/pkg/sink/codec/simple/encoder.go +++ b/pkg/sink/codec/simple/encoder.go @@ -67,17 +67,30 @@ func (e *Encoder) AppendRowChangedEvent(ctx context.Context, _ string, event *co result.IncRowsCount() length := result.Length() - if length <= e.config.MaxMessageBytes { + if length <= e.config.BatchMaxMessageBytes() { + if length > e.config.MaxMessageBytes { + log.Error("Single message is too large for simple", + zap.Int("maxMessageBytes", e.config.MaxMessageBytes), + zap.Int("length", length), + zap.Any("table", event.TableInfo.TableName)) + return errors.ErrMessageTooLarge.GenWithStackByArgs( + event.TableInfo.GetTargetTableName(), length, e.config.MaxMessageBytes) + } e.messages = append(e.messages, result) return nil } if e.config.LargeMessageHandle.Disabled() { - log.Error("Single message is too large for simple", - zap.Int("maxMessageBytes", e.config.MaxMessageBytes), - zap.Int("length", length), - zap.Any("table", event.TableInfo.TableName)) - return errors.ErrMessageTooLarge.GenWithStackByArgs(event.TableInfo.GetTargetTableName(), length, e.config.MaxMessageBytes) + if length > e.config.MaxMessageBytes { + log.Error("Single message is too large for simple", + zap.Int("maxMessageBytes", e.config.MaxMessageBytes), + zap.Int("length", length), + zap.Any("table", event.TableInfo.TableName)) + return errors.ErrMessageTooLarge.GenWithStackByArgs( + event.TableInfo.GetTargetTableName(), length, e.config.MaxMessageBytes) + } + e.messages = append(e.messages, result) + return nil } var claimCheckLocation string @@ -101,6 +114,7 @@ func (e *Encoder) AppendRowChangedEvent(ctx context.Context, _ string, event *co if result.Length() <= e.config.MaxMessageBytes { log.Warn("Single message is too large for simple, only encode handle key columns", + zap.Int("maxBatchMessageBytes", e.config.BatchMaxMessageBytes()), zap.Int("maxMessageBytes", e.config.MaxMessageBytes), zap.Int("originLength", length), zap.Int("length", result.Length()), @@ -113,7 +127,8 @@ func (e *Encoder) AppendRowChangedEvent(ctx context.Context, _ string, event *co zap.Int("maxMessageBytes", e.config.MaxMessageBytes), zap.Int("length", result.Length()), zap.Any("table", event.TableInfo.TableName)) - return errors.ErrMessageTooLarge.GenWithStackByArgs(event.TableInfo.GetTargetTableName(), result.Length(), e.config.MaxMessageBytes) + return errors.ErrMessageTooLarge.GenWithStackByArgs( + event.TableInfo.GetTargetTableName(), result.Length(), e.config.MaxMessageBytes) } // Build implement the RowEventEncoder interface @@ -135,7 +150,15 @@ func (e *Encoder) EncodeCheckpointEvent(ts uint64) (*common.Message, error) { value, err = common.Compress(e.config.ChangefeedID, e.config.LargeMessageHandle.LargeMessageHandleCompression, value) - return common.NewMsg(nil, value), err + if err != nil { + return nil, err + } + result := common.NewMsg(nil, value) + if result.Length() > e.config.MaxMessageBytes { + return nil, errors.ErrMessageTooLarge.GenWithStackByArgs( + "checkpoint", result.Length(), e.config.MaxMessageBytes) + } + return result, nil } // EncodeDDLEvent implement the DDLEventBatchEncoder interface @@ -157,7 +180,8 @@ func (e *Encoder) EncodeDDLEvent(event *commonEvent.DDLEvent) (*common.Message, zap.Int("maxMessageBytes", e.config.MaxMessageBytes), zap.Int("length", result.Length()), zap.String("table", event.GetTargetTableName())) - return nil, errors.ErrMessageTooLarge.GenWithStackByArgs(event.GetTargetTableName(), result.Length(), e.config.MaxMessageBytes) + return nil, errors.ErrMessageTooLarge.GenWithStackByArgs( + event.GetTargetTableName(), result.Length(), e.config.MaxMessageBytes) } return result, nil } diff --git a/pkg/sink/codec/simple/encoder_test.go b/pkg/sink/codec/simple/encoder_test.go index 68170f6f05..839ba10875 100644 --- a/pkg/sink/codec/simple/encoder_test.go +++ b/pkg/sink/codec/simple/encoder_test.go @@ -1592,6 +1592,24 @@ func TestDMLMessageTooLarge(t *testing.T) { } } +func TestDMLLargerThanBatchLimit(t *testing.T) { + _, insertEvent, _, _ := common.NewLargeEvent4Test(t) + + codecConfig := common.NewConfig(config.ProtocolSimple) + codecConfig.MaxMessageBytes = config.DefaultMaxMessageBytes + codecConfig.MaxBatchMessageBytes = 50 + + enc, err := NewEncoder(context.Background(), codecConfig) + require.NoError(t, err) + + err = enc.AppendRowChangedEvent(context.Background(), "", insertEvent) + require.NoError(t, err) + + messages := enc.Build() + require.Len(t, messages, 1) + require.Equal(t, 1, messages[0].GetRowsCount()) +} + func TestLargerMessageHandleClaimCheck(t *testing.T) { ddlEvent, _, updateEvent, _ := common.NewLargeEvent4Test(t) diff --git a/pkg/sink/kafka/options.go b/pkg/sink/kafka/options.go index c9b992814e..cac242f5c7 100644 --- a/pkg/sink/kafka/options.go +++ b/pkg/sink/kafka/options.go @@ -24,6 +24,7 @@ import ( "strings" "time" + "github.com/IBM/sarama" "github.com/gin-gonic/gin/binding" "github.com/imdario/mergo" "github.com/pingcap/errors" @@ -40,13 +41,6 @@ const ( defaultPartitionNum = 3 // defaultMaxRetry is the default retry budget for Kafka producers. defaultMaxRetry = 5 - - // the `max-message-bytes` is set equal to topic's `max.message.bytes`, and is used to check - // whether the message is larger than the max size limit. It's found some message pass the message - // size limit check at the client side and failed at the broker side since message enlarged during - // the network transmission. so we set the `max-message-bytes` to a smaller value to avoid this problem. - // maxMessageBytesOverhead is used to reduce the `max-message-bytes`. - maxMessageBytesOverhead = 128 ) const ( @@ -156,11 +150,14 @@ type options struct { Version string IsAssignedVersion bool RequestVersion int16 - MaxMessageBytes int - MaxRetry int - Compression string - ClientID string - RequiredAcks RequiredAcks + // MaxMessageBytes is the user configured TiCDC batch threshold. + MaxMessageBytes int + // ProducerMaxMessageBytes is the final producer limit derived from Kafka. + ProducerMaxMessageBytes int + MaxRetry int + Compression string + ClientID string + RequiredAcks RequiredAcks // Only for test. User can not set this value. // The current prod default value is 0. MaxMessages int @@ -180,20 +177,20 @@ type options struct { // NewOptions returns a default Kafka configuration func NewOptions() *options { return &options{ - Version: "2.4.0", - // MaxMessageBytes will be used to initialize producer - MaxMessageBytes: config.DefaultMaxMessageBytes, - MaxRetry: defaultMaxRetry, - ReplicationFactor: 1, - Compression: "none", - RequiredAcks: WaitForAll, - Credential: &security.Credential{}, - InsecureSkipVerify: false, - SASL: &security.SASL{}, - AutoCreate: true, - DialTimeout: 10 * time.Second, - WriteTimeout: 10 * time.Second, - ReadTimeout: 10 * time.Second, + Version: "2.4.0", + MaxMessageBytes: config.DefaultMaxMessageBytes, + ProducerMaxMessageBytes: config.DefaultMaxMessageBytes, + MaxRetry: defaultMaxRetry, + ReplicationFactor: 1, + Compression: "none", + RequiredAcks: WaitForAll, + Credential: &security.Credential{}, + InsecureSkipVerify: false, + SASL: &security.SASL{}, + AutoCreate: true, + DialTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + ReadTimeout: 10 * time.Second, } } @@ -601,7 +598,6 @@ func adjustOptions( // once we have found the topic, no matter `auto-create-topic`, // make sure user input parameters are valid. if exists { - // make sure that producer's `MaxMessageBytes` smaller than topic's `max.message.bytes` topicMaxMessageBytesStr, err := getTopicConfig( ctx, admin, info.Name, TopicMaxMessageBytesConfigName, @@ -614,19 +610,8 @@ func adjustOptions( if err != nil { return errors.Trace(err) } - - maxMessageBytes := topicMaxMessageBytes - maxMessageBytesOverhead - if topicMaxMessageBytes <= options.MaxMessageBytes { - log.Warn("topic's `max.message.bytes` less than the `max-message-bytes`,"+ - "use topic's `max.message.bytes` to initialize the Kafka producer", - zap.Int("max.message.bytes", topicMaxMessageBytes), - zap.Int("max-message-bytes", options.MaxMessageBytes), - zap.Int("real-max-message-bytes", maxMessageBytes)) - options.MaxMessageBytes = maxMessageBytes - } else { - if maxMessageBytes < options.MaxMessageBytes { - options.MaxMessageBytes = maxMessageBytes - } + if err = options.setProducerMaxMessageBytes(topicMaxMessageBytes); err != nil { + return err } // no need to create the topic, @@ -655,20 +640,8 @@ func adjustOptions( // when create the topic, `max.message.bytes` is decided by the broker, // it would use broker's `message.max.bytes` to set topic's `max.message.bytes`. - // TiCDC need to make sure that the producer's `MaxMessageBytes` won't larger than - // broker's `message.max.bytes`. - maxMessageBytes := brokerMessageMaxBytes - maxMessageBytesOverhead - if brokerMessageMaxBytes <= options.MaxMessageBytes { - log.Warn("broker's `message.max.bytes` less than the `max-message-bytes`,"+ - "use broker's `message.max.bytes` to initialize the Kafka producer", - zap.Int("message.max.bytes", brokerMessageMaxBytes), - zap.Int("max-message-bytes", options.MaxMessageBytes), - zap.Int("real-max-message-bytes", maxMessageBytes)) - options.MaxMessageBytes = maxMessageBytes - } else { - if maxMessageBytes < options.MaxMessageBytes { - options.MaxMessageBytes = maxMessageBytes - } + if err = options.setProducerMaxMessageBytes(brokerMessageMaxBytes); err != nil { + return err } // topic not exists yet, and user does not specify the `partition-num` in the sink uri. @@ -680,6 +653,16 @@ func adjustOptions( return nil } +func (o *options) setProducerMaxMessageBytes(kafkaMaxMessageBytes int) error { + // Sarama ignores Producer.MaxMessageBytes when it is not smaller than MaxRequestSize. + o.ProducerMaxMessageBytes = min(kafkaMaxMessageBytes, int(sarama.MaxRequestSize)-1) + if o.ProducerMaxMessageBytes <= 0 { + return cerror.ErrKafkaInvalidConfig.GenWithStack( + "invalid Kafka max message bytes %d", kafkaMaxMessageBytes) + } + return nil +} + func validateMinInsyncReplicas( ctx context.Context, admin ClusterAdminClient, diff --git a/pkg/sink/kafka/options_test.go b/pkg/sink/kafka/options_test.go index 0650d3baf2..2b13728b35 100644 --- a/pkg/sink/kafka/options_test.go +++ b/pkg/sink/kafka/options_test.go @@ -158,12 +158,8 @@ func (f *kafkaAdminFixture) setMessageMaxBytes(brokerValue, topicValue string) { f.topicConfig[defaultMockTopicName][TopicMaxMessageBytesConfigName] = topicValue } -func expectedAdjustedMaxMessageBytes(configuredMaxMessageBytes, sourceMaxMessageBytes int) int { - sourceMaxMessageBytes -= maxMessageBytesOverhead - if configuredMaxMessageBytes < sourceMaxMessageBytes { - return configuredMaxMessageBytes - } - return sourceMaxMessageBytes +func expectedProducerMaxMessageBytes(sourceMaxMessageBytes int) int { + return min(sourceMaxMessageBytes, int(sarama.MaxRequestSize)-1) } func (f *kafkaAdminFixture) setMinInsyncReplicas(minInsyncReplicas string) { @@ -381,13 +377,13 @@ func TestAdjustConfigFallsBackToBrokerMessageMaxBytesWhenTopicConfigMissing(t *t }, }, { - name: "uses broker limit when configured value is within overhead", + name: "keeps configured value below broker by one byte", configuredMaxMessageBytes: func(f *kafkaAdminFixture) int { return f.brokerMessageMaxBytes() - 1 }, }, { - name: "uses broker limit when configured value is above broker", + name: "keeps configured value above broker", configuredMaxMessageBytes: func(f *kafkaAdminFixture) int { return f.brokerMessageMaxBytes() + 1 }, @@ -410,11 +406,10 @@ func TestAdjustConfigFallsBackToBrokerMessageMaxBytesWhenTopicConfigMissing(t *t options := NewOptions() options.BrokerEndpoints = []string{"127.0.0.1:9092"} - options.MaxMessageBytes = test.configuredMaxMessageBytes(adminFixture) - expectedMaxMessageBytes := expectedAdjustedMaxMessageBytes( - options.MaxMessageBytes, - adminFixture.brokerMessageMaxBytes(), - ) + configuredMaxMessageBytes := test.configuredMaxMessageBytes(adminFixture) + options.MaxMessageBytes = configuredMaxMessageBytes + expectedProducerLimit := expectedProducerMaxMessageBytes( + adminFixture.brokerMessageMaxBytes()) ctx := context.Background() err = adjustOptions(ctx, adminClient, options, topicName) @@ -423,8 +418,9 @@ func TestAdjustConfigFallsBackToBrokerMessageMaxBytesWhenTopicConfigMissing(t *t saramaConfig, err := newSaramaConfig(ctx, options) require.NoError(t, err) - require.Equal(t, expectedMaxMessageBytes, options.MaxMessageBytes) - require.Equal(t, expectedMaxMessageBytes, saramaConfig.Producer.MaxMessageBytes) + require.Equal(t, configuredMaxMessageBytes, options.MaxMessageBytes) + require.Equal(t, expectedProducerLimit, options.ProducerMaxMessageBytes) + require.Equal(t, expectedProducerLimit, saramaConfig.Producer.MaxMessageBytes) }) } } @@ -559,7 +555,7 @@ func TestConfigurationCombinations(t *testing.T) { mockTopicMessageMaxBytes, }, { - "new topic broker overhead below user", + "new topic broker below user", "kafka://127.0.0.1:9092/%s?max-message-bytes=%s", []any{"not-created-topic", strconv.Itoa(1024*1024 + 1)}, mockBrokerMessageMaxBytes, @@ -625,12 +621,19 @@ func TestConfigurationCombinations(t *testing.T) { strconv.Itoa(config.DefaultMaxMessageBytes + 1), }, { - "existing topic topic overhead below user", + "existing topic topic below user", "kafka://127.0.0.1:9092/%s?max-message-bytes=%s", []any{defaultMockTopicName, strconv.Itoa(1024*1024 + 1)}, mockBrokerMessageMaxBytes, mockTopicMessageMaxBytes, }, + { + "existing topic topic above sarama request limit", + "kafka://127.0.0.1:9092/%s", + []any{defaultMockTopicName}, + mockBrokerMessageMaxBytes, + strconv.Itoa(int(sarama.MaxRequestSize) + 4096), + }, { "existing topic topic below default and user", "kafka://127.0.0.1:9092/%s?max-message-bytes=%s", @@ -677,6 +680,7 @@ func TestConfigurationCombinations(t *testing.T) { options := NewOptions() err = options.Apply(commonType.NewChangefeedID4Test(commonType.DefaultKeyspaceName, "test"), sinkURI, config.GetDefaultReplicaConfig().Sink) require.Nil(t, err) + configuredMaxMessageBytes := options.MaxMessageBytes topic, ok := a.uriParams[0].(string) require.True(t, ok) @@ -686,15 +690,16 @@ func TestConfigurationCombinations(t *testing.T) { if _, exists := adminFixture.topics[topic]; exists { sourceMaxMessageBytes = adminFixture.topicMaxMessageBytes(topic) } - expectedMaxMessageBytes := expectedAdjustedMaxMessageBytes(options.MaxMessageBytes, sourceMaxMessageBytes) + expectedProducerLimit := expectedProducerMaxMessageBytes(sourceMaxMessageBytes) err = adjustOptions(ctx, adminClient, options, topic) require.Nil(t, err) - require.Equal(t, expectedMaxMessageBytes, options.MaxMessageBytes) + require.Equal(t, configuredMaxMessageBytes, options.MaxMessageBytes) + require.Equal(t, expectedProducerLimit, options.ProducerMaxMessageBytes) saramaConfig, err := newSaramaConfig(ctx, options) require.Nil(t, err) - require.Equal(t, expectedMaxMessageBytes, saramaConfig.Producer.MaxMessageBytes) + require.Equal(t, expectedProducerLimit, saramaConfig.Producer.MaxMessageBytes) encoderConfig := common.NewConfig(config.ProtocolOpen) err = encoderConfig.Apply(sinkURI, &config.SinkConfig{ @@ -703,13 +708,19 @@ func TestConfigurationCombinations(t *testing.T) { }, }) require.Nil(t, err) - encoderConfig.WithMaxMessageBytes(options.MaxMessageBytes) + encoderConfig. + WithMaxMessageBytes(options.ProducerMaxMessageBytes). + WithMaxBatchMessageBytes(min(options.MaxMessageBytes, options.ProducerMaxMessageBytes)) err = encoderConfig.Validate() require.Nil(t, err) - // producer's `MaxMessageBytes` = encoder's `MaxMessageBytes`. - require.Equal(t, expectedMaxMessageBytes, encoderConfig.MaxMessageBytes) + require.Equal(t, expectedProducerLimit, encoderConfig.MaxMessageBytes) + require.Equal( + t, + min(configuredMaxMessageBytes, expectedProducerLimit), + encoderConfig.BatchMaxMessageBytes(), + ) adminClient.Close() }) diff --git a/pkg/sink/kafka/sarama_config.go b/pkg/sink/kafka/sarama_config.go index b53dc47b37..9be8050bb5 100644 --- a/pkg/sink/kafka/sarama_config.go +++ b/pkg/sink/kafka/sarama_config.go @@ -70,7 +70,7 @@ func newSaramaConfig(ctx context.Context, o *options) (*sarama.Config, error) { config.Net.ReadTimeout = o.ReadTimeout config.Producer.Partitioner = sarama.NewManualPartitioner - config.Producer.MaxMessageBytes = o.MaxMessageBytes + config.Producer.MaxMessageBytes = o.ProducerMaxMessageBytes config.Producer.Return.Successes = true config.Producer.Return.Errors = true config.Producer.RequiredAcks = sarama.RequiredAcks(o.RequiredAcks) From c2289b6c9166a80f0e2561b32daa8ebac5b028f3 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Wed, 17 Jun 2026 12:27:46 +0800 Subject: [PATCH 02/22] fix code by review --- pkg/sink/codec/common/config.go | 9 +++++++ pkg/sink/codec/common/config_test.go | 35 +++++++++++++++++++++++++++ pkg/sink/codec/simple/encoder.go | 15 +----------- pkg/sink/codec/simple/encoder_test.go | 2 ++ 4 files changed, 47 insertions(+), 14 deletions(-) create mode 100644 pkg/sink/codec/common/config_test.go diff --git a/pkg/sink/codec/common/config.go b/pkg/sink/codec/common/config.go index 2dc181a722..5847f86e46 100644 --- a/pkg/sink/codec/common/config.go +++ b/pkg/sink/codec/common/config.go @@ -437,6 +437,15 @@ func (c *Config) Validate() error { errors.Errorf("invalid max-batch-message-bytes %d", c.MaxBatchMessageBytes), ) } + if c.MaxBatchMessageBytes > c.MaxMessageBytes { + return errors.ErrCodecInvalidConfig.Wrap( + errors.Errorf( + "max-batch-message-bytes %d cannot be greater than max-message-bytes %d", + c.MaxBatchMessageBytes, + c.MaxMessageBytes, + ), + ) + } if c.MaxBatchSize <= 0 { return errors.ErrCodecInvalidConfig.Wrap( diff --git a/pkg/sink/codec/common/config_test.go b/pkg/sink/codec/common/config_test.go new file mode 100644 index 0000000000..5f8ecd4005 --- /dev/null +++ b/pkg/sink/codec/common/config_test.go @@ -0,0 +1,35 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import ( + "testing" + + "github.com/pingcap/ticdc/pkg/config" + "github.com/stretchr/testify/require" +) + +func TestValidateMaxBatchMessageBytes(t *testing.T) { + cfg := NewConfig(config.ProtocolOpen) + cfg.MaxMessageBytes = 100 + cfg.MaxBatchMessageBytes = 101 + + err := cfg.Validate() + require.Error(t, err) + require.ErrorContains( + t, + err, + "max-batch-message-bytes 101 cannot be greater than max-message-bytes 100", + ) +} diff --git a/pkg/sink/codec/simple/encoder.go b/pkg/sink/codec/simple/encoder.go index f7a6e4e6c3..5a6005cb4b 100644 --- a/pkg/sink/codec/simple/encoder.go +++ b/pkg/sink/codec/simple/encoder.go @@ -67,20 +67,7 @@ func (e *Encoder) AppendRowChangedEvent(ctx context.Context, _ string, event *co result.IncRowsCount() length := result.Length() - if length <= e.config.BatchMaxMessageBytes() { - if length > e.config.MaxMessageBytes { - log.Error("Single message is too large for simple", - zap.Int("maxMessageBytes", e.config.MaxMessageBytes), - zap.Int("length", length), - zap.Any("table", event.TableInfo.TableName)) - return errors.ErrMessageTooLarge.GenWithStackByArgs( - event.TableInfo.GetTargetTableName(), length, e.config.MaxMessageBytes) - } - e.messages = append(e.messages, result) - return nil - } - - if e.config.LargeMessageHandle.Disabled() { + if length <= e.config.BatchMaxMessageBytes() || e.config.LargeMessageHandle.Disabled() { if length > e.config.MaxMessageBytes { log.Error("Single message is too large for simple", zap.Int("maxMessageBytes", e.config.MaxMessageBytes), diff --git a/pkg/sink/codec/simple/encoder_test.go b/pkg/sink/codec/simple/encoder_test.go index 839ba10875..ae2f9faf94 100644 --- a/pkg/sink/codec/simple/encoder_test.go +++ b/pkg/sink/codec/simple/encoder_test.go @@ -1607,6 +1607,8 @@ func TestDMLLargerThanBatchLimit(t *testing.T) { messages := enc.Build() require.Len(t, messages, 1) + require.Greater(t, messages[0].Length(), codecConfig.MaxBatchMessageBytes) + require.LessOrEqual(t, messages[0].Length(), codecConfig.MaxMessageBytes) require.Equal(t, 1, messages[0].GetRowsCount()) } From 5d1388b246db7a31fbdcf9ff7ce8a0499827b80c Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Wed, 17 Jun 2026 15:35:36 +0800 Subject: [PATCH 03/22] adjust the kafka configuration --- downstreamadapter/sink/helper/helper.go | 7 +- downstreamadapter/sink/kafka/helper.go | 2 +- downstreamadapter/sink/kafka/sink_test.go | 2 +- pkg/sink/codec/canal/canal_json_encoder.go | 3 +- pkg/sink/codec/common/config.go | 6 +- pkg/sink/codec/open/encoder.go | 2 +- pkg/sink/codec/simple/encoder.go | 3 +- pkg/sink/kafka/options.go | 202 ++++++++++++++------- pkg/sink/kafka/options_test.go | 36 ++-- pkg/sink/kafka/sarama_config.go | 4 +- 10 files changed, 171 insertions(+), 96 deletions(-) diff --git a/downstreamadapter/sink/helper/helper.go b/downstreamadapter/sink/helper/helper.go index 23e443de63..d5621efc39 100644 --- a/downstreamadapter/sink/helper/helper.go +++ b/downstreamadapter/sink/helper/helper.go @@ -50,16 +50,15 @@ func GetEncoderConfig( sinkURI *url.URL, protocol config.Protocol, sinkConfig *config.SinkConfig, - configuredMaxMessageBytes int, - producerMaxMessageBytes int, + maxMessageBytes int, + batchMaxMessageBytes int, ) (*common.Config, error) { encoderConfig := common.NewConfig(protocol) if err := encoderConfig.Apply(sinkURI, sinkConfig); err != nil { return nil, errors.WrapError(errors.ErrSinkInvalidConfig, err) } - batchMaxMessageBytes := min(configuredMaxMessageBytes, producerMaxMessageBytes) encoderConfig = encoderConfig. - WithMaxMessageBytes(producerMaxMessageBytes). + WithMaxMessageBytes(maxMessageBytes). WithMaxBatchMessageBytes(batchMaxMessageBytes). WithChangefeedID(changefeedID) diff --git a/downstreamadapter/sink/kafka/helper.go b/downstreamadapter/sink/kafka/helper.go index 23b3692c3c..cbc5b5fb28 100644 --- a/downstreamadapter/sink/kafka/helper.go +++ b/downstreamadapter/sink/kafka/helper.go @@ -90,7 +90,7 @@ func newKafkaSinkComponentWithFactory(ctx context.Context, encoderConfig, err := helper.GetEncoderConfig( changefeedID, sinkURI, protocol, sinkConfig, - options.MaxMessageBytes, options.ProducerMaxMessageBytes, + options.MaxMessageBytes, options.BatchMaxMessageBytes, ) if err != nil { return kafkaComponent, protocol, errors.Trace(err) diff --git a/downstreamadapter/sink/kafka/sink_test.go b/downstreamadapter/sink/kafka/sink_test.go index b2983f5fa4..7309acfcb8 100644 --- a/downstreamadapter/sink/kafka/sink_test.go +++ b/downstreamadapter/sink/kafka/sink_test.go @@ -99,7 +99,7 @@ func newKafkaSinkForTestWithProducers(ctx context.Context, } encoderConfig, err := helper.GetEncoderConfig( changefeedID, sinkURI, protocol, sinkConfig, - options.MaxMessageBytes, options.ProducerMaxMessageBytes, + options.MaxMessageBytes, options.MaxMessageBytes, ) if err != nil { return nil, err diff --git a/pkg/sink/codec/canal/canal_json_encoder.go b/pkg/sink/codec/canal/canal_json_encoder.go index d1193a0769..215b0e1936 100644 --- a/pkg/sink/codec/canal/canal_json_encoder.go +++ b/pkg/sink/codec/canal/canal_json_encoder.go @@ -473,7 +473,7 @@ func (c *JSONRowEventEncoder) AppendRowChangedEvent( targetTable := e.TableInfo.GetTargetTableName() originLength := m.Length() - if m.Length() > c.config.BatchMaxMessageBytes() && !c.config.LargeMessageHandle.Disabled() { + if m.Length() > c.config.MaxMessageBytes && !c.config.LargeMessageHandle.Disabled() { if c.config.LargeMessageHandle.HandleKeyOnly() { value, err = newJSONMessageForDML(e, c.config, true, "") if err != nil { @@ -497,7 +497,6 @@ func (c *JSONRowEventEncoder) AppendRowChangedEvent( return errors.ErrMessageTooLarge.GenWithStackByArgs(targetTable, length, c.config.MaxMessageBytes) } log.Warn("Single message is too large for canal-json, only encode handle-key columns", - zap.Int("maxBatchMessageBytes", c.config.BatchMaxMessageBytes()), zap.Int("maxMessageBytes", c.config.MaxMessageBytes), zap.Int("originLength", originLength), zap.Int("length", length), diff --git a/pkg/sink/codec/common/config.go b/pkg/sink/codec/common/config.go index 5847f86e46..ab448adeda 100644 --- a/pkg/sink/codec/common/config.go +++ b/pkg/sink/codec/common/config.go @@ -43,7 +43,7 @@ type Config struct { // MaxMessageBytes is the final encoded message size limit. MaxMessageBytes int - // MaxBatchMessageBytes controls batch splitting and large-message handling. + // MaxBatchMessageBytes controls batch splitting. // If it is not set, codecs use MaxMessageBytes to keep the old behavior. MaxBatchMessageBytes int MaxBatchSize int @@ -347,13 +347,13 @@ func (c *Config) WithMaxMessageBytes(bytes int) *Config { return c } -// WithMaxBatchMessageBytes sets the batch splitting and large-message threshold. +// WithMaxBatchMessageBytes sets the batch splitting threshold. func (c *Config) WithMaxBatchMessageBytes(bytes int) *Config { c.MaxBatchMessageBytes = bytes return c } -// BatchMaxMessageBytes returns the batch splitting and large-message threshold. +// BatchMaxMessageBytes returns the batch splitting threshold. func (c *Config) BatchMaxMessageBytes() int { if c.MaxBatchMessageBytes > 0 { return c.MaxBatchMessageBytes diff --git a/pkg/sink/codec/open/encoder.go b/pkg/sink/codec/open/encoder.go index 4f940da41b..3212fb25a8 100644 --- a/pkg/sink/codec/open/encoder.go +++ b/pkg/sink/codec/open/encoder.go @@ -95,7 +95,7 @@ func (d *batchEncoder) AppendRowChangedEvent( return errors.Trace(err) } - if length > d.config.BatchMaxMessageBytes() && !d.config.LargeMessageHandle.Disabled() { + if length > d.config.MaxMessageBytes && !d.config.LargeMessageHandle.Disabled() { if d.config.LargeMessageHandle.EnableClaimCheck() { // send the large message to the external storage first, then // create a new message contains the reference of the large message. diff --git a/pkg/sink/codec/simple/encoder.go b/pkg/sink/codec/simple/encoder.go index 5a6005cb4b..9648f1b42e 100644 --- a/pkg/sink/codec/simple/encoder.go +++ b/pkg/sink/codec/simple/encoder.go @@ -67,7 +67,7 @@ func (e *Encoder) AppendRowChangedEvent(ctx context.Context, _ string, event *co result.IncRowsCount() length := result.Length() - if length <= e.config.BatchMaxMessageBytes() || e.config.LargeMessageHandle.Disabled() { + if length <= e.config.MaxMessageBytes || e.config.LargeMessageHandle.Disabled() { if length > e.config.MaxMessageBytes { log.Error("Single message is too large for simple", zap.Int("maxMessageBytes", e.config.MaxMessageBytes), @@ -101,7 +101,6 @@ func (e *Encoder) AppendRowChangedEvent(ctx context.Context, _ string, event *co if result.Length() <= e.config.MaxMessageBytes { log.Warn("Single message is too large for simple, only encode handle key columns", - zap.Int("maxBatchMessageBytes", e.config.BatchMaxMessageBytes()), zap.Int("maxMessageBytes", e.config.MaxMessageBytes), zap.Int("originLength", length), zap.Int("length", result.Length()), diff --git a/pkg/sink/kafka/options.go b/pkg/sink/kafka/options.go index cac242f5c7..6bef416711 100644 --- a/pkg/sink/kafka/options.go +++ b/pkg/sink/kafka/options.go @@ -137,7 +137,7 @@ type urlConfig struct { InsecureSkipVerify *bool `form:"insecure-skip-verify"` } -// options stores user specified configurations +// options stores Kafka sink configurations type options struct { Topic string BrokerEndpoints []string @@ -150,17 +150,17 @@ type options struct { Version string IsAssignedVersion bool RequestVersion int16 - // MaxMessageBytes is the user configured TiCDC batch threshold. + + // MaxMessageBytes controls the byte size limit of the producer. MaxMessageBytes int - // ProducerMaxMessageBytes is the final producer limit derived from Kafka. - ProducerMaxMessageBytes int - MaxRetry int - Compression string - ClientID string - RequiredAcks RequiredAcks - // Only for test. User can not set this value. - // The current prod default value is 0. - MaxMessages int + // BatchMaxMessageBytes controls the byte size limit when batching messages. + // this is not exposed, and is inferred from the `MaxMessageBytes` and kafka related configurations in the `adjustOption`. + BatchMaxMessageBytes int + + MaxRetry int + Compression string + ClientID string + RequiredAcks RequiredAcks // Credential is used to connect to kafka cluster. EnableTLS bool @@ -177,20 +177,20 @@ type options struct { // NewOptions returns a default Kafka configuration func NewOptions() *options { return &options{ - Version: "2.4.0", - MaxMessageBytes: config.DefaultMaxMessageBytes, - ProducerMaxMessageBytes: config.DefaultMaxMessageBytes, - MaxRetry: defaultMaxRetry, - ReplicationFactor: 1, - Compression: "none", - RequiredAcks: WaitForAll, - Credential: &security.Credential{}, - InsecureSkipVerify: false, - SASL: &security.SASL{}, - AutoCreate: true, - DialTimeout: 10 * time.Second, - WriteTimeout: 10 * time.Second, - ReadTimeout: 10 * time.Second, + Version: "2.4.0", + MaxMessageBytes: config.DefaultMaxMessageBytes, + BatchMaxMessageBytes: config.DefaultMaxMessageBytes, + MaxRetry: defaultMaxRetry, + ReplicationFactor: 1, + Compression: "none", + RequiredAcks: WaitForAll, + Credential: &security.Credential{}, + InsecureSkipVerify: false, + SASL: &security.SASL{}, + AutoCreate: true, + DialTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + ReadTimeout: 10 * time.Second, } } @@ -257,6 +257,7 @@ func (o *options) Apply(changefeedID common.ChangeFeedID, if urlParameter.MaxMessageBytes != nil { o.MaxMessageBytes = *urlParameter.MaxMessageBytes } + o.BatchMaxMessageBytes = o.MaxMessageBytes if urlParameter.MaxRetry != nil && *urlParameter.MaxRetry >= 0 { o.MaxRetry = *urlParameter.MaxRetry @@ -572,7 +573,9 @@ func NewKafkaClientID(captureAddr string, return } -// adjustOptions adjust the `options` and `sarama.Config` by condition. +// adjustOptions adjusts options with Kafka runtime metadata. +// It overwrites MaxMessageBytes with the final producer message limit derived +// from the topic or broker configuration. func adjustOptions( ctx context.Context, admin ClusterAdminClient, @@ -584,63 +587,93 @@ func adjustOptions( return errors.Trace(err) } - // Only check replicationFactor >= minInsyncReplicas when producer's required acks is -1. - // If we don't check it, the producer probably can not send message to the topic. - // Because it will wait for the ack from all replicas. But we do not have enough replicas. - if options.RequiredAcks == WaitForAll { - err = validateMinInsyncReplicas(ctx, admin, topics, topic, int(options.ReplicationFactor)) - if err != nil { - return errors.Trace(err) - } + if err = validateRequiredAcks(ctx, admin, topics, topic, options); err != nil { + return errors.Trace(err) } + return adjustTopicOptions(ctx, admin, options, topic, topics) +} +func adjustTopicOptions( + ctx context.Context, + admin ClusterAdminClient, + options *options, + topic string, + topics map[string]TopicDetail, +) error { + batchMaxBytes := options.MaxMessageBytes info, exists := topics[topic] // once we have found the topic, no matter `auto-create-topic`, // make sure user input parameters are valid. + var err error if exists { - topicMaxMessageBytesStr, err := getTopicConfig( - ctx, admin, info.Name, - TopicMaxMessageBytesConfigName, - BrokerMessageMaxBytesConfigName, - ) - if err != nil { - return errors.Trace(err) - } - topicMaxMessageBytes, err := strconv.Atoi(topicMaxMessageBytesStr) - if err != nil { - return errors.Trace(err) - } - if err = options.setProducerMaxMessageBytes(topicMaxMessageBytes); err != nil { - return err - } - - // no need to create the topic, - // but we would have to log user if they found enter wrong topic name later - if options.AutoCreate { - log.Warn("topic already exist, TiCDC will not create the topic", - zap.String("topic", topic), zap.Any("detail", info)) - } + err = adjustExistingTopicOptions(ctx, admin, options, topic, info) + } else { + err = adjustNewTopicOptions(admin, options, topic) + } + if err != nil { + return err + } - if err = options.setPartitionNum(info.NumPartitions); err != nil { - return errors.Trace(err) - } + options.BatchMaxMessageBytes = min(batchMaxBytes, options.MaxMessageBytes) + return nil +} +func validateRequiredAcks( + ctx context.Context, + admin ClusterAdminClient, + topics map[string]TopicDetail, + topic string, + options *options, +) error { + // Only check replicationFactor >= minInsyncReplicas when producer's required acks is -1. + // If we don't check it, the producer probably can not send message to the topic. + // Because it will wait for the ack from all replicas. But we do not have enough replicas. + if options.RequiredAcks != WaitForAll { return nil } + return validateMinInsyncReplicas(ctx, admin, topics, topic, int(options.ReplicationFactor)) +} - brokerMessageMaxBytesStr, err := admin.GetBrokerConfig(BrokerMessageMaxBytesConfigName) +func adjustExistingTopicOptions( + ctx context.Context, + admin ClusterAdminClient, + options *options, + topic string, + info TopicDetail, +) error { + topicMaxMessageBytes, err := getTopicMaxMessageBytes(ctx, admin, info.Name) if err != nil { - log.Warn("TiCDC cannot find `message.max.bytes` from broker's configuration") - return errors.Trace(err) + return err } - brokerMessageMaxBytes, err := strconv.Atoi(brokerMessageMaxBytesStr) - if err != nil { + if err = options.setMaxMessageBytes(topicMaxMessageBytes); err != nil { + return err + } + + // no need to create the topic, + // but we would have to log user if they found enter wrong topic name later + if options.AutoCreate { + log.Warn("topic already exist, TiCDC will not create the topic", + zap.String("topic", topic), zap.Any("detail", info)) + } + + if err = options.setPartitionNum(info.NumPartitions); err != nil { return errors.Trace(err) } + return nil +} +func adjustNewTopicOptions( + admin ClusterAdminClient, + options *options, + topic string, +) error { // when create the topic, `max.message.bytes` is decided by the broker, // it would use broker's `message.max.bytes` to set topic's `max.message.bytes`. - if err = options.setProducerMaxMessageBytes(brokerMessageMaxBytes); err != nil { + brokerMessageMaxBytes, err := getBrokerMaxMessageBytes(admin) + if err != nil { + return err + } + if err = options.setMaxMessageBytes(brokerMessageMaxBytes); err != nil { return err } @@ -653,10 +686,43 @@ func adjustOptions( return nil } -func (o *options) setProducerMaxMessageBytes(kafkaMaxMessageBytes int) error { +func getTopicMaxMessageBytes( + ctx context.Context, + admin ClusterAdminClient, + topic string, +) (int, error) { + maxMessageBytesStr, err := getTopicConfig( + ctx, admin, topic, + TopicMaxMessageBytesConfigName, + BrokerMessageMaxBytesConfigName, + ) + if err != nil { + return 0, errors.Trace(err) + } + maxMessageBytes, err := strconv.Atoi(maxMessageBytesStr) + if err != nil { + return 0, errors.Trace(err) + } + return maxMessageBytes, nil +} + +func getBrokerMaxMessageBytes(admin ClusterAdminClient) (int, error) { + maxMessageBytesStr, err := admin.GetBrokerConfig(BrokerMessageMaxBytesConfigName) + if err != nil { + log.Warn("TiCDC cannot find `message.max.bytes` from broker's configuration") + return 0, errors.Trace(err) + } + maxMessageBytes, err := strconv.Atoi(maxMessageBytesStr) + if err != nil { + return 0, errors.Trace(err) + } + return maxMessageBytes, nil +} + +func (o *options) setMaxMessageBytes(kafkaMaxMessageBytes int) error { // Sarama ignores Producer.MaxMessageBytes when it is not smaller than MaxRequestSize. - o.ProducerMaxMessageBytes = min(kafkaMaxMessageBytes, int(sarama.MaxRequestSize)-1) - if o.ProducerMaxMessageBytes <= 0 { + o.MaxMessageBytes = min(kafkaMaxMessageBytes, int(sarama.MaxRequestSize)-1) + if o.MaxMessageBytes <= 0 { return cerror.ErrKafkaInvalidConfig.GenWithStack( "invalid Kafka max message bytes %d", kafkaMaxMessageBytes) } diff --git a/pkg/sink/kafka/options_test.go b/pkg/sink/kafka/options_test.go index 2b13728b35..b4b8058aac 100644 --- a/pkg/sink/kafka/options_test.go +++ b/pkg/sink/kafka/options_test.go @@ -158,7 +158,7 @@ func (f *kafkaAdminFixture) setMessageMaxBytes(brokerValue, topicValue string) { f.topicConfig[defaultMockTopicName][TopicMaxMessageBytesConfigName] = topicValue } -func expectedProducerMaxMessageBytes(sourceMaxMessageBytes int) int { +func expectedAdjustedMaxMessageBytes(sourceMaxMessageBytes int) int { return min(sourceMaxMessageBytes, int(sarama.MaxRequestSize)-1) } @@ -189,6 +189,7 @@ func TestCompleteOptions(t *testing.T) { require.Equal(t, int16(3), options.ReplicationFactor) require.Equal(t, "2.6.0", options.Version) require.Equal(t, 4096, options.MaxMessageBytes) + require.Equal(t, 4096, options.BatchMaxMessageBytes) require.Equal(t, WaitForLocal, options.RequiredAcks) require.Equal(t, defaultMaxRetry, options.MaxRetry) @@ -371,19 +372,19 @@ func TestAdjustConfigFallsBackToBrokerMessageMaxBytesWhenTopicConfigMissing(t *t configuredMaxMessageBytes func(*kafkaAdminFixture) int }{ { - name: "keeps configured value below broker limit", + name: "uses broker limit when configured value is below broker", configuredMaxMessageBytes: func(*kafkaAdminFixture) int { return 1024 }, }, { - name: "keeps configured value below broker by one byte", + name: "uses broker limit when configured value is below broker by one byte", configuredMaxMessageBytes: func(f *kafkaAdminFixture) int { return f.brokerMessageMaxBytes() - 1 }, }, { - name: "keeps configured value above broker", + name: "uses broker limit when configured value is above broker", configuredMaxMessageBytes: func(f *kafkaAdminFixture) int { return f.brokerMessageMaxBytes() + 1 }, @@ -408,7 +409,7 @@ func TestAdjustConfigFallsBackToBrokerMessageMaxBytesWhenTopicConfigMissing(t *t options.BrokerEndpoints = []string{"127.0.0.1:9092"} configuredMaxMessageBytes := test.configuredMaxMessageBytes(adminFixture) options.MaxMessageBytes = configuredMaxMessageBytes - expectedProducerLimit := expectedProducerMaxMessageBytes( + expectedProducerLimit := expectedAdjustedMaxMessageBytes( adminFixture.brokerMessageMaxBytes()) ctx := context.Background() @@ -418,8 +419,13 @@ func TestAdjustConfigFallsBackToBrokerMessageMaxBytesWhenTopicConfigMissing(t *t saramaConfig, err := newSaramaConfig(ctx, options) require.NoError(t, err) - require.Equal(t, configuredMaxMessageBytes, options.MaxMessageBytes) - require.Equal(t, expectedProducerLimit, options.ProducerMaxMessageBytes) + require.NotEqual(t, configuredMaxMessageBytes, options.MaxMessageBytes) + require.Equal(t, expectedProducerLimit, options.MaxMessageBytes) + require.Equal( + t, + min(configuredMaxMessageBytes, expectedProducerLimit), + options.BatchMaxMessageBytes, + ) require.Equal(t, expectedProducerLimit, saramaConfig.Producer.MaxMessageBytes) }) } @@ -690,12 +696,16 @@ func TestConfigurationCombinations(t *testing.T) { if _, exists := adminFixture.topics[topic]; exists { sourceMaxMessageBytes = adminFixture.topicMaxMessageBytes(topic) } - expectedProducerLimit := expectedProducerMaxMessageBytes(sourceMaxMessageBytes) + expectedProducerLimit := expectedAdjustedMaxMessageBytes(sourceMaxMessageBytes) err = adjustOptions(ctx, adminClient, options, topic) require.Nil(t, err) - require.Equal(t, configuredMaxMessageBytes, options.MaxMessageBytes) - require.Equal(t, expectedProducerLimit, options.ProducerMaxMessageBytes) + require.Equal(t, expectedProducerLimit, options.MaxMessageBytes) + require.Equal( + t, + min(configuredMaxMessageBytes, expectedProducerLimit), + options.BatchMaxMessageBytes, + ) saramaConfig, err := newSaramaConfig(ctx, options) require.Nil(t, err) @@ -709,8 +719,8 @@ func TestConfigurationCombinations(t *testing.T) { }) require.Nil(t, err) encoderConfig. - WithMaxMessageBytes(options.ProducerMaxMessageBytes). - WithMaxBatchMessageBytes(min(options.MaxMessageBytes, options.ProducerMaxMessageBytes)) + WithMaxMessageBytes(options.MaxMessageBytes). + WithMaxBatchMessageBytes(options.BatchMaxMessageBytes) err = encoderConfig.Validate() require.Nil(t, err) @@ -765,6 +775,7 @@ func TestMerge(t *testing.T) { require.Equal(t, int16(5), c.ReplicationFactor) require.Equal(t, "3.1.2", c.Version) require.Equal(t, 1024*1024, c.MaxMessageBytes) + require.Equal(t, 1024*1024, c.BatchMaxMessageBytes) require.Equal(t, "gzip", c.Compression) require.Equal(t, "test-id", c.ClientID) require.Equal(t, true, c.AutoCreate) @@ -846,6 +857,7 @@ func TestMerge(t *testing.T) { require.Equal(t, int16(5), c.ReplicationFactor) require.Equal(t, "3.1.2", c.Version) require.Equal(t, 1024*1024, c.MaxMessageBytes) + require.Equal(t, 1024*1024, c.BatchMaxMessageBytes) require.Equal(t, "gzip", c.Compression) require.Equal(t, "test-id", c.ClientID) require.Equal(t, true, c.AutoCreate) diff --git a/pkg/sink/kafka/sarama_config.go b/pkg/sink/kafka/sarama_config.go index 9be8050bb5..d18e8b0c56 100644 --- a/pkg/sink/kafka/sarama_config.go +++ b/pkg/sink/kafka/sarama_config.go @@ -62,7 +62,7 @@ func newSaramaConfig(ctx context.Context, o *options) (*sarama.Config, error) { config.Producer.Flush.Bytes = 0 config.Producer.Flush.Messages = 0 config.Producer.Flush.Frequency = time.Duration(0) - config.Producer.Flush.MaxMessages = o.MaxMessages + config.Producer.Flush.MaxMessages = 0 config.Net.MaxOpenRequests = 1 config.Net.DialTimeout = o.DialTimeout @@ -70,7 +70,7 @@ func newSaramaConfig(ctx context.Context, o *options) (*sarama.Config, error) { config.Net.ReadTimeout = o.ReadTimeout config.Producer.Partitioner = sarama.NewManualPartitioner - config.Producer.MaxMessageBytes = o.ProducerMaxMessageBytes + config.Producer.MaxMessageBytes = o.MaxMessageBytes config.Producer.Return.Successes = true config.Producer.Return.Errors = true config.Producer.RequiredAcks = sarama.RequiredAcks(o.RequiredAcks) From b5fe1c9700c89b644cabd4b5169aef1bface216f Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Sat, 4 Jul 2026 01:16:34 +0800 Subject: [PATCH 04/22] remove set max message bytes by the sarama request size --- pkg/sink/kafka/options.go | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/pkg/sink/kafka/options.go b/pkg/sink/kafka/options.go index 6bef416711..f789451f30 100644 --- a/pkg/sink/kafka/options.go +++ b/pkg/sink/kafka/options.go @@ -24,7 +24,6 @@ import ( "strings" "time" - "github.com/IBM/sarama" "github.com/gin-gonic/gin/binding" "github.com/imdario/mergo" "github.com/pingcap/errors" @@ -606,7 +605,7 @@ func adjustTopicOptions( // make sure user input parameters are valid. var err error if exists { - err = adjustExistingTopicOptions(ctx, admin, options, topic, info) + err = adjustExistingTopicOption(ctx, admin, options, topic, info) } else { err = adjustNewTopicOptions(admin, options, topic) } @@ -634,7 +633,7 @@ func validateRequiredAcks( return validateMinInsyncReplicas(ctx, admin, topics, topic, int(options.ReplicationFactor)) } -func adjustExistingTopicOptions( +func adjustExistingTopicOption( ctx context.Context, admin ClusterAdminClient, options *options, @@ -645,9 +644,7 @@ func adjustExistingTopicOptions( if err != nil { return err } - if err = options.setMaxMessageBytes(topicMaxMessageBytes); err != nil { - return err - } + options.MaxMessageBytes = topicMaxMessageBytes // no need to create the topic, // but we would have to log user if they found enter wrong topic name later @@ -673,9 +670,7 @@ func adjustNewTopicOptions( if err != nil { return err } - if err = options.setMaxMessageBytes(brokerMessageMaxBytes); err != nil { - return err - } + options.MaxMessageBytes = brokerMessageMaxBytes // topic not exists yet, and user does not specify the `partition-num` in the sink uri. if options.PartitionNum == 0 { @@ -719,16 +714,6 @@ func getBrokerMaxMessageBytes(admin ClusterAdminClient) (int, error) { return maxMessageBytes, nil } -func (o *options) setMaxMessageBytes(kafkaMaxMessageBytes int) error { - // Sarama ignores Producer.MaxMessageBytes when it is not smaller than MaxRequestSize. - o.MaxMessageBytes = min(kafkaMaxMessageBytes, int(sarama.MaxRequestSize)-1) - if o.MaxMessageBytes <= 0 { - return cerror.ErrKafkaInvalidConfig.GenWithStack( - "invalid Kafka max message bytes %d", kafkaMaxMessageBytes) - } - return nil -} - func validateMinInsyncReplicas( ctx context.Context, admin ClusterAdminClient, From bf3026edac117a6e49d0160ff6bde7faa500a9b9 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Sat, 4 Jul 2026 01:19:41 +0800 Subject: [PATCH 05/22] simplify the code further --- pkg/sink/kafka/options.go | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/pkg/sink/kafka/options.go b/pkg/sink/kafka/options.go index f789451f30..1e29519127 100644 --- a/pkg/sink/kafka/options.go +++ b/pkg/sink/kafka/options.go @@ -26,11 +26,10 @@ import ( "github.com/gin-gonic/gin/binding" "github.com/imdario/mergo" - "github.com/pingcap/errors" "github.com/pingcap/log" "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/config" - cerror "github.com/pingcap/ticdc/pkg/errors" + "github.com/pingcap/ticdc/pkg/errors" "github.com/pingcap/ticdc/pkg/security" "go.uber.org/zap" ) @@ -101,7 +100,7 @@ func requireAcksFromString(acks int) (RequiredAcks, error) { case int(NoResponse): return NoResponse, nil default: - return Unknown, cerror.ErrKafkaInvalidRequiredAcks.GenWithStackByArgs(acks) + return Unknown, errors.ErrKafkaInvalidRequiredAcks.GenWithStackByArgs(acks) } } @@ -215,7 +214,7 @@ func (o *options) setPartitionNum(realPartitionCount int32) error { // the real partition count, since messages would be dispatched to different // partitions, this could prevent potential correctness problems. if o.PartitionNum > realPartitionCount { - return cerror.ErrKafkaInvalidPartitionNum.GenWithStack( + return errors.ErrKafkaInvalidPartitionNum.GenWithStack( "the number of partition (%d) specified in sink-uri is more than that of actual topic (%d)", o.PartitionNum, realPartitionCount) } @@ -232,7 +231,7 @@ func (o *options) Apply(changefeedID common.ChangeFeedID, req := &http.Request{URL: sinkURI} urlParameter := &urlConfig{} if err = binding.Query.Bind(req, urlParameter); err != nil { - return cerror.WrapError(cerror.ErrMySQLInvalidConfig, err) + return errors.WrapError(errors.ErrMySQLInvalidConfig, err) } if urlParameter, err = mergeConfig(sinkConfig, urlParameter); err != nil { return err @@ -240,7 +239,7 @@ func (o *options) Apply(changefeedID common.ChangeFeedID, if urlParameter.PartitionNum != nil { o.PartitionNum = *urlParameter.PartitionNum if o.PartitionNum <= 0 { - return cerror.ErrKafkaInvalidPartitionNum.GenWithStackByArgs(o.PartitionNum) + return errors.ErrKafkaInvalidPartitionNum.GenWithStackByArgs(o.PartitionNum) } } @@ -384,7 +383,7 @@ func (o *options) applyTLS(params *urlConfig) error { if o.Credential != nil && !o.Credential.IsEmpty() && !o.Credential.IsTLSEnabled() { - return cerror.WrapError(cerror.ErrKafkaInvalidConfig, + return errors.WrapError(errors.ErrKafkaInvalidConfig, errors.New("ca, cert and key files should all be supplied")) } @@ -398,7 +397,7 @@ func (o *options) applyTLS(params *urlConfig) error { enableTLS := *params.EnableTLS if o.Credential != nil && o.Credential.IsTLSEnabled() && !enableTLS { - return cerror.WrapError(cerror.ErrKafkaInvalidConfig, + return errors.WrapError(errors.ErrKafkaInvalidConfig, errors.New("credential files are supplied, but 'enable-tls' is set to false")) } o.EnableTLS = enableTLS @@ -428,7 +427,7 @@ func (o *options) applySASL(urlParameter *urlConfig, sinkConfig *config.SinkConf if urlParameter.SASLMechanism != nil && *urlParameter.SASLMechanism != "" { mechanism, err := security.SASLMechanismFromString(*urlParameter.SASLMechanism) if err != nil { - return cerror.WrapError(cerror.ErrKafkaInvalidConfig, err) + return errors.WrapError(errors.ErrKafkaInvalidConfig, err) } o.SASL.SASLMechanism = mechanism } @@ -436,7 +435,7 @@ func (o *options) applySASL(urlParameter *urlConfig, sinkConfig *config.SinkConf if urlParameter.SASLGssAPIAuthType != nil && *urlParameter.SASLGssAPIAuthType != "" { authType, err := security.AuthTypeFromString(*urlParameter.SASLGssAPIAuthType) if err != nil { - return cerror.WrapError(cerror.ErrKafkaInvalidConfig, err) + return errors.WrapError(errors.ErrKafkaInvalidConfig, err) } o.SASL.GSSAPI.AuthType = authType } @@ -474,7 +473,7 @@ func (o *options) applySASL(urlParameter *urlConfig, sinkConfig *config.SinkConf if sinkConfig.KafkaConfig.SASLOAuthClientID != nil { clientID := *sinkConfig.KafkaConfig.SASLOAuthClientID if clientID == "" { - return cerror.ErrKafkaInvalidConfig.GenWithStack("OAuth2 client ID cannot be empty") + return errors.ErrKafkaInvalidConfig.GenWithStack("OAuth2 client ID cannot be empty") } o.SASL.OAuth2.ClientID = clientID } @@ -482,7 +481,7 @@ func (o *options) applySASL(urlParameter *urlConfig, sinkConfig *config.SinkConf if sinkConfig.KafkaConfig.SASLOAuthClientSecret != nil { clientSecret := *sinkConfig.KafkaConfig.SASLOAuthClientSecret if clientSecret == "" { - return cerror.ErrKafkaInvalidConfig.GenWithStack( + return errors.ErrKafkaInvalidConfig.GenWithStack( "OAuth2 client secret cannot be empty") } @@ -490,7 +489,7 @@ func (o *options) applySASL(urlParameter *urlConfig, sinkConfig *config.SinkConf decodedClientSecret, err := base64.StdEncoding.DecodeString(clientSecret) if err != nil { log.Error("OAuth2 client secret is not base64 encoded", zap.Error(err)) - return cerror.ErrKafkaInvalidConfig.GenWithStack( + return errors.ErrKafkaInvalidConfig.GenWithStack( "OAuth2 client secret is not base64 encoded") } o.SASL.OAuth2.ClientSecret = string(decodedClientSecret) @@ -499,7 +498,7 @@ func (o *options) applySASL(urlParameter *urlConfig, sinkConfig *config.SinkConf if sinkConfig.KafkaConfig.SASLOAuthTokenURL != nil { tokenURL := *sinkConfig.KafkaConfig.SASLOAuthTokenURL if tokenURL == "" { - return cerror.ErrKafkaInvalidConfig.GenWithStack( + return errors.ErrKafkaInvalidConfig.GenWithStack( "OAuth2 token URL cannot be empty") } o.SASL.OAuth2.TokenURL = tokenURL @@ -507,13 +506,13 @@ func (o *options) applySASL(urlParameter *urlConfig, sinkConfig *config.SinkConf if o.SASL.OAuth2.IsEnable() { if o.SASL.SASLMechanism != security.OAuthMechanism { - return cerror.ErrKafkaInvalidConfig.GenWithStack( + return errors.ErrKafkaInvalidConfig.GenWithStack( "OAuth2 is only supported with SASL mechanism type OAUTHBEARER, but got %s", o.SASL.SASLMechanism) } if err := o.SASL.OAuth2.Validate(); err != nil { - return cerror.ErrKafkaInvalidConfig.Wrap(err) + return errors.ErrKafkaInvalidConfig.Wrap(err) } o.SASL.OAuth2.SetDefault() } @@ -567,7 +566,7 @@ func NewKafkaClientID(captureAddr string, clientID = commonInvalidChar.ReplaceAllString(clientID, "_") } if !validClientID.MatchString(clientID) { - return "", cerror.ErrKafkaInvalidClientID.GenWithStackByArgs(clientID) + return "", errors.ErrKafkaInvalidClientID.GenWithStackByArgs(clientID) } return } @@ -745,7 +744,7 @@ func validateMinInsyncReplicas( minInsyncReplicasStr, exists, err := minInsyncReplicasConfigGetter() if err != nil { // 'min.insync.replica' is invisible to us in Confluent Cloud Kafka. - if cerror.ErrKafkaConfigNotFound.Equal(err) { + if errors.ErrKafkaConfigNotFound.Equal(err) { log.Warn("TiCDC cannot find `min.insync.replicas` from broker's configuration, " + "please make sure that the replication factor is greater than or equal " + "to the minimum number of in-sync replicas" + @@ -770,7 +769,7 @@ func validateMinInsyncReplicas( MinInsyncReplicasConfigName, configFrom) log.Error(msg, zap.Int("replication-factor", replicationFactor), zap.Int("min.insync.replicas", minInsyncReplicas)) - return cerror.ErrKafkaInvalidConfig.GenWithStack( + return errors.ErrKafkaInvalidConfig.GenWithStack( "TiCDC Kafka sink's `request.required.acks` defaults to -1, "+ "TiCDC cannot deliver messages when the `replication-factor` %d "+ "is smaller than the `min.insync.replicas` %d of %s", From 6ff53bad95e451f72f0fdb2edee35e8071435976 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Tue, 21 Jul 2026 12:30:44 +0800 Subject: [PATCH 06/22] fix build --- downstreamadapter/sink/kafka/helper.go | 2 +- downstreamadapter/sink/kafka/sink.go | 5 +++- downstreamadapter/sink/kafka/sink_test.go | 2 +- pkg/sink/kafka/options.go | 36 +++++++++++------------ pkg/sink/kafka/options_test.go | 12 ++++---- 5 files changed, 30 insertions(+), 27 deletions(-) diff --git a/downstreamadapter/sink/kafka/helper.go b/downstreamadapter/sink/kafka/helper.go index cb5e9a80d1..bd776c5e49 100644 --- a/downstreamadapter/sink/kafka/helper.go +++ b/downstreamadapter/sink/kafka/helper.go @@ -91,7 +91,7 @@ func newKafkaSinkComponent( encoderConfig, err := helper.GetEncoderConfig( changefeedID, sinkURI, protocol, sinkConfig, - options.MaxMessageBytes, options.BatchMaxMessageBytes, + options.MaxMessageBytes, options.MaxBatchedBytes, ) if err != nil { return kafkaComponent, protocol, errors.Trace(err) diff --git a/downstreamadapter/sink/kafka/sink.go b/downstreamadapter/sink/kafka/sink.go index 673d3e9993..9de1070e15 100644 --- a/downstreamadapter/sink/kafka/sink.go +++ b/downstreamadapter/sink/kafka/sink.go @@ -87,7 +87,10 @@ func Verify(ctx context.Context, changefeedID commonType.ChangeFeedID, uri *url. } options.Topic = topic - encoderConfig, err := helper.GetEncoderConfig(changefeedID, uri, protocol, sinkConfig, options.MaxMessageBytes) + encoderConfig, err := helper.GetEncoderConfig( + changefeedID, uri, protocol, sinkConfig, + options.MaxMessageBytes, options.MaxBatchedBytes, + ) if err != nil { return errors.Trace(err) } diff --git a/downstreamadapter/sink/kafka/sink_test.go b/downstreamadapter/sink/kafka/sink_test.go index 410687a410..deaecf5c0a 100644 --- a/downstreamadapter/sink/kafka/sink_test.go +++ b/downstreamadapter/sink/kafka/sink_test.go @@ -99,7 +99,7 @@ func newKafkaSinkForTestWithProducers(ctx context.Context, } encoderConfig, err := helper.GetEncoderConfig( changefeedID, sinkURI, protocol, sinkConfig, - options.MaxMessageBytes, options.MaxMessageBytes, + options.MaxMessageBytes, options.MaxBatchedBytes, ) if err != nil { return nil, err diff --git a/pkg/sink/kafka/options.go b/pkg/sink/kafka/options.go index f24749e298..9fe1c5e6b3 100644 --- a/pkg/sink/kafka/options.go +++ b/pkg/sink/kafka/options.go @@ -151,9 +151,9 @@ type options struct { // MaxMessageBytes controls the byte size limit of the producer. MaxMessageBytes int - // BatchMaxMessageBytes controls the byte size limit when batching messages. + // MaxBatchedBytes controls the byte size limit when batching messages. // this is not exposed, and is inferred from the `MaxMessageBytes` and kafka related configurations in the `adjustOption`. - BatchMaxMessageBytes int + MaxBatchedBytes int MaxRetry int Compression string @@ -175,20 +175,20 @@ type options struct { // NewOptions returns a default Kafka configuration func NewOptions() *options { return &options{ - Version: "2.4.0", - MaxMessageBytes: config.DefaultMaxMessageBytes, - BatchMaxMessageBytes: config.DefaultMaxMessageBytes, - MaxRetry: defaultMaxRetry, - ReplicationFactor: 1, - Compression: "none", - RequiredAcks: WaitForAll, - Credential: &security.Credential{}, - InsecureSkipVerify: false, - SASL: &security.SASL{}, - AutoCreate: true, - DialTimeout: 10 * time.Second, - WriteTimeout: 10 * time.Second, - ReadTimeout: 10 * time.Second, + Version: "2.4.0", + MaxMessageBytes: config.DefaultMaxMessageBytes, + MaxBatchedBytes: config.DefaultMaxMessageBytes, + MaxRetry: defaultMaxRetry, + ReplicationFactor: 1, + Compression: "none", + RequiredAcks: WaitForAll, + Credential: &security.Credential{}, + InsecureSkipVerify: false, + SASL: &security.SASL{}, + AutoCreate: true, + DialTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + ReadTimeout: 10 * time.Second, } } @@ -255,7 +255,7 @@ func (o *options) Apply(changefeedID common.ChangeFeedID, if urlParameter.MaxMessageBytes != nil { o.MaxMessageBytes = *urlParameter.MaxMessageBytes } - o.BatchMaxMessageBytes = o.MaxMessageBytes + o.MaxBatchedBytes = o.MaxMessageBytes if urlParameter.MaxRetry != nil && *urlParameter.MaxRetry >= 0 { o.MaxRetry = *urlParameter.MaxRetry @@ -612,7 +612,7 @@ func adjustTopicOptions( return err } - options.BatchMaxMessageBytes = min(batchMaxBytes, options.MaxMessageBytes) + options.MaxBatchedBytes = min(batchMaxBytes, options.MaxMessageBytes) return nil } diff --git a/pkg/sink/kafka/options_test.go b/pkg/sink/kafka/options_test.go index 0716df39ca..162f6602a9 100644 --- a/pkg/sink/kafka/options_test.go +++ b/pkg/sink/kafka/options_test.go @@ -189,7 +189,7 @@ func TestCompleteOptions(t *testing.T) { require.Equal(t, int16(3), options.ReplicationFactor) require.Equal(t, "2.6.0", options.Version) require.Equal(t, 4096, options.MaxMessageBytes) - require.Equal(t, 4096, options.BatchMaxMessageBytes) + require.Equal(t, 4096, options.MaxBatchedBytes) require.Equal(t, WaitForLocal, options.RequiredAcks) require.Equal(t, defaultMaxRetry, options.MaxRetry) @@ -424,7 +424,7 @@ func TestAdjustConfigFallsBackToBrokerMessageMaxBytesWhenTopicConfigMissing(t *t require.Equal( t, min(configuredMaxMessageBytes, expectedProducerLimit), - options.BatchMaxMessageBytes, + options.MaxBatchedBytes, ) require.Equal(t, expectedProducerLimit, saramaConfig.Producer.MaxMessageBytes) }) @@ -704,7 +704,7 @@ func TestConfigurationCombinations(t *testing.T) { require.Equal( t, min(configuredMaxMessageBytes, expectedProducerLimit), - options.BatchMaxMessageBytes, + options.MaxBatchedBytes, ) saramaConfig, err := newSaramaConfig(ctx, options) @@ -720,7 +720,7 @@ func TestConfigurationCombinations(t *testing.T) { require.Nil(t, err) encoderConfig. WithMaxMessageBytes(options.MaxMessageBytes). - WithMaxBatchMessageBytes(options.BatchMaxMessageBytes) + WithMaxBatchMessageBytes(options.MaxBatchedBytes) err = encoderConfig.Validate() require.Nil(t, err) @@ -775,7 +775,7 @@ func TestMerge(t *testing.T) { require.Equal(t, int16(5), c.ReplicationFactor) require.Equal(t, "3.1.2", c.Version) require.Equal(t, 1024*1024, c.MaxMessageBytes) - require.Equal(t, 1024*1024, c.BatchMaxMessageBytes) + require.Equal(t, 1024*1024, c.MaxBatchedBytes) require.Equal(t, "gzip", c.Compression) require.Equal(t, "test-id", c.ClientID) require.Equal(t, true, c.AutoCreate) @@ -857,7 +857,7 @@ func TestMerge(t *testing.T) { require.Equal(t, int16(5), c.ReplicationFactor) require.Equal(t, "3.1.2", c.Version) require.Equal(t, 1024*1024, c.MaxMessageBytes) - require.Equal(t, 1024*1024, c.BatchMaxMessageBytes) + require.Equal(t, 1024*1024, c.MaxBatchedBytes) require.Equal(t, "gzip", c.Compression) require.Equal(t, "test-id", c.ClientID) require.Equal(t, true, c.AutoCreate) From 30189634c5c7452021391e3c6821de08f1733e73 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Tue, 21 Jul 2026 15:03:22 +0800 Subject: [PATCH 07/22] simplify the code --- pkg/sink/codec/avro/encoder.go | 17 +++-------------- pkg/sink/codec/canal/canal_json_encoder.go | 14 ++------------ pkg/sink/codec/open/encoder.go | 14 ++------------ pkg/sink/codec/simple/encoder.go | 16 +++------------- 4 files changed, 10 insertions(+), 51 deletions(-) diff --git a/pkg/sink/codec/avro/encoder.go b/pkg/sink/codec/avro/encoder.go index bcb0d1f5f6..fdad6c7dbf 100644 --- a/pkg/sink/codec/avro/encoder.go +++ b/pkg/sink/codec/avro/encoder.go @@ -93,8 +93,7 @@ func (a *BatchEncoder) AppendRowChangedEvent( zap.Int("maxMessageBytes", a.config.MaxMessageBytes), zap.Int("length", message.Length()), zap.Any("table", e.TableInfo.TableName)) - return errors.ErrMessageTooLarge.GenWithStackByArgs( - e.TableInfo.GetTargetTableName(), message.Length(), a.config.MaxMessageBytes) + return errors.ErrMessageTooLarge.GenWithStackByArgs(e.TableInfo.GetTargetTableName(), message.Length(), a.config.MaxMessageBytes) } a.result = append(a.result, message) @@ -115,12 +114,7 @@ func (a *BatchEncoder) EncodeCheckpointEvent(ts uint64) (*common.Message, error) } value := buf.Bytes() - message := common.NewMsg(nil, value) - if message.Length() > a.config.MaxMessageBytes { - return nil, errors.ErrMessageTooLarge.GenWithStackByArgs( - "checkpoint", message.Length(), a.config.MaxMessageBytes) - } - return message, nil + return common.NewMsg(nil, value), nil } return nil, nil } @@ -146,12 +140,7 @@ func (a *BatchEncoder) EncodeDDLEvent(e *commonEvent.DDLEvent) (*common.Message, buf.Write(data) value := buf.Bytes() - message := common.NewMsg(nil, value) - if message.Length() > a.config.MaxMessageBytes { - return nil, errors.ErrMessageTooLarge.GenWithStackByArgs( - e.GetTargetTableName(), message.Length(), a.config.MaxMessageBytes) - } - return message, nil + return common.NewMsg(nil, value), nil } return nil, nil diff --git a/pkg/sink/codec/canal/canal_json_encoder.go b/pkg/sink/codec/canal/canal_json_encoder.go index 215b0e1936..c89818e764 100644 --- a/pkg/sink/codec/canal/canal_json_encoder.go +++ b/pkg/sink/codec/canal/canal_json_encoder.go @@ -441,12 +441,7 @@ func (c *JSONRowEventEncoder) EncodeCheckpointEvent(ts uint64) (*common.Message, return nil, errors.WrapError(errors.ErrCanalEncodeFailed, err) } - message := common.NewMsg(nil, value) - if message.Length() > c.config.MaxMessageBytes { - return nil, errors.ErrMessageTooLarge.GenWithStackByArgs( - "checkpoint", message.Length(), c.config.MaxMessageBytes) - } - return message, nil + return common.NewMsg(nil, value), nil } // AppendRowChangedEvent implements the interface EventJSONBatchEncoder @@ -584,12 +579,7 @@ func (c *JSONRowEventEncoder) EncodeDDLEvent(e *commonEvent.DDLEvent) (*common.M return nil, errors.WrapError(errors.ErrCanalEncodeFailed, err) } - result := common.NewMsg(nil, value) - if result.Length() > c.config.MaxMessageBytes { - return nil, errors.ErrMessageTooLarge.GenWithStackByArgs( - e.GetTargetTableName(), result.Length(), c.config.MaxMessageBytes) - } - return result, nil + return common.NewMsg(nil, value), nil } func (c *JSONRowEventEncoder) Clean() { diff --git a/pkg/sink/codec/open/encoder.go b/pkg/sink/codec/open/encoder.go index 3212fb25a8..fe7d77c24d 100644 --- a/pkg/sink/codec/open/encoder.go +++ b/pkg/sink/codec/open/encoder.go @@ -243,12 +243,7 @@ func (d *batchEncoder) EncodeDDLEvent(e *commonEvent.DDLEvent) (*common.Message, return nil, errors.Trace(err) } - message := common.NewMsg(key, value) - if message.Length() > d.config.MaxMessageBytes { - return nil, errors.ErrMessageTooLarge.GenWithStackByArgs( - e.GetTargetTableName(), message.Length(), d.config.MaxMessageBytes) - } - return message, nil + return common.NewMsg(key, value), nil } // EncodeCheckpointEvent implements the RowEventEncoder interface @@ -283,10 +278,5 @@ func (d *batchEncoder) EncodeCheckpointEvent(ts uint64) (*common.Message, error) key = keyOutput.Bytes() value := valueOutput.Bytes() - message := common.NewMsg(key, value) - if message.Length() > d.config.MaxMessageBytes { - return nil, errors.ErrMessageTooLarge.GenWithStackByArgs( - "checkpoint", message.Length(), d.config.MaxMessageBytes) - } - return message, nil + return common.NewMsg(key, value), nil } diff --git a/pkg/sink/codec/simple/encoder.go b/pkg/sink/codec/simple/encoder.go index 9648f1b42e..95d93f1d34 100644 --- a/pkg/sink/codec/simple/encoder.go +++ b/pkg/sink/codec/simple/encoder.go @@ -113,8 +113,7 @@ func (e *Encoder) AppendRowChangedEvent(ctx context.Context, _ string, event *co zap.Int("maxMessageBytes", e.config.MaxMessageBytes), zap.Int("length", result.Length()), zap.Any("table", event.TableInfo.TableName)) - return errors.ErrMessageTooLarge.GenWithStackByArgs( - event.TableInfo.GetTargetTableName(), result.Length(), e.config.MaxMessageBytes) + return errors.ErrMessageTooLarge.GenWithStackByArgs(event.TableInfo.GetTargetTableName(), result.Length(), e.config.MaxMessageBytes) } // Build implement the RowEventEncoder interface @@ -136,15 +135,7 @@ func (e *Encoder) EncodeCheckpointEvent(ts uint64) (*common.Message, error) { value, err = common.Compress(e.config.ChangefeedID, e.config.LargeMessageHandle.LargeMessageHandleCompression, value) - if err != nil { - return nil, err - } - result := common.NewMsg(nil, value) - if result.Length() > e.config.MaxMessageBytes { - return nil, errors.ErrMessageTooLarge.GenWithStackByArgs( - "checkpoint", result.Length(), e.config.MaxMessageBytes) - } - return result, nil + return common.NewMsg(nil, value), err } // EncodeDDLEvent implement the DDLEventBatchEncoder interface @@ -166,8 +157,7 @@ func (e *Encoder) EncodeDDLEvent(event *commonEvent.DDLEvent) (*common.Message, zap.Int("maxMessageBytes", e.config.MaxMessageBytes), zap.Int("length", result.Length()), zap.String("table", event.GetTargetTableName())) - return nil, errors.ErrMessageTooLarge.GenWithStackByArgs( - event.GetTargetTableName(), result.Length(), e.config.MaxMessageBytes) + return nil, errors.ErrMessageTooLarge.GenWithStackByArgs(event.GetTargetTableName(), result.Length(), e.config.MaxMessageBytes) } return result, nil } From 91c3acd74c5650740b2bd5dde8089901cf1cc07e Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Tue, 21 Jul 2026 16:24:32 +0800 Subject: [PATCH 08/22] refactor the adjust options --- downstreamadapter/sink/kafka/sink.go | 16 ++++---- pkg/sink/codec/common/config.go | 16 ++++---- pkg/sink/kafka/options.go | 59 +++++++++++++++------------- pkg/sink/kafka/options_test.go | 25 +++++++----- pkg/sink/kafka/sarama_factory.go | 2 +- 5 files changed, 63 insertions(+), 55 deletions(-) diff --git a/downstreamadapter/sink/kafka/sink.go b/downstreamadapter/sink/kafka/sink.go index 9de1070e15..7c2231c9da 100644 --- a/downstreamadapter/sink/kafka/sink.go +++ b/downstreamadapter/sink/kafka/sink.go @@ -87,14 +87,6 @@ func Verify(ctx context.Context, changefeedID commonType.ChangeFeedID, uri *url. } options.Topic = topic - encoderConfig, err := helper.GetEncoderConfig( - changefeedID, uri, protocol, sinkConfig, - options.MaxMessageBytes, options.MaxBatchedBytes, - ) - if err != nil { - return errors.Trace(err) - } - isAvroLike := protocol == config.ProtocolAvro || protocol == config.ProtocolDebeziumAvro if _, err = eventrouter.NewEventRouter(sinkConfig, topic, false, isAvroLike); err != nil { return errors.Trace(err) @@ -138,6 +130,14 @@ func Verify(ctx context.Context, changefeedID commonType.ChangeFeedID, uri *url. return errors.WrapError(errors.ErrKafkaCreateTopic, err) } + encoderConfig, err := helper.GetEncoderConfig( + changefeedID, uri, protocol, sinkConfig, + options.MaxMessageBytes, options.MaxBatchedBytes, + ) + if err != nil { + return errors.Trace(err) + } + encoder, err := codec.NewEventEncoder(ctx, encoderConfig) if err != nil { return errors.Trace(err) diff --git a/pkg/sink/codec/common/config.go b/pkg/sink/codec/common/config.go index 0a962057a0..8c860bfad4 100644 --- a/pkg/sink/codec/common/config.go +++ b/pkg/sink/codec/common/config.go @@ -117,8 +117,9 @@ func NewConfig(protocol config.Protocol) *Config { return &Config{ Protocol: protocol, - MaxMessageBytes: config.DefaultMaxMessageBytes, - MaxBatchSize: defaultMaxBatchSize, + MaxMessageBytes: config.DefaultMaxMessageBytes, + MaxBatchMessageBytes: config.DefaultMaxMessageBytes, + MaxBatchSize: defaultMaxBatchSize, EnableTiDBExtension: false, EnableRowChecksum: false, @@ -197,7 +198,7 @@ func (c *Config) Apply(sinkURI *url.URL, sinkConfig *config.SinkConfig) error { var err error urlParameter := &urlConfig{} if err = binding.Query.Bind(req, urlParameter); err != nil { - return errors.WrapError(errors.ErrMySQLInvalidConfig, err) + return errors.WrapError(errors.ErrKafkaInvalidConfig, err) } if urlParameter, err = mergeConfig(sinkConfig, urlParameter); err != nil { return err @@ -350,18 +351,15 @@ func (c *Config) WithMaxMessageBytes(bytes int) *Config { return c } -// WithMaxBatchMessageBytes sets the batch splitting threshold. +// WithMaxBatchMessageBytes sets the maximum batched message bytes. func (c *Config) WithMaxBatchMessageBytes(bytes int) *Config { c.MaxBatchMessageBytes = bytes return c } -// BatchMaxMessageBytes returns the batch splitting threshold. +// BatchMaxMessageBytes returns the maximum batched message bytes. func (c *Config) BatchMaxMessageBytes() int { - if c.MaxBatchMessageBytes > 0 { - return c.MaxBatchMessageBytes - } - return c.MaxMessageBytes + return c.MaxBatchMessageBytes } // WithChangefeedID set the `changefeedID` diff --git a/pkg/sink/kafka/options.go b/pkg/sink/kafka/options.go index 9fe1c5e6b3..e611953040 100644 --- a/pkg/sink/kafka/options.go +++ b/pkg/sink/kafka/options.go @@ -152,7 +152,6 @@ type options struct { // MaxMessageBytes controls the byte size limit of the producer. MaxMessageBytes int // MaxBatchedBytes controls the byte size limit when batching messages. - // this is not exposed, and is inferred from the `MaxMessageBytes` and kafka related configurations in the `adjustOption`. MaxBatchedBytes int MaxRetry int @@ -193,11 +192,12 @@ func NewOptions() *options { } // setPartitionNum set the partition-num by the topic's partition count. -func (o *options) setPartitionNum(realPartitionCount int32) error { +func (o *options) setPartitionNum(changefeedID common.ChangeFeedID, realPartitionCount int32) error { // user does not specify the `partition-num` in the sink-uri if o.PartitionNum == 0 { o.PartitionNum = realPartitionCount log.Info("partitionNum is not set, set by topic's partition-num", + zap.String("namespace", changefeedID.Keyspace()), zap.String("changefeed", changefeedID.Name()), zap.Int32("partitionNum", realPartitionCount)) return nil } @@ -205,8 +205,8 @@ func (o *options) setPartitionNum(realPartitionCount int32) error { if o.PartitionNum < realPartitionCount { log.Warn("number of partition specified in sink-uri is less than that of the actual topic. "+ "Some partitions will not have messages dispatched to", - zap.Int32("sinkUriPartitions", o.PartitionNum), - zap.Int32("topicPartitions", realPartitionCount)) + zap.String("namespace", changefeedID.Keyspace()), zap.String("changefeed", changefeedID.Name()), + zap.Int32("sinkUriPartitions", o.PartitionNum), zap.Int32("topicPartitions", realPartitionCount)) return nil } @@ -576,6 +576,7 @@ func NewKafkaClientID(captureAddr string, // from the topic or broker configuration. func adjustOptions( ctx context.Context, + changefeedID common.ChangeFeedID, admin ClusterAdminClient, options *options, topic string, @@ -588,31 +589,31 @@ func adjustOptions( if err = validateRequiredAcks(ctx, admin, topics, topic, options); err != nil { return errors.Trace(err) } - return adjustTopicOptions(ctx, admin, options, topic, topics) + return adjustTopicOptions(ctx, changefeedID, admin, options, topic, topics) } func adjustTopicOptions( ctx context.Context, + changefeedID common.ChangeFeedID, admin ClusterAdminClient, options *options, topic string, topics map[string]TopicDetail, ) error { - batchMaxBytes := options.MaxMessageBytes info, exists := topics[topic] // once we have found the topic, no matter `auto-create-topic`, // make sure user input parameters are valid. var err error if exists { - err = adjustExistingTopicOption(ctx, admin, options, topic, info) + err = adjustExistingTopicOption(ctx, changefeedID, admin, options, topic, info) } else { - err = adjustNewTopicOptions(admin, options, topic) + adjustNewTopicOptions(admin, changefeedID, options, topic) } if err != nil { return err } - options.MaxBatchedBytes = min(batchMaxBytes, options.MaxMessageBytes) + options.MaxBatchedBytes = min(options.MaxBatchedBytes, options.MaxMessageBytes) return nil } @@ -634,26 +635,30 @@ func validateRequiredAcks( func adjustExistingTopicOption( ctx context.Context, + changefeedID common.ChangeFeedID, admin ClusterAdminClient, options *options, topic string, info TopicDetail, ) error { - topicMaxMessageBytes, err := getTopicMaxMessageBytes( - ctx, admin, info.Name, options.MaxMessageBytes) + maxMessageBytes, err := getTopicMaxMessageBytes(ctx, admin, info.Name) if err != nil { - return err + log.Warn("`max.message.bytes` not found from topic's configuration, use the option `MaxMessageBytes` as default", + zap.String("namespace", changefeedID.Keyspace()), zap.String("changefeed", changefeedID.Name()), + zap.Int("maxMessageBytes", options.MaxMessageBytes), zap.Error(err)) + maxMessageBytes = options.MaxMessageBytes } - options.MaxMessageBytes = topicMaxMessageBytes + options.MaxMessageBytes = maxMessageBytes // no need to create the topic, // but we would have to log user if they found enter wrong topic name later if options.AutoCreate { log.Warn("topic already exist, TiCDC will not create the topic", + zap.String("namespace", changefeedID.Keyspace()), zap.String("changefeed", changefeedID.Name()), zap.String("topic", topic), zap.Any("detail", info)) } - if err = options.setPartitionNum(info.NumPartitions); err != nil { + if err = options.setPartitionNum(changefeedID, info.NumPartitions); err != nil { return errors.Trace(err) } return nil @@ -661,31 +666,34 @@ func adjustExistingTopicOption( func adjustNewTopicOptions( admin ClusterAdminClient, + changefeedID common.ChangeFeedID, options *options, topic string, -) error { +) { // when create the topic, `max.message.bytes` is decided by the broker, // it would use broker's `message.max.bytes` to set topic's `max.message.bytes`. - brokerMessageMaxBytes, err := getBrokerMaxMessageBytes(admin, options.MaxMessageBytes) + messageMaxBytes, err := getBrokerMaxMessageBytes(admin) if err != nil { - return err + log.Warn("`message.max.bytes` not found from broker's configuration, use the option `MaxMessageBytes` as default", + zap.String("namespace", changefeedID.Keyspace()), zap.String("changefeed", changefeedID.Name()), + zap.Int("maxMessageBytes", options.MaxMessageBytes), zap.Error(err)) + messageMaxBytes = options.MaxMessageBytes } - options.MaxMessageBytes = brokerMessageMaxBytes + options.MaxMessageBytes = messageMaxBytes // topic not exists yet, and user does not specify the `partition-num` in the sink uri. if options.PartitionNum == 0 { options.PartitionNum = defaultPartitionNum log.Warn("partition-num is not set, use the default partition count", + zap.String("namespace", changefeedID.Keyspace()), zap.String("changefeed", changefeedID.Name()), zap.String("topic", topic), zap.Int32("partitions", options.PartitionNum)) } - return nil } func getTopicMaxMessageBytes( ctx context.Context, admin ClusterAdminClient, topic string, - defaultMaxMessageBytes int, ) (int, error) { maxMessageBytesStr, err := getTopicConfig( ctx, admin, topic, @@ -693,8 +701,7 @@ func getTopicMaxMessageBytes( BrokerMessageMaxBytesConfigName, ) if err != nil { - log.Warn("TiCDC cannot find `max.message.bytes` from topic's configuration, use the option `MaxMessageBytes` as default") - return defaultMaxMessageBytes, nil + return 0, errors.Trace(err) } maxMessageBytes, err := strconv.Atoi(maxMessageBytesStr) if err != nil { @@ -703,14 +710,10 @@ func getTopicMaxMessageBytes( return maxMessageBytes, nil } -func getBrokerMaxMessageBytes( - admin ClusterAdminClient, - defaultMaxMessageBytes int, -) (int, error) { +func getBrokerMaxMessageBytes(admin ClusterAdminClient) (int, error) { maxMessageBytesStr, err := admin.GetBrokerConfig(BrokerMessageMaxBytesConfigName) if err != nil { - log.Warn("TiCDC cannot find `message.max.bytes` from broker's configuration, use the option `MaxMessageBytes` as default") - return defaultMaxMessageBytes, nil + return 0, errors.Trace(err) } maxMessageBytes, err := strconv.Atoi(maxMessageBytesStr) if err != nil { diff --git a/pkg/sink/kafka/options_test.go b/pkg/sink/kafka/options_test.go index 162f6602a9..2815c9c823 100644 --- a/pkg/sink/kafka/options_test.go +++ b/pkg/sink/kafka/options_test.go @@ -288,17 +288,18 @@ func TestCompleteOptions(t *testing.T) { func TestSetPartitionNum(t *testing.T) { options := NewOptions() - err := options.setPartitionNum(2) + changefeedID := commonType.NewChangefeedID4Test(commonType.DefaultKeyspaceName, "test") + err := options.setPartitionNum(changefeedID, 2) require.NoError(t, err) require.Equal(t, int32(2), options.PartitionNum) options.PartitionNum = 1 - err = options.setPartitionNum(2) + err = options.setPartitionNum(changefeedID, 2) require.NoError(t, err) require.Equal(t, int32(1), options.PartitionNum) options.PartitionNum = 3 - err = options.setPartitionNum(2) + err = options.setPartitionNum(changefeedID, 2) require.True(t, errors.ErrKafkaInvalidPartitionNum.Equal(err)) } @@ -392,7 +393,7 @@ func TestAdjustConfigFallsBackToBrokerMessageMaxBytesWhenTopicConfigMissing(t *t } topicName := "test-topic" - + changefeedID := commonType.NewChangefeedID4Test(commonType.DefaultKeyspaceName, "test") for _, test := range tests { t.Run(test.name, func(t *testing.T) { adminFixture := newKafkaAdminFixture(t) @@ -413,7 +414,7 @@ func TestAdjustConfigFallsBackToBrokerMessageMaxBytesWhenTopicConfigMissing(t *t adminFixture.brokerMessageMaxBytes()) ctx := context.Background() - err = adjustOptions(ctx, adminClient, options, topicName) + err = adjustOptions(ctx, changefeedID, adminClient, options, topicName) require.NoError(t, err) saramaConfig, err := newSaramaConfig(ctx, options) @@ -441,10 +442,12 @@ func TestAdjustConfigMinInsyncReplicas(t *testing.T) { // Report an error if the replication-factor is less than min.insync.replicas // when the topic does not exist. adminFixture.setMinInsyncReplicas("2") + changefeedID := commonType.NewChangefeedID4Test(commonType.DefaultKeyspaceName, "test") ctx := context.Background() err := adjustOptions( ctx, + changefeedID, adminClient, options, "create-new-fail-invalid-min-insync-replicas", @@ -458,7 +461,7 @@ func TestAdjustConfigMinInsyncReplicas(t *testing.T) { // topic not exist, and `min.insync.replicas` not found in broker's configuration adminFixture.dropBrokerConfig(MinInsyncReplicasConfigName) topicName := "no-topic-no-min-insync-replicas" - err = adjustOptions(ctx, adminClient, options, "no-topic-no-min-insync-replicas") + err = adjustOptions(ctx, changefeedID, adminClient, options, "no-topic-no-min-insync-replicas") require.Nil(t, err) err = adminClient.CreateTopic(&TopicDetail{ Name: topicName, @@ -477,12 +480,12 @@ func TestAdjustConfigMinInsyncReplicas(t *testing.T) { NumPartitions: 3, }, false) require.Nil(t, err) - err = adjustOptions(ctx, adminClient, options, topicName) + err = adjustOptions(ctx, changefeedID, adminClient, options, topicName) require.Nil(t, err) // topic found, and have `min.insync.replicas`, but set to 2, larger than `replication-factor`. adminFixture.setMinInsyncReplicas("2") - err = adjustOptions(ctx, adminClient, options, defaultMockTopicName) + err = adjustOptions(ctx, changefeedID, adminClient, options, defaultMockTopicName) require.Regexp(t, ".*`replication-factor` 1 is smaller than the `min.insync.replicas` 2 of topic.*", errors.Cause(err), @@ -497,10 +500,13 @@ func TestSkipAdjustConfigMinInsyncReplicasWhenRequiredAcksIsNotWailAll(t *testin options.BrokerEndpoints = []string{"127.0.0.1:9092"} options.RequiredAcks = WaitForLocal + changefeedID := commonType.NewChangefeedID4Test(commonType.DefaultKeyspaceName, "test") + // Do not report an error if the replication-factor is less than min.insync.replicas(1<2). adminFixture.setMinInsyncReplicas("2") err := adjustOptions( context.Background(), + changefeedID, adminClient, options, "skip-check-min-insync-replicas", @@ -698,7 +704,8 @@ func TestConfigurationCombinations(t *testing.T) { } expectedProducerLimit := expectedAdjustedMaxMessageBytes(sourceMaxMessageBytes) - err = adjustOptions(ctx, adminClient, options, topic) + changefeedID := commonType.NewChangefeedID4Test(commonType.DefaultKeyspaceName, "test") + err = adjustOptions(ctx, changefeedID, adminClient, options, topic) require.Nil(t, err) require.Equal(t, expectedProducerLimit, options.MaxMessageBytes) require.Equal( diff --git a/pkg/sink/kafka/sarama_factory.go b/pkg/sink/kafka/sarama_factory.go index 650f346b4c..39c69c0c0b 100644 --- a/pkg/sink/kafka/sarama_factory.go +++ b/pkg/sink/kafka/sarama_factory.go @@ -57,7 +57,7 @@ func NewSaramaFactory( admin.Close() }() - if err = adjustOptions(ctx, admin, o, o.Topic); err != nil { + if err = adjustOptions(ctx, changefeedID, admin, o, o.Topic); err != nil { return nil, errors.Trace(err) } From d9cef9e480f8f615a8339d2928f0dd7f5b507f0d Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Tue, 21 Jul 2026 16:31:34 +0800 Subject: [PATCH 09/22] rename some fields --- downstreamadapter/sink/helper/helper.go | 4 +-- .../codec/canal/canal_json_encoder_test.go | 2 +- pkg/sink/codec/common/config.go | 31 ++++++++----------- pkg/sink/codec/common/config_test.go | 2 +- pkg/sink/codec/open/encoder.go | 2 +- pkg/sink/codec/open/encoder_test.go | 16 +++++----- pkg/sink/codec/simple/encoder_test.go | 4 +-- pkg/sink/kafka/options_test.go | 4 +-- 8 files changed, 30 insertions(+), 35 deletions(-) diff --git a/downstreamadapter/sink/helper/helper.go b/downstreamadapter/sink/helper/helper.go index d14e43217a..f93410f9ab 100644 --- a/downstreamadapter/sink/helper/helper.go +++ b/downstreamadapter/sink/helper/helper.go @@ -51,7 +51,7 @@ func GetEncoderConfig( protocol config.Protocol, sinkConfig *config.SinkConfig, maxMessageBytes int, - batchMaxMessageBytes int, + maxBatchedBytes int, ) (*common.Config, error) { encoderConfig := common.NewConfig(protocol) if err := encoderConfig.Apply(sinkURI, sinkConfig); err != nil { @@ -59,7 +59,7 @@ func GetEncoderConfig( } encoderConfig = encoderConfig. WithMaxMessageBytes(maxMessageBytes). - WithMaxBatchMessageBytes(batchMaxMessageBytes). + WithMaxBatchedBytes(maxBatchedBytes). WithChangefeedID(changefeedID) tz, err := util.GetTimezone(config.GetGlobalServerConfig().TZ) diff --git a/pkg/sink/codec/canal/canal_json_encoder_test.go b/pkg/sink/codec/canal/canal_json_encoder_test.go index ac68162050..e3c597b07b 100644 --- a/pkg/sink/codec/canal/canal_json_encoder_test.go +++ b/pkg/sink/codec/canal/canal_json_encoder_test.go @@ -668,7 +668,7 @@ func TestMaxMessageBytes(t *testing.T) { codecConfig = common.NewConfig(config.ProtocolCanalJSON). WithMaxMessageBytes(maxMessageBytes). - WithMaxBatchMessageBytes(100) + WithMaxBatchedBytes(100) encIface, err = NewJSONRowEventEncoder(ctx, codecConfig) require.NoError(t, err) diff --git a/pkg/sink/codec/common/config.go b/pkg/sink/codec/common/config.go index 8c860bfad4..b38f2e7b3e 100644 --- a/pkg/sink/codec/common/config.go +++ b/pkg/sink/codec/common/config.go @@ -43,10 +43,10 @@ type Config struct { // MaxMessageBytes is the final encoded message size limit. MaxMessageBytes int - // MaxBatchMessageBytes controls batch splitting. + // MaxBatchedBytes controls batch splitting. // If it is not set, codecs use MaxMessageBytes to keep the old behavior. - MaxBatchMessageBytes int - MaxBatchSize int + MaxBatchedBytes int + MaxBatchSize int // DeleteOnlyHandleKeyColumns is true, for the delete event only output the handle key columns. DeleteOnlyHandleKeyColumns bool @@ -117,9 +117,9 @@ func NewConfig(protocol config.Protocol) *Config { return &Config{ Protocol: protocol, - MaxMessageBytes: config.DefaultMaxMessageBytes, - MaxBatchMessageBytes: config.DefaultMaxMessageBytes, - MaxBatchSize: defaultMaxBatchSize, + MaxMessageBytes: config.DefaultMaxMessageBytes, + MaxBatchedBytes: config.DefaultMaxMessageBytes, + MaxBatchSize: defaultMaxBatchSize, EnableTiDBExtension: false, EnableRowChecksum: false, @@ -351,17 +351,12 @@ func (c *Config) WithMaxMessageBytes(bytes int) *Config { return c } -// WithMaxBatchMessageBytes sets the maximum batched message bytes. -func (c *Config) WithMaxBatchMessageBytes(bytes int) *Config { - c.MaxBatchMessageBytes = bytes +// WithMaxBatchedBytes sets the maximum batched message bytes. +func (c *Config) WithMaxBatchedBytes(bytes int) *Config { + c.MaxBatchedBytes = bytes return c } -// BatchMaxMessageBytes returns the maximum batched message bytes. -func (c *Config) BatchMaxMessageBytes() int { - return c.MaxBatchMessageBytes -} - // WithChangefeedID set the `changefeedID` func (c *Config) WithChangefeedID(id common.ChangeFeedID) *Config { c.ChangefeedID = id @@ -476,16 +471,16 @@ func (c *Config) Validate() error { errors.Errorf("invalid max-message-bytes %d", c.MaxMessageBytes), ) } - if c.MaxBatchMessageBytes < 0 { + if c.MaxBatchedBytes < 0 { return errors.ErrCodecInvalidConfig.Wrap( - errors.Errorf("invalid max-batch-message-bytes %d", c.MaxBatchMessageBytes), + errors.Errorf("invalid max-batch-message-bytes %d", c.MaxBatchedBytes), ) } - if c.MaxBatchMessageBytes > c.MaxMessageBytes { + if c.MaxBatchedBytes > c.MaxMessageBytes { return errors.ErrCodecInvalidConfig.Wrap( errors.Errorf( "max-batch-message-bytes %d cannot be greater than max-message-bytes %d", - c.MaxBatchMessageBytes, + c.MaxBatchedBytes, c.MaxMessageBytes, ), ) diff --git a/pkg/sink/codec/common/config_test.go b/pkg/sink/codec/common/config_test.go index cd3d9b5c6d..d9aad8b3fe 100644 --- a/pkg/sink/codec/common/config_test.go +++ b/pkg/sink/codec/common/config_test.go @@ -25,7 +25,7 @@ import ( func TestValidateMaxBatchMessageBytes(t *testing.T) { cfg := NewConfig(config.ProtocolOpen) cfg.MaxMessageBytes = 100 - cfg.MaxBatchMessageBytes = 101 + cfg.MaxBatchedBytes = 101 err := cfg.Validate() require.Error(t, err) diff --git a/pkg/sink/codec/open/encoder.go b/pkg/sink/codec/open/encoder.go index fe7d77c24d..fbf1e51c70 100644 --- a/pkg/sink/codec/open/encoder.go +++ b/pkg/sink/codec/open/encoder.go @@ -173,7 +173,7 @@ func (d *batchEncoder) pushMessage(key, value []byte, callback func()) { binary.BigEndian.PutUint64(keyLenByte[:], uint64(len(key))) binary.BigEndian.PutUint64(valueLenByte[:], uint64(len(value))) - if len(d.messages) == 0 || d.messages[len(d.messages)-1].Length()+length > d.config.BatchMaxMessageBytes() || d.messages[len(d.messages)-1].GetRowsCount() >= d.config.MaxBatchSize { + if len(d.messages) == 0 || d.messages[len(d.messages)-1].Length()+length > d.config.MaxBatchedBytes() || d.messages[len(d.messages)-1].GetRowsCount() >= d.config.MaxBatchSize { d.finalizeCallback() // create a new message versionHead := make([]byte, 8) diff --git a/pkg/sink/codec/open/encoder_test.go b/pkg/sink/codec/open/encoder_test.go index c0aed9ebb2..faeec70950 100644 --- a/pkg/sink/codec/open/encoder_test.go +++ b/pkg/sink/codec/open/encoder_test.go @@ -189,7 +189,7 @@ func TestFloatTypes(t *testing.T) { helper.Tk().MustExec("use test") job := helper.DDL2Job(`create table test.t( id int primary key auto_increment, - a float, b float(10, 3), c float(10), + a float, b float(10, 3), c float(10), d double, e double(20, 3))`) dmlEvent := helper.DML2Event("test", "t", `insert into test.t(a,b,c,d,e) values (1.23, 4.56, 7.89, 10.11, 12.13)`) @@ -533,17 +533,17 @@ func TestOtherTypes(t *testing.T) { helper.Tk().MustExec("use test") job := helper.DDL2Job(`create table test.t( - id int primary key auto_increment, + id int primary key auto_increment, a bool, b bool, c year, - d bit(10), e json, - f decimal(10,2), + d bit(10), e json, + f decimal(10,2), g enum('a','b','c'), h set('a','b','c'))`) tableInfo := helper.GetTableInfo(job) dmlEvent := helper.DML2Event("test", "t", `insert into test.t(a, b, c, d, e, f, g, h) values ( - true, false, 2000, - 0b0101010101, '{"key1": "value1"}', - 153.123, + true, false, 2000, + 0b0101010101, '{"key1": "value1"}', + 153.123, 'a', 'a,b')`) require.NotNil(t, dmlEvent) @@ -889,7 +889,7 @@ func TestMessageLargerThanBatchLimit(t *testing.T) { ctx := context.Background() codecConfig := common.NewConfig(config.ProtocolOpen). WithMaxMessageBytes(400). - WithMaxBatchMessageBytes(100) + WithMaxBatchedBytes(100) encoder, err := NewBatchEncoder(ctx, codecConfig) require.NoError(t, err) diff --git a/pkg/sink/codec/simple/encoder_test.go b/pkg/sink/codec/simple/encoder_test.go index ae2f9faf94..6144050c5f 100644 --- a/pkg/sink/codec/simple/encoder_test.go +++ b/pkg/sink/codec/simple/encoder_test.go @@ -1597,7 +1597,7 @@ func TestDMLLargerThanBatchLimit(t *testing.T) { codecConfig := common.NewConfig(config.ProtocolSimple) codecConfig.MaxMessageBytes = config.DefaultMaxMessageBytes - codecConfig.MaxBatchMessageBytes = 50 + codecConfig.MaxBatchedBytes = 50 enc, err := NewEncoder(context.Background(), codecConfig) require.NoError(t, err) @@ -1607,7 +1607,7 @@ func TestDMLLargerThanBatchLimit(t *testing.T) { messages := enc.Build() require.Len(t, messages, 1) - require.Greater(t, messages[0].Length(), codecConfig.MaxBatchMessageBytes) + require.Greater(t, messages[0].Length(), codecConfig.MaxBatchedBytes) require.LessOrEqual(t, messages[0].Length(), codecConfig.MaxMessageBytes) require.Equal(t, 1, messages[0].GetRowsCount()) } diff --git a/pkg/sink/kafka/options_test.go b/pkg/sink/kafka/options_test.go index 2815c9c823..5f9e1d65a3 100644 --- a/pkg/sink/kafka/options_test.go +++ b/pkg/sink/kafka/options_test.go @@ -727,7 +727,7 @@ func TestConfigurationCombinations(t *testing.T) { require.Nil(t, err) encoderConfig. WithMaxMessageBytes(options.MaxMessageBytes). - WithMaxBatchMessageBytes(options.MaxBatchedBytes) + WithMaxBatchedBytes(options.MaxBatchedBytes) err = encoderConfig.Validate() require.Nil(t, err) @@ -736,7 +736,7 @@ func TestConfigurationCombinations(t *testing.T) { require.Equal( t, min(configuredMaxMessageBytes, expectedProducerLimit), - encoderConfig.BatchMaxMessageBytes(), + encoderConfig.MaxBatchedBytes(), ) adminClient.Close() From b3ffc57dd8318d4b172d77e616ff2e5aafadb54c Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Tue, 21 Jul 2026 19:37:57 +0800 Subject: [PATCH 10/22] fix more code --- downstreamadapter/sink/kafka/sink.go | 16 ++++++++-------- downstreamadapter/sink/kafka/sink_test.go | 13 +++++++++++++ pkg/sink/codec/open/encoder.go | 2 +- pkg/sink/codec/open/encoder_test.go | 4 ++-- pkg/sink/kafka/options_test.go | 2 +- 5 files changed, 25 insertions(+), 12 deletions(-) diff --git a/downstreamadapter/sink/kafka/sink.go b/downstreamadapter/sink/kafka/sink.go index 7c2231c9da..9de1070e15 100644 --- a/downstreamadapter/sink/kafka/sink.go +++ b/downstreamadapter/sink/kafka/sink.go @@ -87,6 +87,14 @@ func Verify(ctx context.Context, changefeedID commonType.ChangeFeedID, uri *url. } options.Topic = topic + encoderConfig, err := helper.GetEncoderConfig( + changefeedID, uri, protocol, sinkConfig, + options.MaxMessageBytes, options.MaxBatchedBytes, + ) + if err != nil { + return errors.Trace(err) + } + isAvroLike := protocol == config.ProtocolAvro || protocol == config.ProtocolDebeziumAvro if _, err = eventrouter.NewEventRouter(sinkConfig, topic, false, isAvroLike); err != nil { return errors.Trace(err) @@ -130,14 +138,6 @@ func Verify(ctx context.Context, changefeedID commonType.ChangeFeedID, uri *url. return errors.WrapError(errors.ErrKafkaCreateTopic, err) } - encoderConfig, err := helper.GetEncoderConfig( - changefeedID, uri, protocol, sinkConfig, - options.MaxMessageBytes, options.MaxBatchedBytes, - ) - if err != nil { - return errors.Trace(err) - } - encoder, err := codec.NewEventEncoder(ctx, encoderConfig) if err != nil { return errors.Trace(err) diff --git a/downstreamadapter/sink/kafka/sink_test.go b/downstreamadapter/sink/kafka/sink_test.go index deaecf5c0a..6d7d337574 100644 --- a/downstreamadapter/sink/kafka/sink_test.go +++ b/downstreamadapter/sink/kafka/sink_test.go @@ -37,6 +37,19 @@ import ( const kafkaSinkTestTopic = "mock_topic" +func TestVerifyValidatesEncoderConfigBeforeKafkaConnection(t *testing.T) { + openProtocol := config.ProtocolOpen.String() + sinkConfig := &config.SinkConfig{Protocol: &openProtocol} + sinkURI, err := url.Parse("kafka://127.0.0.1:1/" + kafkaSinkTestTopic + "?max-batch-size=0") + require.NoError(t, err) + + changefeedID := common.NewChangefeedID4Test("test", "verify-existing-topic") + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + err = Verify(ctx, changefeedID, sinkURI, sinkConfig) + require.ErrorContains(t, err, "invalid max-batch-size 0") +} + func newKafkaSinkForTestWithProducers(ctx context.Context, t *testing.T, ctrl *gomock.Controller, diff --git a/pkg/sink/codec/open/encoder.go b/pkg/sink/codec/open/encoder.go index fbf1e51c70..30378a65b8 100644 --- a/pkg/sink/codec/open/encoder.go +++ b/pkg/sink/codec/open/encoder.go @@ -173,7 +173,7 @@ func (d *batchEncoder) pushMessage(key, value []byte, callback func()) { binary.BigEndian.PutUint64(keyLenByte[:], uint64(len(key))) binary.BigEndian.PutUint64(valueLenByte[:], uint64(len(value))) - if len(d.messages) == 0 || d.messages[len(d.messages)-1].Length()+length > d.config.MaxBatchedBytes() || d.messages[len(d.messages)-1].GetRowsCount() >= d.config.MaxBatchSize { + if len(d.messages) == 0 || d.messages[len(d.messages)-1].Length()+length > d.config.MaxBatchedBytes || d.messages[len(d.messages)-1].GetRowsCount() >= d.config.MaxBatchSize { d.finalizeCallback() // create a new message versionHead := make([]byte, 8) diff --git a/pkg/sink/codec/open/encoder_test.go b/pkg/sink/codec/open/encoder_test.go index faeec70950..c1e5689998 100644 --- a/pkg/sink/codec/open/encoder_test.go +++ b/pkg/sink/codec/open/encoder_test.go @@ -533,7 +533,7 @@ func TestOtherTypes(t *testing.T) { helper.Tk().MustExec("use test") job := helper.DDL2Job(`create table test.t( - id int primary key auto_increment, + id int primary key auto_increment, a bool, b bool, c year, d bit(10), e json, f decimal(10,2), @@ -541,7 +541,7 @@ func TestOtherTypes(t *testing.T) { tableInfo := helper.GetTableInfo(job) dmlEvent := helper.DML2Event("test", "t", `insert into test.t(a, b, c, d, e, f, g, h) values ( - true, false, 2000, + true, false, 2000, 0b0101010101, '{"key1": "value1"}', 153.123, 'a', 'a,b')`) diff --git a/pkg/sink/kafka/options_test.go b/pkg/sink/kafka/options_test.go index 5f9e1d65a3..c5e8933491 100644 --- a/pkg/sink/kafka/options_test.go +++ b/pkg/sink/kafka/options_test.go @@ -736,7 +736,7 @@ func TestConfigurationCombinations(t *testing.T) { require.Equal( t, min(configuredMaxMessageBytes, expectedProducerLimit), - encoderConfig.MaxBatchedBytes(), + encoderConfig.MaxBatchedBytes, ) adminClient.Close() From 19447b15acc71aaa3b9a9d412fa134a64c55bd48 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Tue, 21 Jul 2026 21:54:43 +0800 Subject: [PATCH 11/22] add unit test to cover open protocol batch --- pkg/sink/codec/open/encoder_test.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/pkg/sink/codec/open/encoder_test.go b/pkg/sink/codec/open/encoder_test.go index c1e5689998..55f3f37c86 100644 --- a/pkg/sink/codec/open/encoder_test.go +++ b/pkg/sink/codec/open/encoder_test.go @@ -778,7 +778,9 @@ func TestEncoderMultipleMessage(t *testing.T) { `insert into test.t values (3, 333)`) ctx := context.Background() - codecConfig := common.NewConfig(config.ProtocolOpen).WithMaxMessageBytes(400) + codecConfig := common.NewConfig(config.ProtocolOpen). + WithMaxMessageBytes(1000). + WithMaxBatchedBytes(400) encoder, err := NewBatchEncoder(ctx, codecConfig) require.NoError(t, err) @@ -808,11 +810,13 @@ func TestEncoderMultipleMessage(t *testing.T) { require.Equal(t, 2, len(messages)) require.Equal(t, 2, messages[0].GetRowsCount()) require.Equal(t, 1, messages[1].GetRowsCount()) + require.LessOrEqual(t, messages[0].Length(), codecConfig.MaxBatchedBytes) + require.LessOrEqual(t, messages[1].Length(), codecConfig.MaxBatchedBytes) + require.Equal(t, 0, count) - for _, message := range messages { - message.Callback() - } - + messages[0].Callback() + require.Equal(t, 2, count) + messages[1].Callback() require.Equal(t, 3, count) decoder, err := NewDecoder(ctx, 0, codecConfig, nil) @@ -919,6 +923,8 @@ func TestMessageLargerThanBatchLimit(t *testing.T) { messages := encoder.Build() require.Len(t, messages, 1) require.Equal(t, 1, messages[0].GetRowsCount()) + require.Greater(t, messages[0].Length(), codecConfig.MaxBatchedBytes) + require.LessOrEqual(t, messages[0].Length(), codecConfig.MaxMessageBytes) require.Equal(t, 0, count) messages[0].Callback() From b30e94a8cbe4333107be8e7019c04415a353f00d Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Tue, 21 Jul 2026 22:08:15 +0800 Subject: [PATCH 12/22] add unit test to cover open protocol batch --- pkg/sink/codec/common/config.go | 2 +- pkg/sink/codec/common/config_test.go | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/pkg/sink/codec/common/config.go b/pkg/sink/codec/common/config.go index b38f2e7b3e..d075cf97a1 100644 --- a/pkg/sink/codec/common/config.go +++ b/pkg/sink/codec/common/config.go @@ -198,7 +198,7 @@ func (c *Config) Apply(sinkURI *url.URL, sinkConfig *config.SinkConfig) error { var err error urlParameter := &urlConfig{} if err = binding.Query.Bind(req, urlParameter); err != nil { - return errors.WrapError(errors.ErrKafkaInvalidConfig, err) + return errors.WrapError(errors.ErrSinkInvalidConfig, err) } if urlParameter, err = mergeConfig(sinkConfig, urlParameter); err != nil { return err diff --git a/pkg/sink/codec/common/config_test.go b/pkg/sink/codec/common/config_test.go index d9aad8b3fe..20dab47acc 100644 --- a/pkg/sink/codec/common/config_test.go +++ b/pkg/sink/codec/common/config_test.go @@ -18,10 +18,22 @@ import ( "testing" "github.com/pingcap/ticdc/pkg/config" + "github.com/pingcap/ticdc/pkg/errors" "github.com/pingcap/ticdc/pkg/util" "github.com/stretchr/testify/require" ) +func TestApplyReturnsSinkInvalidConfigForQueryBindingError(t *testing.T) { + cfg := NewConfig(config.ProtocolOpen) + sinkURI, err := url.Parse("kafka://127.0.0.1:9092/topic?max-batch-size=invalid") + require.NoError(t, err) + + err = cfg.Apply(sinkURI, config.GetDefaultReplicaConfig().Sink) + errCode, ok := errors.RFCCode(err) + require.True(t, ok, err) + require.Equal(t, errors.ErrSinkInvalidConfig.RFCCode(), errCode) +} + func TestValidateMaxBatchMessageBytes(t *testing.T) { cfg := NewConfig(config.ProtocolOpen) cfg.MaxMessageBytes = 100 From 6ad02d23530f367d070c7caee4bb63f2e9049480 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Wed, 22 Jul 2026 11:51:16 +0800 Subject: [PATCH 13/22] fix tests --- pkg/sink/kafka/options_test.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/pkg/sink/kafka/options_test.go b/pkg/sink/kafka/options_test.go index c5e8933491..b4070be5f8 100644 --- a/pkg/sink/kafka/options_test.go +++ b/pkg/sink/kafka/options_test.go @@ -406,10 +406,18 @@ func TestAdjustConfigFallsBackToBrokerMessageMaxBytesWhenTopicConfigMissing(t *t err := adminClient.CreateTopic(detail, false) require.NoError(t, err) - options := NewOptions() - options.BrokerEndpoints = []string{"127.0.0.1:9092"} configuredMaxMessageBytes := test.configuredMaxMessageBytes(adminFixture) - options.MaxMessageBytes = configuredMaxMessageBytes + sinkURI, err := url.Parse(fmt.Sprintf( + "kafka://127.0.0.1:9092/%s?max-message-bytes=%d", + topicName, configuredMaxMessageBytes, + )) + require.NoError(t, err) + + options := NewOptions() + err = options.Apply(changefeedID, sinkURI, config.GetDefaultReplicaConfig().Sink) + require.NoError(t, err) + require.Equal(t, configuredMaxMessageBytes, options.MaxMessageBytes) + require.Equal(t, configuredMaxMessageBytes, options.MaxBatchedBytes) expectedProducerLimit := expectedAdjustedMaxMessageBytes( adminFixture.brokerMessageMaxBytes()) From af183a7150204e43ca5aa7c3d623b5b9f2d49289 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Wed, 22 Jul 2026 15:25:13 +0800 Subject: [PATCH 14/22] fix some code --- pkg/config/large_message.go | 20 ++-- pkg/config/large_message_test.go | 12 +++ pkg/sink/codec/canal/canal_json_encoder.go | 19 ++-- pkg/sink/codec/common/config.go | 5 +- pkg/sink/codec/open/encoder.go | 21 ++-- pkg/sink/codec/simple/encoder.go | 18 ++-- pkg/sink/kafka/options.go | 13 ++- pkg/sink/kafka/options_test.go | 109 +++++++++++++++++++++ 8 files changed, 174 insertions(+), 43 deletions(-) diff --git a/pkg/config/large_message.go b/pkg/config/large_message.go index d04584b451..6b19afe260 100644 --- a/pkg/config/large_message.go +++ b/pkg/config/large_message.go @@ -15,7 +15,7 @@ package config import ( "github.com/pingcap/ticdc/pkg/compression" - cerror "github.com/pingcap/ticdc/pkg/errors" + "github.com/pingcap/ticdc/pkg/errors" ) const ( @@ -55,34 +55,39 @@ func (c *LargeMessageHandleConfig) AdjustAndValidate(protocol Protocol, enableTi // compression can be enabled independently if !compression.Supported(c.LargeMessageHandleCompression) { - return cerror.ErrInvalidReplicaConfig.GenWithStack( + return errors.ErrInvalidReplicaConfig.GenWithStack( "large message handle compression is not supported, got %s", c.LargeMessageHandleCompression) } if c.LargeMessageHandleOption == LargeMessageHandleOptionNone { return nil } + if c.LargeMessageHandleOption != LargeMessageHandleOptionClaimCheck && + c.LargeMessageHandleOption != LargeMessageHandleOptionHandleKeyOnly { + return errors.ErrInvalidReplicaConfig.GenWithStack( + "unknown large-message-handle-option %s", c.LargeMessageHandleOption) + } switch protocol { case ProtocolOpen, ProtocolSimple: case ProtocolCanalJSON: if !enableTiDBExtension { - return cerror.ErrInvalidReplicaConfig.GenWithStack( + return errors.ErrInvalidReplicaConfig.GenWithStack( "large message handle is set to %s, protocol is %s, but enable-tidb-extension is false", c.LargeMessageHandleOption, protocol.String()) } default: - return cerror.ErrInvalidReplicaConfig.GenWithStack( + return errors.ErrInvalidReplicaConfig.GenWithStack( "large message handle is set to %s, protocol is %s, it's not supported", c.LargeMessageHandleOption, protocol.String()) } if c.LargeMessageHandleOption == LargeMessageHandleOptionClaimCheck { if c.ClaimCheckStorageURI == "" { - return cerror.ErrInvalidReplicaConfig.GenWithStack( + return errors.ErrInvalidReplicaConfig.GenWithStack( "large message handle is set to claim-check, but the claim-check-storage-uri is empty") } if c.ClaimCheckRawValue && protocol == ProtocolOpen { - return cerror.ErrInvalidReplicaConfig.GenWithStack( + return errors.ErrInvalidReplicaConfig.GenWithStack( "large message handle is set to claim-check, raw value is not supported for the open protocol") } } @@ -106,7 +111,8 @@ func (c *LargeMessageHandleConfig) EnableClaimCheck() bool { return c.LargeMessageHandleOption == LargeMessageHandleOptionClaimCheck } -// Disabled returns true if disable large message handle. +// Disabled returns true only when large message handling is explicitly disabled. +// It returns false for nil and unknown configurations. func (c *LargeMessageHandleConfig) Disabled() bool { if c == nil { return false diff --git a/pkg/config/large_message_test.go b/pkg/config/large_message_test.go index ad3345643e..f0721f4b53 100644 --- a/pkg/config/large_message_test.go +++ b/pkg/config/large_message_test.go @@ -58,6 +58,18 @@ func TestLargeMessageHandle4NotSupportedProtocol(t *testing.T) { require.ErrorIs(t, err, cerror.ErrInvalidReplicaConfig) } +func TestLargeMessageHandleRejectsUnknownOption(t *testing.T) { + t.Parallel() + + largeMessageHandle := NewDefaultLargeMessageHandleConfig() + largeMessageHandle.LargeMessageHandleOption = "unknown" + + require.False(t, largeMessageHandle.Disabled()) + err := largeMessageHandle.AdjustAndValidate(ProtocolOpen, false) + require.ErrorIs(t, err, cerror.ErrInvalidReplicaConfig) + require.ErrorContains(t, err, "unknown large-message-handle-option unknown") +} + func TestHandleKeyOnly4CanalJSON(t *testing.T) { t.Parallel() diff --git a/pkg/sink/codec/canal/canal_json_encoder.go b/pkg/sink/codec/canal/canal_json_encoder.go index c89818e764..7425a666ef 100644 --- a/pkg/sink/codec/canal/canal_json_encoder.go +++ b/pkg/sink/codec/canal/canal_json_encoder.go @@ -468,7 +468,16 @@ func (c *JSONRowEventEncoder) AppendRowChangedEvent( targetTable := e.TableInfo.GetTargetTableName() originLength := m.Length() - if m.Length() > c.config.MaxMessageBytes && !c.config.LargeMessageHandle.Disabled() { + if m.Length() > c.config.MaxMessageBytes { + // for single message that is longer than max-message-bytes, do not send it. + if c.config.LargeMessageHandle.Disabled() { + log.Error("Single message is too large for canal-json", + zap.Int("maxMessageBytes", c.config.MaxMessageBytes), + zap.Int("length", originLength), + zap.Any("table", e.TableInfo.TableName)) + return errors.ErrMessageTooLarge.GenWithStackByArgs(targetTable, originLength, c.config.MaxMessageBytes) + } + if c.config.LargeMessageHandle.HandleKeyOnly() { value, err = newJSONMessageForDML(e, c.config, true, "") if err != nil { @@ -511,14 +520,6 @@ func (c *JSONRowEventEncoder) AppendRowChangedEvent( } } - if m.Length() > c.config.MaxMessageBytes { - log.Error("Single message is too large for canal-json", - zap.Int("maxMessageBytes", c.config.MaxMessageBytes), - zap.Int("length", m.Length()), - zap.Any("table", e.TableInfo.TableName)) - return errors.ErrMessageTooLarge.GenWithStackByArgs(targetTable, m.Length(), c.config.MaxMessageBytes) - } - c.messages = append(c.messages, m) return nil } diff --git a/pkg/sink/codec/common/config.go b/pkg/sink/codec/common/config.go index d075cf97a1..21e96bd06b 100644 --- a/pkg/sink/codec/common/config.go +++ b/pkg/sink/codec/common/config.go @@ -41,10 +41,9 @@ type Config struct { Protocol config.Protocol - // MaxMessageBytes is the final encoded message size limit. + // MaxMessageBytes is the encoded message size limit. MaxMessageBytes int - // MaxBatchedBytes controls batch splitting. - // If it is not set, codecs use MaxMessageBytes to keep the old behavior. + // MaxBatchedBytes controls batched message size limit. MaxBatchedBytes int MaxBatchSize int diff --git a/pkg/sink/codec/open/encoder.go b/pkg/sink/codec/open/encoder.go index 30378a65b8..ee061082e8 100644 --- a/pkg/sink/codec/open/encoder.go +++ b/pkg/sink/codec/open/encoder.go @@ -95,7 +95,17 @@ func (d *batchEncoder) AppendRowChangedEvent( return errors.Trace(err) } - if length > d.config.MaxMessageBytes && !d.config.LargeMessageHandle.Disabled() { + if length > d.config.MaxMessageBytes { + // message len is larger than max-message-bytes + if d.config.LargeMessageHandle.Disabled() { + log.Warn("Single message is too large for open-protocol", + zap.Int("maxMessageBytes", d.config.MaxMessageBytes), + zap.Int("length", length), + zap.Any("table", e.TableInfo.TableName), + zap.Any("key", key)) + return errors.ErrMessageTooLarge.GenWithStackByArgs(e.TableInfo.GetTargetTableName(), length, d.config.MaxMessageBytes) + } + if d.config.LargeMessageHandle.EnableClaimCheck() { // send the large message to the external storage first, then // create a new message contains the reference of the large message. @@ -139,15 +149,6 @@ func (d *batchEncoder) AppendRowChangedEvent( } } - if length > d.config.MaxMessageBytes { - log.Warn("Single message is too large for open-protocol", - zap.Int("maxMessageBytes", d.config.MaxMessageBytes), - zap.Int("length", length), - zap.Any("table", e.TableInfo.TableName), - zap.Any("key", key)) - return errors.ErrMessageTooLarge.GenWithStackByArgs(e.TableInfo.GetTargetTableName(), length, d.config.MaxMessageBytes) - } - d.pushMessage(key, value, e.Callback) return nil } diff --git a/pkg/sink/codec/simple/encoder.go b/pkg/sink/codec/simple/encoder.go index 95d93f1d34..b8ef228561 100644 --- a/pkg/sink/codec/simple/encoder.go +++ b/pkg/sink/codec/simple/encoder.go @@ -67,19 +67,19 @@ func (e *Encoder) AppendRowChangedEvent(ctx context.Context, _ string, event *co result.IncRowsCount() length := result.Length() - if length <= e.config.MaxMessageBytes || e.config.LargeMessageHandle.Disabled() { - if length > e.config.MaxMessageBytes { - log.Error("Single message is too large for simple", - zap.Int("maxMessageBytes", e.config.MaxMessageBytes), - zap.Int("length", length), - zap.Any("table", event.TableInfo.TableName)) - return errors.ErrMessageTooLarge.GenWithStackByArgs( - event.TableInfo.GetTargetTableName(), length, e.config.MaxMessageBytes) - } + if length <= e.config.MaxMessageBytes { e.messages = append(e.messages, result) return nil } + if e.config.LargeMessageHandle.Disabled() { + log.Error("Single message is too large for simple", + zap.Int("maxMessageBytes", e.config.MaxMessageBytes), + zap.Int("length", length), + zap.Any("table", event.TableInfo.TableName)) + return errors.ErrMessageTooLarge.GenWithStackByArgs(event.TableInfo.GetTargetTableName(), length, e.config.MaxMessageBytes) + } + var claimCheckLocation string if e.config.LargeMessageHandle.EnableClaimCheck() { fileName := claimcheck.NewFileName() diff --git a/pkg/sink/kafka/options.go b/pkg/sink/kafka/options.go index e611953040..301471917c 100644 --- a/pkg/sink/kafka/options.go +++ b/pkg/sink/kafka/options.go @@ -253,6 +253,9 @@ func (o *options) Apply(changefeedID common.ChangeFeedID, } if urlParameter.MaxMessageBytes != nil { + if *urlParameter.MaxMessageBytes <= 0 { + return errors.ErrKafkaInvalidConfig.GenWithStack("invalid max-message-bytes %d", *urlParameter.MaxMessageBytes) + } o.MaxMessageBytes = *urlParameter.MaxMessageBytes } o.MaxBatchedBytes = o.MaxMessageBytes @@ -695,7 +698,7 @@ func getTopicMaxMessageBytes( admin ClusterAdminClient, topic string, ) (int, error) { - maxMessageBytesStr, err := getTopicConfig( + raw, err := getTopicConfig( ctx, admin, topic, TopicMaxMessageBytesConfigName, BrokerMessageMaxBytesConfigName, @@ -703,7 +706,7 @@ func getTopicMaxMessageBytes( if err != nil { return 0, errors.Trace(err) } - maxMessageBytes, err := strconv.Atoi(maxMessageBytesStr) + maxMessageBytes, err := strconv.Atoi(raw) if err != nil { return 0, errors.Trace(err) } @@ -711,15 +714,15 @@ func getTopicMaxMessageBytes( } func getBrokerMaxMessageBytes(admin ClusterAdminClient) (int, error) { - maxMessageBytesStr, err := admin.GetBrokerConfig(BrokerMessageMaxBytesConfigName) + raw, err := admin.GetBrokerConfig(BrokerMessageMaxBytesConfigName) if err != nil { return 0, errors.Trace(err) } - maxMessageBytes, err := strconv.Atoi(maxMessageBytesStr) + messageMaxBytes, err := strconv.Atoi(raw) if err != nil { return 0, errors.Trace(err) } - return maxMessageBytes, nil + return messageMaxBytes, nil } func validateMinInsyncReplicas( diff --git a/pkg/sink/kafka/options_test.go b/pkg/sink/kafka/options_test.go index b4070be5f8..4fde4edf8e 100644 --- a/pkg/sink/kafka/options_test.go +++ b/pkg/sink/kafka/options_test.go @@ -286,6 +286,61 @@ func TestCompleteOptions(t *testing.T) { require.Equal(t, defaultMaxRetry, options.MaxRetry) } +func TestApplyRejectsNonPositiveMaxMessageBytes(t *testing.T) { + tests := []struct { + name string + uri string + configValue *int + expected int + }{ + { + name: "zero from URI", + uri: "kafka://127.0.0.1:9092/test-topic?max-message-bytes=0", + expected: 0, + }, + { + name: "negative from URI", + uri: "kafka://127.0.0.1:9092/test-topic?max-message-bytes=-1", + expected: -1, + }, + { + name: "zero from sink config", + uri: "kafka://127.0.0.1:9092/test-topic", + configValue: aws.Int(0), + expected: 0, + }, + { + name: "negative from sink config", + uri: "kafka://127.0.0.1:9092/test-topic", + configValue: aws.Int(-1), + expected: -1, + }, + } + + changefeedID := commonType.NewChangefeedID4Test(commonType.DefaultKeyspaceName, "test") + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + sinkURI, err := url.Parse(test.uri) + require.NoError(t, err) + + sinkConfig := config.GetDefaultReplicaConfig().Sink + if test.configValue != nil { + sinkConfig.KafkaConfig = &config.KafkaConfig{ + MaxMessageBytes: test.configValue, + } + } + + options := NewOptions() + err = options.Apply(changefeedID, sinkURI, sinkConfig) + require.ErrorContains( + t, err, fmt.Sprintf("invalid max-message-bytes %d", test.expected)) + errCode, ok := errors.RFCCode(err) + require.True(t, ok) + require.Equal(t, errors.ErrKafkaInvalidConfig.RFCCode(), errCode) + }) + } +} + func TestSetPartitionNum(t *testing.T) { options := NewOptions() changefeedID := commonType.NewChangefeedID4Test(commonType.DefaultKeyspaceName, "test") @@ -440,6 +495,60 @@ func TestAdjustConfigFallsBackToBrokerMessageMaxBytesWhenTopicConfigMissing(t *t } } +func TestAdjustConfigFallsBackWhenKafkaMaxMessageBytesIsNonPositive(t *testing.T) { + tests := []struct { + name string + topic string + kafkaValue string + }{ + { + name: "existing topic max.message.bytes is zero", + topic: defaultMockTopicName, + kafkaValue: "0", + }, + { + name: "existing topic max.message.bytes is negative", + topic: defaultMockTopicName, + kafkaValue: "-1", + }, + { + name: "new topic broker message.max.bytes is zero", + topic: "not-exist-topic", + kafkaValue: "0", + }, + { + name: "new topic broker message.max.bytes is negative", + topic: "not-exist-topic", + kafkaValue: "-1", + }, + } + + const configuredMaxMessageBytes = 4096 + changefeedID := commonType.NewChangefeedID4Test(commonType.DefaultKeyspaceName, "test") + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + adminFixture := newKafkaAdminFixture(t) + adminFixture.setMessageMaxBytes(test.kafkaValue, test.kafkaValue) + + sinkURI, err := url.Parse(fmt.Sprintf( + "kafka://127.0.0.1:9092/%s?max-message-bytes=%d", + test.topic, configuredMaxMessageBytes, + )) + require.NoError(t, err) + + options := NewOptions() + err = options.Apply(changefeedID, sinkURI, config.GetDefaultReplicaConfig().Sink) + require.NoError(t, err) + + err = adjustOptions( + context.Background(), changefeedID, adminFixture.admin, options, test.topic) + require.NoError(t, err) + require.Equal(t, configuredMaxMessageBytes, options.MaxMessageBytes) + require.Equal(t, configuredMaxMessageBytes, options.MaxBatchedBytes) + }) + } +} + func TestAdjustConfigMinInsyncReplicas(t *testing.T) { adminFixture := newKafkaAdminFixture(t) adminClient := adminFixture.admin From c9477045fbbaf205486a29d0b22b3782509dbe3d Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Wed, 22 Jul 2026 15:59:12 +0800 Subject: [PATCH 15/22] fix some code --- .../codec/canal/canal_json_txn_encoder.go | 3 +- pkg/sink/codec/common/config.go | 22 ++----- pkg/sink/codec/common/config_test.go | 57 +++++++++++++++---- pkg/sink/codec/open/encoder.go | 2 +- pkg/sink/kafka/options_test.go | 54 ------------------ 5 files changed, 53 insertions(+), 85 deletions(-) diff --git a/pkg/sink/codec/canal/canal_json_txn_encoder.go b/pkg/sink/codec/canal/canal_json_txn_encoder.go index 4998378ef7..3e10d98229 100644 --- a/pkg/sink/codec/canal/canal_json_txn_encoder.go +++ b/pkg/sink/codec/canal/canal_json_txn_encoder.go @@ -70,8 +70,7 @@ func (j *JSONTxnEventEncoder) AppendTxnEvent(event *commonEvent.DMLEvent) error zap.Int("maxMessageBytes", j.config.MaxMessageBytes), zap.Int("length", length), zap.Any("table", event.TableInfo.TableName)) - return errors.ErrMessageTooLarge.GenWithStackByArgs( - event.TableInfo.GetTargetTableName(), length, j.config.MaxMessageBytes) + return errors.ErrMessageTooLarge.GenWithStackByArgs(event.TableInfo.GetTargetTableName(), length, j.config.MaxMessageBytes) } j.valueBuf.Write(value) j.valueBuf.Write(j.terminator) diff --git a/pkg/sink/codec/common/config.go b/pkg/sink/codec/common/config.go index 21e96bd06b..991841fbb7 100644 --- a/pkg/sink/codec/common/config.go +++ b/pkg/sink/codec/common/config.go @@ -41,9 +41,7 @@ type Config struct { Protocol config.Protocol - // MaxMessageBytes is the encoded message size limit. MaxMessageBytes int - // MaxBatchedBytes controls batched message size limit. MaxBatchedBytes int MaxBatchSize int @@ -466,29 +464,17 @@ func (c *Config) Validate() error { } if c.MaxMessageBytes <= 0 { - return errors.ErrCodecInvalidConfig.Wrap( - errors.Errorf("invalid max-message-bytes %d", c.MaxMessageBytes), - ) + return errors.ErrCodecInvalidConfig.GenWithStack("invalid max-message-bytes %d", c.MaxMessageBytes) } if c.MaxBatchedBytes < 0 { - return errors.ErrCodecInvalidConfig.Wrap( - errors.Errorf("invalid max-batch-message-bytes %d", c.MaxBatchedBytes), - ) + return errors.ErrCodecInvalidConfig.GenWithStack("invalid max-batch-message-bytes %d", c.MaxBatchedBytes) } if c.MaxBatchedBytes > c.MaxMessageBytes { - return errors.ErrCodecInvalidConfig.Wrap( - errors.Errorf( - "max-batch-message-bytes %d cannot be greater than max-message-bytes %d", - c.MaxBatchedBytes, - c.MaxMessageBytes, - ), - ) + return errors.ErrCodecInvalidConfig.GenWithStack("max-batch-message-bytes %d cannot be greater than max-message-bytes %d", c.MaxBatchedBytes, c.MaxMessageBytes) } if c.MaxBatchSize <= 0 { - return errors.ErrCodecInvalidConfig.Wrap( - errors.Errorf("invalid max-batch-size %d", c.MaxBatchSize), - ) + return errors.ErrCodecInvalidConfig.GenWithStack("invalid max-batch-size %d", c.MaxBatchSize) } if c.LargeMessageHandle != nil { diff --git a/pkg/sink/codec/common/config_test.go b/pkg/sink/codec/common/config_test.go index 20dab47acc..4369de7216 100644 --- a/pkg/sink/codec/common/config_test.go +++ b/pkg/sink/codec/common/config_test.go @@ -35,17 +35,54 @@ func TestApplyReturnsSinkInvalidConfigForQueryBindingError(t *testing.T) { } func TestValidateMaxBatchMessageBytes(t *testing.T) { - cfg := NewConfig(config.ProtocolOpen) - cfg.MaxMessageBytes = 100 - cfg.MaxBatchedBytes = 101 + tests := []struct { + name string + adjust func(*Config) + expected string + }{ + { + name: "non-positive max message bytes", + adjust: func(cfg *Config) { + cfg.MaxMessageBytes = 0 + }, + expected: "invalid max-message-bytes 0", + }, + { + name: "negative max batched bytes", + adjust: func(cfg *Config) { + cfg.MaxBatchedBytes = -1 + }, + expected: "invalid max-batch-message-bytes -1", + }, + { + name: "max batched bytes exceeds max message bytes", + adjust: func(cfg *Config) { + cfg.MaxMessageBytes = 100 + cfg.MaxBatchedBytes = 101 + }, + expected: "max-batch-message-bytes 101 cannot be greater than max-message-bytes 100", + }, + { + name: "non-positive max batch size", + adjust: func(cfg *Config) { + cfg.MaxBatchSize = 0 + }, + expected: "invalid max-batch-size 0", + }, + } - err := cfg.Validate() - require.Error(t, err) - require.ErrorContains( - t, - err, - "max-batch-message-bytes 101 cannot be greater than max-message-bytes 100", - ) + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cfg := NewConfig(config.ProtocolOpen) + test.adjust(cfg) + + err := cfg.Validate() + require.ErrorContains(t, err, test.expected) + errCode, ok := errors.RFCCode(err) + require.True(t, ok, err) + require.Equal(t, errors.ErrCodecInvalidConfig.RFCCode(), errCode) + }) + } } func TestDebeziumAvroSchemaRegistryConfig(t *testing.T) { diff --git a/pkg/sink/codec/open/encoder.go b/pkg/sink/codec/open/encoder.go index ee061082e8..17a323cb7d 100644 --- a/pkg/sink/codec/open/encoder.go +++ b/pkg/sink/codec/open/encoder.go @@ -38,7 +38,7 @@ var ( ) // batchEncoder for open protocol will batch multiple row changed events into a single message. -// One message can contain at most MaxBatchSize events, and the total size of the message cannot exceed BatchMaxMessageBytes. +// One message can contain at most MaxBatchSize events, and the total size of the message cannot exceed MaxBatchedBytes. type batchEncoder struct { messages []*common.Message // buff the callback of the latest message diff --git a/pkg/sink/kafka/options_test.go b/pkg/sink/kafka/options_test.go index 4fde4edf8e..0e8d5ac528 100644 --- a/pkg/sink/kafka/options_test.go +++ b/pkg/sink/kafka/options_test.go @@ -495,60 +495,6 @@ func TestAdjustConfigFallsBackToBrokerMessageMaxBytesWhenTopicConfigMissing(t *t } } -func TestAdjustConfigFallsBackWhenKafkaMaxMessageBytesIsNonPositive(t *testing.T) { - tests := []struct { - name string - topic string - kafkaValue string - }{ - { - name: "existing topic max.message.bytes is zero", - topic: defaultMockTopicName, - kafkaValue: "0", - }, - { - name: "existing topic max.message.bytes is negative", - topic: defaultMockTopicName, - kafkaValue: "-1", - }, - { - name: "new topic broker message.max.bytes is zero", - topic: "not-exist-topic", - kafkaValue: "0", - }, - { - name: "new topic broker message.max.bytes is negative", - topic: "not-exist-topic", - kafkaValue: "-1", - }, - } - - const configuredMaxMessageBytes = 4096 - changefeedID := commonType.NewChangefeedID4Test(commonType.DefaultKeyspaceName, "test") - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - adminFixture := newKafkaAdminFixture(t) - adminFixture.setMessageMaxBytes(test.kafkaValue, test.kafkaValue) - - sinkURI, err := url.Parse(fmt.Sprintf( - "kafka://127.0.0.1:9092/%s?max-message-bytes=%d", - test.topic, configuredMaxMessageBytes, - )) - require.NoError(t, err) - - options := NewOptions() - err = options.Apply(changefeedID, sinkURI, config.GetDefaultReplicaConfig().Sink) - require.NoError(t, err) - - err = adjustOptions( - context.Background(), changefeedID, adminFixture.admin, options, test.topic) - require.NoError(t, err) - require.Equal(t, configuredMaxMessageBytes, options.MaxMessageBytes) - require.Equal(t, configuredMaxMessageBytes, options.MaxBatchedBytes) - }) - } -} - func TestAdjustConfigMinInsyncReplicas(t *testing.T) { adminFixture := newKafkaAdminFixture(t) adminClient := adminFixture.admin From af5f0a8c0b7033314f73da9dfc3ef4c85c4863d3 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Wed, 22 Jul 2026 16:07:45 +0800 Subject: [PATCH 16/22] simplify the code --- .../codec/canal/canal_json_encoder_test.go | 16 --------------- pkg/sink/codec/simple/encoder_test.go | 20 ------------------- pkg/sink/kafka/options_test.go | 18 ++++++----------- 3 files changed, 6 insertions(+), 48 deletions(-) diff --git a/pkg/sink/codec/canal/canal_json_encoder_test.go b/pkg/sink/codec/canal/canal_json_encoder_test.go index e3c597b07b..2953642f89 100644 --- a/pkg/sink/codec/canal/canal_json_encoder_test.go +++ b/pkg/sink/codec/canal/canal_json_encoder_test.go @@ -666,22 +666,6 @@ func TestMaxMessageBytes(t *testing.T) { }) require.NoError(t, err) - codecConfig = common.NewConfig(config.ProtocolCanalJSON). - WithMaxMessageBytes(maxMessageBytes). - WithMaxBatchedBytes(100) - - encIface, err = NewJSONRowEventEncoder(ctx, codecConfig) - require.NoError(t, err) - - encoder = encIface.(*JSONRowEventEncoder) - err = encoder.AppendRowChangedEvent(ctx, topic, &commonEvent.RowEvent{ - TableInfo: dml.TableInfo, - CommitTs: dml.CommitTs, - Event: rc, - ColumnSelector: columnselector.NewDefaultColumnSelector(), - }) - require.NoError(t, err) - // the test message length is larger than max-message-bytes codecConfig = codecConfig.WithMaxMessageBytes(100) diff --git a/pkg/sink/codec/simple/encoder_test.go b/pkg/sink/codec/simple/encoder_test.go index 6144050c5f..68170f6f05 100644 --- a/pkg/sink/codec/simple/encoder_test.go +++ b/pkg/sink/codec/simple/encoder_test.go @@ -1592,26 +1592,6 @@ func TestDMLMessageTooLarge(t *testing.T) { } } -func TestDMLLargerThanBatchLimit(t *testing.T) { - _, insertEvent, _, _ := common.NewLargeEvent4Test(t) - - codecConfig := common.NewConfig(config.ProtocolSimple) - codecConfig.MaxMessageBytes = config.DefaultMaxMessageBytes - codecConfig.MaxBatchedBytes = 50 - - enc, err := NewEncoder(context.Background(), codecConfig) - require.NoError(t, err) - - err = enc.AppendRowChangedEvent(context.Background(), "", insertEvent) - require.NoError(t, err) - - messages := enc.Build() - require.Len(t, messages, 1) - require.Greater(t, messages[0].Length(), codecConfig.MaxBatchedBytes) - require.LessOrEqual(t, messages[0].Length(), codecConfig.MaxMessageBytes) - require.Equal(t, 1, messages[0].GetRowsCount()) -} - func TestLargerMessageHandleClaimCheck(t *testing.T) { ddlEvent, _, updateEvent, _ := common.NewLargeEvent4Test(t) diff --git a/pkg/sink/kafka/options_test.go b/pkg/sink/kafka/options_test.go index 0e8d5ac528..2c22da54e9 100644 --- a/pkg/sink/kafka/options_test.go +++ b/pkg/sink/kafka/options_test.go @@ -158,10 +158,6 @@ func (f *kafkaAdminFixture) setMessageMaxBytes(brokerValue, topicValue string) { f.topicConfig[defaultMockTopicName][TopicMaxMessageBytesConfigName] = topicValue } -func expectedAdjustedMaxMessageBytes(sourceMaxMessageBytes int) int { - return sourceMaxMessageBytes -} - func (f *kafkaAdminFixture) setMinInsyncReplicas(minInsyncReplicas string) { f.topicConfig[defaultMockTopicName][MinInsyncReplicasConfigName] = minInsyncReplicas f.brokerConfig[MinInsyncReplicasConfigName] = minInsyncReplicas @@ -473,8 +469,7 @@ func TestAdjustConfigFallsBackToBrokerMessageMaxBytesWhenTopicConfigMissing(t *t require.NoError(t, err) require.Equal(t, configuredMaxMessageBytes, options.MaxMessageBytes) require.Equal(t, configuredMaxMessageBytes, options.MaxBatchedBytes) - expectedProducerLimit := expectedAdjustedMaxMessageBytes( - adminFixture.brokerMessageMaxBytes()) + expectedProducerLimit := adminFixture.brokerMessageMaxBytes() ctx := context.Background() err = adjustOptions(ctx, changefeedID, adminClient, options, topicName) @@ -765,21 +760,20 @@ func TestConfigurationCombinations(t *testing.T) { if _, exists := adminFixture.topics[topic]; exists { sourceMaxMessageBytes = adminFixture.topicMaxMessageBytes(topic) } - expectedProducerLimit := expectedAdjustedMaxMessageBytes(sourceMaxMessageBytes) changefeedID := commonType.NewChangefeedID4Test(commonType.DefaultKeyspaceName, "test") err = adjustOptions(ctx, changefeedID, adminClient, options, topic) require.Nil(t, err) - require.Equal(t, expectedProducerLimit, options.MaxMessageBytes) + require.Equal(t, sourceMaxMessageBytes, options.MaxMessageBytes) require.Equal( t, - min(configuredMaxMessageBytes, expectedProducerLimit), + min(configuredMaxMessageBytes, sourceMaxMessageBytes), options.MaxBatchedBytes, ) saramaConfig, err := newSaramaConfig(ctx, options) require.Nil(t, err) - require.Equal(t, expectedProducerLimit, saramaConfig.Producer.MaxMessageBytes) + require.Equal(t, sourceMaxMessageBytes, saramaConfig.Producer.MaxMessageBytes) encoderConfig := common.NewConfig(config.ProtocolOpen) err = encoderConfig.Apply(sinkURI, &config.SinkConfig{ @@ -795,10 +789,10 @@ func TestConfigurationCombinations(t *testing.T) { err = encoderConfig.Validate() require.Nil(t, err) - require.Equal(t, expectedProducerLimit, encoderConfig.MaxMessageBytes) + require.Equal(t, sourceMaxMessageBytes, encoderConfig.MaxMessageBytes) require.Equal( t, - min(configuredMaxMessageBytes, expectedProducerLimit), + min(configuredMaxMessageBytes, sourceMaxMessageBytes), encoderConfig.MaxBatchedBytes, ) From 384391dd978b6f5298759631e5b4a1000ee26362 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Wed, 22 Jul 2026 16:32:28 +0800 Subject: [PATCH 17/22] enhance the integration tests --- pkg/sink/kafka/options_test.go | 33 --------- pkg/sink/kafka/sarama_config_test.go | 1 + tests/integration_tests/_utils/kafka_topic | 12 ++++ .../canal_json_claim_check/run.sh | 9 ++- .../canal_json_handle_key_only/run.sh | 1 + tests/integration_tests/kafka_messages/run.sh | 18 ++++- .../kafka_simple_claim_check/run.sh | 7 ++ .../kafka_simple_claim_check_avro/run.sh | 7 ++ .../kafka_simple_handle_key_only/run.sh | 1 + .../kafka_simple_handle_key_only_avro/run.sh | 1 + .../open_protocol_claim_check/run.sh | 9 ++- .../open_protocol_handle_key_only/run.sh | 1 + tests/utils/kafka_topic/main.go | 68 +++++++++++++++++++ 13 files changed, 132 insertions(+), 36 deletions(-) create mode 100755 tests/integration_tests/_utils/kafka_topic create mode 100644 tests/utils/kafka_topic/main.go diff --git a/pkg/sink/kafka/options_test.go b/pkg/sink/kafka/options_test.go index 2c22da54e9..ea5d2e7147 100644 --- a/pkg/sink/kafka/options_test.go +++ b/pkg/sink/kafka/options_test.go @@ -28,7 +28,6 @@ import ( commonType "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/config" "github.com/pingcap/ticdc/pkg/errors" - "github.com/pingcap/ticdc/pkg/sink/codec/common" "github.com/stretchr/testify/require" ) @@ -697,13 +696,6 @@ func TestConfigurationCombinations(t *testing.T) { mockBrokerMessageMaxBytes, mockTopicMessageMaxBytes, }, - { - "existing topic topic above sarama request limit is preserved", - "kafka://127.0.0.1:9092/%s", - []any{defaultMockTopicName}, - mockBrokerMessageMaxBytes, - strconv.Itoa(int(sarama.MaxRequestSize) + 4096), - }, { "existing topic topic below default and user", "kafka://127.0.0.1:9092/%s?max-message-bytes=%s", @@ -771,31 +763,6 @@ func TestConfigurationCombinations(t *testing.T) { options.MaxBatchedBytes, ) - saramaConfig, err := newSaramaConfig(ctx, options) - require.Nil(t, err) - require.Equal(t, sourceMaxMessageBytes, saramaConfig.Producer.MaxMessageBytes) - - encoderConfig := common.NewConfig(config.ProtocolOpen) - err = encoderConfig.Apply(sinkURI, &config.SinkConfig{ - KafkaConfig: &config.KafkaConfig{ - LargeMessageHandle: config.NewDefaultLargeMessageHandleConfig(), - }, - }) - require.Nil(t, err) - encoderConfig. - WithMaxMessageBytes(options.MaxMessageBytes). - WithMaxBatchedBytes(options.MaxBatchedBytes) - - err = encoderConfig.Validate() - require.Nil(t, err) - - require.Equal(t, sourceMaxMessageBytes, encoderConfig.MaxMessageBytes) - require.Equal( - t, - min(configuredMaxMessageBytes, sourceMaxMessageBytes), - encoderConfig.MaxBatchedBytes, - ) - adminClient.Close() }) } diff --git a/pkg/sink/kafka/sarama_config_test.go b/pkg/sink/kafka/sarama_config_test.go index d12595bfe9..89f3e004c6 100644 --- a/pkg/sink/kafka/sarama_config_test.go +++ b/pkg/sink/kafka/sarama_config_test.go @@ -58,6 +58,7 @@ func TestNewSaramaConfig(t *testing.T) { cfg, err := newSaramaConfig(ctx, options) require.NoError(t, err) require.Equal(t, defaultMaxRetry, cfg.Producer.Retry.Max) + require.Equal(t, options.MaxMessageBytes, cfg.Producer.MaxMessageBytes) options.EnableTLS = true options.Credential = &security.Credential{ diff --git a/tests/integration_tests/_utils/kafka_topic b/tests/integration_tests/_utils/kafka_topic new file mode 100755 index 0000000000..9f9a79a1cd --- /dev/null +++ b/tests/integration_tests/_utils/kafka_topic @@ -0,0 +1,12 @@ +#!/bin/bash + +set -eu + +CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +UTILITY_DIR="$CUR/../../utils/kafka_topic" + +if [ ! -f "$UTILITY_DIR/kafka_topic" ]; then + (cd "$UTILITY_DIR" && GO111MODULE=on go build) +fi + +"$UTILITY_DIR/kafka_topic" "$@" diff --git a/tests/integration_tests/canal_json_claim_check/run.sh b/tests/integration_tests/canal_json_claim_check/run.sh index 10f822c98b..a354dc589a 100755 --- a/tests/integration_tests/canal_json_claim_check/run.sh +++ b/tests/integration_tests/canal_json_claim_check/run.sh @@ -18,7 +18,10 @@ function run() { start_tidb_cluster --workdir $WORK_DIR - TOPIC_NAME="canal-json-claim-check" + TOPIC_NAME="canal-json-claim-check-$RANDOM" + CLAIM_CHECK_DIR="/tmp/canal-json-claim-check" + rm -rf "$CLAIM_CHECK_DIR" + kafka_topic --topic "$TOPIC_NAME" --max-message-bytes 1000 # record tso before we create tables to skip the system table DDLs start_ts=$(run_cdc_cli_tso_query ${UP_PD_HOST_1} ${UP_PD_PORT_1}) @@ -36,6 +39,10 @@ function run() { # sync_diff can't check non-exist table, so we check expected tables are created in downstream first check_table_exists test.finish_mark ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} 200 check_sync_diff $WORK_DIR $CUR/conf/diff_config.toml + if ! find "$CLAIM_CHECK_DIR" -type f -print -quit | grep -q .; then + echo "claim-check did not write any file to $CLAIM_CHECK_DIR" + exit 1 + fi cleanup_process $CDC_BINARY } diff --git a/tests/integration_tests/canal_json_handle_key_only/run.sh b/tests/integration_tests/canal_json_handle_key_only/run.sh index 372dff24fe..93ae5adfba 100755 --- a/tests/integration_tests/canal_json_handle_key_only/run.sh +++ b/tests/integration_tests/canal_json_handle_key_only/run.sh @@ -19,6 +19,7 @@ function run() { start_tidb_cluster --workdir $WORK_DIR TOPIC_NAME="canal-json-handle-key-only-$RANDOM" + kafka_topic --topic "$TOPIC_NAME" --max-message-bytes 1000 # record tso before we create tables to skip the system table DDLs start_ts=$(run_cdc_cli_tso_query ${UP_PD_HOST_1} ${UP_PD_PORT_1}) diff --git a/tests/integration_tests/kafka_messages/run.sh b/tests/integration_tests/kafka_messages/run.sh index 71ac18774c..168cd3dbdf 100755 --- a/tests/integration_tests/kafka_messages/run.sh +++ b/tests/integration_tests/kafka_messages/run.sh @@ -28,7 +28,8 @@ function run_length_limit() { # Test if TiCDC automatically uses the max-message-bytes of the broker. # See: https://github.com/PingCAP-QE/ci/blob/ddde195ebf4364a0028d53405d1194aa37a4d853/jenkins/pipelines/ci/ticdc/cdc_ghpr_kafka_integration_test.groovy#L178 SINK_URI="kafka://127.0.0.1:9092/$TOPIC_NAME?protocol=open-protocol&partition-num=4&kafka-version=${KAFKA_VERSION}&max-message-bytes=12582912" - cdc_cli_changefeed create --start-ts=$start_ts --sink-uri="$SINK_URI" + changefeed_id="kafka-message-length-limit" + cdc_cli_changefeed create --start-ts=$start_ts --sink-uri="$SINK_URI" -c "$changefeed_id" if [ "$SINK_TYPE" == "kafka" ]; then run_kafka_consumer $WORK_DIR "kafka://127.0.0.1:9092/$TOPIC_NAME?protocol=open-protocol&partition-num=4&version=${KAFKA_VERSION}" fi @@ -59,6 +60,21 @@ function run_length_limit() { check_sync_diff $WORK_DIR $CUR/conf/diff_config.toml + # Verify that max-message-bytes only limits Open Protocol batches. The + # encoded 64 KiB row is larger than T (1 KiB), but smaller than K (256 KiB). + cdc_cli_changefeed pause -c "$changefeed_id" + kafka_topic --topic "$TOPIC_NAME" --max-message-bytes 262144 --alter + SINK_URI="kafka://127.0.0.1:9092/$TOPIC_NAME?protocol=open-protocol&partition-num=4&kafka-version=${KAFKA_VERSION}&max-message-bytes=1024" + cdc_cli_changefeed update -c "$changefeed_id" --sink-uri="$SINK_URI" --no-confirm + cdc_cli_changefeed resume -c "$changefeed_id" + + run_sql "create table kafka_message.usertable_decoupled(id int primary key, payload longtext)" ${UP_TIDB_HOST} ${UP_TIDB_PORT} + run_sql "insert into kafka_message.usertable_decoupled values (1, repeat('x', 65536)), (2, 'small-message-1'), (3, 'small-message-2')" ${UP_TIDB_HOST} ${UP_TIDB_PORT} + run_sql "create table kafka_message.check5(id int primary key)" ${UP_TIDB_HOST} ${UP_TIDB_PORT} + check_table_exists "kafka_message.usertable_decoupled" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} 90 + check_table_exists "kafka_message.check5" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} 90 + check_sync_diff $WORK_DIR $CUR/conf/diff_config.toml + cleanup_process $CDC_BINARY stop_tidb_cluster } diff --git a/tests/integration_tests/kafka_simple_claim_check/run.sh b/tests/integration_tests/kafka_simple_claim_check/run.sh index c8414aee56..a0ff8de268 100755 --- a/tests/integration_tests/kafka_simple_claim_check/run.sh +++ b/tests/integration_tests/kafka_simple_claim_check/run.sh @@ -24,6 +24,8 @@ function run() { run_cdc_server --workdir $WORK_DIR --binary $CDC_BINARY TOPIC_NAME="kafka-simple-claim-check-$RANDOM" + CLAIM_CHECK_DIR="/tmp/kafka-simple-claim-check" + rm -rf "$CLAIM_CHECK_DIR" # record tso before we create tables to skip the system table DDLs start_ts=$(run_cdc_cli_tso_query ${UP_PD_HOST_1} ${UP_PD_PORT_1}) @@ -37,6 +39,7 @@ function run() { cdc_cli_changefeed pause -c ${changefeed_id} + kafka_topic --topic "$TOPIC_NAME" --max-message-bytes 2048 --alter SINK_URI="kafka://127.0.0.1:9092/$TOPIC_NAME?protocol=simple&max-message-bytes=2048" cdc_cli_changefeed update -c ${changefeed_id} --sink-uri="$SINK_URI" --config="$CUR/conf/changefeed.toml" --no-confirm cdc_cli_changefeed resume -c ${changefeed_id} @@ -48,6 +51,10 @@ function run() { # sync_diff can't check non-exist table, so we check expected tables are created in downstream first check_table_exists test.finish_mark ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} 200 check_sync_diff $WORK_DIR $CUR/conf/diff_config.toml + if ! find "$CLAIM_CHECK_DIR" -type f -print -quit | grep -q .; then + echo "claim-check did not write any file to $CLAIM_CHECK_DIR" + exit 1 + fi cleanup_process $CDC_BINARY } diff --git a/tests/integration_tests/kafka_simple_claim_check_avro/run.sh b/tests/integration_tests/kafka_simple_claim_check_avro/run.sh index 259d526d59..ac3626634e 100755 --- a/tests/integration_tests/kafka_simple_claim_check_avro/run.sh +++ b/tests/integration_tests/kafka_simple_claim_check_avro/run.sh @@ -24,6 +24,8 @@ function run() { run_cdc_server --workdir $WORK_DIR --binary $CDC_BINARY TOPIC_NAME="kafka-simple-claim-check-avro-$RANDOM" + CLAIM_CHECK_DIR="/tmp/kafka-simple-avro-claim-check" + rm -rf "$CLAIM_CHECK_DIR" # record tso before we create tables to skip the system table DDLs start_ts=$(run_cdc_cli_tso_query ${UP_PD_HOST_1} ${UP_PD_PORT_1}) @@ -37,6 +39,7 @@ function run() { cdc_cli_changefeed pause -c ${changefeed_id} + kafka_topic --topic "$TOPIC_NAME" --max-message-bytes 2048 --alter SINK_URI="kafka://127.0.0.1:9092/$TOPIC_NAME?protocol=simple&encoding-format=avro&max-message-bytes=2048" cdc_cli_changefeed update -c ${changefeed_id} --sink-uri="$SINK_URI" --config="$CUR/conf/changefeed.toml" --no-confirm cdc_cli_changefeed resume -c ${changefeed_id} @@ -48,6 +51,10 @@ function run() { # sync_diff can't check non-exist table, so we check expected tables are created in downstream first check_table_exists test.finish_mark ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} 200 check_sync_diff $WORK_DIR $CUR/conf/diff_config.toml + if ! find "$CLAIM_CHECK_DIR" -type f -print -quit | grep -q .; then + echo "claim-check did not write any file to $CLAIM_CHECK_DIR" + exit 1 + fi cleanup_process $CDC_BINARY } diff --git a/tests/integration_tests/kafka_simple_handle_key_only/run.sh b/tests/integration_tests/kafka_simple_handle_key_only/run.sh index 32f7ecc6e3..e7b9f38884 100755 --- a/tests/integration_tests/kafka_simple_handle_key_only/run.sh +++ b/tests/integration_tests/kafka_simple_handle_key_only/run.sh @@ -36,6 +36,7 @@ function run() { cdc_cli_changefeed pause -c ${changefeed_id} + kafka_topic --topic "$TOPIC_NAME" --max-message-bytes 700 --alter SINK_URI="kafka://127.0.0.1:9092/$TOPIC_NAME?protocol=simple&max-message-bytes=700" cdc_cli_changefeed update -c ${changefeed_id} --sink-uri="$SINK_URI" --config="$CUR/conf/changefeed.toml" --no-confirm cdc_cli_changefeed resume -c ${changefeed_id} diff --git a/tests/integration_tests/kafka_simple_handle_key_only_avro/run.sh b/tests/integration_tests/kafka_simple_handle_key_only_avro/run.sh index 717d3924d3..94ea60d97b 100755 --- a/tests/integration_tests/kafka_simple_handle_key_only_avro/run.sh +++ b/tests/integration_tests/kafka_simple_handle_key_only_avro/run.sh @@ -36,6 +36,7 @@ function run() { cdc_cli_changefeed pause -c ${changefeed_id} + kafka_topic --topic "$TOPIC_NAME" --max-message-bytes 650 --alter SINK_URI="kafka://127.0.0.1:9092/$TOPIC_NAME?protocol=simple&encoding-format=avro&max-message-bytes=650" cdc_cli_changefeed update -c ${changefeed_id} --sink-uri="$SINK_URI" --config="$CUR/conf/changefeed.toml" --no-confirm cdc_cli_changefeed resume -c ${changefeed_id} diff --git a/tests/integration_tests/open_protocol_claim_check/run.sh b/tests/integration_tests/open_protocol_claim_check/run.sh index 2b262fd3df..be05a800f0 100755 --- a/tests/integration_tests/open_protocol_claim_check/run.sh +++ b/tests/integration_tests/open_protocol_claim_check/run.sh @@ -18,7 +18,10 @@ function run() { start_tidb_cluster --workdir $WORK_DIR - TOPIC_NAME="open-protocol-claim-check" + TOPIC_NAME="open-protocol-claim-check-$RANDOM" + CLAIM_CHECK_DIR="/tmp/open-protocol-claim-check" + rm -rf "$CLAIM_CHECK_DIR" + kafka_topic --topic "$TOPIC_NAME" --max-message-bytes 800 # record tso before we create tables to skip the system table DDLs start_ts=$(run_cdc_cli_tso_query ${UP_PD_HOST_1} ${UP_PD_PORT_1}) @@ -40,6 +43,10 @@ function run() { # sync_diff can't check non-exist table, so we check expected tables are created in downstream first check_table_exists test.finish_mark ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} 200 check_sync_diff $WORK_DIR $CUR/conf/diff_config.toml + if ! find "$CLAIM_CHECK_DIR" -type f -print -quit | grep -q .; then + echo "claim-check did not write any file to $CLAIM_CHECK_DIR" + exit 1 + fi cleanup_process $CDC_BINARY } diff --git a/tests/integration_tests/open_protocol_handle_key_only/run.sh b/tests/integration_tests/open_protocol_handle_key_only/run.sh index a416274e1d..3b714e6761 100755 --- a/tests/integration_tests/open_protocol_handle_key_only/run.sh +++ b/tests/integration_tests/open_protocol_handle_key_only/run.sh @@ -19,6 +19,7 @@ function run() { start_tidb_cluster --workdir $WORK_DIR TOPIC_NAME="open-protocol-handle-key-only-$RANDOM" + kafka_topic --topic "$TOPIC_NAME" --max-message-bytes 800 # record tso before we create tables to skip the system table DDLs start_ts=$(run_cdc_cli_tso_query ${UP_PD_HOST_1} ${UP_PD_PORT_1}) diff --git a/tests/utils/kafka_topic/main.go b/tests/utils/kafka_topic/main.go new file mode 100644 index 0000000000..6227492cf9 --- /dev/null +++ b/tests/utils/kafka_topic/main.go @@ -0,0 +1,68 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "flag" + "log" + "strconv" + "strings" + + "github.com/IBM/sarama" +) + +func main() { + brokers := flag.String("brokers", "127.0.0.1:9092", "Comma-separated Kafka broker addresses.") + topic := flag.String("topic", "", "Kafka topic name.") + maxMessageBytes := flag.Int("max-message-bytes", 0, "Topic max.message.bytes value.") + alter := flag.Bool("alter", false, "Alter an existing topic instead of creating it.") + flag.Parse() + + if *topic == "" { + log.Fatal("topic must not be empty") + } + if *maxMessageBytes <= 0 { + log.Fatal("max-message-bytes must be greater than zero") + } + + value := strconv.Itoa(*maxMessageBytes) + config := sarama.NewConfig() + config.ClientID = "ticdc-integration-test-kafka-topic" + admin, err := sarama.NewClusterAdmin(strings.Split(*brokers, ","), config) + if err != nil { + log.Fatalf("create Kafka admin client: %v", err) + } + defer func() { + if err := admin.Close(); err != nil { + log.Printf("close Kafka admin client: %v", err) + } + }() + + configEntries := map[string]*string{"max.message.bytes": &value} + if *alter { + if err := admin.AlterConfig(sarama.TopicResource, *topic, configEntries, false); err != nil { + log.Fatalf("alter Kafka topic %s: %v", *topic, err) + } + return + } + + detail := &sarama.TopicDetail{ + NumPartitions: 1, + ReplicationFactor: 1, + ConfigEntries: configEntries, + } + if err := admin.CreateTopic(*topic, detail, false); err != nil { + log.Fatalf("create Kafka topic %s: %v", *topic, err) + } +} From 5f3e4a1b41c2180e872101b3f4f0a675ec348082 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Wed, 22 Jul 2026 17:50:22 +0800 Subject: [PATCH 18/22] refactor the integration tests --- .../kafka_big_messages/conf/diff_config.toml | 2 +- .../kafka_big_messages/run.sh | 188 +++++++++++++++--- tests/integration_tests/kafka_messages/run.sh | 18 +- 3 files changed, 158 insertions(+), 50 deletions(-) diff --git a/tests/integration_tests/kafka_big_messages/conf/diff_config.toml b/tests/integration_tests/kafka_big_messages/conf/diff_config.toml index 0082a37028..bf73beb595 100644 --- a/tests/integration_tests/kafka_big_messages/conf/diff_config.toml +++ b/tests/integration_tests/kafka_big_messages/conf/diff_config.toml @@ -13,7 +13,7 @@ source-instances = ["mysql1"] target-instance = "tidb0" -target-check-tables = ["kafka_big_messages.test"] +target-check-tables = ["database_name.*"] [data-sources] [data-sources.mysql1] diff --git a/tests/integration_tests/kafka_big_messages/run.sh b/tests/integration_tests/kafka_big_messages/run.sh index 0628eaa92e..178f1bf9e0 100755 --- a/tests/integration_tests/kafka_big_messages/run.sh +++ b/tests/integration_tests/kafka_big_messages/run.sh @@ -3,53 +3,177 @@ set -eu CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -source $CUR/../_utils/test_prepare +source "$CUR/../_utils/test_prepare" WORK_DIR=$OUT_DIR/$TEST_NAME CDC_BINARY=cdc.test SINK_TYPE=$1 -function run() { - # test kafka sink only in this case - if [ "$SINK_TYPE" != "kafka" ]; then +MAX_RETRIES=60 +BATCH_LIMIT=262144 +SMALL_TOPIC_LIMIT=524288 +LARGE_TOPIC_LIMIT=2097152 +ROW_BYTES=1048576 +SCHEMA_REGISTRY_URI=http://127.0.0.1:8088 +GENERATOR_DIR=$CUR/../../utils/gen_kafka_big_messages +consumer_pid="" + +function start_schema_registry() { + if curl -o /dev/null -s "$SCHEMA_REGISTRY_URI"; then return fi - rm -rf $WORK_DIR && mkdir -p $WORK_DIR - start_tidb_cluster --workdir $WORK_DIR + echo "Starting schema registry..." + ./bin/bin/schema-registry-start -daemon ./bin/etc/schema-registry/schema-registry.properties + local i=0 + while ! curl -o /dev/null -s "$SCHEMA_REGISTRY_URI"; do + i=$((i + 1)) + if [ "$i" -gt 30 ]; then + echo "Failed to start schema registry" + exit 1 + fi + sleep 2 + done + curl -X PUT -H "Content-Type: application/vnd.schemaregistry.v1+json" --data '{"compatibility": "NONE"}' "$SCHEMA_REGISTRY_URI/config" +} - TOPIC_NAME="big-message-test-$RANDOM" +function build_message_generator() { + if [ ! -f "$GENERATOR_DIR/gen_kafka_big_messages" ]; then + (cd "$GENERATOR_DIR" && GO111MODULE=on go build) + fi +} - # record tso before we create tables to skip the system table DDLs - start_ts=$(run_cdc_cli_tso_query $UP_PD_HOST_1 $UP_PD_PORT_1) +function kafka_sink_uri() { + local topic_name=$1 + local protocol=$2 + local extra_params=$3 + local sink_uri="kafka://127.0.0.1:9092/${topic_name}?protocol=${protocol}&partition-num=1&kafka-version=${KAFKA_VERSION}&max-message-bytes=${BATCH_LIMIT}" + if [ "$extra_params" != "" ]; then + sink_uri="${sink_uri}&${extra_params}" + fi + echo "$sink_uri" +} - run_cdc_server --workdir $WORK_DIR --binary $CDC_BINARY +function start_kafka_consumer() { + local work_dir=$1 + local sink_uri=$2 + local schema_registry_uri=$3 + local protocol_case=$4 + local downstream_uri="mysql://root@${DOWN_TIDB_HOST}:${DOWN_TIDB_PORT}/?safe-mode=true&batch-dml-enable=false&enable-ddl-ts=false" + local args=( + --log-file "$work_dir/cdc_kafka_consumer.log" + --log-level debug + --upstream-uri "$sink_uri" + --downstream-uri "$downstream_uri" + ) + if [ "$schema_registry_uri" != "" ]; then + args+=(--schema-registry-uri "$schema_registry_uri") + fi + if [[ "$protocol_case" == simple_* ]]; then + args+=(--upstream-tidb-dsn "root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/?") + fi - # Use a max-message-bytes parameter that is larger than the kafka topic max message bytes. - # Test if TiCDC automatically uses the max-message-bytes of the topic. - # See: https://github.com/PingCAP-QE/ci/blob/ddde195ebf4364a0028d53405d1194aa37a4d853/jenkins/pipelines/ci/ticdc/cdc_ghpr_kafka_integration_test.groovy#L178 - # Use a topic that has already been created. - # See: https://github.com/PingCAP-QE/ci/blob/ddde195ebf4364a0028d53405d1194aa37a4d853/jenkins/pipelines/ci/ticdc/cdc_ghpr_kafka_integration_test.groovy#L180 - SINK_URI="kafka://127.0.0.1:9092/$TOPIC_NAME?protocol=open-protocol&partition-num=1&kafka-version=${KAFKA_VERSION}&max-message-bytes=12582912" - cdc_cli_changefeed create --start-ts=$start_ts --sink-uri="$SINK_URI" - run_kafka_consumer $WORK_DIR "kafka://127.0.0.1:9092/$TOPIC_NAME?protocol=open-protocol&partition-num=1&version=${KAFKA_VERSION}" + cdc_kafka_consumer "${args[@]}" >>"$work_dir/cdc_kafka_consumer_stdout.log" 2>&1 & + consumer_pid=$! +} - echo "Starting generate kafka big messages..." - cd $CUR/../../utils/gen_kafka_big_messages - if [ ! -f ./gen_kafka_big_messages ]; then - GO111MODULE=on go build +function stop_kafka_consumer() { + if [ "$consumer_pid" != "" ]; then + kill "$consumer_pid" 2>/dev/null || true + wait "$consumer_pid" 2>/dev/null || true + consumer_pid="" fi - # Generate data larger than kafka broker max.message.bytes. We can send this data correctly. - ./gen_kafka_big_messages --row-count=15 --sql-file-path=$CUR/test.sql +} + +function render_diff_config() { + local work_dir=$1 + local database_name=$2 + local diff_config=$3 + + sed -e "s/database_name/${database_name}/g" \ + -e "s|/tmp/tidb_cdc_test/kafka_big_messages/sync_diff/output|${work_dir}/sync_diff/output|g" \ + "$CUR/conf/diff_config.toml" >"$diff_config" +} + +function run_protocol_case() { + local protocol_case=$1 + local protocol=$2 + local schema_registry_uri=$3 + local extra_params=$4 + local topic_case=${protocol_case//_/-} + local topic_name="big-message-${topic_case}-${RANDOM}" + local changefeed_id="kafka-big-messages-${topic_case}" + local database_name="kafka_big_messages_${protocol_case}" + local work_dir="$WORK_DIR/$protocol_case" + local sql_file="$work_dir/test.sql" + local diff_config="$work_dir/diff_config.toml" + local pd_addr="http://${UP_PD_HOST_1}:${UP_PD_PORT_1}" + local sink_uri + + mkdir -p "$work_dir" + render_diff_config "$work_dir" "$database_name" "$diff_config" + kafka_topic --topic "$topic_name" --max-message-bytes "$SMALL_TOPIC_LIMIT" + local start_ts + start_ts=$(run_cdc_cli_tso_query "$UP_PD_HOST_1" "$UP_PD_PORT_1") + sink_uri=$(kafka_sink_uri "$topic_name" "$protocol" "$extra_params") + + if [ "$schema_registry_uri" != "" ]; then + cdc_cli_changefeed create --start-ts="$start_ts" --sink-uri="$sink_uri" --schema-registry="$schema_registry_uri" -c "$changefeed_id" + else + cdc_cli_changefeed create --start-ts="$start_ts" --sink-uri="$sink_uri" -c "$changefeed_id" + fi + start_kafka_consumer "$work_dir" "$sink_uri" "$schema_registry_uri" "$protocol_case" + ensure "$MAX_RETRIES" check_changefeed_state "$pd_addr" "$changefeed_id" "normal" "null" "" + + "$GENERATOR_DIR/gen_kafka_big_messages" --row-bytes="$ROW_BYTES" --row-count=1 --database-name="$database_name" --table-name=test --sql-file-path="$sql_file" + run_sql_file "$sql_file" "$UP_TIDB_HOST" "$UP_TIDB_PORT" + run_sql "CREATE TABLE ${database_name}.finish_mark(id INT PRIMARY KEY)" "$UP_TIDB_HOST" "$UP_TIDB_PORT" + + # The encoded row is larger than the topic limit, so the changefeed must + # enter the retryable warning state with ErrMessageTooLarge. + ensure "$MAX_RETRIES" check_changefeed_state "$pd_addr" "$changefeed_id" "warning" "ErrMessageTooLarge" "" + + # Only increase Kafka's topic limit. TiCDC must recreate the sink, read the + # new limit, and resume without updating, pausing, or resuming the changefeed. + kafka_topic --topic "$topic_name" --max-message-bytes "$LARGE_TOPIC_LIMIT" --alter + ensure "$MAX_RETRIES" check_changefeed_state "$pd_addr" "$changefeed_id" "normal" "null" "" + check_table_exists "${database_name}.finish_mark" "$DOWN_TIDB_HOST" "$DOWN_TIDB_PORT" 200 + check_sync_diff "$work_dir" "$diff_config" + + cdc_cli_changefeed remove -c "$changefeed_id" + stop_kafka_consumer +} + +function run() { + # Test Kafka sink only in this case. + if [ "$SINK_TYPE" != "kafka" ]; then + return + fi + + local cases=( + "canal_json|canal-json||enable-tidb-extension=true" + "open_protocol|open-protocol||" + "simple_json|simple||" + "simple_avro|simple||encoding-format=avro" + "avro|avro|$SCHEMA_REGISTRY_URI|enable-tidb-extension=true&avro-enable-watermark=true&avro-decimal-handling-mode=string&avro-bigint-unsigned-handling-mode=string" + ) + + rm -rf "$WORK_DIR" && mkdir -p "$WORK_DIR" + start_schema_registry + build_message_generator + start_tidb_cluster --workdir "$WORK_DIR" + run_cdc_server --workdir "$WORK_DIR" --binary "$CDC_BINARY" - run_sql_file $CUR/test.sql ${UP_TIDB_HOST} ${UP_TIDB_PORT} - table="kafka_big_messages.test" - check_table_exists $table ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} - check_sync_diff $WORK_DIR $CUR/conf/diff_config.toml + local case_entry + for case_entry in "${cases[@]}"; do + local protocol_case protocol schema_registry_uri extra_params + IFS='|' read -r protocol_case protocol schema_registry_uri extra_params <<<"$case_entry" + run_protocol_case "$protocol_case" "$protocol" "$schema_registry_uri" "$extra_params" + done - cleanup_process $CDC_BINARY + cleanup_process "$CDC_BINARY" } -trap 'stop_test $WORK_DIR' EXIT -run $* -check_logs $WORK_DIR +trap 'stop_kafka_consumer; stop_test "$WORK_DIR"' EXIT +run "$@" +check_logs "$WORK_DIR" echo "[$(date)] <<<<<< run test case $TEST_NAME success! >>>>>>" diff --git a/tests/integration_tests/kafka_messages/run.sh b/tests/integration_tests/kafka_messages/run.sh index 168cd3dbdf..71ac18774c 100755 --- a/tests/integration_tests/kafka_messages/run.sh +++ b/tests/integration_tests/kafka_messages/run.sh @@ -28,8 +28,7 @@ function run_length_limit() { # Test if TiCDC automatically uses the max-message-bytes of the broker. # See: https://github.com/PingCAP-QE/ci/blob/ddde195ebf4364a0028d53405d1194aa37a4d853/jenkins/pipelines/ci/ticdc/cdc_ghpr_kafka_integration_test.groovy#L178 SINK_URI="kafka://127.0.0.1:9092/$TOPIC_NAME?protocol=open-protocol&partition-num=4&kafka-version=${KAFKA_VERSION}&max-message-bytes=12582912" - changefeed_id="kafka-message-length-limit" - cdc_cli_changefeed create --start-ts=$start_ts --sink-uri="$SINK_URI" -c "$changefeed_id" + cdc_cli_changefeed create --start-ts=$start_ts --sink-uri="$SINK_URI" if [ "$SINK_TYPE" == "kafka" ]; then run_kafka_consumer $WORK_DIR "kafka://127.0.0.1:9092/$TOPIC_NAME?protocol=open-protocol&partition-num=4&version=${KAFKA_VERSION}" fi @@ -60,21 +59,6 @@ function run_length_limit() { check_sync_diff $WORK_DIR $CUR/conf/diff_config.toml - # Verify that max-message-bytes only limits Open Protocol batches. The - # encoded 64 KiB row is larger than T (1 KiB), but smaller than K (256 KiB). - cdc_cli_changefeed pause -c "$changefeed_id" - kafka_topic --topic "$TOPIC_NAME" --max-message-bytes 262144 --alter - SINK_URI="kafka://127.0.0.1:9092/$TOPIC_NAME?protocol=open-protocol&partition-num=4&kafka-version=${KAFKA_VERSION}&max-message-bytes=1024" - cdc_cli_changefeed update -c "$changefeed_id" --sink-uri="$SINK_URI" --no-confirm - cdc_cli_changefeed resume -c "$changefeed_id" - - run_sql "create table kafka_message.usertable_decoupled(id int primary key, payload longtext)" ${UP_TIDB_HOST} ${UP_TIDB_PORT} - run_sql "insert into kafka_message.usertable_decoupled values (1, repeat('x', 65536)), (2, 'small-message-1'), (3, 'small-message-2')" ${UP_TIDB_HOST} ${UP_TIDB_PORT} - run_sql "create table kafka_message.check5(id int primary key)" ${UP_TIDB_HOST} ${UP_TIDB_PORT} - check_table_exists "kafka_message.usertable_decoupled" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} 90 - check_table_exists "kafka_message.check5" ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} 90 - check_sync_diff $WORK_DIR $CUR/conf/diff_config.toml - cleanup_process $CDC_BINARY stop_tidb_cluster } From d8d366d4cad389a453998ad5ea5ecb2145e8c6af Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Wed, 22 Jul 2026 18:26:45 +0800 Subject: [PATCH 19/22] fix the tests --- .../kafka_simple_claim_check/data/data.sql | 6 ++++-- .../kafka_simple_claim_check_avro/data/data.sql | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/integration_tests/kafka_simple_claim_check/data/data.sql b/tests/integration_tests/kafka_simple_claim_check/data/data.sql index 2730d96d71..c753734f2e 100644 --- a/tests/integration_tests/kafka_simple_claim_check/data/data.sql +++ b/tests/integration_tests/kafka_simple_claim_check/data/data.sql @@ -1,4 +1,6 @@ use test; +-- Keep the encoded row larger than max-message-bytes after Snappy compression, +-- so this case exercises claim-check instead of sending the full row to Kafka. insert into t values ( 1, 1, 2, 3, 4, 5, @@ -7,7 +9,7 @@ insert into t values ( 3.1415, 2.7182, 8000, 179394.233, '2020-02-20', '2020-02-20 02:20:20', '2020-02-20 02:20:20', '02:20:20', '2020', '89504E470D0A1A0A', '89504E470D0A1A0A', '89504E470D0A1A0A', '89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A', - x'89504E470D0A1A0A', x'89504E470D0A1A0A', x'89504E470D0A1A0A', x'89504E470D0A1A0A', + x'89504E470D0A1A0A', RANDOM_BYTES(1024), RANDOM_BYTES(1024), RANDOM_BYTES(1024), '89504E470D0A1A0A', '89504E470D0A1A0A', x'89504E470D0A1A0A', x'89504E470D0A1A0A', 'b', 'b,c', b'1000001', '{ "key1": "value1", @@ -28,7 +30,7 @@ insert into t values ( 3.1415, 2.7182, 8000, 179394.233, '2020-02-20', '2020-02-20 02:20:20', '2020-02-20 02:20:20', '02:20:20', '2020', '89504E470D0A1A0A', '89504E470D0A1A0A', '89504E470D0A1A0A', '89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A', - x'89504E470D0A1A0A', x'89504E470D0A1A0A', x'89504E470D0A1A0A', x'89504E470D0A1A0A', + x'89504E470D0A1A0A', RANDOM_BYTES(1024), RANDOM_BYTES(1024), RANDOM_BYTES(1024), '89504E470D0A1A0A', '89504E470D0A1A0A', x'89504E470D0A1A0A', x'89504E470D0A1A0A', 'b', 'b,c', b'1000001', '{ "key1": "value1", diff --git a/tests/integration_tests/kafka_simple_claim_check_avro/data/data.sql b/tests/integration_tests/kafka_simple_claim_check_avro/data/data.sql index 2730d96d71..c753734f2e 100644 --- a/tests/integration_tests/kafka_simple_claim_check_avro/data/data.sql +++ b/tests/integration_tests/kafka_simple_claim_check_avro/data/data.sql @@ -1,4 +1,6 @@ use test; +-- Keep the encoded row larger than max-message-bytes after Snappy compression, +-- so this case exercises claim-check instead of sending the full row to Kafka. insert into t values ( 1, 1, 2, 3, 4, 5, @@ -7,7 +9,7 @@ insert into t values ( 3.1415, 2.7182, 8000, 179394.233, '2020-02-20', '2020-02-20 02:20:20', '2020-02-20 02:20:20', '02:20:20', '2020', '89504E470D0A1A0A', '89504E470D0A1A0A', '89504E470D0A1A0A', '89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A', - x'89504E470D0A1A0A', x'89504E470D0A1A0A', x'89504E470D0A1A0A', x'89504E470D0A1A0A', + x'89504E470D0A1A0A', RANDOM_BYTES(1024), RANDOM_BYTES(1024), RANDOM_BYTES(1024), '89504E470D0A1A0A', '89504E470D0A1A0A', x'89504E470D0A1A0A', x'89504E470D0A1A0A', 'b', 'b,c', b'1000001', '{ "key1": "value1", @@ -28,7 +30,7 @@ insert into t values ( 3.1415, 2.7182, 8000, 179394.233, '2020-02-20', '2020-02-20 02:20:20', '2020-02-20 02:20:20', '02:20:20', '2020', '89504E470D0A1A0A', '89504E470D0A1A0A', '89504E470D0A1A0A', '89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A89504E470D0A1A0A', - x'89504E470D0A1A0A', x'89504E470D0A1A0A', x'89504E470D0A1A0A', x'89504E470D0A1A0A', + x'89504E470D0A1A0A', RANDOM_BYTES(1024), RANDOM_BYTES(1024), RANDOM_BYTES(1024), '89504E470D0A1A0A', '89504E470D0A1A0A', x'89504E470D0A1A0A', x'89504E470D0A1A0A', 'b', 'b,c', b'1000001', '{ "key1": "value1", From 7dd9f7232328deeeb895a00650e25542ebcfb07b Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Thu, 23 Jul 2026 10:38:46 +0800 Subject: [PATCH 20/22] fix claim check tests --- tests/integration_tests/open_protocol_claim_check/data/data.sql | 1 + tests/integration_tests/open_protocol_claim_check/run.sh | 2 +- .../open_protocol_handle_key_only/data/data.sql | 1 + tests/integration_tests/open_protocol_handle_key_only/run.sh | 2 +- 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/integration_tests/open_protocol_claim_check/data/data.sql b/tests/integration_tests/open_protocol_claim_check/data/data.sql index 14ae2db77e..dbacc55cf2 100644 --- a/tests/integration_tests/open_protocol_claim_check/data/data.sql +++ b/tests/integration_tests/open_protocol_claim_check/data/data.sql @@ -93,6 +93,7 @@ insert into t values ( ); update t set c_float = 3.1415, c_double = 2.7182, c_decimal = 8000, c_decimal_2 = 179394.233 where id = 2; +update t set c_longblob = concat(random_bytes(1024), random_bytes(1024), random_bytes(1024)) where id = 2; begin; diff --git a/tests/integration_tests/open_protocol_claim_check/run.sh b/tests/integration_tests/open_protocol_claim_check/run.sh index be05a800f0..7f9f94e3be 100755 --- a/tests/integration_tests/open_protocol_claim_check/run.sh +++ b/tests/integration_tests/open_protocol_claim_check/run.sh @@ -21,7 +21,7 @@ function run() { TOPIC_NAME="open-protocol-claim-check-$RANDOM" CLAIM_CHECK_DIR="/tmp/open-protocol-claim-check" rm -rf "$CLAIM_CHECK_DIR" - kafka_topic --topic "$TOPIC_NAME" --max-message-bytes 800 + kafka_topic --topic "$TOPIC_NAME" --max-message-bytes 2048 # record tso before we create tables to skip the system table DDLs start_ts=$(run_cdc_cli_tso_query ${UP_PD_HOST_1} ${UP_PD_PORT_1}) diff --git a/tests/integration_tests/open_protocol_handle_key_only/data/data.sql b/tests/integration_tests/open_protocol_handle_key_only/data/data.sql index 2977b9aa12..2c79413baf 100644 --- a/tests/integration_tests/open_protocol_handle_key_only/data/data.sql +++ b/tests/integration_tests/open_protocol_handle_key_only/data/data.sql @@ -93,6 +93,7 @@ insert into t values ( ); update t set c_float = 3.1415, c_double = 2.7182, c_decimal = 8000, c_decimal_2 = 179394.233 where id = 2; +update t set c_longblob = concat(random_bytes(1024), random_bytes(1024), random_bytes(1024)) where id = 2; create table finish_mark ( diff --git a/tests/integration_tests/open_protocol_handle_key_only/run.sh b/tests/integration_tests/open_protocol_handle_key_only/run.sh index 3b714e6761..01aab001dc 100755 --- a/tests/integration_tests/open_protocol_handle_key_only/run.sh +++ b/tests/integration_tests/open_protocol_handle_key_only/run.sh @@ -19,7 +19,7 @@ function run() { start_tidb_cluster --workdir $WORK_DIR TOPIC_NAME="open-protocol-handle-key-only-$RANDOM" - kafka_topic --topic "$TOPIC_NAME" --max-message-bytes 800 + kafka_topic --topic "$TOPIC_NAME" --max-message-bytes 2048 # record tso before we create tables to skip the system table DDLs start_ts=$(run_cdc_cli_tso_query ${UP_PD_HOST_1} ${UP_PD_PORT_1}) From 8655de61f6d8c2eca84d9381cdd8dd15dde8640a Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Thu, 23 Jul 2026 12:48:30 +0800 Subject: [PATCH 21/22] fix integration tests --- .../kafka_big_messages/run.sh | 33 +++++++++++++++---- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/tests/integration_tests/kafka_big_messages/run.sh b/tests/integration_tests/kafka_big_messages/run.sh index 178f1bf9e0..7225decaaa 100755 --- a/tests/integration_tests/kafka_big_messages/run.sh +++ b/tests/integration_tests/kafka_big_messages/run.sh @@ -8,7 +8,9 @@ WORK_DIR=$OUT_DIR/$TEST_NAME CDC_BINARY=cdc.test SINK_TYPE=$1 -MAX_RETRIES=60 +STATE_WAIT_TIMEOUT_SECONDS=30 +STATE_CHECK_INTERVAL_SECONDS=1 +TABLE_CHECK_RETRIES=15 BATCH_LIMIT=262144 SMALL_TOPIC_LIMIT=524288 LARGE_TOPIC_LIMIT=2097152 @@ -78,12 +80,31 @@ function start_kafka_consumer() { function stop_kafka_consumer() { if [ "$consumer_pid" != "" ]; then - kill "$consumer_pid" 2>/dev/null || true + kill -9 "$consumer_pid" 2>/dev/null || true wait "$consumer_pid" 2>/dev/null || true consumer_pid="" fi } +function wait_changefeed_state() { + local pd_addr=$1 + local changefeed_id=$2 + local expected_state=$3 + local expected_error=$4 + local deadline=$((SECONDS + STATE_WAIT_TIMEOUT_SECONDS)) + + while true; do + if check_changefeed_state "$pd_addr" "$changefeed_id" "$expected_state" "$expected_error" ""; then + return + fi + if [ "$SECONDS" -ge "$deadline" ]; then + echo "changefeed $changefeed_id did not reach state $expected_state within ${STATE_WAIT_TIMEOUT_SECONDS}s" + return 1 + fi + sleep "$STATE_CHECK_INTERVAL_SECONDS" + done +} + function render_diff_config() { local work_dir=$1 local database_name=$2 @@ -122,7 +143,7 @@ function run_protocol_case() { cdc_cli_changefeed create --start-ts="$start_ts" --sink-uri="$sink_uri" -c "$changefeed_id" fi start_kafka_consumer "$work_dir" "$sink_uri" "$schema_registry_uri" "$protocol_case" - ensure "$MAX_RETRIES" check_changefeed_state "$pd_addr" "$changefeed_id" "normal" "null" "" + wait_changefeed_state "$pd_addr" "$changefeed_id" "normal" "null" "$GENERATOR_DIR/gen_kafka_big_messages" --row-bytes="$ROW_BYTES" --row-count=1 --database-name="$database_name" --table-name=test --sql-file-path="$sql_file" run_sql_file "$sql_file" "$UP_TIDB_HOST" "$UP_TIDB_PORT" @@ -130,13 +151,13 @@ function run_protocol_case() { # The encoded row is larger than the topic limit, so the changefeed must # enter the retryable warning state with ErrMessageTooLarge. - ensure "$MAX_RETRIES" check_changefeed_state "$pd_addr" "$changefeed_id" "warning" "ErrMessageTooLarge" "" + wait_changefeed_state "$pd_addr" "$changefeed_id" "warning" "ErrMessageTooLarge" # Only increase Kafka's topic limit. TiCDC must recreate the sink, read the # new limit, and resume without updating, pausing, or resuming the changefeed. kafka_topic --topic "$topic_name" --max-message-bytes "$LARGE_TOPIC_LIMIT" --alter - ensure "$MAX_RETRIES" check_changefeed_state "$pd_addr" "$changefeed_id" "normal" "null" "" - check_table_exists "${database_name}.finish_mark" "$DOWN_TIDB_HOST" "$DOWN_TIDB_PORT" 200 + wait_changefeed_state "$pd_addr" "$changefeed_id" "normal" "null" + check_table_exists "${database_name}.finish_mark" "$DOWN_TIDB_HOST" "$DOWN_TIDB_PORT" "$TABLE_CHECK_RETRIES" check_sync_diff "$work_dir" "$diff_config" cdc_cli_changefeed remove -c "$changefeed_id" From 56ac99ed64c763bc293553d5334e1f2074323983 Mon Sep 17 00:00:00 2001 From: 3AceShowHand Date: Thu, 23 Jul 2026 16:05:23 +0800 Subject: [PATCH 22/22] Add comment --- pkg/sink/codec/common/config.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/sink/codec/common/config.go b/pkg/sink/codec/common/config.go index 991841fbb7..4f5cee429d 100644 --- a/pkg/sink/codec/common/config.go +++ b/pkg/sink/codec/common/config.go @@ -42,8 +42,11 @@ type Config struct { Protocol config.Protocol MaxMessageBytes int + + // MaxBatchedBytes controls open-protocol encoder's maximum number of bytes for a batched message. MaxBatchedBytes int - MaxBatchSize int + // MaxBatchedBytes controls open-protocol encoder's maximum number of events for a batched message. + MaxBatchSize int // DeleteOnlyHandleKeyColumns is true, for the delete event only output the handle key columns. DeleteOnlyHandleKeyColumns bool