From bb168220908372c204451629af84e28b4c5ee489 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Mon, 10 Aug 2026 22:00:34 +0800 Subject: [PATCH 01/10] schemastore: fix old table metadata for rename table --- .../persist_storage_ddl_handlers.go | 48 +++++++--- .../schemastore/persist_storage_test.go | 87 +++++++++++++++++++ 2 files changed, 122 insertions(+), 13 deletions(-) diff --git a/logservice/schemastore/persist_storage_ddl_handlers.go b/logservice/schemastore/persist_storage_ddl_handlers.go index 2439fb2974..4efed56659 100644 --- a/logservice/schemastore/persist_storage_ddl_handlers.go +++ b/logservice/schemastore/persist_storage_ddl_handlers.go @@ -798,9 +798,9 @@ func buildPersistedDDLEventForRenameTable(args buildPersistedDDLEventFuncArgs) P // Note: schema id/schema name/table name may be changed or not // table id does not change, we use it to get the table's prev schema id/name and table name event.ExtraSchemaID = getSchemaID(args.tableMap, event.TableID) - // TODO: check how ExtraTableName will be used later event.ExtraTableName = getTableName(args.tableMap, event.TableID) event.ExtraSchemaName = getSchemaName(args.databaseMap, event.ExtraSchemaID) + snapshotExtraSchemaID := event.ExtraSchemaID event.SchemaName = getSchemaName(args.databaseMap, event.SchemaID) // get the table's current table name from the ddl job event.TableName = event.TableInfo.Name.O @@ -811,17 +811,18 @@ func buildPersistedDDLEventForRenameTable(args buildPersistedDDLEventFuncArgs) P // table `test.t`, DDL `rename table t to test2.t;`, commit ts = 100 // snapshot at ts = 99 already shows `t` under `test2` // => event.ExtraSchemaName becomes `test2`, which is wrong for the old name - // SchemaStore can still use ExtraSchemaID to update internal state, - // but the emitted event.Query must carry the correct old names. - // Rebuild them with the following precedence: + // Both Query and the Extra fields must carry the correct old identity, because + // the Extra fields are later used for filtering, blocking, table-name changes, + // and cross-schema updates. Rebuild them with the following precedence: // 1. InvolvingSchemaInfo provides a fallback old schema/table pair, but names may be normalized. - // 2. RenameTableArgs.OldSchemaName overrides the fallback when available. - // It is reliable in TiDB >= v8.5, but can be missing in older versions. + // 2. RenameTableArgs.OldSchemaID identifies the old schema. OldSchemaName overrides the fallback + // when available. OldSchemaName is reliable in TiDB >= v8.5, but can be missing in older versions. // 3. The original query (if it specifies old schema) has the highest priority for identifier case. - // 4. If the query omits old schema and ExtraSchemaID differs from SchemaID, use ExtraSchemaID to + // 4. If the query omits old schema and the snapshot ExtraSchemaID differs from SchemaID, use it to // recover the old schema name from the schema store. oldSchemaName := "" oldTableName := "" + renameArgsOldSchemaID := int64(0) oldSchemaSource := "unknown" if len(args.job.InvolvingSchemaInfo) > 0 { oldSchemaName = args.job.InvolvingSchemaInfo[0].Database @@ -832,6 +833,7 @@ func buildPersistedDDLEventForRenameTable(args buildPersistedDDLEventFuncArgs) P } if args.job.Version == model.JobVersion1 || args.job.Version == model.JobVersion2 { if renameArgs, err := model.GetRenameTableArgs(args.job); err == nil { + renameArgsOldSchemaID = renameArgs.OldSchemaID if renameArgs.OldSchemaName.O != "" { oldSchemaName = renameArgs.OldSchemaName.O oldSchemaSource = "rename_table_args" @@ -854,14 +856,34 @@ func buildPersistedDDLEventForRenameTable(args buildPersistedDDLEventFuncArgs) P oldSchemaSource = "query" } } - // ExtraSchemaID can be incorrect due to snapshot timing, so only use it if the query - // does not specify the old schema. - if !queryProvidedOldSchema && event.ExtraSchemaID != 0 && event.ExtraSchemaID != event.SchemaID { - if extraName := getSchemaName(args.databaseMap, event.ExtraSchemaID); extraName != "" { - oldSchemaName = extraName - oldSchemaSource = "extra_schema_id" + oldSchemaID := int64(0) + if queryProvidedOldSchema { + oldSchemaID, _ = findSchemaIDByName(args.databaseMap, oldSchemaName) + } else if oldSchema, ok := args.databaseMap[renameArgsOldSchemaID]; renameArgsOldSchemaID != 0 && ok { + oldSchemaID = renameArgsOldSchemaID + oldSchemaName = oldSchema.Name + oldSchemaSource = "rename_table_args_old_schema_id" + } else if oldSchema, ok := args.databaseMap[snapshotExtraSchemaID]; snapshotExtraSchemaID != 0 && snapshotExtraSchemaID != event.SchemaID && ok { + oldSchemaID = snapshotExtraSchemaID + oldSchemaName = oldSchema.Name + oldSchemaSource = "extra_schema_id" + } else { + oldSchemaID, _ = findSchemaIDByName(args.databaseMap, oldSchemaName) + } + if !queryProvidedOldSchema { + if oldSchema, ok := args.databaseMap[oldSchemaID]; ok { + // Prefer the schema-store spelling unless the original query explicitly + // provides the old schema identifier. + oldSchemaName = oldSchema.Name } } + if oldSchemaID != 0 && oldSchemaName != "" && oldTableName != "" { + // Keep all representations of the old table identity consistent. In particular, + // do not leave the post-rename snapshot values in the Extra fields. + event.ExtraSchemaID = oldSchemaID + event.ExtraSchemaName = oldSchemaName + event.ExtraTableName = oldTableName + } if oldSchemaName != "" && oldTableName != "" { log.Info("rebuild rename table query", zap.Int64("jobID", event.ID), diff --git a/logservice/schemastore/persist_storage_test.go b/logservice/schemastore/persist_storage_test.go index 378ea0154b..71a677f86d 100644 --- a/logservice/schemastore/persist_storage_test.go +++ b/logservice/schemastore/persist_storage_test.go @@ -3197,6 +3197,93 @@ func TestRenameTable(t *testing.T) { assert.Equal(t, "RENAME TABLE `SalesDB`.`t1` TO `ArchiveDB`.`t1`", ddl.Query) } +func TestRenameTableRepairsOldTableMetadata(t *testing.T) { + t.Run("same schema", func(t *testing.T) { + job := buildRenameTableJobForTest(100, 101, "t2", 100, &model.InvolvingSchemaInfo{ + Database: "test", + Table: "t1", + }) + job.Query = "RENAME TABLE t1 TO t2" + rawEvent := buildPersistedDDLEventForRenameTable(buildPersistedDDLEventFuncArgs{ + job: job, + databaseMap: map[int64]*BasicDatabaseInfo{ + 100: {Name: "test", Tables: map[int64]bool{101: true}}, + }, + // Simulate a snapshot that already contains the post-rename table name. + tableMap: map[int64]*BasicTableInfo{ + 101: {SchemaID: 100, Name: "t2"}, + }, + }) + + require.Equal(t, "RENAME TABLE `test`.`t1` TO `test`.`t2`", rawEvent.Query) + require.Equal(t, int64(100), rawEvent.ExtraSchemaID) + require.Equal(t, "test", rawEvent.ExtraSchemaName) + require.Equal(t, "t1", rawEvent.ExtraTableName) + + ddlEvent, ok, err := buildDDLEventForRenameTable(&rawEvent, nil, 0) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, []commonEvent.SchemaTableName{{SchemaName: "test", TableName: "t1"}}, ddlEvent.BlockedTableNames) + require.Equal(t, &commonEvent.TableNameChange{ + AddName: []commonEvent.SchemaTableName{{SchemaName: "test", TableName: "t2"}}, + DropName: []commonEvent.SchemaTableName{{SchemaName: "test", TableName: "t1"}}, + }, ddlEvent.TableNameChange) + }) + + t.Run("cross schema with old TiDB job args", func(t *testing.T) { + job := buildRenameTableJobForTest(100, 101, "t1", 100, nil) + job.Version = model.JobVersion1 + job.FillArgs(&model.RenameTableArgs{ + OldSchemaID: 200, + NewTableName: ast.NewCIStr("t1"), + }) + _, err := job.Encode(true) + require.NoError(t, err) + job.Query = "RENAME TABLE t1 TO target_db.t1" + rawEvent := buildPersistedDDLEventForRenameTable(buildPersistedDDLEventFuncArgs{ + job: job, + databaseMap: map[int64]*BasicDatabaseInfo{ + 100: {Name: "target_db", Tables: map[int64]bool{101: true}}, + 200: {Name: "source_db", Tables: map[int64]bool{}}, + }, + // Simulate a snapshot that has already moved the table to the new schema. + tableMap: map[int64]*BasicTableInfo{ + 101: {SchemaID: 100, Name: "t1"}, + }, + }) + + require.Equal(t, "RENAME TABLE `source_db`.`t1` TO `target_db`.`t1`", rawEvent.Query) + require.Equal(t, int64(200), rawEvent.ExtraSchemaID) + require.Equal(t, "source_db", rawEvent.ExtraSchemaName) + require.Equal(t, "t1", rawEvent.ExtraTableName) + + ddlEvent, ok, err := buildDDLEventForRenameTable(&rawEvent, nil, 0) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, []commonEvent.SchemaIDChange{{ + TableID: 101, + OldSchemaID: 200, + NewSchemaID: 100, + }}, ddlEvent.UpdatedSchemas) + require.Equal(t, &commonEvent.TableNameChange{ + AddName: []commonEvent.SchemaTableName{{SchemaName: "target_db", TableName: "t1"}}, + DropName: []commonEvent.SchemaTableName{{SchemaName: "source_db", TableName: "t1"}}, + }, ddlEvent.TableNameChange) + + ddlEvent, ok, err = buildDDLEventForRenameTable( + &rawEvent, buildTableFilterByNameForTest("source_db", "*"), 0) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, &commonEvent.InfluencedTables{ + InfluenceType: commonEvent.InfluenceTypeNormal, + TableIDs: []int64{101}, + }, ddlEvent.NeedDroppedTables) + require.Equal(t, &commonEvent.TableNameChange{ + DropName: []commonEvent.SchemaTableName{{SchemaName: "source_db", TableName: "t1"}}, + }, ddlEvent.TableNameChange) + }) +} + func TestBuildPersistedDDLEventForRenameTablesFallbackOldTableName(t *testing.T) { job := buildRenameTablesJobForTest( []int64{100, 100}, From ac4f9927fb3e42f99d0c85de3c6b157e9482fe25 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Tue, 11 Aug 2026 12:56:58 +0800 Subject: [PATCH 02/10] more refactor --- .../persist_storage_ddl_handlers.go | 118 +++++++++++------- 1 file changed, 70 insertions(+), 48 deletions(-) diff --git a/logservice/schemastore/persist_storage_ddl_handlers.go b/logservice/schemastore/persist_storage_ddl_handlers.go index 4efed56659..80758c11e3 100644 --- a/logservice/schemastore/persist_storage_ddl_handlers.go +++ b/logservice/schemastore/persist_storage_ddl_handlers.go @@ -795,35 +795,44 @@ func buildPersistedDDLEventForTruncateTable(args buildPersistedDDLEventFuncArgs) func buildPersistedDDLEventForRenameTable(args buildPersistedDDLEventFuncArgs) PersistedDDLEvent { event := buildPersistedDDLEventCommon(args) - // Note: schema id/schema name/table name may be changed or not - // table id does not change, we use it to get the table's prev schema id/name and table name - event.ExtraSchemaID = getSchemaID(args.tableMap, event.TableID) - event.ExtraTableName = getTableName(args.tableMap, event.TableID) - event.ExtraSchemaName = getSchemaName(args.databaseMap, event.ExtraSchemaID) - snapshotExtraSchemaID := event.ExtraSchemaID event.SchemaName = getSchemaName(args.databaseMap, event.SchemaID) - // get the table's current table name from the ddl job event.TableName = event.TableInfo.Name.O - // The old schema/table names cannot rely on ExtraSchemaName/ExtraTableName, - // because the snapshot used by schema store may already reflect the post-rename state. - // Example (after https://github.com/pingcap/tidb/pull/43341): - // table `test.t`, DDL `rename table t to test2.t;`, commit ts = 100 - // snapshot at ts = 99 already shows `t` under `test2` - // => event.ExtraSchemaName becomes `test2`, which is wrong for the old name - // Both Query and the Extra fields must carry the correct old identity, because - // the Extra fields are later used for filtering, blocking, table-name changes, - // and cross-schema updates. Rebuild them with the following precedence: - // 1. InvolvingSchemaInfo provides a fallback old schema/table pair, but names may be normalized. - // 2. RenameTableArgs.OldSchemaID identifies the old schema. OldSchemaName overrides the fallback - // when available. OldSchemaName is reliable in TiDB >= v8.5, but can be missing in older versions. - // 3. The original query (if it specifies old schema) has the highest priority for identifier case. - // 4. If the query omits old schema and the snapshot ExtraSchemaID differs from SchemaID, use it to - // recover the old schema name from the schema store. + // Why the old table identity must be recovered instead of being read directly from tableMap: + // suppose `RENAME TABLE test.t1 TO test2.t2` commits at ts=100 and SchemaStore starts + // from ts=99. SchemaStore first loads a TiDB metadata snapshot at ts=99, then replays + // DDL jobs after that snapshot. Since https://github.com/pingcap/tidb/pull/43341, the + // snapshot at ts=99 may already contain the post-rename table `test2.t2`. Therefore, + // tableMap[event.TableID] is not guaranteed to describe the table before this DDL. + // Using it directly would make ExtraSchemaID/ExtraSchemaName/ExtraTableName describe + // the new table, although these fields are consumed as the old table identity by DDL + // filtering, barrier construction, table-name changes, and cross-schema updates. + // + // Recover the old identity from independent fields in the DDL job: + // 1. InvolvingSchemaInfo[0] contains the old schema/table names. TiDB deliberately + // stores these values in lower case (Schema.Name.L and Table.Name.L), so they are + // useful for lookup but do not preserve the original identifier capitalization. + // 2. RenameTableArgs.OldSchemaID identifies the old schema. OldSchemaName preserves + // its original capitalization. TiDB has included both fields in rename-table job + // args since v5.3.0, so they are available in every supported TiDB version (v7.5.0+). + // 3. TiDB always records the original rename SQL in job.Query. It always contains the + // old table name, but the old schema name is optional. Prefer names parsed from SQL + // because they preserve the identifier capitalization written by the user. + // 4. Complete the old schema identity with databaseMap: look up the schema ID when only + // its name is known, or look up its name when only the ID is known. If the original + // SQL explicitly specifies the old schema name, preserve that spelling. + // + // Rename jobs from supported TiDB versions provide enough information to recover all three + // old identity fields. tableMap is only a defensive fallback for a malformed or unsupported + // job whose args cannot be decoded or whose SQL cannot be parsed. If this fallback is used + // with a post-rename snapshot, the Extra fields may describe the new table and cause incorrect + // filter or table-name-store behavior. + oldSchemaID := int64(0) oldSchemaName := "" oldTableName := "" - renameArgsOldSchemaID := int64(0) oldSchemaSource := "unknown" + + // Start with the lower-case old schema/table names recorded for DDL dependency checks. if len(args.job.InvolvingSchemaInfo) > 0 { oldSchemaName = args.job.InvolvingSchemaInfo[0].Database oldTableName = args.job.InvolvingSchemaInfo[0].Table @@ -831,9 +840,14 @@ func buildPersistedDDLEventForRenameTable(args buildPersistedDDLEventFuncArgs) P oldSchemaSource = "involving_schema_info" } } + + // Recover the authoritative old schema ID and its case-preserving name from job args. if args.job.Version == model.JobVersion1 || args.job.Version == model.JobVersion2 { if renameArgs, err := model.GetRenameTableArgs(args.job); err == nil { - renameArgsOldSchemaID = renameArgs.OldSchemaID + oldSchemaID = renameArgs.OldSchemaID + if oldSchemaID != 0 { + oldSchemaSource = "rename_table_args_old_schema_id" + } if renameArgs.OldSchemaName.O != "" { oldSchemaName = renameArgs.OldSchemaName.O oldSchemaSource = "rename_table_args" @@ -845,45 +859,53 @@ func buildPersistedDDLEventForRenameTable(args buildPersistedDDLEventFuncArgs) P zap.Error(err)) } } - queryProvidedOldSchema := false - if queryInfo, parsed := parseRenameTableQueryInfo(args.job.Query); parsed { + + // Recover the old table name from SQL, and prefer its schema spelling when explicitly present. + queryInfo, parsed := parseRenameTableQueryInfo(args.job.Query) + if parsed { if queryInfo.oldTableName != "" { oldTableName = queryInfo.oldTableName } if queryInfo.oldSchemaName != "" { oldSchemaName = queryInfo.oldSchemaName - queryProvidedOldSchema = true oldSchemaSource = "query" } } - oldSchemaID := int64(0) - if queryProvidedOldSchema { - oldSchemaID, _ = findSchemaIDByName(args.databaseMap, oldSchemaName) - } else if oldSchema, ok := args.databaseMap[renameArgsOldSchemaID]; renameArgsOldSchemaID != 0 && ok { - oldSchemaID = renameArgsOldSchemaID - oldSchemaName = oldSchema.Name - oldSchemaSource = "rename_table_args_old_schema_id" - } else if oldSchema, ok := args.databaseMap[snapshotExtraSchemaID]; snapshotExtraSchemaID != 0 && snapshotExtraSchemaID != event.SchemaID && ok { - oldSchemaID = snapshotExtraSchemaID - oldSchemaName = oldSchema.Name - oldSchemaSource = "extra_schema_id" - } else { + + // Complete a missing old schema ID or name through databaseMap. + if oldSchemaID == 0 && oldSchemaName != "" { oldSchemaID, _ = findSchemaIDByName(args.databaseMap, oldSchemaName) } - if !queryProvidedOldSchema { + if queryInfo.oldSchemaName == "" { if oldSchema, ok := args.databaseMap[oldSchemaID]; ok { - // Prefer the schema-store spelling unless the original query explicitly - // provides the old schema identifier. + // SQL does not provide the old schema spelling. Use databaseMap to replace + // the lower-case InvolvingSchemaInfo name with its original capitalization. oldSchemaName = oldSchema.Name } } - if oldSchemaID != 0 && oldSchemaName != "" && oldTableName != "" { - // Keep all representations of the old table identity consistent. In particular, - // do not leave the post-rename snapshot values in the Extra fields. - event.ExtraSchemaID = oldSchemaID - event.ExtraSchemaName = oldSchemaName - event.ExtraTableName = oldTableName + + // Fall back to snapshot metadata only when the DDL job cannot provide a complete identity. + if oldSchemaID == 0 || oldSchemaName == "" || oldTableName == "" { + snapshotSchemaID := getSchemaID(args.tableMap, event.TableID) + snapshotTableName := getTableName(args.tableMap, event.TableID) + if oldSchemaID == 0 { + oldSchemaID = snapshotSchemaID + oldSchemaSource = "table_map_fallback" + } + if oldSchemaName == "" { + oldSchemaName = getSchemaName(args.databaseMap, oldSchemaID) + } + if oldTableName == "" { + oldTableName = snapshotTableName + } } + + // Persist the recovered old identity for downstream filtering and coordination. + event.ExtraSchemaID = oldSchemaID + event.ExtraSchemaName = oldSchemaName + event.ExtraTableName = oldTableName + + // Keep the executable query consistent with the recovered structured metadata. if oldSchemaName != "" && oldTableName != "" { log.Info("rebuild rename table query", zap.Int64("jobID", event.ID), From f3a6285707194d66e3859fa38c7ddfa49c5e28d9 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Tue, 11 Aug 2026 18:57:10 +0800 Subject: [PATCH 03/10] address comment --- .../persist_storage_ddl_handlers.go | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/logservice/schemastore/persist_storage_ddl_handlers.go b/logservice/schemastore/persist_storage_ddl_handlers.go index 80758c11e3..4543cd9f9d 100644 --- a/logservice/schemastore/persist_storage_ddl_handlers.go +++ b/logservice/schemastore/persist_storage_ddl_handlers.go @@ -884,19 +884,21 @@ func buildPersistedDDLEventForRenameTable(args buildPersistedDDLEventFuncArgs) P } } - // Fall back to snapshot metadata only when the DDL job cannot provide a complete identity. - if oldSchemaID == 0 || oldSchemaName == "" || oldTableName == "" { - snapshotSchemaID := getSchemaID(args.tableMap, event.TableID) - snapshotTableName := getTableName(args.tableMap, event.TableID) - if oldSchemaID == 0 { - oldSchemaID = snapshotSchemaID - oldSchemaSource = "table_map_fallback" - } - if oldSchemaName == "" { - oldSchemaName = getSchemaName(args.databaseMap, oldSchemaID) - } - if oldTableName == "" { - oldTableName = snapshotTableName + // TiDB v7.5+ rename jobs provide the old schema through job args and the old table + // through SQL, so they do not normally need the fallbacks below. Keep them only for + // malformed jobs or jobs created by unsupported TiDB versions. + if oldSchemaID == 0 { + oldSchemaID = getSchemaID(args.tableMap, event.TableID) + oldSchemaSource = "table_map_fallback" + } + if oldTableName == "" { + oldTableName = getTableName(args.tableMap, event.TableID) + } + if oldSchemaName == "" { + // The old schema may not be tracked. Keep the name empty instead of calling + // getSchemaName, which panics for an unknown schema ID. + if oldSchema, ok := args.databaseMap[oldSchemaID]; ok { + oldSchemaName = oldSchema.Name } } From 72a8515739149160afee2ccb7ae5b77a7b383f35 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Wed, 12 Aug 2026 12:19:49 +0800 Subject: [PATCH 04/10] refactor --- .../persist_storage_ddl_handlers.go | 59 ++++++------- .../schemastore/persist_storage_test.go | 82 ++++++++++++++++++- 2 files changed, 107 insertions(+), 34 deletions(-) diff --git a/logservice/schemastore/persist_storage_ddl_handlers.go b/logservice/schemastore/persist_storage_ddl_handlers.go index 4543cd9f9d..2e1589b8f4 100644 --- a/logservice/schemastore/persist_storage_ddl_handlers.go +++ b/logservice/schemastore/persist_storage_ddl_handlers.go @@ -830,27 +830,19 @@ func buildPersistedDDLEventForRenameTable(args buildPersistedDDLEventFuncArgs) P oldSchemaID := int64(0) oldSchemaName := "" oldTableName := "" - oldSchemaSource := "unknown" // Start with the lower-case old schema/table names recorded for DDL dependency checks. if len(args.job.InvolvingSchemaInfo) > 0 { oldSchemaName = args.job.InvolvingSchemaInfo[0].Database oldTableName = args.job.InvolvingSchemaInfo[0].Table - if oldSchemaName != "" { - oldSchemaSource = "involving_schema_info" - } } // Recover the authoritative old schema ID and its case-preserving name from job args. if args.job.Version == model.JobVersion1 || args.job.Version == model.JobVersion2 { if renameArgs, err := model.GetRenameTableArgs(args.job); err == nil { oldSchemaID = renameArgs.OldSchemaID - if oldSchemaID != 0 { - oldSchemaSource = "rename_table_args_old_schema_id" - } if renameArgs.OldSchemaName.O != "" { oldSchemaName = renameArgs.OldSchemaName.O - oldSchemaSource = "rename_table_args" } } else { log.Warn("failed to get rename table args from ddl job", @@ -860,15 +852,28 @@ func buildPersistedDDLEventForRenameTable(args buildPersistedDDLEventFuncArgs) P } } - // Recover the old table name from SQL, and prefer its schema spelling when explicitly present. - queryInfo, parsed := parseRenameTableQueryInfo(args.job.Query) + // Recover the old table name from the normalized SQL, and prefer its schema spelling + // when it identifies the same schema as the DDL job args. event.Query has already been + // parsed with job.SQLMode and restored by buildPersistedDDLEventCommon. + queryInfo, parsed := parseRenameTableQueryInfo(event.Query) if parsed { if queryInfo.oldTableName != "" { oldTableName = queryInfo.oldTableName } if queryInfo.oldSchemaName != "" { - oldSchemaName = queryInfo.oldSchemaName - oldSchemaSource = "query" + queryOldSchemaID, _ := findSchemaIDByName(args.databaseMap, queryInfo.oldSchemaName) + if oldSchemaID != 0 && oldSchemaID != queryOldSchemaID { + log.Warn("rename table old schema is inconsistent between job args and query", + zap.Int64("jobID", args.job.ID), + zap.Int64("argsOldSchemaID", oldSchemaID), + zap.Int64("queryOldSchemaID", queryOldSchemaID), + zap.String("queryOldSchemaName", queryInfo.oldSchemaName), + zap.String("query", event.Query)) + oldSchemaName = getSchemaName(args.databaseMap, oldSchemaID) + } else { + oldSchemaID = queryOldSchemaID + oldSchemaName = queryInfo.oldSchemaName + } } } @@ -877,30 +882,21 @@ func buildPersistedDDLEventForRenameTable(args buildPersistedDDLEventFuncArgs) P oldSchemaID, _ = findSchemaIDByName(args.databaseMap, oldSchemaName) } if queryInfo.oldSchemaName == "" { - if oldSchema, ok := args.databaseMap[oldSchemaID]; ok { + if oldSchemaID != 0 { // SQL does not provide the old schema spelling. Use databaseMap to replace // the lower-case InvolvingSchemaInfo name with its original capitalization. - oldSchemaName = oldSchema.Name + oldSchemaName = getSchemaName(args.databaseMap, oldSchemaID) } } // TiDB v7.5+ rename jobs provide the old schema through job args and the old table - // through SQL, so they do not normally need the fallbacks below. Keep them only for - // malformed jobs or jobs created by unsupported TiDB versions. - if oldSchemaID == 0 { + // through SQL. For malformed or unsupported jobs, fall back to the complete snapshot + // identity instead of combining fields from different sources. + if oldSchemaID == 0 || oldSchemaName == "" || oldTableName == "" { oldSchemaID = getSchemaID(args.tableMap, event.TableID) - oldSchemaSource = "table_map_fallback" - } - if oldTableName == "" { + oldSchemaName = getSchemaName(args.databaseMap, oldSchemaID) oldTableName = getTableName(args.tableMap, event.TableID) } - if oldSchemaName == "" { - // The old schema may not be tracked. Keep the name empty instead of calling - // getSchemaName, which panics for an unknown schema ID. - if oldSchema, ok := args.databaseMap[oldSchemaID]; ok { - oldSchemaName = oldSchema.Name - } - } // Persist the recovered old identity for downstream filtering and coordination. event.ExtraSchemaID = oldSchemaID @@ -908,7 +904,7 @@ func buildPersistedDDLEventForRenameTable(args buildPersistedDDLEventFuncArgs) P event.ExtraTableName = oldTableName // Keep the executable query consistent with the recovered structured metadata. - if oldSchemaName != "" && oldTableName != "" { + if event.ExtraSchemaName != "" && event.ExtraTableName != "" { log.Info("rebuild rename table query", zap.Int64("jobID", event.ID), zap.String("query", event.Query), @@ -917,12 +913,9 @@ func buildPersistedDDLEventForRenameTable(args buildPersistedDDLEventFuncArgs) P zap.String("tableName", event.TableName), zap.Int64("extraSchemaID", event.ExtraSchemaID), zap.String("extraSchemaName", event.ExtraSchemaName), - zap.String("extraTableName", event.ExtraTableName), - zap.String("oldSchemaName", oldSchemaName), - zap.String("oldTableName", oldTableName), - zap.String("oldSchemaSource", oldSchemaSource)) + zap.String("extraTableName", event.ExtraTableName)) event.Query = fmt.Sprintf("RENAME TABLE %s TO %s", - common.QuoteSchema(oldSchemaName, oldTableName), + common.QuoteSchema(event.ExtraSchemaName, event.ExtraTableName), common.QuoteSchema(event.SchemaName, event.TableName)) } return event diff --git a/logservice/schemastore/persist_storage_test.go b/logservice/schemastore/persist_storage_test.go index 71a677f86d..ad5d2719a6 100644 --- a/logservice/schemastore/persist_storage_test.go +++ b/logservice/schemastore/persist_storage_test.go @@ -29,6 +29,7 @@ import ( "github.com/pingcap/tidb/pkg/meta/model" "github.com/pingcap/tidb/pkg/parser/ast" "github.com/pingcap/tidb/pkg/parser/charset" + "github.com/pingcap/tidb/pkg/parser/mysql" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap" @@ -3282,6 +3283,84 @@ func TestRenameTableRepairsOldTableMetadata(t *testing.T) { DropName: []commonEvent.SchemaTableName{{SchemaName: "source_db", TableName: "t1"}}, }, ddlEvent.TableNameChange) }) + + t.Run("parse normalized query with ANSI quotes", func(t *testing.T) { + job := buildRenameTableJobForTest(100, 101, "NewTable", 100, &model.InvolvingSchemaInfo{ + Database: "sourcedb", + Table: "oldtable", + }) + job.Version = model.JobVersion2 + job.SQLMode = mysql.ModeANSIQuotes + job.FillArgs(&model.RenameTableArgs{ + OldSchemaID: 200, + OldSchemaName: ast.NewCIStr("SourceDB"), + NewTableName: ast.NewCIStr("NewTable"), + }) + job.Query = `RENAME TABLE "SourceDB"."OldTable" TO "TargetDB"."NewTable"` + + rawEvent := buildPersistedDDLEventForRenameTable(buildPersistedDDLEventFuncArgs{ + job: job, + databaseMap: map[int64]*BasicDatabaseInfo{ + 100: {Name: "TargetDB", Tables: map[int64]bool{101: true}}, + 200: {Name: "SourceDB", Tables: map[int64]bool{}}, + }, + tableMap: map[int64]*BasicTableInfo{ + 101: {SchemaID: 100, Name: "NewTable"}, + }, + }) + + require.Equal(t, int64(200), rawEvent.ExtraSchemaID) + require.Equal(t, "SourceDB", rawEvent.ExtraSchemaName) + require.Equal(t, "OldTable", rawEvent.ExtraTableName) + require.Equal(t, "RENAME TABLE `SourceDB`.`OldTable` TO `TargetDB`.`NewTable`", rawEvent.Query) + }) + + t.Run("prefer job args when query schema ID is inconsistent", func(t *testing.T) { + job := buildRenameTableJobForTest(100, 101, "target_t", 100, nil) + job.Version = model.JobVersion2 + job.FillArgs(&model.RenameTableArgs{ + OldSchemaID: 200, + OldSchemaName: ast.NewCIStr("source_db"), + NewTableName: ast.NewCIStr("target_t"), + }) + job.Query = "RENAME TABLE wrong_db.source_t TO target_db.target_t" + + rawEvent := buildPersistedDDLEventForRenameTable(buildPersistedDDLEventFuncArgs{ + job: job, + databaseMap: map[int64]*BasicDatabaseInfo{ + 100: {Name: "target_db", Tables: map[int64]bool{101: true}}, + 200: {Name: "source_db", Tables: map[int64]bool{}}, + 300: {Name: "wrong_db", Tables: map[int64]bool{}}, + }, + // The snapshot contains the post-rename identity and must not be mixed in. + tableMap: map[int64]*BasicTableInfo{ + 101: {SchemaID: 100, Name: "target_t"}, + }, + }) + + require.Equal(t, int64(200), rawEvent.ExtraSchemaID) + require.Equal(t, "source_db", rawEvent.ExtraSchemaName) + require.Equal(t, "source_t", rawEvent.ExtraTableName) + require.Equal(t, "RENAME TABLE `source_db`.`source_t` TO `target_db`.`target_t`", rawEvent.Query) + }) + + t.Run("fall back to complete snapshot identity", func(t *testing.T) { + job := buildRenameTableJobForTest(100, 101, "target_t", 100, nil) + + rawEvent := buildPersistedDDLEventForRenameTable(buildPersistedDDLEventFuncArgs{ + job: job, + databaseMap: map[int64]*BasicDatabaseInfo{ + 100: {Name: "snapshot_db", Tables: map[int64]bool{101: true}}, + }, + tableMap: map[int64]*BasicTableInfo{ + 101: {SchemaID: 100, Name: "snapshot_t"}, + }, + }) + + require.Equal(t, int64(100), rawEvent.ExtraSchemaID) + require.Equal(t, "snapshot_db", rawEvent.ExtraSchemaName) + require.Equal(t, "snapshot_t", rawEvent.ExtraTableName) + }) } func TestBuildPersistedDDLEventForRenameTablesFallbackOldTableName(t *testing.T) { @@ -3484,9 +3563,10 @@ func TestBuildPersistedDDLEventEscapesIdentifiers(t *testing.T) { job: job, databaseMap: map[int64]*BasicDatabaseInfo{ 100: {Name: "target`db", Tables: map[int64]bool{101: true}}, + 200: {Name: "source`db", Tables: map[int64]bool{}}, }, tableMap: map[int64]*BasicTableInfo{ - 101: {SchemaID: 100, Name: "source`t"}, + 101: {SchemaID: 200, Name: "source`t"}, }, }) From b60fbf50c8244c03a94653c5798c127b16afa5f0 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Wed, 12 Aug 2026 12:52:04 +0800 Subject: [PATCH 05/10] small fix --- .../schemastore/persist_storage_ddl_handlers.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/logservice/schemastore/persist_storage_ddl_handlers.go b/logservice/schemastore/persist_storage_ddl_handlers.go index 2e1589b8f4..50fff85a84 100644 --- a/logservice/schemastore/persist_storage_ddl_handlers.go +++ b/logservice/schemastore/persist_storage_ddl_handlers.go @@ -830,11 +830,15 @@ func buildPersistedDDLEventForRenameTable(args buildPersistedDDLEventFuncArgs) P oldSchemaID := int64(0) oldSchemaName := "" oldTableName := "" + oldSchemaSource := "unknown" // Start with the lower-case old schema/table names recorded for DDL dependency checks. if len(args.job.InvolvingSchemaInfo) > 0 { oldSchemaName = args.job.InvolvingSchemaInfo[0].Database oldTableName = args.job.InvolvingSchemaInfo[0].Table + if oldSchemaName != "" { + oldSchemaSource = "involving_schema_info" + } } // Recover the authoritative old schema ID and its case-preserving name from job args. @@ -844,6 +848,9 @@ func buildPersistedDDLEventForRenameTable(args buildPersistedDDLEventFuncArgs) P if renameArgs.OldSchemaName.O != "" { oldSchemaName = renameArgs.OldSchemaName.O } + if oldSchemaID != 0 || renameArgs.OldSchemaName.O != "" { + oldSchemaSource = "rename_table_args" + } } else { log.Warn("failed to get rename table args from ddl job", zap.Int64("jobID", args.job.ID), @@ -873,6 +880,7 @@ func buildPersistedDDLEventForRenameTable(args buildPersistedDDLEventFuncArgs) P } else { oldSchemaID = queryOldSchemaID oldSchemaName = queryInfo.oldSchemaName + oldSchemaSource = "query" } } } @@ -896,6 +904,7 @@ func buildPersistedDDLEventForRenameTable(args buildPersistedDDLEventFuncArgs) P oldSchemaID = getSchemaID(args.tableMap, event.TableID) oldSchemaName = getSchemaName(args.databaseMap, oldSchemaID) oldTableName = getTableName(args.tableMap, event.TableID) + oldSchemaSource = "table_map_fallback" } // Persist the recovered old identity for downstream filtering and coordination. @@ -913,7 +922,8 @@ func buildPersistedDDLEventForRenameTable(args buildPersistedDDLEventFuncArgs) P zap.String("tableName", event.TableName), zap.Int64("extraSchemaID", event.ExtraSchemaID), zap.String("extraSchemaName", event.ExtraSchemaName), - zap.String("extraTableName", event.ExtraTableName)) + zap.String("extraTableName", event.ExtraTableName), + zap.String("oldSchemaSource", oldSchemaSource)) event.Query = fmt.Sprintf("RENAME TABLE %s TO %s", common.QuoteSchema(event.ExtraSchemaName, event.ExtraTableName), common.QuoteSchema(event.SchemaName, event.TableName)) From fb7496c3d43ea6966d80f7271890e2bab521ce1e Mon Sep 17 00:00:00 2001 From: lidezhu Date: Wed, 12 Aug 2026 16:45:48 +0800 Subject: [PATCH 06/10] add integration test --- .../conf/changefeed.toml | 6 ++ .../rename_table_start_ts/run.sh | 91 +++++++++++++++++++ .../rename_table_start_ts/set_gc_safepoint.go | 44 +++++++++ tests/integration_tests/run_light_it_in_ci.sh | 2 +- 4 files changed, 142 insertions(+), 1 deletion(-) create mode 100644 tests/integration_tests/rename_table_start_ts/conf/changefeed.toml create mode 100755 tests/integration_tests/rename_table_start_ts/run.sh create mode 100644 tests/integration_tests/rename_table_start_ts/set_gc_safepoint.go diff --git a/tests/integration_tests/rename_table_start_ts/conf/changefeed.toml b/tests/integration_tests/rename_table_start_ts/conf/changefeed.toml new file mode 100644 index 0000000000..766dc8af2c --- /dev/null +++ b/tests/integration_tests/rename_table_start_ts/conf/changefeed.toml @@ -0,0 +1,6 @@ +[filter] +rules = ["rename_table_start_ts_source.*", "rename_table_start_ts_target.*"] + +[[filter.event-filters]] +matcher = ["rename_table_start_ts_source.t"] +ignore-event = ["rename table"] diff --git a/tests/integration_tests/rename_table_start_ts/run.sh b/tests/integration_tests/rename_table_start_ts/run.sh new file mode 100755 index 0000000000..ef210d8902 --- /dev/null +++ b/tests/integration_tests/rename_table_start_ts/run.sh @@ -0,0 +1,91 @@ +#!/bin/bash + +set -eu + +CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +source "$CUR/../_utils/test_prepare" +WORK_DIR="$OUT_DIR/$TEST_NAME" +CDC_BINARY=cdc.test +SINK_TYPE=$1 + +function run() { + if [ "$SINK_TYPE" != "mysql" ]; then + return + fi + + rm -rf "$WORK_DIR" + mkdir -p "$WORK_DIR" + start_tidb_cluster --workdir "$WORK_DIR" + + local source_db=rename_table_start_ts_source + local target_db=rename_table_start_ts_target + local table_name=t + local changefeed_id=rename-table-start-ts + local gc_worker_key + local gc_worker_value + local pd_cluster_id + local rename_finished_ts + local start_ts + local table_id + + run_sql "CREATE DATABASE $source_db; CREATE DATABASE $target_db; CREATE TABLE $source_db.$table_name (id INT PRIMARY KEY); INSERT INTO $source_db.$table_name VALUES (1);" "$UP_TIDB_HOST" "$UP_TIDB_PORT" + run_sql "CREATE DATABASE $source_db; CREATE DATABASE $target_db; CREATE TABLE $source_db.$table_name (id INT PRIMARY KEY); INSERT INTO $source_db.$table_name VALUES (1);" "$DOWN_TIDB_HOST" "$DOWN_TIDB_PORT" + + table_id=$(get_table_id "$source_db" "$table_name") + run_sql "RENAME TABLE $source_db.$table_name TO $target_db.$table_name;" "$UP_TIDB_HOST" "$UP_TIDB_PORT" + + run_cdc_server --workdir "$WORK_DIR" --binary "$CDC_BINARY" --logsuffix _probe + ensure 30 "grep 'write ddl event' '$WORK_DIR/cdc_probe.log' | grep 'tableID=$table_id' | grep -q 'RENAME TABLE'" + rename_finished_ts=$(grep "write ddl event" "$WORK_DIR/cdc_probe.log" | + grep "tableID=$table_id" | + grep "RENAME TABLE" | + head -n 1 | + grep -oE 'finishedTs=[0-9]+' | + cut -d= -f2) + if ! [[ "$rename_finished_ts" =~ ^[0-9]+$ ]]; then + echo "failed to get rename table finishedTs" + exit 1 + fi + start_ts=$((rename_finished_ts - 1)) + cleanup_process "$CDC_BINARY" + + # Force SchemaStore to initialize from the same snapshot used by the + # changefeed. Restore the GC worker safepoint before creating the changefeed + # so its start-ts is still considered readable. + pd_cluster_id=$(curl -s "http://$UP_PD_HOST_1:$UP_PD_PORT_1/pd/api/v1/cluster" | + grep -oE '"id":[[:space:]]*[0-9]+' | + grep -oE '[0-9]+') + gc_worker_key="/pd/$pd_cluster_id/gc/safe_point/service/gc_worker" + gc_worker_value=$(curl -fsS "http://$UP_PD_HOST_1:$UP_PD_PORT_1/pd/api/v1/gc/safepoint" | + jq -cer '.service_gc_safe_points[] | select(.service_id == "gc_worker")') + if [ -z "$gc_worker_value" ]; then + echo "failed to get gc_worker service safepoint" + exit 1 + fi + GO111MODULE=on go run "$CUR/set_gc_safepoint.go" "$UP_PD_HOST_1:$UP_PD_PORT_1" "$gc_worker_key" \ + "{\"service_id\":\"gc_worker\",\"expired_at\":9223372036854775807,\"safe_point\":$start_ts}" + + run_cdc_server --workdir "$WORK_DIR" --binary "$CDC_BINARY" + ensure 30 "grep 'schema store initialized' '$WORK_DIR/cdc.log' | grep -q 'resolvedTs=$start_ts'" + GO111MODULE=on go run "$CUR/set_gc_safepoint.go" "$UP_PD_HOST_1:$UP_PD_PORT_1" "$gc_worker_key" "$gc_worker_value" + + cdc_cli_changefeed create -c "$changefeed_id" --start-ts="$start_ts" \ + --sink-uri="mysql://normal:123456@$DOWN_TIDB_HOST:$DOWN_TIDB_PORT/" \ + --config="$CUR/conf/changefeed.toml" + + # The event filter matches the table name before the rename, so the downstream + # table must keep its original name. + run_sql "CREATE TABLE $source_db.finish_mark (id INT PRIMARY KEY);" "$UP_TIDB_HOST" "$UP_TIDB_PORT" + check_table_exists "$source_db.finish_mark" "$DOWN_TIDB_HOST" "$DOWN_TIDB_PORT" 60 + check_table_exists "$source_db.$table_name" "$DOWN_TIDB_HOST" "$DOWN_TIDB_PORT" 60 + check_table_not_exists "$target_db.$table_name" "$DOWN_TIDB_HOST" "$DOWN_TIDB_PORT" 60 + ensure 30 "run_sql 'SELECT id FROM $source_db.$table_name;' '$DOWN_TIDB_HOST' '$DOWN_TIDB_PORT' && check_contains 'id: 1'" + check_changefeed_state "http://$UP_PD_HOST_1:$UP_PD_PORT_1" "$changefeed_id" "normal" "null" "" + + cleanup_process "$CDC_BINARY" +} + +trap 'stop_test "$WORK_DIR"' EXIT +run "$@" +check_logs "$WORK_DIR" +echo "[$(date)] <<<<<< run test case $TEST_NAME success! >>>>>>" diff --git a/tests/integration_tests/rename_table_start_ts/set_gc_safepoint.go b/tests/integration_tests/rename_table_start_ts/set_gc_safepoint.go new file mode 100644 index 0000000000..d4df6a9276 --- /dev/null +++ b/tests/integration_tests/rename_table_start_ts/set_gc_safepoint.go @@ -0,0 +1,44 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "log" + "os" + "time" + + clientv3 "go.etcd.io/etcd/client/v3" +) + +func main() { + if len(os.Args) != 4 { + log.Fatalf("usage: %s ", os.Args[0]) + } + + client, err := clientv3.New(clientv3.Config{ + Endpoints: []string{os.Args[1]}, + DialTimeout: 5 * time.Second, + }) + if err != nil { + log.Fatal(err) + } + defer client.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if _, err = client.Put(ctx, os.Args[2], os.Args[3]); err != nil { + log.Fatal(err) + } +} diff --git a/tests/integration_tests/run_light_it_in_ci.sh b/tests/integration_tests/run_light_it_in_ci.sh index 789310dd61..cc2df33b48 100755 --- a/tests/integration_tests/run_light_it_in_ci.sh +++ b/tests/integration_tests/run_light_it_in_ci.sh @@ -50,7 +50,7 @@ mysql_groups=( # G08 'capture_session_done_during_task changefeed_dup_error_restart mysql_sink_retry fail_over_ddl_I table_route' # G09 - 'sequence cdc_server_tips ddl_sequence server_config_compatibility log_redaction fail_over_ddl_J' + 'sequence cdc_server_tips ddl_sequence rename_table_start_ts server_config_compatibility log_redaction fail_over_ddl_J' # G10 'overwrite_resume_with_syncpoint restart_changefeed changefeed_error bdr_mode fail_over_ddl_K split_table_check' # G11 From 3b7c228b94faeb9e36c25bf54c539037270da017 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Wed, 12 Aug 2026 17:03:17 +0800 Subject: [PATCH 07/10] fix integration test --- .../conf/changefeed.toml | 6 +-- .../rename_table_start_ts/run.sh | 32 +++----------- .../rename_table_start_ts/set_gc_safepoint.go | 44 ------------------- 3 files changed, 6 insertions(+), 76 deletions(-) delete mode 100644 tests/integration_tests/rename_table_start_ts/set_gc_safepoint.go diff --git a/tests/integration_tests/rename_table_start_ts/conf/changefeed.toml b/tests/integration_tests/rename_table_start_ts/conf/changefeed.toml index 766dc8af2c..8449ec10df 100644 --- a/tests/integration_tests/rename_table_start_ts/conf/changefeed.toml +++ b/tests/integration_tests/rename_table_start_ts/conf/changefeed.toml @@ -1,6 +1,2 @@ [filter] -rules = ["rename_table_start_ts_source.*", "rename_table_start_ts_target.*"] - -[[filter.event-filters]] -matcher = ["rename_table_start_ts_source.t"] -ignore-event = ["rename table"] +rules = ["rename_table_start_ts_source.*"] diff --git a/tests/integration_tests/rename_table_start_ts/run.sh b/tests/integration_tests/rename_table_start_ts/run.sh index ef210d8902..d32e7b162f 100755 --- a/tests/integration_tests/rename_table_start_ts/run.sh +++ b/tests/integration_tests/rename_table_start_ts/run.sh @@ -21,9 +21,6 @@ function run() { local target_db=rename_table_start_ts_target local table_name=t local changefeed_id=rename-table-start-ts - local gc_worker_key - local gc_worker_value - local pd_cluster_id local rename_finished_ts local start_ts local table_id @@ -49,37 +46,18 @@ function run() { start_ts=$((rename_finished_ts - 1)) cleanup_process "$CDC_BINARY" - # Force SchemaStore to initialize from the same snapshot used by the - # changefeed. Restore the GC worker safepoint before creating the changefeed - # so its start-ts is still considered readable. - pd_cluster_id=$(curl -s "http://$UP_PD_HOST_1:$UP_PD_PORT_1/pd/api/v1/cluster" | - grep -oE '"id":[[:space:]]*[0-9]+' | - grep -oE '[0-9]+') - gc_worker_key="/pd/$pd_cluster_id/gc/safe_point/service/gc_worker" - gc_worker_value=$(curl -fsS "http://$UP_PD_HOST_1:$UP_PD_PORT_1/pd/api/v1/gc/safepoint" | - jq -cer '.service_gc_safe_points[] | select(.service_id == "gc_worker")') - if [ -z "$gc_worker_value" ]; then - echo "failed to get gc_worker service safepoint" - exit 1 - fi - GO111MODULE=on go run "$CUR/set_gc_safepoint.go" "$UP_PD_HOST_1:$UP_PD_PORT_1" "$gc_worker_key" \ - "{\"service_id\":\"gc_worker\",\"expired_at\":9223372036854775807,\"safe_point\":$start_ts}" - run_cdc_server --workdir "$WORK_DIR" --binary "$CDC_BINARY" - ensure 30 "grep 'schema store initialized' '$WORK_DIR/cdc.log' | grep -q 'resolvedTs=$start_ts'" - GO111MODULE=on go run "$CUR/set_gc_safepoint.go" "$UP_PD_HOST_1:$UP_PD_PORT_1" "$gc_worker_key" "$gc_worker_value" - cdc_cli_changefeed create -c "$changefeed_id" --start-ts="$start_ts" \ --sink-uri="mysql://normal:123456@$DOWN_TIDB_HOST:$DOWN_TIDB_PORT/" \ --config="$CUR/conf/changefeed.toml" - # The event filter matches the table name before the rename, so the downstream - # table must keep its original name. + # The table is in the filter before the rename and outside it afterwards. The + # rename DDL must still be replicated to downstream. run_sql "CREATE TABLE $source_db.finish_mark (id INT PRIMARY KEY);" "$UP_TIDB_HOST" "$UP_TIDB_PORT" check_table_exists "$source_db.finish_mark" "$DOWN_TIDB_HOST" "$DOWN_TIDB_PORT" 60 - check_table_exists "$source_db.$table_name" "$DOWN_TIDB_HOST" "$DOWN_TIDB_PORT" 60 - check_table_not_exists "$target_db.$table_name" "$DOWN_TIDB_HOST" "$DOWN_TIDB_PORT" 60 - ensure 30 "run_sql 'SELECT id FROM $source_db.$table_name;' '$DOWN_TIDB_HOST' '$DOWN_TIDB_PORT' && check_contains 'id: 1'" + check_table_not_exists "$source_db.$table_name" "$DOWN_TIDB_HOST" "$DOWN_TIDB_PORT" 60 + check_table_exists "$target_db.$table_name" "$DOWN_TIDB_HOST" "$DOWN_TIDB_PORT" 60 + ensure 30 "run_sql 'SELECT id FROM $target_db.$table_name;' '$DOWN_TIDB_HOST' '$DOWN_TIDB_PORT' && check_contains 'id: 1'" check_changefeed_state "http://$UP_PD_HOST_1:$UP_PD_PORT_1" "$changefeed_id" "normal" "null" "" cleanup_process "$CDC_BINARY" diff --git a/tests/integration_tests/rename_table_start_ts/set_gc_safepoint.go b/tests/integration_tests/rename_table_start_ts/set_gc_safepoint.go deleted file mode 100644 index d4df6a9276..0000000000 --- a/tests/integration_tests/rename_table_start_ts/set_gc_safepoint.go +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2026 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "context" - "log" - "os" - "time" - - clientv3 "go.etcd.io/etcd/client/v3" -) - -func main() { - if len(os.Args) != 4 { - log.Fatalf("usage: %s ", os.Args[0]) - } - - client, err := clientv3.New(clientv3.Config{ - Endpoints: []string{os.Args[1]}, - DialTimeout: 5 * time.Second, - }) - if err != nil { - log.Fatal(err) - } - defer client.Close() - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - if _, err = client.Put(ctx, os.Args[2], os.Args[3]); err != nil { - log.Fatal(err) - } -} From 83127f976d66befdb0b067188f85e6ab9ea8594f Mon Sep 17 00:00:00 2001 From: lidezhu Date: Wed, 12 Aug 2026 18:20:37 +0800 Subject: [PATCH 08/10] fix test --- .../conf/changefeed.toml | 6 ++- .../rename_table_start_ts/run.sh | 32 ++++++++++--- .../rename_table_start_ts/set_gc_safepoint.go | 45 +++++++++++++++++++ 3 files changed, 77 insertions(+), 6 deletions(-) create mode 100644 tests/integration_tests/rename_table_start_ts/set_gc_safepoint.go diff --git a/tests/integration_tests/rename_table_start_ts/conf/changefeed.toml b/tests/integration_tests/rename_table_start_ts/conf/changefeed.toml index 8449ec10df..766dc8af2c 100644 --- a/tests/integration_tests/rename_table_start_ts/conf/changefeed.toml +++ b/tests/integration_tests/rename_table_start_ts/conf/changefeed.toml @@ -1,2 +1,6 @@ [filter] -rules = ["rename_table_start_ts_source.*"] +rules = ["rename_table_start_ts_source.*", "rename_table_start_ts_target.*"] + +[[filter.event-filters]] +matcher = ["rename_table_start_ts_source.t"] +ignore-event = ["rename table"] diff --git a/tests/integration_tests/rename_table_start_ts/run.sh b/tests/integration_tests/rename_table_start_ts/run.sh index d32e7b162f..ef210d8902 100755 --- a/tests/integration_tests/rename_table_start_ts/run.sh +++ b/tests/integration_tests/rename_table_start_ts/run.sh @@ -21,6 +21,9 @@ function run() { local target_db=rename_table_start_ts_target local table_name=t local changefeed_id=rename-table-start-ts + local gc_worker_key + local gc_worker_value + local pd_cluster_id local rename_finished_ts local start_ts local table_id @@ -46,18 +49,37 @@ function run() { start_ts=$((rename_finished_ts - 1)) cleanup_process "$CDC_BINARY" + # Force SchemaStore to initialize from the same snapshot used by the + # changefeed. Restore the GC worker safepoint before creating the changefeed + # so its start-ts is still considered readable. + pd_cluster_id=$(curl -s "http://$UP_PD_HOST_1:$UP_PD_PORT_1/pd/api/v1/cluster" | + grep -oE '"id":[[:space:]]*[0-9]+' | + grep -oE '[0-9]+') + gc_worker_key="/pd/$pd_cluster_id/gc/safe_point/service/gc_worker" + gc_worker_value=$(curl -fsS "http://$UP_PD_HOST_1:$UP_PD_PORT_1/pd/api/v1/gc/safepoint" | + jq -cer '.service_gc_safe_points[] | select(.service_id == "gc_worker")') + if [ -z "$gc_worker_value" ]; then + echo "failed to get gc_worker service safepoint" + exit 1 + fi + GO111MODULE=on go run "$CUR/set_gc_safepoint.go" "$UP_PD_HOST_1:$UP_PD_PORT_1" "$gc_worker_key" \ + "{\"service_id\":\"gc_worker\",\"expired_at\":9223372036854775807,\"safe_point\":$start_ts}" + run_cdc_server --workdir "$WORK_DIR" --binary "$CDC_BINARY" + ensure 30 "grep 'schema store initialized' '$WORK_DIR/cdc.log' | grep -q 'resolvedTs=$start_ts'" + GO111MODULE=on go run "$CUR/set_gc_safepoint.go" "$UP_PD_HOST_1:$UP_PD_PORT_1" "$gc_worker_key" "$gc_worker_value" + cdc_cli_changefeed create -c "$changefeed_id" --start-ts="$start_ts" \ --sink-uri="mysql://normal:123456@$DOWN_TIDB_HOST:$DOWN_TIDB_PORT/" \ --config="$CUR/conf/changefeed.toml" - # The table is in the filter before the rename and outside it afterwards. The - # rename DDL must still be replicated to downstream. + # The event filter matches the table name before the rename, so the downstream + # table must keep its original name. run_sql "CREATE TABLE $source_db.finish_mark (id INT PRIMARY KEY);" "$UP_TIDB_HOST" "$UP_TIDB_PORT" check_table_exists "$source_db.finish_mark" "$DOWN_TIDB_HOST" "$DOWN_TIDB_PORT" 60 - check_table_not_exists "$source_db.$table_name" "$DOWN_TIDB_HOST" "$DOWN_TIDB_PORT" 60 - check_table_exists "$target_db.$table_name" "$DOWN_TIDB_HOST" "$DOWN_TIDB_PORT" 60 - ensure 30 "run_sql 'SELECT id FROM $target_db.$table_name;' '$DOWN_TIDB_HOST' '$DOWN_TIDB_PORT' && check_contains 'id: 1'" + check_table_exists "$source_db.$table_name" "$DOWN_TIDB_HOST" "$DOWN_TIDB_PORT" 60 + check_table_not_exists "$target_db.$table_name" "$DOWN_TIDB_HOST" "$DOWN_TIDB_PORT" 60 + ensure 30 "run_sql 'SELECT id FROM $source_db.$table_name;' '$DOWN_TIDB_HOST' '$DOWN_TIDB_PORT' && check_contains 'id: 1'" check_changefeed_state "http://$UP_PD_HOST_1:$UP_PD_PORT_1" "$changefeed_id" "normal" "null" "" cleanup_process "$CDC_BINARY" diff --git a/tests/integration_tests/rename_table_start_ts/set_gc_safepoint.go b/tests/integration_tests/rename_table_start_ts/set_gc_safepoint.go new file mode 100644 index 0000000000..3ee0686b05 --- /dev/null +++ b/tests/integration_tests/rename_table_start_ts/set_gc_safepoint.go @@ -0,0 +1,45 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "log" + "os" + "time" + + clientv3 "go.etcd.io/etcd/client/v3" +) + +func main() { + if len(os.Args) != 4 { + log.Fatalf("usage: %s ", os.Args[0]) + } + + client, err := clientv3.New(clientv3.Config{ + Endpoints: []string{os.Args[1]}, + DialTimeout: 5 * time.Second, + }) + if err != nil { + log.Fatal(err) + } + defer client.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if _, err = client.Put(ctx, os.Args[2], os.Args[3]); err != nil { + log.Fatal(err) + } +} From 5d8a3901288181abe801d07696ab8a9c4e8a951a Mon Sep 17 00:00:00 2001 From: lidezhu Date: Wed, 12 Aug 2026 18:57:40 +0800 Subject: [PATCH 09/10] remove test --- .../conf/changefeed.toml | 6 -- .../rename_table_start_ts/run.sh | 91 ------------------- .../rename_table_start_ts/set_gc_safepoint.go | 45 --------- 3 files changed, 142 deletions(-) delete mode 100644 tests/integration_tests/rename_table_start_ts/conf/changefeed.toml delete mode 100755 tests/integration_tests/rename_table_start_ts/run.sh delete mode 100644 tests/integration_tests/rename_table_start_ts/set_gc_safepoint.go diff --git a/tests/integration_tests/rename_table_start_ts/conf/changefeed.toml b/tests/integration_tests/rename_table_start_ts/conf/changefeed.toml deleted file mode 100644 index 766dc8af2c..0000000000 --- a/tests/integration_tests/rename_table_start_ts/conf/changefeed.toml +++ /dev/null @@ -1,6 +0,0 @@ -[filter] -rules = ["rename_table_start_ts_source.*", "rename_table_start_ts_target.*"] - -[[filter.event-filters]] -matcher = ["rename_table_start_ts_source.t"] -ignore-event = ["rename table"] diff --git a/tests/integration_tests/rename_table_start_ts/run.sh b/tests/integration_tests/rename_table_start_ts/run.sh deleted file mode 100755 index ef210d8902..0000000000 --- a/tests/integration_tests/rename_table_start_ts/run.sh +++ /dev/null @@ -1,91 +0,0 @@ -#!/bin/bash - -set -eu - -CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -source "$CUR/../_utils/test_prepare" -WORK_DIR="$OUT_DIR/$TEST_NAME" -CDC_BINARY=cdc.test -SINK_TYPE=$1 - -function run() { - if [ "$SINK_TYPE" != "mysql" ]; then - return - fi - - rm -rf "$WORK_DIR" - mkdir -p "$WORK_DIR" - start_tidb_cluster --workdir "$WORK_DIR" - - local source_db=rename_table_start_ts_source - local target_db=rename_table_start_ts_target - local table_name=t - local changefeed_id=rename-table-start-ts - local gc_worker_key - local gc_worker_value - local pd_cluster_id - local rename_finished_ts - local start_ts - local table_id - - run_sql "CREATE DATABASE $source_db; CREATE DATABASE $target_db; CREATE TABLE $source_db.$table_name (id INT PRIMARY KEY); INSERT INTO $source_db.$table_name VALUES (1);" "$UP_TIDB_HOST" "$UP_TIDB_PORT" - run_sql "CREATE DATABASE $source_db; CREATE DATABASE $target_db; CREATE TABLE $source_db.$table_name (id INT PRIMARY KEY); INSERT INTO $source_db.$table_name VALUES (1);" "$DOWN_TIDB_HOST" "$DOWN_TIDB_PORT" - - table_id=$(get_table_id "$source_db" "$table_name") - run_sql "RENAME TABLE $source_db.$table_name TO $target_db.$table_name;" "$UP_TIDB_HOST" "$UP_TIDB_PORT" - - run_cdc_server --workdir "$WORK_DIR" --binary "$CDC_BINARY" --logsuffix _probe - ensure 30 "grep 'write ddl event' '$WORK_DIR/cdc_probe.log' | grep 'tableID=$table_id' | grep -q 'RENAME TABLE'" - rename_finished_ts=$(grep "write ddl event" "$WORK_DIR/cdc_probe.log" | - grep "tableID=$table_id" | - grep "RENAME TABLE" | - head -n 1 | - grep -oE 'finishedTs=[0-9]+' | - cut -d= -f2) - if ! [[ "$rename_finished_ts" =~ ^[0-9]+$ ]]; then - echo "failed to get rename table finishedTs" - exit 1 - fi - start_ts=$((rename_finished_ts - 1)) - cleanup_process "$CDC_BINARY" - - # Force SchemaStore to initialize from the same snapshot used by the - # changefeed. Restore the GC worker safepoint before creating the changefeed - # so its start-ts is still considered readable. - pd_cluster_id=$(curl -s "http://$UP_PD_HOST_1:$UP_PD_PORT_1/pd/api/v1/cluster" | - grep -oE '"id":[[:space:]]*[0-9]+' | - grep -oE '[0-9]+') - gc_worker_key="/pd/$pd_cluster_id/gc/safe_point/service/gc_worker" - gc_worker_value=$(curl -fsS "http://$UP_PD_HOST_1:$UP_PD_PORT_1/pd/api/v1/gc/safepoint" | - jq -cer '.service_gc_safe_points[] | select(.service_id == "gc_worker")') - if [ -z "$gc_worker_value" ]; then - echo "failed to get gc_worker service safepoint" - exit 1 - fi - GO111MODULE=on go run "$CUR/set_gc_safepoint.go" "$UP_PD_HOST_1:$UP_PD_PORT_1" "$gc_worker_key" \ - "{\"service_id\":\"gc_worker\",\"expired_at\":9223372036854775807,\"safe_point\":$start_ts}" - - run_cdc_server --workdir "$WORK_DIR" --binary "$CDC_BINARY" - ensure 30 "grep 'schema store initialized' '$WORK_DIR/cdc.log' | grep -q 'resolvedTs=$start_ts'" - GO111MODULE=on go run "$CUR/set_gc_safepoint.go" "$UP_PD_HOST_1:$UP_PD_PORT_1" "$gc_worker_key" "$gc_worker_value" - - cdc_cli_changefeed create -c "$changefeed_id" --start-ts="$start_ts" \ - --sink-uri="mysql://normal:123456@$DOWN_TIDB_HOST:$DOWN_TIDB_PORT/" \ - --config="$CUR/conf/changefeed.toml" - - # The event filter matches the table name before the rename, so the downstream - # table must keep its original name. - run_sql "CREATE TABLE $source_db.finish_mark (id INT PRIMARY KEY);" "$UP_TIDB_HOST" "$UP_TIDB_PORT" - check_table_exists "$source_db.finish_mark" "$DOWN_TIDB_HOST" "$DOWN_TIDB_PORT" 60 - check_table_exists "$source_db.$table_name" "$DOWN_TIDB_HOST" "$DOWN_TIDB_PORT" 60 - check_table_not_exists "$target_db.$table_name" "$DOWN_TIDB_HOST" "$DOWN_TIDB_PORT" 60 - ensure 30 "run_sql 'SELECT id FROM $source_db.$table_name;' '$DOWN_TIDB_HOST' '$DOWN_TIDB_PORT' && check_contains 'id: 1'" - check_changefeed_state "http://$UP_PD_HOST_1:$UP_PD_PORT_1" "$changefeed_id" "normal" "null" "" - - cleanup_process "$CDC_BINARY" -} - -trap 'stop_test "$WORK_DIR"' EXIT -run "$@" -check_logs "$WORK_DIR" -echo "[$(date)] <<<<<< run test case $TEST_NAME success! >>>>>>" diff --git a/tests/integration_tests/rename_table_start_ts/set_gc_safepoint.go b/tests/integration_tests/rename_table_start_ts/set_gc_safepoint.go deleted file mode 100644 index 3ee0686b05..0000000000 --- a/tests/integration_tests/rename_table_start_ts/set_gc_safepoint.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2026 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "context" - "log" - "os" - "time" - - clientv3 "go.etcd.io/etcd/client/v3" -) - -func main() { - if len(os.Args) != 4 { - log.Fatalf("usage: %s ", os.Args[0]) - } - - client, err := clientv3.New(clientv3.Config{ - Endpoints: []string{os.Args[1]}, - DialTimeout: 5 * time.Second, - }) - if err != nil { - log.Fatal(err) - } - defer client.Close() - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - if _, err = client.Put(ctx, os.Args[2], os.Args[3]); err != nil { - log.Fatal(err) - } -} From ead92871d1157ce866844919149af55fad167a16 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Wed, 12 Aug 2026 18:58:12 +0800 Subject: [PATCH 10/10] f --- tests/integration_tests/run_light_it_in_ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration_tests/run_light_it_in_ci.sh b/tests/integration_tests/run_light_it_in_ci.sh index cc2df33b48..789310dd61 100755 --- a/tests/integration_tests/run_light_it_in_ci.sh +++ b/tests/integration_tests/run_light_it_in_ci.sh @@ -50,7 +50,7 @@ mysql_groups=( # G08 'capture_session_done_during_task changefeed_dup_error_restart mysql_sink_retry fail_over_ddl_I table_route' # G09 - 'sequence cdc_server_tips ddl_sequence rename_table_start_ts server_config_compatibility log_redaction fail_over_ddl_J' + 'sequence cdc_server_tips ddl_sequence server_config_compatibility log_redaction fail_over_ddl_J' # G10 'overwrite_resume_with_syncpoint restart_changefeed changefeed_error bdr_mode fail_over_ddl_K split_table_check' # G11