sync-diff-inspector: calculate checksum only with export-fix-sql = false
#12531
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -150,11 +150,46 @@ type Checkpoint struct { | |
| hp *nodeHeap | ||
| } | ||
|
|
||
| // ChecksumSourceState stores one source's checksum progress in global-checksum mode. | ||
| type ChecksumSourceState struct { | ||
| LastRange *chunk.Range `json:"last-range,omitempty"` | ||
| Checksum uint64 `json:"checksum"` | ||
|
Contributor
There was a problem hiding this comment. should we add omitempty for other fields too
Contributor
Author
There was a problem hiding this comment. Done in joechenrh@faf8fd2 . Added |
||
| Count int64 `json:"count"` | ||
| Done bool `json:"done"` | ||
|
D3Hunter marked this conversation as resolved.
|
||
| } | ||
|
|
||
| // ChecksumState stores checkpoint progress for global-checksum mode. | ||
| type ChecksumState struct { | ||
| TableIndex int `json:"table-index"` | ||
| Upstream *ChecksumSourceState `json:"upstream,omitempty"` | ||
| Downstream *ChecksumSourceState `json:"downstream,omitempty"` | ||
| } | ||
|
|
||
| // NewChecksumState returns an initialized checksum state for one table index. | ||
| func NewChecksumState(tableIndex int) *ChecksumState { | ||
| return &ChecksumState{ | ||
| TableIndex: tableIndex, | ||
| Upstream: &ChecksumSourceState{}, | ||
| Downstream: &ChecksumSourceState{}, | ||
| } | ||
| } | ||
|
|
||
| func (s *ChecksumState) init() { | ||
| if s.Upstream == nil { | ||
| s.Upstream = &ChecksumSourceState{} | ||
| } | ||
| if s.Downstream == nil { | ||
| s.Downstream = &ChecksumSourceState{} | ||
| } | ||
| } | ||
|
|
||
| // SavedState stores mode-dependent checkpoint state plus report state for resume. | ||
| // Depending on the execution mode, it contains either Chunk (chunk diff mode) | ||
| // or Checksum (global-checksum mode). | ||
| type SavedState struct { | ||
|
D3Hunter marked this conversation as resolved.
joechenrh marked this conversation as resolved.
|
||
| Chunk *Node `json:"chunk-info,omitempty"` | ||
| Checksum *ChecksumState `json:"checksum-info,omitempty"` | ||
| Report *report.Report `json:"report-info"` | ||
| } | ||
|
|
||
| // InitCurrentSavedID the method is only used in initialization without lock, be cautious | ||
|
|
@@ -258,16 +293,56 @@ func (cp *Checkpoint) SaveChunk(ctx context.Context, fileName string, cur *Node, | |
| return cur.GetID(), nil | ||
| } | ||
|
|
||
| // SaveChecksumState saves global-checksum checkpoint state to file. | ||
| func (cp *Checkpoint) SaveChecksumState( | ||
| ctx context.Context, | ||
| fileName string, | ||
| state *ChecksumState, | ||
| reportInfo *report.Report, | ||
| ) error { | ||
| savedState := &SavedState{ | ||
| Checksum: state, | ||
| Report: reportInfo, | ||
| } | ||
| checkpointData, err := json.Marshal(savedState) | ||
| if err != nil { | ||
| log.Warn("fail to save checksum checkpoint", zap.Error(err)) | ||
| return errors.Trace(err) | ||
| } | ||
|
|
||
| return writeFileAtomic(fileName, checkpointData, config.LocalFilePerm) | ||
| } | ||
|
|
||
| func loadSavedState(fileName string) (*SavedState, error) { | ||
| bytes, err := os.ReadFile(fileName) | ||
| if err != nil { | ||
| return nil, errors.Trace(err) | ||
| } | ||
| n := &SavedState{} | ||
| err = json.Unmarshal(bytes, n) | ||
| if err != nil { | ||
| return nil, errors.Trace(err) | ||
| } | ||
| if n.Checksum != nil { | ||
| n.Checksum.init() | ||
| } | ||
| return n, nil | ||
| } | ||
|
|
||
| // LoadChunk loads chunk info from file `chunk` | ||
| func (cp *Checkpoint) LoadChunk(fileName string) (*Node, *report.Report, error) { | ||
| n, err := loadSavedState(fileName) | ||
| if err != nil { | ||
| return nil, nil, errors.Trace(err) | ||
| } | ||
| return n.Chunk, n.Report, nil | ||
| } | ||
|
|
||
| // LoadChecksumState loads checksum checkpoint state from file. | ||
| func (cp *Checkpoint) LoadChecksumState(fileName string) (*ChecksumState, *report.Report, error) { | ||
| n, err := loadSavedState(fileName) | ||
| if err != nil { | ||
| return nil, nil, errors.Trace(err) | ||
| } | ||
| return n.Checksum, n.Report, nil | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
does checksum-only mode the
global checksum?Uh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Yep — when
export-fix-sql=falseand both sources implementChecksumCapableSource, the tool routes intoequalByGlobalChecksumwhich computes a single aggregated checksum per table (XOR of per-chunk CRC32), comparing the final value between upstream and downstream. Unified the terminology to "global checksum" throughout the codebase in joechenrh@faf8fd2 👍