diff --git a/.agents/skills/tidb-test-guidelines/references/executor-case-map.md b/.agents/skills/tidb-test-guidelines/references/executor-case-map.md index 1a14fcc5a97cd..3bac8b89d3bcb 100644 --- a/.agents/skills/tidb-test-guidelines/references/executor-case-map.md +++ b/.agents/skills/tidb-test-guidelines/references/executor-case-map.md @@ -33,7 +33,6 @@ - `pkg/executor/explain_unit_test.go` - executor: Tests explain analyze invoke next and close. - `pkg/executor/explainfor_test.go` - executor: Tests explain for. - `pkg/executor/grant_test.go` - executor: Tests grant global. -- `pkg/executor/historical_stats_test.go` - executor: Tests record history stats after analyze. - `pkg/executor/hot_regions_history_table_test.go` - executor: Tests TiDB hot regions history. - `pkg/executor/import_into_test.go` - executor: Tests security enhanced mode. - `pkg/executor/infoschema_cluster_table_test.go` - executor: Tests skip empty IP nodes for TiDB-type coprocessor. @@ -393,7 +392,7 @@ ### Tests - `pkg/executor/test/planreplayer/main_test.go` - Configures default goleak settings and registers testdata. -- `pkg/executor/test/planreplayer/plan_replayer_test.go` - executor/test/planreplayer: Tests plan replayer. +- `pkg/executor/test/planreplayer/plan_replayer_test.go` - executor/test/planreplayer: Tests plan replayer dump, load, capture, and legacy syntax rejection. ## pkg/executor/test/seqtest diff --git a/br/pkg/restore/snap_client/pipeline_items.go b/br/pkg/restore/snap_client/pipeline_items.go index 3869abe6391a2..cab2a14bc33b1 100644 --- a/br/pkg/restore/snap_client/pipeline_items.go +++ b/br/pkg/restore/snap_client/pipeline_items.go @@ -480,7 +480,7 @@ func (buffer *statsMetaItemBuffer) saveMetaToStorageWithRetry( ) error { state := utils.InitialRetryState(8, 500*time.Millisecond, 500*time.Millisecond) err := utils.WithRetry(ctx, func() error { - if err := statsHandler.SaveMetaToStorage("br restore", false, metaUpdates...); err != nil { + if err := statsHandler.SaveMetaToStorage(false, metaUpdates...); err != nil { log.Error("failed to save meta to storage", zap.Error(err)) return errors.Trace(err) } diff --git a/br/pkg/restore/snap_client/pipeline_items_test.go b/br/pkg/restore/snap_client/pipeline_items_test.go index c36150b0c5ee6..fb610ddb03583 100644 --- a/br/pkg/restore/snap_client/pipeline_items_test.go +++ b/br/pkg/restore/snap_client/pipeline_items_test.go @@ -121,7 +121,7 @@ type mockStatsReadWriter struct { rows map[int64]int64 } -func (m *mockStatsReadWriter) SaveMetaToStorage(_ string, _ bool, metaUpdates ...statstypes.MetaUpdate) (err error) { +func (m *mockStatsReadWriter) SaveMetaToStorage(_ bool, metaUpdates ...statstypes.MetaUpdate) (err error) { for _, metaUpdate := range metaUpdates { m.rows[metaUpdate.PhysicalID] += metaUpdate.Count } diff --git a/docs/tidb_http_api.md b/docs/tidb_http_api.md index f94ed4e4ee961..945cba24e8b28 100644 --- a/docs/tidb_http_api.md +++ b/docs/tidb_http_api.md @@ -609,17 +609,7 @@ timezone.* curl http://{TiDBIP}:10080/stats/dump/{db}/{table} ``` -30. Get statistics data of specific table and timestamp. - - ```shell - curl http://{TiDBIP}:10080/stats/dump/{db}/{table}/{yyyyMMddHHmmss} - ``` - - ```shell - curl http://{TiDBIP}:10080/stats/dump/{db}/{table}/{yyyy-MM-dd HH:mm:ss} - ``` - -31. Resume the binlog writing when Pump is recovered. +30. Resume the binlog writing when Pump is recovered. ```shell curl http://{TiDBIP}:10080/binlog/recover @@ -646,28 +636,28 @@ timezone.* - op=reset: reset `SkippedCommitterCounter` to 0 to avoid the problem that `SkippedCommitterCounter` is not cleared due to some unusual cases. - op=status: Get the current status of binlog recovery. -32. Enable/disable async commit feature +31. Enable/disable async commit feature ```shell curl -X POST -d "tidb_enable_async_commit=1" http://{TiDBIP}:10080/settings curl -X POST -d "tidb_enable_async_commit=0" http://{TiDBIP}:10080/settings ``` -33. Enable/disable one-phase commit feature +32. Enable/disable one-phase commit feature ```shell curl -X POST -d "tidb_enable_1pc=1" http://{TiDBIP}:10080/settings curl -X POST -d "tidb_enable_1pc=0" http://{TiDBIP}:10080/settings ``` -34. Enable/disable the mutation checker +33. Enable/disable the mutation checker ```shell curl -X POST -d "tidb_enable_mutation_checker=1" http://{TiDBIP}:10080/settings curl -X POST -d "tidb_enable_mutation_checker=0" http://{TiDBIP}:10080/settings ``` -35. Get/Set the size of the Ballast Object +34. Get/Set the size of the Ballast Object ```shell # get current size of the ballast object @@ -676,19 +666,19 @@ timezone.* curl -v -X POST -d "2147483648" http://{TiDBIP}:10080/debug/ballast-object-sz ``` -36. Set deadlock history table capacity +35. Set deadlock history table capacity ```shell curl -X POST -d "deadlock_history_capacity={number}" http://{TiDBIP}:10080/settings ``` -37. Set whether deadlock history (`DEADLOCKS`) collect retryable deadlocks +36. Set whether deadlock history (`DEADLOCKS`) collect retryable deadlocks ```shell curl -X POST -d "deadlock_history_collect_retryable={bool_val}" http://{TiDBIP}:10080/settings ``` -38. Set transaction_id to digest mapping minimum duration threshold, only transactions which last longer than this threshold will be collected into `TRX_SUMMARY`. +37. Set transaction_id to digest mapping minimum duration threshold, only transactions which last longer than this threshold will be collected into `TRX_SUMMARY`. ```shell curl -X POST -d "transaction_id_digest_min_duration={number}" http://{TiDBIP}:10080/settings @@ -696,13 +686,13 @@ timezone.* Unit of duration here is ms. -39. Set transaction summary table (`TRX_SUMMARY`) capacity +38. Set transaction summary table (`TRX_SUMMARY`) capacity ```shell curl -X POST -d "transaction_summary_capacity={number}" http://{TiDBIP}:10080/settings ``` -40. The commands are used to handle smooth upgrade mode(refer to the [TiDB Smooth Upgrade](https://github.com/pingcap/docs/blob/4aa0b1d5078617cc06bd1957c5c93e86efb4668d/smooth-upgrade-tidb.md) for details) operations. We can send these upgrade operations to the cluster. The operations here include `start`, `finish` and `show`. +39. The commands are used to handle smooth upgrade mode(refer to the [TiDB Smooth Upgrade](https://github.com/pingcap/docs/blob/4aa0b1d5078617cc06bd1957c5c93e86efb4668d/smooth-upgrade-tidb.md) for details) operations. We can send these upgrade operations to the cluster. The operations here include `start`, `finish` and `show`. ```shell curl -X POST http://{TiDBIP}:10080/upgrade/{op} @@ -713,7 +703,7 @@ timezone.* "success!" ``` -41. Set split & scatter regions concurrency before ingest, and ingest request concurrency. Value ranges: +40. Set split & scatter regions concurrency before ingest, and ingest request concurrency. Value ranges: - `max-batch-split-ranges`: `[1, 9223372036854775807]`, default `2048` - `max-split-ranges-per-sec`: `[0, 9223372036854775807]`, default `0` (no limit) - `max-ingest-per-sec`: `[0, 9223372036854775807]`, default `0` (no limit) @@ -733,7 +723,7 @@ timezone.* curl http://{TiDBIP}:10080/ingest/max-ingest-inflight -X POST -d "{\"value\": 2}" ``` -42. Get TiDB transaction GC states: +41. Get TiDB transaction GC states: ```shell curl http://{TiDBIP}:10080/txn-gc-states diff --git a/pkg/bindinfo/tests/BUILD.bazel b/pkg/bindinfo/tests/BUILD.bazel index 732e64505debd..a1a162f48d39b 100644 --- a/pkg/bindinfo/tests/BUILD.bazel +++ b/pkg/bindinfo/tests/BUILD.bazel @@ -11,7 +11,7 @@ go_test( ], flaky = True, race = "on", - shard_count = 25, + shard_count = 26, deps = [ "//pkg/bindinfo", "//pkg/domain", diff --git a/pkg/domain/BUILD.bazel b/pkg/domain/BUILD.bazel index 3b18f8d2dce0f..fbe49573e3f2b 100644 --- a/pkg/domain/BUILD.bazel +++ b/pkg/domain/BUILD.bazel @@ -7,7 +7,6 @@ go_library( "domain_sysvars.go", "domainctx.go", "extract.go", - "historical_stats.go", "optimize_trace.go", "plan_replayer.go", "plan_replayer_dump.go", diff --git a/pkg/domain/domain.go b/pkg/domain/domain.go index bd9d348f58d49..55f0ee98aec1d 100644 --- a/pkg/domain/domain.go +++ b/pkg/domain/domain.go @@ -205,11 +205,10 @@ type Domain struct { expiredTimeStamp types.Time } - brOwnerMgr owner.Manager - logBackupAdvancer *daemon.OwnerDaemon - historicalStatsWorker *HistoricalStatsWorker - ttlJobManager atomic.Pointer[ttlworker.JobManager] - runawayManager *runaway.Manager + brOwnerMgr owner.Manager + logBackupAdvancer *daemon.OwnerDaemon + ttlJobManager atomic.Pointer[ttlworker.JobManager] + runawayManager *runaway.Manager // resourceGroupsController can be changed via `SetResourceGroupsController` // in unit test. resourceGroupsController atomic.Pointer[rmclient.ResourceGroupsController] @@ -1757,14 +1756,6 @@ func (do *Domain) GetRUVersion() rmclient.RUVersion { return rmclient.DefaultRUVersion } -// SetupHistoricalStatsWorker setups worker -func (do *Domain) SetupHistoricalStatsWorker(ctx sessionctx.Context) { - do.historicalStatsWorker = &HistoricalStatsWorker{ - tblCH: make(chan int64, 16), - sctx: ctx, - } -} - // SetupDumpFileGCChecker setup sctx func (do *Domain) SetupDumpFileGCChecker(ctx sessionctx.Context) { do.dumpFileGcChecker.setupSctx(ctx) @@ -1780,7 +1771,6 @@ var planReplayerHandleLease atomic.Uint64 func init() { planReplayerHandleLease.Store(uint64(10 * time.Second)) - enableDumpHistoricalStats.Store(true) } // DisablePlanReplayerBackgroundJob4Test disable plan replayer handle for test @@ -1788,11 +1778,6 @@ func DisablePlanReplayerBackgroundJob4Test() { planReplayerHandleLease.Store(0) } -// DisableDumpHistoricalStats4Test disable historical dump worker for test -func DisableDumpHistoricalStats4Test() { - enableDumpHistoricalStats.Store(false) -} - // StartPlanReplayerHandle start plan replayer handle job func (do *Domain) StartPlanReplayerHandle() { lease := planReplayerHandleLease.Load() @@ -1873,44 +1858,6 @@ func (do *Domain) DumpFileGcCheckerLoop() { }, "dumpFileGcChecker") } -// GetHistoricalStatsWorker gets historical workers -func (do *Domain) GetHistoricalStatsWorker() *HistoricalStatsWorker { - return do.historicalStatsWorker -} - -// EnableDumpHistoricalStats used to control whether enable dump stats for unit test -var enableDumpHistoricalStats atomic.Bool - -// StartHistoricalStatsWorker start historical workers running -func (do *Domain) StartHistoricalStatsWorker() { - if !enableDumpHistoricalStats.Load() { - return - } - do.wg.Run(func() { - logutil.BgLogger().Info("HistoricalStatsWorker started") - defer func() { - logutil.BgLogger().Info("HistoricalStatsWorker exited.") - }() - defer util.Recover(metrics.LabelDomain, "HistoricalStatsWorkerLoop", nil, false) - - for { - select { - case <-do.exit: - close(do.historicalStatsWorker.tblCH) - return - case tblID, ok := <-do.historicalStatsWorker.tblCH: - if !ok { - return - } - err := do.historicalStatsWorker.DumpHistoricalStats(tblID, do.StatsHandle()) - if err != nil { - logutil.BgLogger().Warn("dump historical stats failed", zap.Error(err), zap.Int64("tableID", tblID)) - } - } - } - }, "HistoricalStatsWorker") -} - // StatsHandle returns the statistic handle. func (do *Domain) StatsHandle() *handle.Handle { return do.statsHandle.Load() diff --git a/pkg/domain/extract.go b/pkg/domain/extract.go index 067e90321d81d..a9fc498b3bbc1 100644 --- a/pkg/domain/extract.go +++ b/pkg/domain/extract.go @@ -393,7 +393,7 @@ func (w *extractWorker) dumpExtractPlanPackage(ctx context.Context, task *Extrac } // Dump stats if !task.SkipStats { - if _, err = dumpStats(zw, p.tables, GetDomain(w.sctx), 0); err != nil { + if err = dumpStats(zw, p.tables, GetDomain(w.sctx)); err != nil { return "", err } } diff --git a/pkg/domain/historical_stats.go b/pkg/domain/historical_stats.go deleted file mode 100644 index 22569fbc7d1d8..0000000000000 --- a/pkg/domain/historical_stats.go +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2022 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package domain - -import ( - "context" - - "github.com/pingcap/errors" - "github.com/pingcap/failpoint" - domain_metrics "github.com/pingcap/tidb/pkg/domain/metrics" - "github.com/pingcap/tidb/pkg/infoschema" - "github.com/pingcap/tidb/pkg/meta/model" - "github.com/pingcap/tidb/pkg/sessionctx" - "github.com/pingcap/tidb/pkg/statistics/handle" - "github.com/pingcap/tidb/pkg/util/logutil" - "go.uber.org/zap" -) - -// HistoricalStatsWorker indicates for dump historical stats -type HistoricalStatsWorker struct { - tblCH chan int64 - sctx sessionctx.Context -} - -// SendTblToDumpHistoricalStats send tableID to worker to dump historical stats -func (w *HistoricalStatsWorker) SendTblToDumpHistoricalStats(tableID int64) { - send := enableDumpHistoricalStats.Load() - failpoint.Inject("sendHistoricalStats", func(val failpoint.Value) { - if val.(bool) { - send = true - } - }) - if !send { - return - } - select { - case w.tblCH <- tableID: - return - default: - logutil.BgLogger().Warn("discard dump historical stats task", zap.Int64("table-id", tableID)) - } -} - -// DumpHistoricalStats dump stats by given tableID -func (w *HistoricalStatsWorker) DumpHistoricalStats(tableID int64, statsHandle *handle.Handle) error { - historicalStatsEnabled, err := statsHandle.CheckHistoricalStatsEnable() - if err != nil { - return errors.Errorf("check tidb_enable_historical_stats failed: %v", err) - } - if !historicalStatsEnabled { - return nil - } - sctx := w.sctx - is := GetDomain(sctx).InfoSchema() - isPartition := false - var tblInfo *model.TableInfo - tbl, existed := is.TableByID(context.Background(), tableID) - if !existed { - tbl, db, p := is.FindTableByPartitionID(tableID) - if !(tbl != nil && db != nil && p != nil) { - return errors.Errorf("cannot get table by id %d", tableID) - } - isPartition = true - tblInfo = tbl.Meta() - } else { - tblInfo = tbl.Meta() - } - dbInfo, existed := infoschema.SchemaByTable(is, tblInfo) - if !existed { - return errors.Errorf("cannot get DBInfo by TableID %d", tableID) - } - if _, err := statsHandle.RecordHistoricalStatsToStorage(dbInfo.Name.O, tblInfo, tableID, isPartition); err != nil { - domain_metrics.GenerateHistoricalStatsFailedCounter.Inc() - return errors.Errorf("record table %s.%s's historical stats failed, err:%v", dbInfo.Name.O, tblInfo.Name.O, err) - } - domain_metrics.GenerateHistoricalStatsSuccessCounter.Inc() - return nil -} - -// GetOneHistoricalStatsTable gets one tableID from channel, only used for test -func (w *HistoricalStatsWorker) GetOneHistoricalStatsTable() int64 { - select { - case tblID := <-w.tblCH: - return tblID - default: - return -1 - } -} diff --git a/pkg/domain/metrics/metrics.go b/pkg/domain/metrics/metrics.go index 2b19aaba82742..01e71ac4a5d5f 100644 --- a/pkg/domain/metrics/metrics.go +++ b/pkg/domain/metrics/metrics.go @@ -21,9 +21,6 @@ import ( // domain metrics vars var ( - GenerateHistoricalStatsSuccessCounter prometheus.Counter - GenerateHistoricalStatsFailedCounter prometheus.Counter - PlanReplayerDumpTaskSuccess prometheus.Counter PlanReplayerDumpTaskFailed prometheus.Counter @@ -39,9 +36,6 @@ func init() { // InitMetricsVars init domain metrics vars. func InitMetricsVars() { - GenerateHistoricalStatsSuccessCounter = metrics.HistoricalStatsCounter.WithLabelValues("generate", "success") - GenerateHistoricalStatsFailedCounter = metrics.HistoricalStatsCounter.WithLabelValues("generate", "fail") - PlanReplayerDumpTaskSuccess = metrics.PlanReplayerTaskCounter.WithLabelValues("dump", "success") PlanReplayerDumpTaskFailed = metrics.PlanReplayerTaskCounter.WithLabelValues("dump", "fail") diff --git a/pkg/domain/plan_replayer.go b/pkg/domain/plan_replayer.go index deb53c78df734..b892b3e2e6f44 100644 --- a/pkg/domain/plan_replayer.go +++ b/pkg/domain/plan_replayer.go @@ -36,7 +36,6 @@ import ( "github.com/pingcap/tidb/pkg/parser/terror" "github.com/pingcap/tidb/pkg/planner/extstore" "github.com/pingcap/tidb/pkg/sessionctx" - "github.com/pingcap/tidb/pkg/sessionctx/vardef" "github.com/pingcap/tidb/pkg/sessionctx/variable" "github.com/pingcap/tidb/pkg/util" "github.com/pingcap/tidb/pkg/util/chunk" @@ -470,7 +469,7 @@ func (w *planReplayerTaskDumpWorker) HandleTask(task *PlanReplayerDumpTask) (suc zap.Error(err)) return false } - file, fileName, err := replayer.GeneratePlanReplayerFile(w.ctx, storage, task.IsCapture, task.IsContinuesCapture, vardef.EnableHistoricalStatsForCapture.Load()) + file, fileName, err := replayer.GeneratePlanReplayerFile(w.ctx, storage, task.IsCapture, task.IsContinuesCapture) if err != nil { logutil.BgLogger().Warn("generate task file failed", zap.String("category", "plan-replayer-capture"), zap.String("sqlDigest", taskKey.SQLDigest), @@ -576,14 +575,13 @@ type PlanReplayerDumpTask struct { TblStats map[int64]any // variables used to dump the plan - StartTS uint64 - SessionBindings [][]*bindinfo.Binding - EncodedPlan string - SessionVars *variable.SessionVars - ExecStmts []ast.StmtNode - Analyze bool - HistoricalStatsTS uint64 - DebugTrace []any + StartTS uint64 + SessionBindings [][]*bindinfo.Binding + EncodedPlan string + SessionVars *variable.SessionVars + ExecStmts []ast.StmtNode + Analyze bool + DebugTrace []any FileName string PresignedURL string diff --git a/pkg/domain/plan_replayer_dump.go b/pkg/domain/plan_replayer_dump.go index eede314ea1a2c..64fb802cfbbc3 100644 --- a/pkg/domain/plan_replayer_dump.go +++ b/pkg/domain/plan_replayer_dump.go @@ -27,7 +27,6 @@ import ( "github.com/BurntSushi/toml" "github.com/pingcap/errors" - "github.com/pingcap/failpoint" "github.com/pingcap/tidb/pkg/bindinfo" "github.com/pingcap/tidb/pkg/config" domain_metrics "github.com/pingcap/tidb/pkg/domain/metrics" @@ -79,10 +78,6 @@ const ( PlanReplayerTaskMetaSQLDigest = "sqlDigest" // PlanReplayerTaskMetaPlanDigest indicates the plan digest of this task PlanReplayerTaskMetaPlanDigest = "planDigest" - // PlanReplayerTaskEnableHistoricalStats indicates whether the task is using historical stats - PlanReplayerTaskEnableHistoricalStats = "enableHistoricalStats" - // PlanReplayerHistoricalStatsTS indicates the expected TS of the historical stats if it's specified by the user. - PlanReplayerHistoricalStatsTS = "historicalStatsTS" ) type tableNamePair struct { @@ -325,34 +320,9 @@ func DumpPlanReplayerInfo(ctx context.Context, sctx sessionctx.Context, return err } - // For continuous capture task, we dump stats in storage only if EnableHistoricalStatsForCapture is disabled. - // For manual plan replayer dump command or capture, we directly dump stats in storage - if task.IsCapture && task.IsContinuesCapture { - if !vardef.EnableHistoricalStatsForCapture.Load() { - // Dump stats - fallbackMsg, err := dumpStats(zw, pairs, do, 0) - if err != nil { - return err - } - if len(fallbackMsg) > 0 { - errMsgs = append(errMsgs, fallbackMsg) - } - } else { - failpoint.Inject("shouldDumpStats", func(val failpoint.Value) { - if val.(bool) { - panic("shouldDumpStats") - } - }) - } - } else { - // Dump stats - fallbackMsg, err := dumpStats(zw, pairs, do, task.HistoricalStatsTS) - if err != nil { - return err - } - if len(fallbackMsg) > 0 { - errMsgs = append(errMsgs, fallbackMsg) - } + // Dump stats + if err = dumpStats(zw, pairs, do); err != nil { + return err } if err = dumpStatsMemStatus(zw, pairs, do); err != nil { @@ -461,10 +431,6 @@ func dumpSQLMeta(zw *zip.Writer, task *PlanReplayerDumpTask) error { varMap[PlanReplayerTaskMetaIsContinues] = strconv.FormatBool(task.IsContinuesCapture) varMap[PlanReplayerTaskMetaSQLDigest] = task.SQLDigest varMap[PlanReplayerTaskMetaPlanDigest] = task.PlanDigest - varMap[PlanReplayerTaskEnableHistoricalStats] = strconv.FormatBool(vardef.EnableHistoricalStatsForCapture.Load()) - if task.HistoricalStatsTS > 0 { - varMap[PlanReplayerHistoricalStatsTS] = strconv.FormatUint(task.HistoricalStatsTS, 10) - } if err := toml.NewEncoder(cf).Encode(varMap); err != nil { return errors.AddStack(err) } @@ -582,35 +548,29 @@ func dumpStatsMemStatus(zw *zip.Writer, pairs map[tableNamePair]struct{}, do *Do return nil } -func dumpStats(zw *zip.Writer, pairs map[tableNamePair]struct{}, do *Domain, historyStatsTS uint64) (string, error) { - allFallBackTbls := make([]string, 0) +func dumpStats(zw *zip.Writer, pairs map[tableNamePair]struct{}, do *Domain) error { for pair := range pairs { if pair.IsView { continue } - jsonTbl, fallBackTbls, err := getStatsForTable(do, pair, historyStatsTS) + jsonTbl, err := getStatsForTable(do, pair) if err != nil { - return "", err + return err } statsFw, err := zw.Create(fmt.Sprintf("stats/%v.%v.json", pair.DBName, pair.TableName)) if err != nil { - return "", errors.AddStack(err) + return errors.AddStack(err) } data, err := json.Marshal(jsonTbl) if err != nil { - return "", errors.AddStack(err) + return errors.AddStack(err) } _, err = statsFw.Write(data) if err != nil { - return "", errors.AddStack(err) + return errors.AddStack(err) } - allFallBackTbls = append(allFallBackTbls, fallBackTbls...) - } - var msg string - if len(allFallBackTbls) > 0 { - msg = "Historical stats for " + strings.Join(allFallBackTbls, ", ") + " are unavailable, fallback to latest stats" } - return msg, nil + return nil } func dumpSQLs(execStmts []ast.StmtNode, zw *zip.Writer) error { @@ -852,18 +812,15 @@ func extractTableNames(ctx context.Context, sctx sessionctx.Context, return tableExtractor.getTablesAndViews() } -func getStatsForTable(do *Domain, pair tableNamePair, historyStatsTS uint64) (*util.JSONTable, []string, error) { +func getStatsForTable(do *Domain, pair tableNamePair) (*util.JSONTable, error) { is := do.InfoSchema() h := do.StatsHandle() tbl, err := is.TableByName(context.Background(), ast.NewCIStr(pair.DBName), ast.NewCIStr(pair.TableName)) if err != nil { - return nil, nil, err - } - if historyStatsTS > 0 { - return h.DumpHistoricalStatsBySnapshot(pair.DBName, tbl.Meta(), historyStatsTS) + return nil, err } jt, err := h.DumpStatsToJSON(pair.DBName, tbl.Meta(), nil, true) - return jt, nil, err + return jt, err } func getShowCreateTable(pair tableNamePair, zw *zip.Writer, ctx sessionctx.Context) error { diff --git a/pkg/domain/plan_replayer_test.go b/pkg/domain/plan_replayer_test.go index 5781ae7076785..55ae80df784b2 100644 --- a/pkg/domain/plan_replayer_test.go +++ b/pkg/domain/plan_replayer_test.go @@ -42,7 +42,7 @@ func TestPlanReplayerDifferentGC(t *testing.T) { time1 := time.Now().Add(-7 * 25 * time.Hour).UnixNano() require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/util/replayer/InjectPlanReplayerFileNameTimeField", fmt.Sprintf("return(%d)", time1))) - file1, fileName1, err := replayer.GeneratePlanReplayerFile(ctx, storage, true, false, false) + file1, fileName1, err := replayer.GeneratePlanReplayerFile(ctx, storage, true, false) require.NoError(t, err) require.NoError(t, file1.Close()) filePath1 := filepath.Join(dirName, fileName1) @@ -50,7 +50,7 @@ func TestPlanReplayerDifferentGC(t *testing.T) { time2 := time.Now().Add(-7 * 23 * time.Hour).UnixNano() require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/util/replayer/InjectPlanReplayerFileNameTimeField", fmt.Sprintf("return(%d)", time2))) - file2, fileName2, err := replayer.GeneratePlanReplayerFile(ctx, storage, true, false, false) + file2, fileName2, err := replayer.GeneratePlanReplayerFile(ctx, storage, true, false) require.NoError(t, err) require.NoError(t, file2.Close()) filePath2 := filepath.Join(dirName, fileName2) @@ -58,7 +58,7 @@ func TestPlanReplayerDifferentGC(t *testing.T) { time3 := time.Now().Add(-2 * time.Hour).UnixNano() require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/util/replayer/InjectPlanReplayerFileNameTimeField", fmt.Sprintf("return(%d)", time3))) - file3, fileName3, err := replayer.GeneratePlanReplayerFile(ctx, storage, false, false, false) + file3, fileName3, err := replayer.GeneratePlanReplayerFile(ctx, storage, false, false) require.NoError(t, err) require.NoError(t, file3.Close()) filePath3 := filepath.Join(dirName, fileName3) @@ -66,7 +66,7 @@ func TestPlanReplayerDifferentGC(t *testing.T) { time4 := time.Now().UnixNano() require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/util/replayer/InjectPlanReplayerFileNameTimeField", fmt.Sprintf("return(%d)", time4))) - file4, fileName4, err := replayer.GeneratePlanReplayerFile(ctx, storage, false, false, false) + file4, fileName4, err := replayer.GeneratePlanReplayerFile(ctx, storage, false, false) require.NoError(t, err) require.NoError(t, file4.Close()) filePath4 := filepath.Join(dirName, fileName4) @@ -118,42 +118,22 @@ func TestDumpGCFileParseTime(t *testing.T) { require.NoError(t, err) var pName string - pName, err = replayer.GeneratePlanReplayerFileName(false, false, false) + pName, err = replayer.GeneratePlanReplayerFileName(false, false) require.NoError(t, err) _, err = parseTime(pName) require.NoError(t, err) - pName, err = replayer.GeneratePlanReplayerFileName(true, false, false) + pName, err = replayer.GeneratePlanReplayerFileName(true, false) require.NoError(t, err) _, err = parseTime(pName) require.NoError(t, err) - pName, err = replayer.GeneratePlanReplayerFileName(false, true, false) + pName, err = replayer.GeneratePlanReplayerFileName(false, true) require.NoError(t, err) _, err = parseTime(pName) require.NoError(t, err) - pName, err = replayer.GeneratePlanReplayerFileName(true, true, false) - require.NoError(t, err) - _, err = parseTime(pName) - require.NoError(t, err) - - pName, err = replayer.GeneratePlanReplayerFileName(false, false, true) - require.NoError(t, err) - _, err = parseTime(pName) - require.NoError(t, err) - - pName, err = replayer.GeneratePlanReplayerFileName(true, false, true) - require.NoError(t, err) - _, err = parseTime(pName) - require.NoError(t, err) - - pName, err = replayer.GeneratePlanReplayerFileName(false, true, true) - require.NoError(t, err) - _, err = parseTime(pName) - require.NoError(t, err) - - pName, err = replayer.GeneratePlanReplayerFileName(true, true, true) + pName, err = replayer.GeneratePlanReplayerFileName(true, true) require.NoError(t, err) _, err = parseTime(pName) require.NoError(t, err) diff --git a/pkg/executor/BUILD.bazel b/pkg/executor/BUILD.bazel index dded565b45a74..1e342d9cfdaca 100644 --- a/pkg/executor/BUILD.bazel +++ b/pkg/executor/BUILD.bazel @@ -369,7 +369,6 @@ go_test( "explain_unit_test.go", "explainfor_test.go", "grant_test.go", - "historical_stats_test.go", "hot_regions_history_table_test.go", "import_into_test.go", "infoschema_cluster_table_test.go", @@ -492,8 +491,6 @@ go_test( "//pkg/sessionctx/variable", "//pkg/sessiontxn", "//pkg/statistics", - "//pkg/statistics/handle/storage", - "//pkg/statistics/util", "//pkg/store/copr", "//pkg/store/driver/error", "//pkg/store/helper", diff --git a/pkg/executor/analyze.go b/pkg/executor/analyze.go index 30addfe0a457b..7cd9682462676 100644 --- a/pkg/executor/analyze.go +++ b/pkg/executor/analyze.go @@ -625,20 +625,6 @@ func (e *AnalyzeExec) saveAnalyzeOptions() error { return nil } -func recordHistoricalStats(sctx sessionctx.Context, tableID int64) error { - statsHandle := domain.GetDomain(sctx).StatsHandle() - historicalStatsEnabled, err := statsHandle.CheckHistoricalStatsEnable() - if err != nil { - return errors.Errorf("check tidb_enable_historical_stats failed: %v", err) - } - if !historicalStatsEnabled { - return nil - } - historicalStatsWorker := domain.GetDomain(sctx).GetHistoricalStatsWorker() - historicalStatsWorker.SendTblToDumpHistoricalStats(tableID) - return nil -} - // handleResultsError will handle the error fetch from resultsCh and record it in log func (e *AnalyzeExec) handleResultsError( buildStatsConcurrency int, @@ -696,7 +682,6 @@ func (e *AnalyzeExec) handleResultsErrorWithConcurrency( worker.run(ctx1, statsHandle, enableAnalyzeSnapshot) }) } - tableIDs := map[int64]struct{}{} panicCnt := 0 var err error // Only if all the analyze workers exit can we close the saveResultsCh. @@ -720,7 +705,6 @@ func (e *AnalyzeExec) handleResultsErrorWithConcurrency( continue } handleGlobalStats(needGlobalStats, globalStatsMap, results) - tableIDs[results.TableID.GetStatisticsID()] = struct{}{} failpoint.InjectCall("analyzeBeforeSendToSaveResults") saveResultsCh <- results } @@ -736,12 +720,6 @@ func (e *AnalyzeExec) handleResultsErrorWithConcurrency( errMsg := slices.Collect(maps.Keys(errSet)) err = errors.New(strings.Join(errMsg, ",")) } - for tableID := range tableIDs { - // Dump stats to historical storage. - if err := recordHistoricalStats(e.Ctx(), tableID); err != nil { - statslogutil.StatsErrVerboseLogger().Error("record historical stats failed", zap.Error(err)) - } - } return err } diff --git a/pkg/executor/analyze_global_stats.go b/pkg/executor/analyze_global_stats.go index dc9846f3602c5..df893218d5725 100644 --- a/pkg/executor/analyze_global_stats.go +++ b/pkg/executor/analyze_global_stats.go @@ -43,9 +43,7 @@ func (e *AnalyzeExec) handleGlobalStats(statsHandle *handle.Handle, globalStatsM globalStatsTableIDs[globalStatsID.tableID] = struct{}{} } - tableIDs := make(map[int64]struct{}, len(globalStatsTableIDs)) for tableID := range globalStatsTableIDs { - tableIDs[tableID] = struct{}{} for globalStatsID, info := range globalStatsMap { if globalStatsID.tableID != tableID { continue @@ -81,13 +79,6 @@ func (e *AnalyzeExec) handleGlobalStats(statsHandle *handle.Handle, globalStatsM } } - for tableID := range tableIDs { - // Dump stats to historical storage. - if err := recordHistoricalStats(e.Ctx(), tableID); err != nil { - logutil.BgLogger().Error("record historical stats failed", zap.Error(err)) - } - } - return nil } diff --git a/pkg/executor/analyze_worker.go b/pkg/executor/analyze_worker.go index c4038941bdece..831ce24afb26e 100644 --- a/pkg/executor/analyze_worker.go +++ b/pkg/executor/analyze_worker.go @@ -20,7 +20,6 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/tidb/pkg/statistics" "github.com/pingcap/tidb/pkg/statistics/handle" - "github.com/pingcap/tidb/pkg/statistics/handle/util" "github.com/pingcap/tidb/pkg/util/logutil" "github.com/pingcap/tidb/pkg/util/sqlkiller" "go.uber.org/zap" @@ -84,7 +83,7 @@ func (worker *analyzeSaveStatsWorker) run(ctx context.Context, statsHandle *hand } continue } - err := statsHandle.SaveAnalyzeResultToStorage(results, analyzeSnapshot, util.StatsMetaHistorySourceAnalyze) + err := statsHandle.SaveAnalyzeResultToStorage(results, analyzeSnapshot) if err != nil { logutil.Logger(ctx).Warn("save table stats to storage failed", zap.Error(err)) finishJobWithLog(statsHandle, results.Job, err) diff --git a/pkg/executor/builder.go b/pkg/executor/builder.go index 2e174a2628998..9b7c4c488c7ff 100644 --- a/pkg/executor/builder.go +++ b/pkg/executor/builder.go @@ -1197,10 +1197,9 @@ func (b *executorBuilder) buildPlanReplayer(v *plannercore.PlanReplayer) exec.Ex e := &PlanReplayerExec{ BaseExecutor: exec.NewBaseExecutor(b.sctx, v.Schema(), v.ID()), DumpInfo: &PlanReplayerDumpInfo{ - Analyze: v.Analyze, - Path: v.File, - ctx: b.sctx, - HistoricalStatsTS: v.HistoricalStatsTS, + Analyze: v.Analyze, + Path: v.File, + ctx: b.sctx, }, } if len(v.StmtList) > 0 { diff --git a/pkg/executor/historical_stats_test.go b/pkg/executor/historical_stats_test.go deleted file mode 100644 index 1bf08a0df16ae..0000000000000 --- a/pkg/executor/historical_stats_test.go +++ /dev/null @@ -1,436 +0,0 @@ -// Copyright 2022 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package executor_test - -import ( - "context" - "encoding/json" - "fmt" - "strconv" - "testing" - "time" - - "github.com/pingcap/failpoint" - "github.com/pingcap/tidb/pkg/parser/ast" - "github.com/pingcap/tidb/pkg/sessionctx/vardef" - "github.com/pingcap/tidb/pkg/statistics/handle/storage" - "github.com/pingcap/tidb/pkg/statistics/util" - "github.com/pingcap/tidb/pkg/testkit" - "github.com/stretchr/testify/require" - "github.com/tikv/client-go/v2/oracle" -) - -func TestRecordHistoryStatsAfterAnalyze(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/pkg/domain/sendHistoricalStats", "return(true)") - defer failpoint.Disable("github.com/pingcap/tidb/pkg/domain/sendHistoricalStats") - store, dom := testkit.CreateMockStoreAndDomain(t) - - tk := testkit.NewTestKit(t, store) - tk.MustExec("set @@tidb_analyze_version = 2") - tk.MustExec("set global tidb_enable_historical_stats = 0") - tk.MustExec("use test") - tk.MustExec("drop table if exists t") - tk.MustExec("create table t(a int, b varchar(10), index idx(a, b))") - - h := dom.StatsHandle() - is := dom.InfoSchema() - tableInfo, err := is.TableByName(context.Background(), ast.NewCIStr("test"), ast.NewCIStr("t")) - require.NoError(t, err) - - // 1. switch off the tidb_enable_historical_stats, and there is no records in table `mysql.stats_history` - rows := tk.MustQuery(fmt.Sprintf("select count(*) from mysql.stats_history where table_id = '%d'", tableInfo.Meta().ID)).Rows() - num, _ := strconv.Atoi(rows[0][0].(string)) - require.Equal(t, num, 0) - - tk.MustExec("analyze table t with 2 topn") - rows = tk.MustQuery(fmt.Sprintf("select count(*) from mysql.stats_history where table_id = '%d'", tableInfo.Meta().ID)).Rows() - num, _ = strconv.Atoi(rows[0][0].(string)) - require.Equal(t, num, 0) - - // 2. switch on the tidb_enable_historical_stats and do analyze - tk.MustExec("set global tidb_enable_historical_stats = 1") - defer tk.MustExec("set global tidb_enable_historical_stats = 0") - tk.MustExec("analyze table t with 2 topn") - // dump historical stats - hsWorker := dom.GetHistoricalStatsWorker() - tblID := hsWorker.GetOneHistoricalStatsTable() - err = hsWorker.DumpHistoricalStats(tblID, h) - require.Nil(t, err) - rows = tk.MustQuery(fmt.Sprintf("select count(*) from mysql.stats_history where table_id = '%d'", tableInfo.Meta().ID)).Rows() - num, _ = strconv.Atoi(rows[0][0].(string)) - require.GreaterOrEqual(t, num, 1) - - // 3. dump current stats json - dumpJSONTable, err := h.DumpStatsToJSON("test", tableInfo.Meta(), nil, true) - require.NoError(t, err) - dumpJSONTable.Sort() - jsOrigin, _ := json.Marshal(dumpJSONTable) - - // 4. get the historical stats json - rows = tk.MustQuery(fmt.Sprintf("select * from mysql.stats_history where table_id = '%d' and create_time = ("+ - "select create_time from mysql.stats_history where table_id = '%d' order by create_time desc limit 1) "+ - "order by seq_no", tableInfo.Meta().ID, tableInfo.Meta().ID)).Rows() - num = len(rows) - require.GreaterOrEqual(t, num, 1) - data := make([][]byte, num) - for i, row := range rows { - data[i] = []byte(row[1].(string)) - } - jsonTbl, err := storage.BlocksToJSONTable(data) - require.NoError(t, err) - jsonTbl.Sort() - jsCur, err := json.Marshal(jsonTbl) - require.NoError(t, err) - // 5. historical stats must be equal to the current stats - require.JSONEq(t, string(jsOrigin), string(jsCur)) -} - -func TestRecordHistoryStatsMetaAfterAnalyze(t *testing.T) { - store, dom := testkit.CreateMockStoreAndDomain(t) - - tk := testkit.NewTestKit(t, store) - tk.MustExec("set @@tidb_analyze_version = 2") - tk.MustExec("set global tidb_enable_historical_stats = 0") - tk.MustExec("use test") - tk.MustExec("drop table if exists t") - tk.MustExec("create table t(a int, b int, index idx(a, b))") - tk.MustExec("analyze table test.t") - - is := dom.InfoSchema() - tableInfo, err := is.TableByName(context.Background(), ast.NewCIStr("test"), ast.NewCIStr("t")) - require.NoError(t, err) - - // 1. switch off the tidb_enable_historical_stats, and there is no record in table `mysql.stats_meta_history` - tk.MustQuery(fmt.Sprintf("select count(*) from mysql.stats_meta_history where table_id = '%d'", tableInfo.Meta().ID)).Check(testkit.Rows("0")) - // insert demo tuples, and there is no record either. - insertNums := 5 - for range insertNums { - tk.MustExec("insert into test.t (a,b) values (1,1), (2,2), (3,3)") - tk.MustExec("flush stats_delta *.*") - } - tk.MustQuery(fmt.Sprintf("select count(*) from mysql.stats_meta_history where table_id = '%d'", tableInfo.Meta().ID)).Check(testkit.Rows("0")) - - // 2. switch on the tidb_enable_historical_stats and insert tuples to produce count/modifyCount delta change. - tk.MustExec("set global tidb_enable_historical_stats = 1") - defer tk.MustExec("set global tidb_enable_historical_stats = 0") - - for range insertNums { - tk.MustExec("insert into test.t (a,b) values (1,1), (2,2), (3,3)") - tk.MustExec("flush stats_delta *.*") - } - tk.MustQuery(fmt.Sprintf("select modify_count, count from mysql.stats_meta_history where table_id = '%d' order by create_time", tableInfo.Meta().ID)).Sort().Check( - testkit.Rows("18 18", "21 21", "24 24", "27 27", "30 30")) - tk.MustQuery(fmt.Sprintf("select distinct source from mysql.stats_meta_history where table_id = '%d'", tableInfo.Meta().ID)).Sort().Check(testkit.Rows("flush stats")) - - // assert delete - tk.MustExec("delete from test.t where test.t.a = 1") - tk.MustExec("flush stats_delta *.*") - tk.MustQuery(fmt.Sprintf("select modify_count, count from mysql.stats_meta where table_id = '%d'", tableInfo.Meta().ID)).Sort().Check( - testkit.Rows("40 20")) - tk.MustQuery(fmt.Sprintf("select modify_count, count from mysql.stats_meta_history where table_id = '%d' order by create_time desc limit 1", tableInfo.Meta().ID)).Sort().Check( - testkit.Rows("40 20")) - - // assert update - tk.MustExec("update test.t set test.t.b = 4 where test.t.a = 2") - tk.MustExec("flush stats_delta *.*") - tk.MustQuery(fmt.Sprintf("select modify_count, count from mysql.stats_meta where table_id = '%d'", tableInfo.Meta().ID)).Sort().Check( - testkit.Rows("50 20")) - tk.MustQuery(fmt.Sprintf("select modify_count, count from mysql.stats_meta_history where table_id = '%d' order by create_time desc limit 1", tableInfo.Meta().ID)).Sort().Check( - testkit.Rows("50 20")) -} - -func TestGCHistoryStatsAfterDropTable(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/pkg/domain/sendHistoricalStats", "return(true)") - defer failpoint.Disable("github.com/pingcap/tidb/pkg/domain/sendHistoricalStats") - store, dom := testkit.CreateMockStoreAndDomain(t) - tk := testkit.NewTestKit(t, store) - tk.MustExec("set global tidb_enable_historical_stats = 1") - tk.MustExec("use test") - tk.MustExec("drop table if exists t") - tk.MustExec("create table t(a int, b varchar(10), index idx(a, b))") - tk.MustExec("analyze table test.t") - is := dom.InfoSchema() - tableInfo, err := is.TableByName(context.Background(), ast.NewCIStr("test"), ast.NewCIStr("t")) - require.NoError(t, err) - // dump historical stats - h := dom.StatsHandle() - hsWorker := dom.GetHistoricalStatsWorker() - tblID := hsWorker.GetOneHistoricalStatsTable() - err = hsWorker.DumpHistoricalStats(tblID, h) - require.Nil(t, err) - - // assert the records of history stats table - tk.MustQuery(fmt.Sprintf("select count(*) from mysql.stats_meta_history where table_id = '%d' order by create_time", - tableInfo.Meta().ID)).Check(testkit.Rows("1")) - tk.MustQuery(fmt.Sprintf("select count(*) from mysql.stats_history where table_id = '%d'", - tableInfo.Meta().ID)).Check(testkit.Rows("1")) - // drop the table and gc stats - tk.MustExec("drop table t") - is = dom.InfoSchema() - h.GCStats(is, 0) - - // assert stats_history tables delete the record of dropped table - tk.MustQuery(fmt.Sprintf("select count(*) from mysql.stats_meta_history where table_id = '%d' order by create_time", - tableInfo.Meta().ID)).Check(testkit.Rows("0")) - tk.MustQuery(fmt.Sprintf("select count(*) from mysql.stats_history where table_id = '%d'", - tableInfo.Meta().ID)).Check(testkit.Rows("0")) -} - -func TestAssertHistoricalStatsAfterAlterTable(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/pkg/domain/sendHistoricalStats", "return(true)") - defer failpoint.Disable("github.com/pingcap/tidb/pkg/domain/sendHistoricalStats") - store, dom := testkit.CreateMockStoreAndDomain(t) - tk := testkit.NewTestKit(t, store) - tk.MustExec("set global tidb_enable_historical_stats = 1") - tk.MustExec("use test") - tk.MustExec("drop table if exists t") - tk.MustExec("create table t(a int, b varchar(10),c int, KEY `idx` (`c`))") - tk.MustExec("analyze table test.t") - is := dom.InfoSchema() - tableInfo, err := is.TableByName(context.Background(), ast.NewCIStr("test"), ast.NewCIStr("t")) - require.NoError(t, err) - // dump historical stats - h := dom.StatsHandle() - hsWorker := dom.GetHistoricalStatsWorker() - tblID := hsWorker.GetOneHistoricalStatsTable() - err = hsWorker.DumpHistoricalStats(tblID, h) - require.Nil(t, err) - - time.Sleep(1 * time.Second) - snapshot := oracle.GoTimeToTS(time.Now()) - jsTable, _, err := h.DumpHistoricalStatsBySnapshot("test", tableInfo.Meta(), snapshot) - require.NoError(t, err) - require.NotNil(t, jsTable) - require.NotEqual(t, jsTable.Version, uint64(0)) - originVersion := jsTable.Version - - // assert historical stats non-change after drop column - tk.MustExec("alter table t drop column b") - h.GCStats(is, 0) - snapshot = oracle.GoTimeToTS(time.Now()) - jsTable, _, err = h.DumpHistoricalStatsBySnapshot("test", tableInfo.Meta(), snapshot) - require.NoError(t, err) - require.NotNil(t, jsTable) - require.Equal(t, jsTable.Version, originVersion) - - // assert historical stats non-change after drop index - tk.MustExec("alter table t drop index idx") - h.GCStats(is, 0) - snapshot = oracle.GoTimeToTS(time.Now()) - jsTable, _, err = h.DumpHistoricalStatsBySnapshot("test", tableInfo.Meta(), snapshot) - require.NoError(t, err) - require.NotNil(t, jsTable) - require.Equal(t, jsTable.Version, originVersion) -} - -func TestGCOutdatedHistoryStats(t *testing.T) { - require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/domain/sendHistoricalStats", "return(true)")) - defer func() { - require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/domain/sendHistoricalStats")) - }() - store, dom := testkit.CreateMockStoreAndDomain(t) - tk := testkit.NewTestKit(t, store) - tk.MustExec("set global tidb_enable_historical_stats = 1") - tk.MustExec("use test") - tk.MustExec("drop table if exists t") - tk.MustExec("create table t(a int, b varchar(10), index idx(a, b))") - tk.MustExec("analyze table test.t") - is := dom.InfoSchema() - tableInfo, err := is.TableByName(context.Background(), ast.NewCIStr("test"), ast.NewCIStr("t")) - require.NoError(t, err) - // dump historical stats - h := dom.StatsHandle() - hsWorker := dom.GetHistoricalStatsWorker() - tblID := hsWorker.GetOneHistoricalStatsTable() - err = hsWorker.DumpHistoricalStats(tblID, h) - require.Nil(t, err) - - // assert the records of history stats table - tk.MustQuery(fmt.Sprintf("select count(*) from mysql.stats_meta_history where table_id = '%d' order by create_time", - tableInfo.Meta().ID)).Check(testkit.Rows("1")) - tk.MustQuery(fmt.Sprintf("select count(*) from mysql.stats_history where table_id = '%d'", - tableInfo.Meta().ID)).Check(testkit.Rows("1")) - - tk.MustExec("set @@global.tidb_historical_stats_duration = '1s'") - duration := vardef.HistoricalStatsDuration.Load() - fmt.Println(duration.String()) - time.Sleep(2 * time.Second) - err = dom.StatsHandle().ClearOutdatedHistoryStats() - require.NoError(t, err) - // assert the records of history stats table - tk.MustQuery(fmt.Sprintf("select count(*) from mysql.stats_meta_history where table_id = '%d' order by create_time", - tableInfo.Meta().ID)).Check(testkit.Rows("0")) - tk.MustQuery(fmt.Sprintf("select count(*) from mysql.stats_history where table_id = '%d'", - tableInfo.Meta().ID)).Check(testkit.Rows("0")) -} - -func TestPartitionTableHistoricalStats(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/pkg/domain/sendHistoricalStats", "return(true)") - defer failpoint.Disable("github.com/pingcap/tidb/pkg/domain/sendHistoricalStats") - store, dom := testkit.CreateMockStoreAndDomain(t) - tk := testkit.NewTestKit(t, store) - tk.MustExec("set global tidb_enable_historical_stats = 1") - tk.MustExec("use test") - tk.MustExec("drop table if exists t") - tk.MustExec(`CREATE TABLE t (a int, b int, index idx(b)) -PARTITION BY RANGE ( a ) ( -PARTITION p0 VALUES LESS THAN (6) -)`) - tk.MustExec("delete from mysql.stats_history") - - tk.MustExec("analyze table test.t") - // dump historical stats - h := dom.StatsHandle() - hsWorker := dom.GetHistoricalStatsWorker() - - // assert global table and partition table be dumped - tblID := hsWorker.GetOneHistoricalStatsTable() - err := hsWorker.DumpHistoricalStats(tblID, h) - require.NoError(t, err) - tblID = hsWorker.GetOneHistoricalStatsTable() - err = hsWorker.DumpHistoricalStats(tblID, h) - require.NoError(t, err) - tk.MustQuery("select count(*) from mysql.stats_history").Check(testkit.Rows("2")) -} - -func TestDumpHistoricalStatsByTable(t *testing.T) { - failpoint.Enable("github.com/pingcap/tidb/pkg/domain/sendHistoricalStats", "return(true)") - defer failpoint.Disable("github.com/pingcap/tidb/pkg/domain/sendHistoricalStats") - store, dom := testkit.CreateMockStoreAndDomain(t) - tk := testkit.NewTestKit(t, store) - tk.MustExec("set global tidb_enable_historical_stats = 1") - tk.MustExec("set @@tidb_partition_prune_mode='static'") - tk.MustExec("use test") - tk.MustExec("drop table if exists t") - tk.MustExec(`CREATE TABLE t (a int, b int, index idx(b)) -PARTITION BY RANGE ( a ) ( -PARTITION p0 VALUES LESS THAN (6) -)`) - // dump historical stats - h := dom.StatsHandle() - - tk.MustExec("analyze table t") - is := dom.InfoSchema() - tbl, err := is.TableByName(context.Background(), ast.NewCIStr("test"), ast.NewCIStr("t")) - require.NoError(t, err) - require.NotNil(t, tbl) - - // dump historical stats - hsWorker := dom.GetHistoricalStatsWorker() - // only partition p0 stats will be dumped in static mode - tblID := hsWorker.GetOneHistoricalStatsTable() - require.NotEqual(t, tblID, -1) - err = hsWorker.DumpHistoricalStats(tblID, h) - require.NoError(t, err) - tblID = hsWorker.GetOneHistoricalStatsTable() - require.Equal(t, tblID, int64(-1)) - - time.Sleep(1 * time.Second) - snapshot := oracle.GoTimeToTS(time.Now()) - jsTable, _, err := h.DumpHistoricalStatsBySnapshot("test", tbl.Meta(), snapshot) - require.NoError(t, err) - require.NotNil(t, jsTable) - // only has p0 stats - require.NotNil(t, jsTable.Partitions["p0"]) - require.Nil(t, jsTable.Partitions[util.TiDBGlobalStats]) - - // change static to dynamic then assert - tk.MustExec("set @@tidb_partition_prune_mode='dynamic'") - tk.MustExec("analyze table t") - require.NoError(t, err) - // global and p0's stats will be dumped - tblID = hsWorker.GetOneHistoricalStatsTable() - require.NotEqual(t, tblID, -1) - err = hsWorker.DumpHistoricalStats(tblID, h) - require.NoError(t, err) - tblID = hsWorker.GetOneHistoricalStatsTable() - require.NotEqual(t, tblID, -1) - err = hsWorker.DumpHistoricalStats(tblID, h) - require.NoError(t, err) - time.Sleep(1 * time.Second) - snapshot = oracle.GoTimeToTS(time.Now()) - jsTable, _, err = h.DumpHistoricalStatsBySnapshot("test", tbl.Meta(), snapshot) - require.NoError(t, err) - require.NotNil(t, jsTable) - // has both global and p0 stats - require.NotNil(t, jsTable.Partitions["p0"]) - require.NotNil(t, jsTable.Partitions[util.TiDBGlobalStats]) -} - -func TestDumpHistoricalStatsFallback(t *testing.T) { - store, dom := testkit.CreateMockStoreAndDomain(t) - tk := testkit.NewTestKit(t, store) - tk.MustExec("set global tidb_enable_historical_stats = 0") - tk.MustExec("use test") - tk.MustExec("drop table if exists t") - tk.MustExec(`CREATE TABLE t (a int, b int, index idx(b)) -PARTITION BY RANGE ( a ) ( -PARTITION p0 VALUES LESS THAN (6) -)`) - // dump historical stats - tk.MustExec("analyze table t") - is := dom.InfoSchema() - tbl, err := is.TableByName(context.Background(), ast.NewCIStr("test"), ast.NewCIStr("t")) - require.NoError(t, err) - require.NotNil(t, tbl) - - // dump historical stats - hsWorker := dom.GetHistoricalStatsWorker() - tblID := hsWorker.GetOneHistoricalStatsTable() - // assert no historical stats task generated - require.Equal(t, tblID, int64(-1)) - tk.MustExec("set global tidb_enable_historical_stats = 1") - h := dom.StatsHandle() - jt, _, err := h.DumpHistoricalStatsBySnapshot("test", tbl.Meta(), oracle.GoTimeToTS(time.Now())) - require.NoError(t, err) - require.NotNil(t, jt) - require.False(t, jt.IsHistoricalStats) -} - -func TestDumpHistoricalStatsMetaForMultiTables(t *testing.T) { - store, dom := testkit.CreateMockStoreAndDomain(t) - tk := testkit.NewTestKit(t, store) - tk.MustExec("set global tidb_enable_historical_stats = 1") - tk.MustExec("use test") - tk.MustExec("drop table if exists t") - tk.MustExec("create table t1(a int, b varchar(10), index idx(a, b))") - tk.MustExec("create table t2(a int, b varchar(10), index idx(a, b))") - // Insert some data. - tk.MustExec("insert into t1 values (1, 'a'), (2, 'b'), (3, 'c')") - tk.MustExec("insert into t2 values (1, 'a'), (2, 'b'), (3, 'c')") - // Analyze the tables. - tk.MustExec("analyze table t1") - tk.MustExec("analyze table t2") - h := dom.StatsHandle() - // Update the stats cache. - require.NoError(t, h.Update(context.Background(), dom.InfoSchema())) - - // Insert more data. - tk.MustExec("insert into t1 values (4, 'd'), (5, 'e'), (6, 'f')") - tk.MustExec("insert into t2 values (4, 'd'), (5, 'e'), (6, 'f')") - // Dump stats delta to kv. - tk.MustExec("flush stats_delta *.*") - - // Check historical stats meta. - tbl1, err := dom.InfoSchema().TableByName(context.Background(), ast.NewCIStr("test"), ast.NewCIStr("t1")) - require.NoError(t, err) - tbl2, err := dom.InfoSchema().TableByName(context.Background(), ast.NewCIStr("test"), ast.NewCIStr("t2")) - require.NoError(t, err) - rows := tk.MustQuery("select version from mysql.stats_meta_history where table_id = ? order by version desc limit 1", tbl1.Meta().ID).Rows() - version1 := rows[0][0].(string) - rows = tk.MustQuery("select version from mysql.stats_meta_history where table_id = ? order by version desc limit 1", tbl2.Meta().ID).Rows() - version2 := rows[0][0].(string) - require.Equal(t, version1, version2) -} diff --git a/pkg/executor/plan_replayer.go b/pkg/executor/plan_replayer.go index 1526ae29dd543..6382261cdc86b 100644 --- a/pkg/executor/plan_replayer.go +++ b/pkg/executor/plan_replayer.go @@ -69,14 +69,13 @@ type PlanReplayerCaptureInfo struct { // PlanReplayerDumpInfo indicates dump info type PlanReplayerDumpInfo struct { - ExecStmts []ast.StmtNode - Analyze bool - HistoricalStatsTS uint64 - StartTS uint64 - Path string - File io.WriteCloser - FileName string - ctx sessionctx.Context + ExecStmts []ast.StmtNode + Analyze bool + StartTS uint64 + Path string + File io.WriteCloser + FileName string + ctx sessionctx.Context } // Next implements the Executor Next interface. @@ -213,7 +212,7 @@ func (e *PlanReplayerExec) createFile(ctx context.Context) error { if err != nil { return err } - e.DumpInfo.File, e.DumpInfo.FileName, err = replayer.GeneratePlanReplayerFile(ctx, storage, false, false, false) + e.DumpInfo.File, e.DumpInfo.FileName, err = replayer.GeneratePlanReplayerFile(ctx, storage, false, false) if err != nil { return err } @@ -224,14 +223,13 @@ func (e *PlanReplayerDumpInfo) dump(ctx context.Context) (err error) { fileName := e.FileName zf := e.File task := &domain.PlanReplayerDumpTask{ - StartTS: e.StartTS, - FileName: fileName, - Zf: zf, - SessionVars: e.ctx.GetSessionVars(), - TblStats: nil, - ExecStmts: e.ExecStmts, - Analyze: e.Analyze, - HistoricalStatsTS: e.HistoricalStatsTS, + StartTS: e.StartTS, + FileName: fileName, + Zf: zf, + SessionVars: e.ctx.GetSessionVars(), + TblStats: nil, + ExecStmts: e.ExecStmts, + Analyze: e.Analyze, } err = domain.DumpPlanReplayerInfo(ctx, e.ctx, task) if err != nil { diff --git a/pkg/executor/test/loaddatatest/load_data_test.go b/pkg/executor/test/loaddatatest/load_data_test.go index a0fa99eed22c8..e7b0f938b6983 100644 --- a/pkg/executor/test/loaddatatest/load_data_test.go +++ b/pkg/executor/test/loaddatatest/load_data_test.go @@ -504,7 +504,6 @@ func prepareFix56408Store() func() { vardef.SetSchemaLease(500 * time.Millisecond) session.DisableStats4Test() domain.DisablePlanReplayerBackgroundJob4Test() - domain.DisableDumpHistoricalStats4Test() dom, err := session.BootstrapSession(store) if err != nil { _ = store.Close() diff --git a/pkg/executor/test/planreplayer/plan_replayer_test.go b/pkg/executor/test/planreplayer/plan_replayer_test.go index 3d488bb22561a..2042dcec4f662 100644 --- a/pkg/executor/test/planreplayer/plan_replayer_test.go +++ b/pkg/executor/test/planreplayer/plan_replayer_test.go @@ -141,6 +141,11 @@ func TestPlanReplayer(t *testing.T) { tk.MustQuery("plan replayer dump explain select * from v2") require.True(t, len(tk.Session().GetSessionVars().LastPlanReplayerToken) > 0) + // Keep accepting the legacy syntax, but reject its execution because the + // historical stats feature has been removed. + tk.MustGetErrMsg("plan replayer dump with stats as of timestamp '2023-06-28 12:34:00' explain select * from t where a=10", + "WITH STATS AS OF TIMESTAMP is no longer supported because the historical stats feature has been removed") + // clear the status table and assert tk.MustExec("delete from mysql.plan_replayer_status") tk.MustQuery("plan replayer dump explain select * from v2") @@ -239,12 +244,9 @@ func TestPlanReplayerCapture(t *testing.T) { _, sqlDigest := tk.Session().GetSessionVars().StmtCtx.SQLDigest() _, planDigest := tk.Session().GetSessionVars().StmtCtx.GetPlanDigest() tk.MustExec("SET @@tidb_enable_plan_replayer_capture = ON;") - tk.MustExec("SET @@global.tidb_enable_historical_stats_for_capture='ON'") tk.MustExec(fmt.Sprintf("plan replayer capture '%v' '%v'", sqlDigest.String(), planDigest.String())) err := dom.GetPlanReplayerHandle().CollectPlanReplayerTask() require.NoError(t, err) - require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/domain/shouldDumpStats", "return(true)")) - defer require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/domain/shouldDumpStats")) tk.MustExec("execute stmt using @number,@number") task := dom.GetPlanReplayerHandle().DrainTask() require.NotNil(t, task) @@ -290,12 +292,6 @@ func TestPlanReplayerContinuesCapture(t *testing.T) { store, dom := testkit.CreateMockStoreAndDomain(t) tk := testkit.NewTestKit(t, store) - tk.MustExec("set @@global.tidb_enable_historical_stats='OFF'") - _, err = tk.Exec("set @@global.tidb_enable_plan_replayer_continuous_capture='ON'") - require.Error(t, err) - require.Equal(t, err.Error(), "tidb_enable_historical_stats should be enabled before enabling tidb_enable_plan_replayer_continuous_capture") - - tk.MustExec("set @@global.tidb_enable_historical_stats='ON'") tk.MustExec("set @@global.tidb_enable_plan_replayer_continuous_capture='ON'") prHandle := dom.GetPlanReplayerHandle() diff --git a/pkg/metrics/grafana/tidb.json b/pkg/metrics/grafana/tidb.json index 7ab7d8f049bf5..5ffaea35d4a52 100644 --- a/pkg/metrics/grafana/tidb.json +++ b/pkg/metrics/grafana/tidb.json @@ -19694,116 +19694,6 @@ "alignLevel": null } }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 41 - }, - "hiddenSeries": false, - "id": 237, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "sum(rate(tidb_statistics_historical_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"generate\"}[1m])) by (result)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "generate-{{result}}", - "refId": "A", - "step": 30 - }, - { - "exemplar": true, - "expr": "sum(rate(tidb_statistics_historical_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"dump\"}[1m])) by (result)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "dump-{{result}}", - "refId": "B", - "step": 30 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Historical Stats OPM", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, { "aliasColors": {}, "bars": false, diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 3bc6342ba4581..d3c5087f1cc58 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -303,7 +303,6 @@ func RegisterMetrics() { prometheus.MustRegister(EMACPUUsageGauge) prometheus.MustRegister(PoolConcurrencyCounter) - prometheus.MustRegister(HistoricalStatsCounter) prometheus.MustRegister(PlanReplayerTaskCounter) prometheus.MustRegister(PlanReplayerRegisterTaskGauge) diff --git a/pkg/metrics/nextgengrafana/tidb_with_keyspace_name.json b/pkg/metrics/nextgengrafana/tidb_with_keyspace_name.json index 0b1ee09b5b917..18f3a006b61c9 100644 --- a/pkg/metrics/nextgengrafana/tidb_with_keyspace_name.json +++ b/pkg/metrics/nextgengrafana/tidb_with_keyspace_name.json @@ -19565,116 +19565,6 @@ "alignLevel": null } }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 41 - }, - "hiddenSeries": false, - "id": 237, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "sum(rate(tidb_statistics_historical_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type=~\"generate\"}[1m])) by (result)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "generate-{{result}}", - "refId": "A", - "step": 30 - }, - { - "exemplar": true, - "expr": "sum(rate(tidb_statistics_historical_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", type=~\"dump\"}[1m])) by (result)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "dump-{{result}}", - "refId": "B", - "step": 30 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Historical Stats OPM", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, { "aliasColors": {}, "bars": false, diff --git a/pkg/metrics/nextgengrafana/tidb_worker.json b/pkg/metrics/nextgengrafana/tidb_worker.json index 2ff5d338ccd5a..86fda7e65be11 100644 --- a/pkg/metrics/nextgengrafana/tidb_worker.json +++ b/pkg/metrics/nextgengrafana/tidb_worker.json @@ -18456,116 +18456,6 @@ "alignLevel": null } }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_TEST-CLUSTER}", - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 41 - }, - "hiddenSeries": false, - "id": 237, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.11", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "sum(rate(tidb_statistics_historical_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"generate\"}[1m])) by (result)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "generate-{{result}}", - "refId": "A", - "step": 30 - }, - { - "exemplar": true, - "expr": "sum(rate(tidb_statistics_historical_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"dump\"}[1m])) by (result)", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "dump-{{result}}", - "refId": "B", - "step": 30 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Historical Stats OPM", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, { "aliasColors": {}, "bars": false, diff --git a/pkg/metrics/stats.go b/pkg/metrics/stats.go index 3190fb120ab92..03a34c8d2fd3a 100644 --- a/pkg/metrics/stats.go +++ b/pkg/metrics/stats.go @@ -38,7 +38,6 @@ var ( StatsDeltaUpdateHistogram prometheus.Histogram StatsUsageUpdateHistogram prometheus.Histogram - HistoricalStatsCounter *prometheus.CounterVec PlanReplayerTaskCounter *prometheus.CounterVec PlanReplayerRegisterTaskGauge prometheus.Gauge ) @@ -135,13 +134,6 @@ func InitStatsMetrics() { Help: "Gauge of stats healthy", }, []string{LblType}) - HistoricalStatsCounter = metricscommon.NewCounterVec(prometheus.CounterOpts{ - Namespace: "tidb", - Subsystem: "statistics", - Name: "historical_stats", - Help: "counter of the historical stats operation", - }, []string{LblType, LblResult}) - PlanReplayerTaskCounter = metricscommon.NewCounterVec(prometheus.CounterOpts{ Namespace: "tidb", Subsystem: "plan_replayer", diff --git a/pkg/planner/core/BUILD.bazel b/pkg/planner/core/BUILD.bazel index 5eddff4b5f903..2a90bb30c8d6b 100644 --- a/pkg/planner/core/BUILD.bazel +++ b/pkg/planner/core/BUILD.bazel @@ -207,7 +207,6 @@ go_library( "@com_github_pingcap_kvproto//pkg/diagnosticspb", "@com_github_pingcap_tipb//go-tipb", "@com_github_tikv_client_go_v2//kv", - "@com_github_tikv_client_go_v2//oracle", "@org_uber_go_atomic//:atomic", "@org_uber_go_zap//:zap", ], diff --git a/pkg/planner/core/common_plans.go b/pkg/planner/core/common_plans.go index ae124a35819e5..f5c37c41009dc 100644 --- a/pkg/planner/core/common_plans.go +++ b/pkg/planner/core/common_plans.go @@ -486,12 +486,11 @@ type UnlockStats struct { // PlanReplayer represents a plan replayer plan. type PlanReplayer struct { physicalop.SimpleSchemaProducer - ExecStmt ast.StmtNode - StmtList []string // For PLAN REPLAYER DUMP EXPLAIN ( "sql1", "sql2", ... ) - Analyze bool - Load bool - File string - HistoricalStatsTS uint64 + ExecStmt ast.StmtNode + StmtList []string // For PLAN REPLAYER DUMP EXPLAIN ( "sql1", "sql2", ... ) + Analyze bool + Load bool + File string Capture bool Remove bool diff --git a/pkg/planner/core/planbuilder.go b/pkg/planner/core/planbuilder.go index 7fb1a89a804ff..a53688d0d6671 100644 --- a/pkg/planner/core/planbuilder.go +++ b/pkg/planner/core/planbuilder.go @@ -83,7 +83,6 @@ import ( sem "github.com/pingcap/tidb/pkg/util/sem/compat" semv2 "github.com/pingcap/tidb/pkg/util/sem/v2" "github.com/pingcap/tidb/pkg/util/sqlexec" - "github.com/tikv/client-go/v2/oracle" "go.uber.org/zap" ) @@ -632,7 +631,7 @@ func (b *PlanBuilder) Build(ctx context.Context, node *resolve.NodeW) (base.Plan case *ast.UnlockStatsStmt: return b.buildUnlockStats(x), nil case *ast.PlanReplayerStmt: - return b.buildPlanReplayer(x), nil + return b.buildPlanReplayer(x) case *ast.TrafficStmt: return b.buildTraffic(x), nil case *ast.PrepareStmt: @@ -6138,59 +6137,20 @@ func convert2OutputSchemasAndNames(names []string, ftypes []byte, flags []uint) return } -func (b *PlanBuilder) buildPlanReplayer(pc *ast.PlanReplayerStmt) base.Plan { - p := &PlanReplayer{ExecStmt: pc.Stmt, StmtList: pc.StmtList, Analyze: pc.Analyze, Load: pc.Load, File: pc.File, - Capture: pc.Capture, Remove: pc.Remove, SQLDigest: pc.SQLDigest, PlanDigest: pc.PlanDigest} - +func (*PlanBuilder) buildPlanReplayer(pc *ast.PlanReplayerStmt) (base.Plan, error) { if pc.HistoricalStatsInfo != nil { - p.HistoricalStatsTS = calcTSForPlanReplayer(b.ctx, pc.HistoricalStatsInfo.TsExpr) + return nil, errors.New("WITH STATS AS OF TIMESTAMP is no longer supported because the historical stats feature has been removed") } + p := &PlanReplayer{ExecStmt: pc.Stmt, StmtList: pc.StmtList, Analyze: pc.Analyze, Load: pc.Load, File: pc.File, + Capture: pc.Capture, Remove: pc.Remove, SQLDigest: pc.SQLDigest, PlanDigest: pc.PlanDigest} + schema := newColumnsWithNames(2) schema.Append(buildColumnWithName("", "Item", mysql.TypeVarchar, 32)) schema.Append(buildColumnWithName("", "Value", mysql.TypeVarchar, mysql.MaxBlobWidth)) p.SetSchema(schema.col2Schema()) p.SetOutputNames(schema.names) - return p -} - -func calcTSForPlanReplayer(sctx base.PlanContext, tsExpr ast.ExprNode) uint64 { - tsVal, err := evalAstExprWithPlanCtx(sctx, tsExpr) - if err != nil { - sctx.GetSessionVars().StmtCtx.AppendWarning(err) - return 0 - } - // mustn't be NULL - if tsVal.IsNull() { - return 0 - } - - // first, treat it as a TSO - tpLonglong := types.NewFieldType(mysql.TypeLonglong) - tpLonglong.SetFlag(mysql.UnsignedFlag) - // We need a strict check, which means no truncate or any other warnings/errors, or it will wrongly try to parse - // a date/time string into a TSO. - // To achieve this, we create a new type context without re-using the one in statement context. - tso, err := tsVal.ConvertTo(types.DefaultStmtNoWarningContext.WithLocation(sctx.GetSessionVars().Location()), tpLonglong) - if err == nil { - return tso.GetUint64() - } - - // if failed, treat it as a date/time - // this part is similar to CalculateAsOfTsExpr - tpDateTime := types.NewFieldType(mysql.TypeDatetime) - tpDateTime.SetDecimal(6) - timestamp, err := tsVal.ConvertTo(sctx.GetSessionVars().StmtCtx.TypeCtx(), tpDateTime) - if err != nil { - sctx.GetSessionVars().StmtCtx.AppendWarning(err) - return 0 - } - goTime, err := timestamp.GetMysqlTime().GoTime(sctx.GetSessionVars().Location()) - if err != nil { - sctx.GetSessionVars().StmtCtx.AppendWarning(err) - return 0 - } - return oracle.GoTimeToTS(goTime) + return p, nil } func buildChecksumTableSchema() (*expression.Schema, []*types.FieldName) { diff --git a/pkg/server/BUILD.bazel b/pkg/server/BUILD.bazel index 1dbc1d28d9e11..60624ef6204a7 100644 --- a/pkg/server/BUILD.bazel +++ b/pkg/server/BUILD.bazel @@ -80,7 +80,6 @@ go_library( "//pkg/sessionctx/vardef", "//pkg/sessionctx/variable", "//pkg/sessiontxn", - "//pkg/statistics/handle", "//pkg/statistics/handle/initstats", "//pkg/statistics/handle/util", "//pkg/store", diff --git a/pkg/server/handler/optimizor/BUILD.bazel b/pkg/server/handler/optimizor/BUILD.bazel index b18245da698a0..8a919e9cd1710 100644 --- a/pkg/server/handler/optimizor/BUILD.bazel +++ b/pkg/server/handler/optimizor/BUILD.bazel @@ -12,24 +12,13 @@ go_library( deps = [ "//pkg/domain", "//pkg/domain/infosync", - "//pkg/infoschema", - "//pkg/meta/model", "//pkg/parser/ast", - "//pkg/parser/mysql", "//pkg/planner/extstore", "//pkg/server/handler", - "//pkg/sessionctx/vardef", - "//pkg/statistics/handle", - "//pkg/statistics/util", - "//pkg/table", - "//pkg/types", "//pkg/util", "//pkg/util/logutil", "//pkg/util/replayer", - "@com_github_burntsushi_toml//:toml", "@com_github_gorilla_mux//:mux", - "@com_github_pingcap_errors//:errors", - "@com_github_tikv_client_go_v2//oracle", "@org_uber_go_zap//:zap", ], ) @@ -43,14 +32,13 @@ go_test( "statistics_handler_test.go", ], flaky = True, - shard_count = 10, + shard_count = 9, deps = [ ":optimizor", "//pkg/config", "//pkg/domain", "//pkg/kv", "//pkg/metrics", - "//pkg/parser/ast", "//pkg/server", "//pkg/server/internal/testserverclient", "//pkg/server/internal/testutil", @@ -64,12 +52,10 @@ go_test( "//pkg/testkit/testsetup", "//pkg/util/replayer", "//pkg/util/topsql/state", - "@com_github_burntsushi_toml//:toml", "@com_github_go_sql_driver_mysql//:mysql", "@com_github_gorilla_mux//:mux", "@com_github_pingcap_failpoint//:failpoint", "@com_github_stretchr_testify//require", - "@com_github_tikv_client_go_v2//oracle", "@com_github_tikv_client_go_v2//tikv", "@org_uber_go_goleak//:goleak", ], diff --git a/pkg/server/handler/optimizor/optimize_trace.go b/pkg/server/handler/optimizor/optimize_trace.go index f7becf5bf1091..fe1da78a6a10b 100644 --- a/pkg/server/handler/optimizor/optimize_trace.go +++ b/pkg/server/handler/optimizor/optimize_trace.go @@ -47,7 +47,6 @@ func (oth OptimizeTraceHandler) ServeHTTP(w http.ResponseWriter, req *http.Reque name := params[handler.FileName] handler := downloadFileHandler{ filePath: filepath.Join(domain.GetOptimizerTraceDirName(), name), - fileName: name, infoGetter: oth.infoGetter, address: oth.address, statusPort: oth.statusPort, diff --git a/pkg/server/handler/optimizor/plan_replayer.go b/pkg/server/handler/optimizor/plan_replayer.go index 68ba1527d8e09..055ca62aefce2 100644 --- a/pkg/server/handler/optimizor/plan_replayer.go +++ b/pkg/server/handler/optimizor/plan_replayer.go @@ -15,10 +15,6 @@ package optimizor import ( - "archive/zip" - "bytes" - "context" - "encoding/json" "fmt" "io" "net" @@ -27,18 +23,10 @@ import ( "strconv" "strings" - "github.com/BurntSushi/toml" "github.com/gorilla/mux" - "github.com/pingcap/errors" - "github.com/pingcap/tidb/pkg/domain" "github.com/pingcap/tidb/pkg/domain/infosync" - "github.com/pingcap/tidb/pkg/infoschema" - "github.com/pingcap/tidb/pkg/meta/model" - "github.com/pingcap/tidb/pkg/parser/ast" "github.com/pingcap/tidb/pkg/planner/extstore" "github.com/pingcap/tidb/pkg/server/handler" - "github.com/pingcap/tidb/pkg/statistics/handle" - util2 "github.com/pingcap/tidb/pkg/statistics/util" "github.com/pingcap/tidb/pkg/util" "github.com/pingcap/tidb/pkg/util/logutil" "github.com/pingcap/tidb/pkg/util/replayer" @@ -47,21 +35,17 @@ import ( // PlanReplayerHandler is the handler for dumping plan replayer file. type PlanReplayerHandler struct { - is infoschema.InfoSchema - statsHandle *handle.Handle - infoGetter *infosync.InfoSyncer - address string - statusPort uint + infoGetter *infosync.InfoSyncer + address string + statusPort uint } // NewPlanReplayerHandler creates a new PlanReplayerHandler. -func NewPlanReplayerHandler(is infoschema.InfoSchema, statsHandle *handle.Handle, infoGetter *infosync.InfoSyncer, address string, statusPort uint) *PlanReplayerHandler { +func NewPlanReplayerHandler(infoGetter *infosync.InfoSyncer, address string, statusPort uint) *PlanReplayerHandler { return &PlanReplayerHandler{ - is: is, - statsHandle: statsHandle, - infoGetter: infoGetter, - address: address, - statusPort: statusPort, + infoGetter: infoGetter, + address: address, + statusPort: statusPort, } } @@ -69,17 +53,21 @@ func NewPlanReplayerHandler(is infoschema.InfoSchema, statsHandle *handle.Handle func (prh PlanReplayerHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { params := mux.Vars(req) name := params[handler.FileName] + if strings.HasPrefix(name, "capture_replayer_") { + // Older TiDB versions used this prefix both for bundles with embedded + // statistics and for bundles that expected statistics to be injected at + // download time. The filename alone cannot distinguish the two cases. + logutil.BgLogger().Warn("serving a legacy plan replayer capture file; statistics may be missing because download-time historical statistics injection is no longer supported", + zap.String("filename", name)) + } handler := downloadFileHandler{ filePath: filepath.Join(replayer.GetPlanReplayerDirName(), name), - fileName: name, infoGetter: prh.infoGetter, address: prh.address, statusPort: prh.statusPort, urlPath: fmt.Sprintf("plan_replayer/dump/%s", name), downloadedFilename: "plan_replayer", scheme: util.InternalHTTPSchema(), - statsHandle: prh.statsHandle, - is: prh.is, } handleDownloadFile(handler, w, req) } @@ -110,36 +98,14 @@ func handleDownloadFile(dfHandler downloadFileHandler, w http.ResponseWriter, re } defer fileReader.Close() - // For capture_replayer files, we need to read all content to process it - if dfHandler.downloadedFilename == "plan_replayer" && strings.HasPrefix(dfHandler.fileName, "capture_replayer") { - content, err := io.ReadAll(fileReader) - if err != nil { - handler.WriteError(w, err) - return - } - content, err = handlePlanReplayerCaptureFile(content, dfHandler) - if err != nil { - handler.WriteError(w, err) - return - } - // Set headers BEFORE writing body - w.Header().Set("Content-Type", "application/zip") - w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s.zip\"", dfHandler.downloadedFilename)) - _, err = w.Write(content) - if err != nil { - handler.WriteError(w, err) - return - } - } else { - // Set headers BEFORE writing body - w.Header().Set("Content-Type", "application/zip") - w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s.zip\"", dfHandler.downloadedFilename)) - // Use streaming io.Copy instead of io.ReadAll to avoid memory bloat - _, err = io.Copy(w, fileReader) - if err != nil { - handler.WriteError(w, err) - return - } + // Set headers BEFORE writing body + w.Header().Set("Content-Type", "application/zip") + w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s.zip\"", dfHandler.downloadedFilename)) + // Use streaming io.Copy instead of io.ReadAll to avoid memory bloat. + _, err = io.Copy(w, fileReader) + if err != nil { + handler.WriteError(w, err) + return } logutil.BgLogger().Info("return dump file successfully", zap.String("filename", name), zap.String("address", localAddr), zap.Bool("forwarded", isForwarded)) @@ -206,148 +172,9 @@ func handleDownloadFile(dfHandler downloadFileHandler, w http.ResponseWriter, re type downloadFileHandler struct { scheme string filePath string - fileName string infoGetter *infosync.InfoSyncer address string statusPort uint urlPath string downloadedFilename string - - statsHandle *handle.Handle - is infoschema.InfoSchema -} - -// handlePlanReplayerCaptureFile handles capture_replayer files by adding historical stats. -// This function is called only when the file is a capture_replayer file (already checked by caller). -func handlePlanReplayerCaptureFile(content []byte, handler downloadFileHandler) ([]byte, error) { - b := bytes.NewReader(content) - zr, err := zip.NewReader(b, int64(len(content))) - if err != nil { - return nil, err - } - startTS, err := loadSQLMetaFile(zr) - if err != nil { - return nil, err - } - if startTS == 0 { - return content, nil - } - tbls, err := loadSchemaMeta(zr, handler.is) - if err != nil { - return nil, err - } - for _, tbl := range tbls { - jsonStats, _, err := handler.statsHandle.DumpHistoricalStatsBySnapshot(tbl.dbName, tbl.info, startTS) - if err != nil { - return nil, err - } - tbl.jsonStats = jsonStats - } - // Create a new zip with the additional stats in memory instead of writing to local filesystem - return dumpJSONStatsIntoZipInMemory(tbls, content) -} - -func loadSQLMetaFile(z *zip.Reader) (uint64, error) { - for _, zipFile := range z.File { - if zipFile.Name == domain.PlanReplayerSQLMetaFile { - varMap := make(map[string]string) - v, err := zipFile.Open() - if err != nil { - return 0, errors.AddStack(err) - } - //nolint: errcheck,all_revive,revive - defer v.Close() - _, err = toml.NewDecoder(v).Decode(&varMap) - if err != nil { - return 0, errors.AddStack(err) - } - startTS, err := strconv.ParseUint(varMap[domain.PlanReplayerSQLMetaStartTS], 10, 64) - if err != nil { - return 0, err - } - return startTS, nil - } - } - return 0, nil -} - -func loadSchemaMeta(z *zip.Reader, is infoschema.InfoSchema) (map[int64]*tblInfo, error) { - r := make(map[int64]*tblInfo, 0) - for _, zipFile := range z.File { - if zipFile.Name == fmt.Sprintf("schema/%v", domain.PlanReplayerSchemaMetaFile) { - v, err := zipFile.Open() - if err != nil { - return nil, errors.AddStack(err) - } - //nolint: errcheck,all_revive,revive - defer v.Close() - buf := new(bytes.Buffer) - _, err = buf.ReadFrom(v) - if err != nil { - return nil, errors.AddStack(err) - } - rows := strings.Split(buf.String(), "\n") - for _, row := range rows { - s := strings.Split(row, ";") - databaseName := s[0] - tableName := s[1] - t, err := is.TableByName(context.Background(), ast.NewCIStr(databaseName), ast.NewCIStr(tableName)) - if err != nil { - return nil, err - } - r[t.Meta().ID] = &tblInfo{ - info: t.Meta(), - dbName: databaseName, - tblName: tableName, - } - } - break - } - } - return r, nil -} - -// dumpJSONStatsIntoZipInMemory creates a new zip with additional stats in memory. -func dumpJSONStatsIntoZipInMemory(tbls map[int64]*tblInfo, content []byte) ([]byte, error) { - zr, err := zip.NewReader(bytes.NewReader(content), int64(len(content))) - if err != nil { - return nil, err - } - // Create new zip in memory - var buf bytes.Buffer - zw := zip.NewWriter(&buf) - for _, f := range zr.File { - err = zw.Copy(f) - if err != nil { - logutil.BgLogger().Warn("copy plan replayer zip file failed", zap.Error(err)) - return nil, err - } - } - for _, tbl := range tbls { - w, err := zw.Create(fmt.Sprintf("stats/%v.%v.json", tbl.dbName, tbl.tblName)) - if err != nil { - return nil, err - } - data, err := json.Marshal(tbl.jsonStats) - if err != nil { - return nil, err - } - _, err = w.Write(data) - if err != nil { - return nil, err - } - } - err = zw.Close() - if err != nil { - logutil.BgLogger().Warn("Closing zip writer failed", zap.Error(err)) - return nil, err - } - return buf.Bytes(), nil -} - -type tblInfo struct { - info *model.TableInfo - jsonStats *util2.JSONTable - dbName string - tblName string } diff --git a/pkg/server/handler/optimizor/plan_replayer_test.go b/pkg/server/handler/optimizor/plan_replayer_test.go index ac1271bc13b4f..cad13b0c1ba76 100644 --- a/pkg/server/handler/optimizor/plan_replayer_test.go +++ b/pkg/server/handler/optimizor/plan_replayer_test.go @@ -17,37 +17,30 @@ package optimizor_test import ( "archive/zip" "bytes" - "context" "database/sql" - "encoding/json" "fmt" "io" "os" "path/filepath" "slices" - "strconv" "strings" "testing" "time" - "github.com/BurntSushi/toml" "github.com/go-sql-driver/mysql" "github.com/pingcap/failpoint" "github.com/pingcap/tidb/pkg/config" "github.com/pingcap/tidb/pkg/domain" "github.com/pingcap/tidb/pkg/kv" - "github.com/pingcap/tidb/pkg/parser/ast" "github.com/pingcap/tidb/pkg/server" "github.com/pingcap/tidb/pkg/server/internal/testserverclient" "github.com/pingcap/tidb/pkg/server/internal/testutil" "github.com/pingcap/tidb/pkg/server/internal/util" "github.com/pingcap/tidb/pkg/session" statstestutil "github.com/pingcap/tidb/pkg/statistics/handle/ddl/testutil" - util2 "github.com/pingcap/tidb/pkg/statistics/util" "github.com/pingcap/tidb/pkg/testkit" "github.com/pingcap/tidb/pkg/util/replayer" "github.com/stretchr/testify/require" - "github.com/tikv/client-go/v2/oracle" ) var expectedFilesInReplayer = []string{ @@ -94,16 +87,6 @@ func requirePlanReplayerFileTokenFromRows(t *testing.T, rows *sql.Rows) string { return filename } -func requirePlanReplayerFileTokenFromResult(t *testing.T, rows [][]any) string { - require.Len(t, rows, 1) - require.Len(t, rows[0], 2) - require.Equal(t, "File token", rows[0][0]) - filename, ok := rows[0][1].(string) - require.True(t, ok) - require.NotEmpty(t, filename) - return filename -} - func requireSingleStringFromRows(t *testing.T, rows *sql.Rows) string { require.True(t, rows.Next(), "unexpected data") var value string @@ -867,206 +850,3 @@ func forEachFileInZipBytes(t *testing.T, b []byte, fn func(file *zip.File)) { fn(f) } } - -func fetchZipFromPlanReplayerAPI(t *testing.T, client *testserverclient.TestServerClient, filename string) *zip.Reader { - resp0, err := client.FetchStatus(filepath.Join("/plan_replayer/dump/", filename)) - require.NoError(t, err) - defer func() { - require.NoError(t, resp0.Body.Close()) - }() - body, err := io.ReadAll(resp0.Body) - require.NoError(t, err) - b := bytes.NewReader(body) - z, err := zip.NewReader(b, int64(len(body))) - require.NoError(t, err) - return z -} - -func getInfoFromPlanReplayerZip( - t *testing.T, - z *zip.Reader, -) ( - jsonTbls []*util2.JSONTable, - metas []map[string]string, - errMsgs []string, -) { - for _, zipFile := range z.File { - if strings.HasPrefix(zipFile.Name, "stats/") { - jsonTbl := &util2.JSONTable{} - r, err := zipFile.Open() - require.NoError(t, err) - //nolint: all_revive - defer func() { - require.NoError(t, r.Close()) - }() - buf := new(bytes.Buffer) - _, err = buf.ReadFrom(r) - require.NoError(t, err) - err = json.Unmarshal(buf.Bytes(), jsonTbl) - require.NoError(t, err) - - jsonTbls = append(jsonTbls, jsonTbl) - } else if zipFile.Name == "sql_meta.toml" { - meta := make(map[string]string) - r, err := zipFile.Open() - require.NoError(t, err) - //nolint: all_revive - defer func() { - require.NoError(t, r.Close()) - }() - _, err = toml.NewDecoder(r).Decode(&meta) - require.NoError(t, err) - - metas = append(metas, meta) - } else if zipFile.Name == "errors.txt" { - r, err := zipFile.Open() - require.NoError(t, err) - //nolint: all_revive - defer func() { - require.NoError(t, r.Close()) - }() - content, err := io.ReadAll(r) - require.NoError(t, err) - errMsgs = strings.Split(string(content), "\n") - } - } - return -} - -func TestDumpPlanReplayerAPIWithHistoryStats(t *testing.T) { - require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/domain/sendHistoricalStats", "return(true)")) - defer func() { - require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/domain/sendHistoricalStats")) - }() - store := testkit.CreateMockStore(t) - dom, err := session.GetDomain(store) - require.NoError(t, err) - server, client := prepareServerAndClientForTest(t, store, dom) - defer server.Close() - statsHandle := dom.StatsHandle() - hsWorker := dom.GetHistoricalStatsWorker() - - // 1. prepare test data - - // time1, ts1: before everything starts - tk := testkit.NewTestKit(t, store) - tk.MustExec("set global tidb_enable_historical_stats = 1") - defer tk.MustExec("set global tidb_enable_historical_stats = 0") - time1 := time.Now() - ts1 := oracle.GoTimeToTS(time1) - - tk.MustExec("use test") - tk.MustExec("create table t(a int, b int, c int, index ia(a))") - is := dom.InfoSchema() - tbl, err := is.TableByName(context.Background(), ast.NewCIStr("test"), ast.NewCIStr("t")) - require.NoError(t, err) - tblInfo := tbl.Meta() - - // 1-1. first insert and first analyze, trigger first dump history stats - tk.MustExec("insert into t value(1,1,1), (2,2,2), (3,3,3)") - tk.MustExec("analyze table t with 1 samplerate") - tblID := hsWorker.GetOneHistoricalStatsTable() - err = hsWorker.DumpHistoricalStats(tblID, statsHandle) - require.NoError(t, err) - - // time2, stats1: after first analyze - time2 := time.Now() - ts2 := oracle.GoTimeToTS(time2) - stats1, err := statsHandle.DumpStatsToJSON("test", tblInfo, nil, true) - require.NoError(t, err) - stats1.Sort() - - // 1-2. second insert and second analyze, trigger second dump history stats - tk.MustExec("insert into t value(4,4,4), (5,5,5), (6,6,6)") - tk.MustExec("analyze table t with 1 samplerate") - tblID = hsWorker.GetOneHistoricalStatsTable() - err = hsWorker.DumpHistoricalStats(tblID, statsHandle) - require.NoError(t, err) - - // time3, stats2: after second analyze - time3 := time.Now() - ts3 := oracle.GoTimeToTS(time3) - stats2, err := statsHandle.DumpStatsToJSON("test", tblInfo, nil, true) - require.NoError(t, err) - stats2.Sort() - - // 2. get the plan replayer and assert - - template := "plan replayer dump with stats as of timestamp '%s' explain %s" - query := "select * from t where a > 1" - - // 2-1. specify time1 to get the plan replayer - filename1 := requirePlanReplayerFileTokenFromResult(t, tk.MustQuery( - fmt.Sprintf(template, strconv.FormatUint(ts1, 10), query), - ).Rows()) - zip1 := fetchZipFromPlanReplayerAPI(t, client, filename1) - jsonTbls1, metas1, errMsg1 := getInfoFromPlanReplayerZip(t, zip1) - - // the TS is recorded in the plan replayer, and it's the same as the TS we calculated above - require.Len(t, metas1, 1) - require.Contains(t, metas1[0], "historicalStatsTS") - tsInReplayerMeta1, err := strconv.ParseUint(metas1[0]["historicalStatsTS"], 10, 64) - require.NoError(t, err) - require.Equal(t, ts1, tsInReplayerMeta1) - - // the result is the same as stats2, and IsHistoricalStats is false. - require.Len(t, jsonTbls1, 1) - require.False(t, jsonTbls1[0].IsHistoricalStats) - jsonTbls1[0].Sort() - require.Equal(t, jsonTbls1[0], stats2) - - // because we failed to get historical stats, there's an error message. - require.Equal(t, []string{"Historical stats for test.t are unavailable, fallback to latest stats", ""}, errMsg1) - - // 2-2. specify time2 to get the plan replayer - filename2 := requirePlanReplayerFileTokenFromResult(t, tk.MustQuery( - fmt.Sprintf(template, time2.Format("2006-01-02 15:04:05.000000"), query), - ).Rows()) - zip2 := fetchZipFromPlanReplayerAPI(t, client, filename2) - jsonTbls2, metas2, errMsg2 := getInfoFromPlanReplayerZip(t, zip2) - - // the TS is recorded in the plan replayer, and it's the same as the TS we calculated above - require.Len(t, metas2, 1) - require.Contains(t, metas2[0], "historicalStatsTS") - tsInReplayerMeta2, err := strconv.ParseUint(metas2[0]["historicalStatsTS"], 10, 64) - require.NoError(t, err) - require.Equal(t, ts2, tsInReplayerMeta2) - - // the result is the same as stats1, and IsHistoricalStats is true. - require.Len(t, jsonTbls2, 1) - require.True(t, jsonTbls2[0].IsHistoricalStats) - jsonTbls2[0].IsHistoricalStats = false - jsonTbls2[0].Sort() - require.Equal(t, jsonTbls2[0], stats1) - - // succeeded to get historical stats, there should be no error message. - require.Empty(t, errMsg2) - - // 2-3. specify time3 to get the plan replayer - filename3 := requirePlanReplayerFileTokenFromResult(t, tk.MustQuery( - fmt.Sprintf(template, time3.Format("2006-01-02T15:04:05.000000Z07:00"), query), - ).Rows()) - zip3 := fetchZipFromPlanReplayerAPI(t, client, filename3) - jsonTbls3, metas3, errMsg3 := getInfoFromPlanReplayerZip(t, zip3) - - // the TS is recorded in the plan replayer, and it's the same as the TS we calculated above - require.Len(t, metas3, 1) - require.Contains(t, metas3[0], "historicalStatsTS") - tsInReplayerMeta3, err := strconv.ParseUint(metas3[0]["historicalStatsTS"], 10, 64) - require.NoError(t, err) - require.Equal(t, ts3, tsInReplayerMeta3) - - // the result is the same as stats2, and IsHistoricalStats is true. - require.Len(t, jsonTbls3, 1) - require.True(t, jsonTbls3[0].IsHistoricalStats) - jsonTbls3[0].IsHistoricalStats = false - jsonTbls3[0].Sort() - require.Equal(t, jsonTbls3[0], stats2) - - // succeeded to get historical stats, there should be no error message. - require.Empty(t, errMsg3) - - // 3. remove the plan replayer files generated during the test - gcHandler := dom.GetDumpFileGCChecker() - gcHandler.GCDumpFiles(context.Background(), 0, 0) -} diff --git a/pkg/server/handler/optimizor/statistics_handler.go b/pkg/server/handler/optimizor/statistics_handler.go index 87ecf51c87251..5498d8623f69e 100644 --- a/pkg/server/handler/optimizor/statistics_handler.go +++ b/pkg/server/handler/optimizor/statistics_handler.go @@ -16,22 +16,13 @@ package optimizor import ( "context" - "fmt" "net/http" "strconv" - "time" "github.com/gorilla/mux" "github.com/pingcap/tidb/pkg/domain" "github.com/pingcap/tidb/pkg/parser/ast" - "github.com/pingcap/tidb/pkg/parser/mysql" "github.com/pingcap/tidb/pkg/server/handler" - "github.com/pingcap/tidb/pkg/sessionctx/vardef" - "github.com/pingcap/tidb/pkg/table" - "github.com/pingcap/tidb/pkg/types" - "github.com/pingcap/tidb/pkg/util/logutil" - "github.com/tikv/client-go/v2/oracle" - "go.uber.org/zap" ) // StatsHandler is the handler for dumping statistics. @@ -80,71 +71,6 @@ func (sh StatsHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { } } -// StatsHistoryHandler is the handler for dumping statistics. -type StatsHistoryHandler struct { - do *domain.Domain -} - -// NewStatsHistoryHandler creates a new StatsHistoryHandler. -func NewStatsHistoryHandler(do *domain.Domain) *StatsHistoryHandler { - return &StatsHistoryHandler{do: do} -} - -func (sh StatsHistoryHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { - w.Header().Set("Content-Type", "application/json") - - params := mux.Vars(req) - enabeld, err := sh.do.StatsHandle().CheckHistoricalStatsEnable() - if err != nil { - handler.WriteError(w, err) - return - } - if !enabeld { - handler.WriteError(w, fmt.Errorf("%v should be enabled", vardef.TiDBEnableHistoricalStats)) - return - } - - typeCtx := types.DefaultStmtNoWarningContext - typeCtx = typeCtx.WithLocation(time.Local) - t, err := types.ParseTime(typeCtx, params[handler.Snapshot], mysql.TypeTimestamp, 6) - if err != nil { - handler.WriteError(w, err) - return - } - t1, err := t.GoTime(time.Local) - if err != nil { - handler.WriteError(w, err) - return - } - snapshot := oracle.GoTimeToTS(t1) - tbl, err := getSnapshotTableInfo(sh.do, snapshot, params[handler.DBName], params[handler.TableName]) - if err != nil { - logutil.BgLogger().Info("fail to get snapshot TableInfo in historical stats API, switch to use latest infoschema", zap.Error(err)) - is := sh.do.InfoSchema() - tbl, err = is.TableByName(context.Background(), ast.NewCIStr(params[handler.DBName]), ast.NewCIStr(params[handler.TableName])) - if err != nil { - handler.WriteError(w, err) - return - } - } - - h := sh.do.StatsHandle() - js, _, err := h.DumpHistoricalStatsBySnapshot(params[handler.DBName], tbl.Meta(), snapshot) - if err != nil { - handler.WriteError(w, err) - } else { - handler.WriteData(w, js) - } -} - -func getSnapshotTableInfo(dom *domain.Domain, snapshot uint64, dbName, tblName string) (table.Table, error) { - is, err := dom.GetSnapshotInfoSchema(snapshot) - if err != nil { - return nil, err - } - return is.TableByName(context.Background(), ast.NewCIStr(dbName), ast.NewCIStr(tblName)) -} - // StatsPriorityQueueHandler is the handler for dumping the stats priority queue snapshot. type StatsPriorityQueueHandler struct { do *domain.Domain diff --git a/pkg/server/handler/optimizor/statistics_handler_test.go b/pkg/server/handler/optimizor/statistics_handler_test.go index 1440ece6719f1..25f8e5b3e61e0 100644 --- a/pkg/server/handler/optimizor/statistics_handler_test.go +++ b/pkg/server/handler/optimizor/statistics_handler_test.go @@ -27,7 +27,6 @@ import ( "github.com/go-sql-driver/mysql" "github.com/gorilla/mux" - "github.com/pingcap/tidb/pkg/parser/ast" server2 "github.com/pingcap/tidb/pkg/server" "github.com/pingcap/tidb/pkg/server/handler/optimizor" "github.com/pingcap/tidb/pkg/server/internal/testserverclient" @@ -50,7 +49,7 @@ func TestDumpStatsAPI(t *testing.T) { cfg.Port = client.Port cfg.Status.StatusPort = client.StatusPort cfg.Status.ReportStatus = true - cfg.Socket = filepath.Join(tmp, fmt.Sprintf("tidb-mock-%d.sock", time.Now().UnixNano())) + cfg.Socket = newTestSocket(t) // RunInGoTestChan is a global channel and will be closed after the first server starts. // Recreate it to avoid racing on subsequent server starts in the same test binary. @@ -74,10 +73,6 @@ func TestDumpStatsAPI(t *testing.T) { statsHandler := optimizor.NewStatsHandler(dom) prepareData(t, client, statsHandler) - tableInfo, err := dom.InfoSchema().TableByName(context.Background(), ast.NewCIStr("tidb"), ast.NewCIStr("test")) - require.NoError(t, err) - err = dom.GetHistoricalStatsWorker().DumpHistoricalStats(tableInfo.Meta().ID, dom.StatsHandle()) - require.NoError(t, err) router := mux.NewRouter() router.Handle("/stats/dump/{db}/{table}", statsHandler) @@ -104,13 +99,9 @@ func TestDumpStatsAPI(t *testing.T) { checkData(t, path, client) checkCorrelation(t, client) - // sleep for 1 seconds to ensure the existence of tidb.test - time.Sleep(time.Second) - timeBeforeDropStats := time.Now() - snapshot := timeBeforeDropStats.Format("20060102150405") - prepare4DumpHistoryStats(t, client) + prepareTableWithoutStats(t, client) - // test dump history stats + // test dump stats for a table without stats resp1, err := client.FetchStatus("/stats/dump/tidb/test") require.NoError(t, err) defer func() { @@ -120,29 +111,17 @@ func TestDumpStatsAPI(t *testing.T) { require.NoError(t, err) require.Equal(t, "null", string(js)) - path1 := filepath.Join(tmp, "stats_history.json") - fp1, err := os.Create(path1) - require.NoError(t, err) - require.NotNil(t, fp1) - defer func() { - require.NoError(t, fp1.Close()) - require.NoError(t, os.Remove(path1)) - }() - - resp2, err := client.FetchStatus("/stats/dump/tidb/test/" + snapshot) - require.NoError(t, err) - defer func() { - require.NoError(t, resp2.Body.Close()) - }() - js, err = io.ReadAll(resp2.Body) - require.NoError(t, err) - _, err = fp1.Write(js) - require.NoError(t, err) - checkData(t, path1, client) - testDumpPartitionTableStats(t, client, statsHandler) } +func newTestSocket(t *testing.T) string { + socket := filepath.Join(os.TempDir(), fmt.Sprintf("tidb-%d.sock", time.Now().UnixNano())) + t.Cleanup(func() { + _ = os.Remove(socket) + }) + return socket +} + func prepareData(t *testing.T, client *testserverclient.TestServerClient, statHandle *optimizor.StatsHandler) { db, err := sql.Open("mysql", client.GetDSN()) require.NoError(t, err, "Error connecting") @@ -162,7 +141,6 @@ func prepareData(t *testing.T, client *testserverclient.TestServerClient, statHa tk.MustExec("insert test values (1, 's')") tk.MustExec("flush stats_delta *.*") tk.MustExec("analyze table test") - tk.MustExec("set global tidb_enable_historical_stats = 1") tk.MustExec("insert into test(a,b) values (1, 'v'),(3, 'vvv'),(5, 'vv')") is := statHandle.Domain().InfoSchema() tk.MustExec("flush stats_delta *.*") @@ -211,7 +189,7 @@ func preparePartitionData(t *testing.T, client *testserverclient.TestServerClien require.NoError(t, h.Update(context.Background(), is)) } -func prepare4DumpHistoryStats(t *testing.T, client *testserverclient.TestServerClient) { +func prepareTableWithoutStats(t *testing.T, client *testserverclient.TestServerClient) { db, err := sql.Open("mysql", client.GetDSN()) require.NoError(t, err, "Error connecting") defer func() { @@ -220,15 +198,6 @@ func prepare4DumpHistoryStats(t *testing.T, client *testserverclient.TestServerC }() tk := testkit.NewDBTestKit(t, db) - - safePointName := "tikv_gc_safe_point" - safePointValue := "20060102-15:04:05 -0700" - safePointComment := "All versions after safe point can be accessed. (DO NOT EDIT)" - updateSafePoint := fmt.Sprintf(`INSERT INTO mysql.tidb VALUES ('%[1]s', '%[2]s', '%[3]s') - ON DUPLICATE KEY - UPDATE variable_value = '%[2]s', comment = '%[3]s'`, safePointName, safePointValue, safePointComment) - tk.MustExec(updateSafePoint) - tk.MustExec("drop table tidb.test") tk.MustExec("create table tidb.test (a int, b varchar(20))") } @@ -297,7 +266,6 @@ func checkData(t *testing.T, path string, client *testserverclient.TestServerCli } func TestStatsPriorityQueueAPI(t *testing.T) { - tmp := t.TempDir() store := testkit.CreateMockStore(t) driver := server2.NewTiDBDriver(store) client := testserverclient.NewTestServerClient() @@ -305,7 +273,7 @@ func TestStatsPriorityQueueAPI(t *testing.T) { cfg.Port = client.Port cfg.Status.StatusPort = client.StatusPort cfg.Status.ReportStatus = true - cfg.Socket = filepath.Join(tmp, fmt.Sprintf("tidb-mock-%d.sock", time.Now().UnixNano())) + cfg.Socket = newTestSocket(t) // RunInGoTestChan is a global channel and will be closed after the first server starts. // Recreate it to avoid racing on subsequent server starts in the same test binary. @@ -368,7 +336,7 @@ func TestLoadNullStatsFile(t *testing.T) { cfg.Port = client.Port cfg.Status.StatusPort = client.StatusPort cfg.Status.ReportStatus = true - cfg.Socket = filepath.Join(tmp, fmt.Sprintf("tidb-mock-%d.sock", time.Now().UnixNano())) + cfg.Socket = newTestSocket(t) // Creating and running the server // RunInGoTestChan is a global channel and will be closed after the first server starts. diff --git a/pkg/server/handler/util.go b/pkg/server/handler/util.go index bd206c683bb1f..dbbbb3c8973b5 100644 --- a/pkg/server/handler/util.go +++ b/pkg/server/handler/util.go @@ -38,7 +38,6 @@ const ( ColumnFlag = "colFlag" ColumnLen = "colLen" RowBin = "rowBin" - Snapshot = "snapshot" FileName = "filename" DumpPartitionStats = "dumpPartitionStats" Begin = "begin" diff --git a/pkg/server/http_handler.go b/pkg/server/http_handler.go index 0bc01c83570ad..b91a2946c357b 100644 --- a/pkg/server/http_handler.go +++ b/pkg/server/http_handler.go @@ -17,10 +17,8 @@ package server import ( "github.com/pingcap/tidb/pkg/config" "github.com/pingcap/tidb/pkg/domain/infosync" - "github.com/pingcap/tidb/pkg/infoschema" "github.com/pingcap/tidb/pkg/server/handler" "github.com/pingcap/tidb/pkg/server/handler/optimizor" - "github.com/pingcap/tidb/pkg/statistics/handle" "github.com/pingcap/tidb/pkg/store/helper" ) @@ -55,13 +53,5 @@ func (s *Server) newPlanReplayerHandler() *optimizor.PlanReplayerHandler { if s.dom != nil && s.dom.InfoSyncer() != nil { infoGetter = s.dom.InfoSyncer() } - var is infoschema.InfoSchema - if s.dom != nil && s.dom.InfoSchema() != nil { - is = s.dom.InfoSchema() - } - var statsHandle *handle.Handle - if s.dom != nil && s.dom.StatsHandle() != nil { - statsHandle = s.dom.StatsHandle() - } - return optimizor.NewPlanReplayerHandler(is, statsHandle, infoGetter, cfg.AdvertiseAddress, cfg.Status.StatusPort) + return optimizor.NewPlanReplayerHandler(infoGetter, cfg.AdvertiseAddress, cfg.Status.StatusPort) } diff --git a/pkg/server/http_status.go b/pkg/server/http_status.go index 6b7b95f5fa947..a2401979c1e67 100644 --- a/pkg/server/http_status.go +++ b/pkg/server/http_status.go @@ -224,8 +224,6 @@ func (s *Server) startHTTPServer() { // HTTP path for dump statistics. router.Handle("/stats/dump/{db}/{table}", s.newStatsHandler()). Name("StatsDump") - router.Handle("/stats/dump/{db}/{table}/{snapshot}", s.newStatsHistoryHandler()). - Name("StatsHistoryDump") router.Handle("/stats/priority-queue", s.newStatsPriorityQueueHandler()). Name("StatsPriorityQueue") @@ -728,19 +726,6 @@ func (s *Server) newStatsHandler() *optimizor.StatsHandler { return optimizor.NewStatsHandler(do) } -func (s *Server) newStatsHistoryHandler() *optimizor.StatsHistoryHandler { - store, ok := s.driver.(*TiDBDriver) - if !ok { - panic("Illegal driver") - } - - do, err := session.GetDomain(store.store) - if err != nil { - panic("Failed to get domain") - } - return optimizor.NewStatsHistoryHandler(do) -} - func (s *Server) newStatsPriorityQueueHandler() *optimizor.StatsPriorityQueueHandler { store, ok := s.driver.(*TiDBDriver) if !ok { diff --git a/pkg/session/session.go b/pkg/session/session.go index b98fbd90717b4..69a2b1adaf892 100644 --- a/pkg/session/session.go +++ b/pkg/session/session.go @@ -4460,7 +4460,7 @@ func bootstrapSessionImpl(ctx context.Context, store kv.Storage, createSessionsI if concurrency < 0 { // it is only for test, in the production, negative value is illegal. concurrency = 0 } - ses, err := createSessionsImpl(store, 10) + ses, err := createSessionsImpl(store, 9) if err != nil { return nil, err } @@ -4472,8 +4472,7 @@ func bootstrapSessionImpl(ctx context.Context, store kv.Storage, createSessionsI // ses[5]: telemetry, expression pushdown // ses[6]: plan replayer collector // ses[7]: dump file GC - // ses[8]: historical stats - // ses[9]: bootstrap SQL file + // ses[8]: bootstrap SQL file for i := range ses { ses[i].GetSessionVars().InRestrictedSQL = true } @@ -4560,16 +4559,13 @@ func bootstrapSessionImpl(ctx context.Context, store kv.Storage, createSessionsI // setup dumpFileGcChecker dom.SetupDumpFileGCChecker(ses[7]) dom.DumpFileGcCheckerLoop() - // setup historical stats worker - dom.SetupHistoricalStatsWorker(ses[8]) - dom.StartHistoricalStatsWorker() failToLoadOrParseSQLFile := false // only used for unit test if runBootstrapSQLFile { pm := &privileges.UserPrivileges{ Handle: dom.PrivilegeHandle(), } - privilege.BindPrivilegeManager(ses[9], pm) - if err := doBootstrapSQLFile(ses[9]); err != nil && intest.EnableInternalCheck { + privilege.BindPrivilegeManager(ses[8], pm) + if err := doBootstrapSQLFile(ses[8]); err != nil && intest.EnableInternalCheck { failToLoadOrParseSQLFile = true } } diff --git a/pkg/sessionctx/vardef/tidb_vars.go b/pkg/sessionctx/vardef/tidb_vars.go index 1ab8cc5e66476..e5ee6d6fb75b9 100644 --- a/pkg/sessionctx/vardef/tidb_vars.go +++ b/pkg/sessionctx/vardef/tidb_vars.go @@ -1159,7 +1159,7 @@ const ( TiDBGCMaxWaitTime = "tidb_gc_max_wait_time" // TiDBEnableEnhancedSecurity restricts SUPER users from certain operations. TiDBEnableEnhancedSecurity = "tidb_enable_enhanced_security" - // TiDBEnableHistoricalStats enables the historical statistics feature (default off) + // TiDBEnableHistoricalStats keeps compatibility for the removed historical statistics feature. TiDBEnableHistoricalStats = "tidb_enable_historical_stats" // TiDBPersistAnalyzeOptions persists analyze options for later analyze and auto-analyze TiDBPersistAnalyzeOptions = "tidb_persist_analyze_options" @@ -1286,9 +1286,11 @@ const ( PasswordReuseHistory = "password_history" // PasswordReuseTime limit how long passwords can be reused. PasswordReuseTime = "password_reuse_interval" - // TiDBHistoricalStatsDuration indicates the duration to remain tidb historical stats + // TiDBHistoricalStatsDuration is still load-bearing after the historical stats feature removal: + // it controls the retention window that stats GC uses to drain legacy rows from + // mysql.stats_history/mysql.stats_meta_history on upgraded clusters. TiDBHistoricalStatsDuration = "tidb_historical_stats_duration" - // TiDBEnableHistoricalStatsForCapture indicates whether use historical stats in plan replayer capture + // TiDBEnableHistoricalStatsForCapture keeps compatibility for the removed historical statistics feature. TiDBEnableHistoricalStatsForCapture = "tidb_enable_historical_stats_for_capture" // TiDBEnableResourceControl indicates whether resource control feature is enabled TiDBEnableResourceControl = "tidb_enable_resource_control" @@ -1770,7 +1772,6 @@ const ( DefMaxUserConnections = 0 DefTiDBStoreBatchSize = 4 DefTiDBHistoricalStatsDuration = 7 * 24 * time.Hour - DefTiDBEnableHistoricalStatsForCapture = false DefTiDBTTLJobScheduleWindowStartTime = "00:00 +0000" DefTiDBTTLJobScheduleWindowEndTime = "23:59 +0000" DefTiDBTTLScanWorkerCount = 4 @@ -1964,16 +1965,15 @@ var ( DefTiDBTTLJobScheduleWindowEndTime, ), ) - TTLScanWorkerCount = atomic.NewInt32(DefTiDBTTLScanWorkerCount) - TTLDeleteWorkerCount = atomic.NewInt32(DefTiDBTTLDeleteWorkerCount) - PasswordHistory = atomic.NewInt64(DefPasswordReuseHistory) - PasswordReuseInterval = atomic.NewInt64(DefPasswordReuseTime) - IsSandBoxModeEnabled = atomic.NewBool(false) - MaxUserConnectionsValue = atomic.NewUint32(DefMaxUserConnections) - MaxPreparedStmtCountValue = atomic.NewInt64(DefMaxPreparedStmtCount) - HistoricalStatsDuration = atomic.NewDuration(DefTiDBHistoricalStatsDuration) - EnableHistoricalStatsForCapture = atomic.NewBool(DefTiDBEnableHistoricalStatsForCapture) - TTLRunningTasks = atomic.NewInt32(DefTiDBTTLRunningTasks) + TTLScanWorkerCount = atomic.NewInt32(DefTiDBTTLScanWorkerCount) + TTLDeleteWorkerCount = atomic.NewInt32(DefTiDBTTLDeleteWorkerCount) + PasswordHistory = atomic.NewInt64(DefPasswordReuseHistory) + PasswordReuseInterval = atomic.NewInt64(DefPasswordReuseTime) + IsSandBoxModeEnabled = atomic.NewBool(false) + MaxUserConnectionsValue = atomic.NewUint32(DefMaxUserConnections) + MaxPreparedStmtCountValue = atomic.NewInt64(DefMaxPreparedStmtCount) + HistoricalStatsDuration = atomic.NewDuration(DefTiDBHistoricalStatsDuration) + TTLRunningTasks = atomic.NewInt32(DefTiDBTTLRunningTasks) // always set the default value to false because the resource control in kv-client is not inited // It will be initialized to the right value after the first call of `rebuildSysVarCache` EnableResourceControl = atomic.NewBool(false) diff --git a/pkg/sessionctx/variable/session.go b/pkg/sessionctx/variable/session.go index 5e0b3a45cafc1..10bd8b2fabfbd 100644 --- a/pkg/sessionctx/variable/session.go +++ b/pkg/sessionctx/variable/session.go @@ -1499,9 +1499,6 @@ type SessionVars struct { // UseHashJoinV2 indicates whether to use hash join v2. UseHashJoinV2 bool - // EnableHistoricalStats indicates whether to enable historical statistics. - EnableHistoricalStats bool - // EnableIndexMergeJoin indicates whether to enable index merge join. EnableIndexMergeJoin bool diff --git a/pkg/sessionctx/variable/sysvar.go b/pkg/sessionctx/variable/sysvar.go index 9df474aefe435..24ba0d5d954a1 100644 --- a/pkg/sessionctx/variable/sysvar.go +++ b/pkg/sessionctx/variable/sysvar.go @@ -1055,7 +1055,13 @@ var defaultSysVars = []*SysVar{ return origin, nil }}, {Scope: vardef.ScopeGlobal, Name: vardef.TiDBEnableTelemetry, Value: BoolToOnOff(vardef.DefTiDBEnableTelemetry), Type: vardef.TypeBool}, - {Scope: vardef.ScopeGlobal, Name: vardef.TiDBEnableHistoricalStats, Value: vardef.Off, Type: vardef.TypeBool, Depended: true}, + {Scope: vardef.ScopeGlobal, Name: vardef.TiDBEnableHistoricalStats, Value: vardef.Off, Type: vardef.TypeBool, + Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope vardef.ScopeFlag) (string, error) { + if TiDBOptOn(normalizedValue) { + vars.StmtCtx.AppendWarning(errors.NewNoStackError("the historical stats feature has been removed, so this will have no effect")) + } + return normalizedValue, nil + }}, /* tikv gc metrics */ {Scope: vardef.ScopeGlobal, Name: vardef.TiDBGCEnable, Value: vardef.On, Type: vardef.TypeBool, GetGlobal: func(_ context.Context, s *SessionVars) (string, error) { return getTiDBTableValue(s, "tikv_gc_enable", vardef.On) @@ -1815,15 +1821,13 @@ var defaultSysVars = []*SysVar{ vardef.PasswordReuseInterval.Store(TidbOptInt64(val, vardef.DefPasswordReuseTime)) return nil }}, - {Scope: vardef.ScopeGlobal, Name: vardef.TiDBEnableHistoricalStatsForCapture, Value: BoolToOnOff(vardef.DefTiDBEnableHistoricalStatsForCapture), Type: vardef.TypeBool, - SetGlobal: func(ctx context.Context, vars *SessionVars, s string) error { - vardef.EnableHistoricalStatsForCapture.Store(TiDBOptOn(s)) - return nil - }, - GetGlobal: func(ctx context.Context, vars *SessionVars) (string, error) { - return BoolToOnOff(vardef.EnableHistoricalStatsForCapture.Load()), nil - }, - }, + {Scope: vardef.ScopeGlobal, Name: vardef.TiDBEnableHistoricalStatsForCapture, Value: vardef.Off, Type: vardef.TypeBool, + Validation: func(vars *SessionVars, normalizedValue string, originalValue string, scope vardef.ScopeFlag) (string, error) { + if TiDBOptOn(normalizedValue) { + vars.StmtCtx.AppendWarning(errors.NewNoStackError("the historical stats feature has been removed, so this will have no effect")) + } + return normalizedValue, nil + }}, {Scope: vardef.ScopeGlobal, Name: vardef.TiDBHistoricalStatsDuration, Value: vardef.DefTiDBHistoricalStatsDuration.String(), Type: vardef.TypeDuration, MinValue: int64(time.Second), MaxValue: uint64(time.Hour * 24 * 365), GetGlobal: func(ctx context.Context, vars *SessionVars) (string, error) { return vardef.HistoricalStatsDuration.Load().String(), nil @@ -1859,29 +1863,12 @@ var defaultSysVars = []*SysVar{ /* The system variables below have GLOBAL and SESSION scope */ {Scope: vardef.ScopeGlobal | vardef.ScopeSession, Name: vardef.TiDBEnablePlanReplayerContinuousCapture, Value: BoolToOnOff(false), Type: vardef.TypeBool, SetSession: func(s *SessionVars, val string) error { - historicalStatsEnabled, err := s.GlobalVarsAccessor.GetGlobalSysVar(vardef.TiDBEnableHistoricalStats) - if err != nil { - return err - } - if !TiDBOptOn(historicalStatsEnabled) && TiDBOptOn(val) { - return errors.Errorf("%v should be enabled before enabling %v", vardef.TiDBEnableHistoricalStats, vardef.TiDBEnablePlanReplayerContinuousCapture) - } s.EnablePlanReplayedContinuesCapture = TiDBOptOn(val) return nil }, GetSession: func(vars *SessionVars) (string, error) { return BoolToOnOff(vars.EnablePlanReplayedContinuesCapture), nil }, - Validation: func(vars *SessionVars, s string, s2 string, flag vardef.ScopeFlag) (string, error) { - historicalStatsEnabled, err := vars.GlobalVarsAccessor.GetGlobalSysVar(vardef.TiDBEnableHistoricalStats) - if err != nil { - return "", err - } - if !TiDBOptOn(historicalStatsEnabled) && TiDBOptOn(s) { - return "", errors.Errorf("%v should be enabled before enabling %v", vardef.TiDBEnableHistoricalStats, vardef.TiDBEnablePlanReplayerContinuousCapture) - } - return s, nil - }, }, {Scope: vardef.ScopeGlobal | vardef.ScopeSession, Name: vardef.TiDBEnablePlanReplayerCapture, Value: BoolToOnOff(vardef.DefTiDBEnablePlanReplayerCapture), Type: vardef.TypeBool, SetSession: func(s *SessionVars, val string) error { diff --git a/pkg/sessionctx/variable/sysvar_test.go b/pkg/sessionctx/variable/sysvar_test.go index b61b658538ba2..57cd4af249498 100644 --- a/pkg/sessionctx/variable/sysvar_test.go +++ b/pkg/sessionctx/variable/sysvar_test.go @@ -2107,3 +2107,25 @@ func TestSkipInitIsUsed(t *testing.T) { } } } + +func TestRemovedHistoricalStatsVarsWarn(t *testing.T) { + // The historical stats feature was removed; its switches are kept as no-op + // variables for compatibility and warn when enabled. + for _, name := range []string{vardef.TiDBEnableHistoricalStats, vardef.TiDBEnableHistoricalStatsForCapture} { + sv := GetSysVar(name) + vars := NewSessionVars(nil) + val, err := sv.Validate(vars, "ON", vardef.ScopeGlobal) + require.NoError(t, err) + require.Equal(t, "ON", val) + warns := vars.StmtCtx.GetWarnings() + require.Len(t, warns, 1) + require.Contains(t, warns[0].Err.Error(), "the historical stats feature has been removed") + + // Turning the switch off must not warn. + vars = NewSessionVars(nil) + val, err = sv.Validate(vars, "OFF", vardef.ScopeGlobal) + require.NoError(t, err) + require.Equal(t, "OFF", val) + require.Len(t, vars.StmtCtx.GetWarnings(), 0) + } +} diff --git a/pkg/sessionctx/variable/tests/variable_test.go b/pkg/sessionctx/variable/tests/variable_test.go index c82771d2ccf07..12d0483fb3b5b 100644 --- a/pkg/sessionctx/variable/tests/variable_test.go +++ b/pkg/sessionctx/variable/tests/variable_test.go @@ -711,21 +711,17 @@ func TestOrderByDependency(t *testing.T) { // Some other exceptions: // - tidb_snapshot and tidb_read_staleness can not be set at the same time. It doesn't affect dependency. vars := map[string]string{ - "unknown": "1", - vardef.TxReadOnly: "1", - vardef.SQLAutoIsNull: "1", - vardef.TiDBEnableNoopFuncs: "1", - vardef.TiDBEnforceMPPExecution: "1", - vardef.TiDBAllowMPPExecution: "1", - vardef.TiDBEnableLocalTxn: "1", - vardef.TiDBEnablePlanReplayerContinuousCapture: "1", - vardef.TiDBEnableHistoricalStats: "1", + "unknown": "1", + vardef.TxReadOnly: "1", + vardef.SQLAutoIsNull: "1", + vardef.TiDBEnableNoopFuncs: "1", + vardef.TiDBEnforceMPPExecution: "1", + vardef.TiDBAllowMPPExecution: "1", + vardef.TiDBEnableLocalTxn: "1", } names := variable.OrderByDependency(vars) require.Greater(t, slices.Index(names, vardef.TxReadOnly), slices.Index(names, vardef.TiDBEnableNoopFuncs)) require.Greater(t, slices.Index(names, vardef.SQLAutoIsNull), slices.Index(names, vardef.TiDBEnableNoopFuncs)) require.Greater(t, slices.Index(names, vardef.TiDBEnforceMPPExecution), slices.Index(names, vardef.TiDBAllowMPPExecution)) - // Depended variables below are global variables, so actually it doesn't matter. - require.Greater(t, slices.Index(names, vardef.TiDBEnablePlanReplayerContinuousCapture), slices.Index(names, vardef.TiDBEnableHistoricalStats)) require.Contains(t, names, "unknown") } diff --git a/pkg/statistics/BUILD.bazel b/pkg/statistics/BUILD.bazel index e193b34420584..3a612d315b468 100644 --- a/pkg/statistics/BUILD.bazel +++ b/pkg/statistics/BUILD.bazel @@ -81,7 +81,7 @@ go_test( data = glob(["testdata/**"]), embed = [":statistics"], flaky = True, - shard_count = 45, + shard_count = 44, deps = [ "//pkg/config", "//pkg/meta/model", diff --git a/pkg/statistics/handle/BUILD.bazel b/pkg/statistics/handle/BUILD.bazel index 7cc4d3ca3fc5c..2f14ba1ff9747 100644 --- a/pkg/statistics/handle/BUILD.bazel +++ b/pkg/statistics/handle/BUILD.bazel @@ -25,7 +25,6 @@ go_library( "//pkg/statistics/handle/cache", "//pkg/statistics/handle/ddl", "//pkg/statistics/handle/globalstats", - "//pkg/statistics/handle/history", "//pkg/statistics/handle/initstats", "//pkg/statistics/handle/lockstats", "//pkg/statistics/handle/logutil", diff --git a/pkg/statistics/handle/ddl/BUILD.bazel b/pkg/statistics/handle/ddl/BUILD.bazel index 34f701e7488bb..9c1ee54ee9686 100644 --- a/pkg/statistics/handle/ddl/BUILD.bazel +++ b/pkg/statistics/handle/ddl/BUILD.bazel @@ -15,7 +15,6 @@ go_library( "//pkg/sessionctx", "//pkg/sessionctx/vardef", "//pkg/sessionctx/variable", - "//pkg/statistics/handle/history", "//pkg/statistics/handle/lockstats", "//pkg/statistics/handle/logutil", "//pkg/statistics/handle/storage", diff --git a/pkg/statistics/handle/ddl/ddl.go b/pkg/statistics/handle/ddl/ddl.go index 520486c48817d..2a25b0614e58c 100644 --- a/pkg/statistics/handle/ddl/ddl.go +++ b/pkg/statistics/handle/ddl/ddl.go @@ -47,7 +47,7 @@ func NewDDLHandler( ddlEventCh: make(chan *notifier.SchemaChangeEvent, 1000), statsWriter: statsWriter, statsHandler: statsHandler, - sub: newSubscriber(statsHandler), + sub: newSubscriber(), } } diff --git a/pkg/statistics/handle/ddl/subscriber.go b/pkg/statistics/handle/ddl/subscriber.go index d2d3406ff3358..d95ce28abbc6c 100644 --- a/pkg/statistics/handle/ddl/subscriber.go +++ b/pkg/statistics/handle/ddl/subscriber.go @@ -24,26 +24,19 @@ import ( "github.com/pingcap/tidb/pkg/sessionctx" "github.com/pingcap/tidb/pkg/sessionctx/vardef" "github.com/pingcap/tidb/pkg/sessionctx/variable" - "github.com/pingcap/tidb/pkg/statistics/handle/history" "github.com/pingcap/tidb/pkg/statistics/handle/lockstats" "github.com/pingcap/tidb/pkg/statistics/handle/logutil" "github.com/pingcap/tidb/pkg/statistics/handle/storage" - "github.com/pingcap/tidb/pkg/statistics/handle/types" "github.com/pingcap/tidb/pkg/statistics/handle/util" "github.com/pingcap/tidb/pkg/util/intest" "go.uber.org/zap" ) -type subscriber struct { - statsCache types.StatsCache -} +type subscriber struct{} // newSubscriber creates a new subscriber. -func newSubscriber( - statsCache types.StatsCache, -) *subscriber { - h := subscriber{statsCache: statsCache} - return &h +func newSubscriber() *subscriber { + return &subscriber{} } func (h subscriber) handle( @@ -284,73 +277,31 @@ func (h subscriber) handle( return nil } -func (h subscriber) insertStats4PhysicalID( +func (subscriber) insertStats4PhysicalID( ctx context.Context, sctx sessionctx.Context, info *model.TableInfo, id int64, ) error { - startTS, err := storage.InsertTableStats2KV(ctx, sctx, info, id) - if err != nil { - return errors.Trace(err) - } - return errors.Trace(h.recordHistoricalStatsMeta(ctx, sctx, id, startTS)) -} - -func (h subscriber) recordHistoricalStatsMeta( - ctx context.Context, - sctx sessionctx.Context, - id int64, - startTS uint64, -) error { - if startTS == 0 { - return nil - } - enableHistoricalStats, err2 := getEnableHistoricalStats(sctx) - if err2 != nil { - return err2 - } - if !enableHistoricalStats { - return nil - } - - tbl, ok := h.statsCache.Get(id) - if !ok || !tbl.IsInitialized() { - return nil - } - - return history.RecordHistoricalStatsMeta( - ctx, - sctx, - startTS, - util.StatsMetaHistorySourceSchemaChange, - id, - ) + return errors.Trace(storage.InsertTableStats2KV(ctx, sctx, info, id)) } -func (h subscriber) delayedDeleteStats4PhysicalID( +func (subscriber) delayedDeleteStats4PhysicalID( ctx context.Context, sctx sessionctx.Context, id int64, ) error { - startTS, err2 := storage.UpdateStatsMetaVerAndLastHistUpdateVer(ctx, sctx, id) - if err2 != nil { - return errors.Trace(err2) - } - return errors.Trace(h.recordHistoricalStatsMeta(ctx, sctx, id, startTS)) + _, err := storage.UpdateStatsMetaVerAndLastHistUpdateVer(ctx, sctx, id) + return errors.Trace(err) } -func (h subscriber) insertStats4Col( +func (subscriber) insertStats4Col( ctx context.Context, sctx sessionctx.Context, physicalID int64, colInfos []*model.ColumnInfo, ) error { - startTS, err := storage.InsertColStats2KV(ctx, sctx, physicalID, colInfos) - if err != nil { - return errors.Trace(err) - } - return errors.Trace(h.recordHistoricalStatsMeta(ctx, sctx, physicalID, startTS)) + return errors.Trace(storage.InsertColStats2KV(ctx, sctx, physicalID, colInfos)) } func getPhysicalIDs( @@ -384,15 +335,6 @@ func getCurrentPruneMode( return variable.PartitionPruneMode(pruneMode), errors.Trace(err) } -func getEnableHistoricalStats( - sctx sessionctx.Context, -) (bool, error) { - val, err := sctx.GetSessionVars(). - GlobalVarsAccessor. - GetGlobalSysVar(vardef.TiDBEnableHistoricalStats) - return variable.TiDBOptOn(val), errors.Trace(err) -} - func updateGlobalTableStats4DropPartition( ctx context.Context, sctx sessionctx.Context, diff --git a/pkg/statistics/handle/globalstats/global_stats.go b/pkg/statistics/handle/globalstats/global_stats.go index c8d735bdbbd63..cc301833af70a 100644 --- a/pkg/statistics/handle/globalstats/global_stats.go +++ b/pkg/statistics/handle/globalstats/global_stats.go @@ -25,7 +25,6 @@ import ( "github.com/pingcap/tidb/pkg/statistics" statslogutil "github.com/pingcap/tidb/pkg/statistics/handle/logutil" statstypes "github.com/pingcap/tidb/pkg/statistics/handle/types" - "github.com/pingcap/tidb/pkg/statistics/handle/util" "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util/logutil" "github.com/tiancaiamao/gp" @@ -378,7 +377,6 @@ func WriteGlobalStatsToStorage(statsHandle statstypes.StatsHandle, globalStats * topN, info.StatsVersion, true, - util.StatsMetaHistorySourceAnalyze, ) if err != nil { statslogutil.StatsLogger().Warn("save global-level stats to storage failed", diff --git a/pkg/statistics/handle/handle.go b/pkg/statistics/handle/handle.go index dc3cb93805992..c61b8eed641f2 100644 --- a/pkg/statistics/handle/handle.go +++ b/pkg/statistics/handle/handle.go @@ -31,7 +31,6 @@ import ( "github.com/pingcap/tidb/pkg/statistics/handle/cache" "github.com/pingcap/tidb/pkg/statistics/handle/ddl" "github.com/pingcap/tidb/pkg/statistics/handle/globalstats" - "github.com/pingcap/tidb/pkg/statistics/handle/history" "github.com/pingcap/tidb/pkg/statistics/handle/lockstats" statslogutil "github.com/pingcap/tidb/pkg/statistics/handle/logutil" "github.com/pingcap/tidb/pkg/statistics/handle/storage" @@ -86,9 +85,6 @@ type Handle struct { // StatsUsage is used to track the usage of column / index statistics. types.StatsUsage - // StatsHistory is used to manage historical stats. - types.StatsHistory - // StatsAnalyze is used to handle auto-analyze and manage analyze jobs. types.StatsAnalyze @@ -166,7 +162,6 @@ func NewHandle( handle.AutoAnalyzeProcIDGenerator = util.NewGenerator(autoAnalyzeProcIDGetter, releaseAutoAnalyzeProcID) handle.LeaseGetter = util.NewLeaseGetter(lease) handle.StatsCache = statsCache - handle.StatsHistory = history.NewStatsHistory(handle) handle.StatsUsage = usage.NewStatsUsageImpl(handle) handle.StatsAnalyze = autoanalyze.NewStatsAnalyze(ctx, handle, tracker, ddlNotifier) handle.StatsSyncLoad = syncload.NewStatsSyncLoad(handle) diff --git a/pkg/statistics/handle/handletest/BUILD.bazel b/pkg/statistics/handle/handletest/BUILD.bazel index 4a8f726492849..b37397b2ae93d 100644 --- a/pkg/statistics/handle/handletest/BUILD.bazel +++ b/pkg/statistics/handle/handletest/BUILD.bazel @@ -8,7 +8,7 @@ go_test( "main_test.go", ], flaky = True, - shard_count = 30, + shard_count = 29, deps = [ "//pkg/config", "//pkg/domain", diff --git a/pkg/statistics/handle/handletest/handle_test.go b/pkg/statistics/handle/handletest/handle_test.go index b04b4f7b9c470..21094b28d68af 100644 --- a/pkg/statistics/handle/handletest/handle_test.go +++ b/pkg/statistics/handle/handletest/handle_test.go @@ -17,7 +17,6 @@ package handletest import ( "context" "fmt" - "strconv" "strings" "testing" "time" @@ -692,31 +691,6 @@ func TestFlushPendingStatsDeltaBeforeAnalyze(t *testing.T) { )) } -func TestRecordHistoricalStatsToStorage(t *testing.T) { - store, dom := testkit.CreateMockStoreAndDomain(t) - tk := testkit.NewTestKit(t, store) - tk.MustExec("set @@tidb_analyze_version = 2") - tk.MustExec("use test") - tk.MustExec("drop table if exists t") - tk.MustExec("create table t(a int, b varchar(10))") - tk.MustExec("insert into t value(1, 'aaa'), (3, 'aab'), (5, 'bba'), (2, 'bbb'), (4, 'cca'), (6, 'ccc')") - // mark column stats as needed - tk.MustExec("select * from t where a = 3") - tk.MustExec("select * from t where b = 'bbb'") - tk.MustExec("alter table t add index single(a)") - tk.MustExec("alter table t add index multi(a, b)") - tk.MustExec("analyze table t with 2 topn") - - tableInfo, err := dom.InfoSchema().TableByName(context.Background(), ast.NewCIStr("test"), ast.NewCIStr("t")) - require.NoError(t, err) - version, err := dom.StatsHandle().RecordHistoricalStatsToStorage("t", tableInfo.Meta(), tableInfo.Meta().ID, false) - require.NoError(t, err) - - rows := tk.MustQuery(fmt.Sprintf("select count(*) from mysql.stats_history where version = '%d'", version)).Rows() - num, _ := strconv.Atoi(rows[0][0].(string)) - require.GreaterOrEqual(t, num, 1) -} - func TestEvictedColumnLoadedStatus(t *testing.T) { t.Skip("skip this test because it is useless") restore := config.RestoreFunc() diff --git a/pkg/statistics/handle/handletest/lockstats/BUILD.bazel b/pkg/statistics/handle/handletest/lockstats/BUILD.bazel index 3844e60219027..b69e956b5c0a0 100644 --- a/pkg/statistics/handle/handletest/lockstats/BUILD.bazel +++ b/pkg/statistics/handle/handletest/lockstats/BUILD.bazel @@ -19,7 +19,6 @@ go_test( "//pkg/statistics", "//pkg/testkit", "//pkg/testkit/testsetup", - "@com_github_pingcap_failpoint//:failpoint", "@com_github_stretchr_testify//require", "@org_uber_go_goleak//:goleak", ], diff --git a/pkg/statistics/handle/handletest/lockstats/lock_table_stats_test.go b/pkg/statistics/handle/handletest/lockstats/lock_table_stats_test.go index c4966a46c506e..bcd9246c2c822 100644 --- a/pkg/statistics/handle/handletest/lockstats/lock_table_stats_test.go +++ b/pkg/statistics/handle/handletest/lockstats/lock_table_stats_test.go @@ -22,7 +22,6 @@ import ( "testing" "time" - "github.com/pingcap/failpoint" "github.com/pingcap/tidb/pkg/config" "github.com/pingcap/tidb/pkg/domain" "github.com/pingcap/tidb/pkg/kv" @@ -63,12 +62,6 @@ func TestLockAndUnlockTableStats(t *testing.T) { // Insert a new row to the table. tk.MustExec("insert into t(a, b) values(3,'c')") - // Enable the failpoint to test the historical stats meta is not recorded. - err = failpoint.Enable( - "github.com/pingcap/tidb/pkg/statistics/handle/usage/panic-when-record-historical-stats-meta", - "1*return(true)", - ) - require.NoError(t, err) // Flush stats delta. tk.MustExec("flush stats_delta *.*") diff --git a/pkg/statistics/handle/history/BUILD.bazel b/pkg/statistics/handle/history/BUILD.bazel deleted file mode 100644 index 0e7955cfddd87..0000000000000 --- a/pkg/statistics/handle/history/BUILD.bazel +++ /dev/null @@ -1,20 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") - -go_library( - name = "history", - srcs = ["history_stats.go"], - importpath = "github.com/pingcap/tidb/pkg/statistics/handle/history", - visibility = ["//visibility:public"], - deps = [ - "//pkg/meta/model", - "//pkg/sessionctx", - "//pkg/statistics/handle/logutil", - "//pkg/statistics/handle/storage", - "//pkg/statistics/handle/types", - "//pkg/statistics/handle/util", - "//pkg/statistics/util", - "//pkg/util/intest", - "@com_github_pingcap_errors//:errors", - "@org_uber_go_zap//:zap", - ], -) diff --git a/pkg/statistics/handle/history/history_stats.go b/pkg/statistics/handle/history/history_stats.go deleted file mode 100644 index 2b76d7e7a7aee..0000000000000 --- a/pkg/statistics/handle/history/history_stats.go +++ /dev/null @@ -1,210 +0,0 @@ -// Copyright 2023 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package history - -import ( - "context" - "time" - - "github.com/pingcap/errors" - "github.com/pingcap/tidb/pkg/meta/model" - "github.com/pingcap/tidb/pkg/sessionctx" - statslogutil "github.com/pingcap/tidb/pkg/statistics/handle/logutil" - "github.com/pingcap/tidb/pkg/statistics/handle/storage" - "github.com/pingcap/tidb/pkg/statistics/handle/types" - handleutil "github.com/pingcap/tidb/pkg/statistics/handle/util" - statsutil "github.com/pingcap/tidb/pkg/statistics/util" - "github.com/pingcap/tidb/pkg/util/intest" - "go.uber.org/zap" -) - -// statsHistoryImpl implements util.StatsHistory. -type statsHistoryImpl struct { - statsHandle types.StatsHandle -} - -// NewStatsHistory creates a new StatsHistory. -func NewStatsHistory(statsHandle types.StatsHandle, -) types.StatsHistory { - return &statsHistoryImpl{ - statsHandle: statsHandle, - } -} - -// RecordHistoricalStatsToStorage records the given table's stats data to mysql.stats_history -func (sh *statsHistoryImpl) RecordHistoricalStatsToStorage(dbName string, tableInfo *model.TableInfo, physicalID int64, isPartition bool) (uint64, error) { - var js *statsutil.JSONTable - var err error - if isPartition { - js, err = sh.statsHandle.TableStatsToJSON(dbName, tableInfo, physicalID, 0) - } else { - js, err = sh.statsHandle.DumpStatsToJSON(dbName, tableInfo, nil, true) - } - if err != nil { - return 0, errors.Trace(err) - } - if js == nil { - statslogutil.StatsLogger().Warn("no stats data to record", zap.String("dbName", dbName), zap.String("tableName", tableInfo.Name.O)) - return 0, nil - } - var version uint64 - err = handleutil.CallWithSCtx(sh.statsHandle.SPool(), func(sctx sessionctx.Context) error { - version, err = RecordHistoricalStatsToStorage(sctx, physicalID, js) - return err - }, handleutil.FlagWrapTxn) - return version, err -} - -// RecordHistoricalStatsMeta records the historical stats meta in mysql.stats_meta_history one by one. -func (sh *statsHistoryImpl) RecordHistoricalStatsMeta(version uint64, source string, enforce bool, tableIDs ...int64) { - if version == 0 { - return - } - - var targetedTableIDs []int64 - if enforce { - targetedTableIDs = tableIDs - } else { - targetedTableIDs = make([]int64, 0, len(tableIDs)) - for _, tableID := range tableIDs { - tbl, ok := sh.statsHandle.Get(tableID) - if tableID == 0 || !ok || !tbl.IsInitialized() { - continue - } - targetedTableIDs = append(targetedTableIDs, tableID) - } - } - shouldSkipHistoricalStats := false - err := handleutil.CallWithSCtx(sh.statsHandle.SPool(), func(sctx sessionctx.Context) error { - if !sctx.GetSessionVars().EnableHistoricalStats { - shouldSkipHistoricalStats = true - return nil - } - return nil - }, handleutil.FlagWrapTxn) - if err != nil { - statslogutil.StatsLogger().Error("failed to check historical stats enable status", - zap.Uint64("version", version), - zap.String("source", source), - zap.Int64s("tableIDs", tableIDs), - zap.Int64s("targetedTableIDs", targetedTableIDs), - zap.Error(err)) - return - } - if !shouldSkipHistoricalStats { - for _, tableID := range targetedTableIDs { - err := handleutil.CallWithSCtx(sh.statsHandle.SPool(), func(sctx sessionctx.Context) error { - return RecordHistoricalStatsMeta(handleutil.StatsCtx, sctx, version, source, tableID) - }, handleutil.FlagWrapTxn) - if err != nil { - statslogutil.StatsLogger().Warn("record historical stats meta failed", - zap.Uint64("version", version), - zap.String("source", source), - zap.Int64("tableID", tableID), - zap.Error(err)) - } - } - } -} - -// CheckHistoricalStatsEnable checks whether historical stats is enabled. -func (sh *statsHistoryImpl) CheckHistoricalStatsEnable() (enable bool, err error) { - err = handleutil.CallWithSCtx(sh.statsHandle.SPool(), func(sctx sessionctx.Context) error { - enable = sctx.GetSessionVars().EnableHistoricalStats - return nil - }) - return -} - -// RecordHistoricalStatsMeta records the historical stats meta in mysql.stats_meta_history with the given version and source. -func RecordHistoricalStatsMeta( - ctx context.Context, - sctx sessionctx.Context, - version uint64, - source string, - tableID int64, -) error { - intest.Assert(version != 0, "version should not be zero") - intest.Assert(tableID != 0, "tableID should not be zero") - if tableID == 0 || version == 0 { - return errors.Errorf("tableID %d, version %d are invalid", tableID, version) - } - - rows, _, err := handleutil.ExecRowsWithCtx( - ctx, - sctx, - "SELECT modify_count, count FROM mysql.stats_meta WHERE table_id = %? and version = %? FOR UPDATE", - tableID, - version, - ) - if err != nil { - return errors.Trace(err) - } - intest.Assert(len(rows) != 0, "no historical meta stats can be recorded") - if len(rows) == 0 { - return errors.New("no historical meta stats can be recorded") - } - - modifyCount, count := rows[0].GetInt64(0), rows[0].GetInt64(1) - // Use NOW(6) to match the datetime(6) precision of the create_time column. - // Plain NOW() truncates to seconds, so rows written within the same second - // share an indistinguishable create_time. In-tree readers key off `version`, - // but full precision keeps create_time usable for ordering and auditing. - const sql = "REPLACE INTO mysql.stats_meta_history(table_id, modify_count, count, version, source, create_time) VALUES (%?, %?, %?, %?, %?, NOW(6))" - if _, err := handleutil.ExecWithCtx( - ctx, - sctx, - sql, - tableID, - modifyCount, - count, - version, - source, - ); err != nil { - return errors.Trace(err) - } - - return nil -} - -// Max column size is 6MB. Refer https://docs.pingcap.com/tidb/dev/tidb-limitations/#limitation-on-a-single-column -// but here is less than 6MB, because stats_history has other info except stats_data. -const maxColumnSize = 5 << 20 - -// RecordHistoricalStatsToStorage records the given table's stats data to mysql.stats_history -func RecordHistoricalStatsToStorage(sctx sessionctx.Context, physicalID int64, js *statsutil.JSONTable) (uint64, error) { - version := uint64(0) - if len(js.Partitions) == 0 { - version = js.Version - } else { - for _, p := range js.Partitions { - version = max(version, p.Version) - } - } - blocks, err := storage.JSONTableToBlocks(js, maxColumnSize) - if err != nil { - return version, errors.Trace(err) - } - - ts := time.Now().Format("2006-01-02 15:04:05.999999") - const sql = "INSERT INTO mysql.stats_history(table_id, stats_data, seq_no, version, create_time) VALUES (%?, %?, %?, %?, %?)" + - "ON DUPLICATE KEY UPDATE stats_data=%?, create_time=%?" - for i := range blocks { - if _, err = handleutil.Exec(sctx, sql, physicalID, blocks[i], i, version, ts, blocks[i], ts); err != nil { - return 0, errors.Trace(err) - } - } - return version, err -} diff --git a/pkg/statistics/handle/metrics/metrics.go b/pkg/statistics/handle/metrics/metrics.go index 58368d90ad8c6..00fb3d5bcdf34 100644 --- a/pkg/statistics/handle/metrics/metrics.go +++ b/pkg/statistics/handle/metrics/metrics.go @@ -62,9 +62,7 @@ var HealthyBucketConfigs = []HealthyBucketConfig{ // statistics metrics vars var ( - StatsHealthyGauges []prometheus.Gauge - DumpHistoricalStatsSuccessCounter prometheus.Counter - DumpHistoricalStatsFailedCounter prometheus.Counter + StatsHealthyGauges []prometheus.Gauge ) func init() { @@ -80,7 +78,4 @@ func InitMetricsVars() { for idx, cfg := range HealthyBucketConfigs { StatsHealthyGauges[idx] = metrics.StatsHealthyGauge.WithLabelValues(cfg.Label) } - - DumpHistoricalStatsSuccessCounter = metrics.HistoricalStatsCounter.WithLabelValues("dump", "success") - DumpHistoricalStatsFailedCounter = metrics.HistoricalStatsCounter.WithLabelValues("dump", "fail") } diff --git a/pkg/statistics/handle/storage/BUILD.bazel b/pkg/statistics/handle/storage/BUILD.bazel index 2eaed0a5e560f..d4cfea3d99265 100644 --- a/pkg/statistics/handle/storage/BUILD.bazel +++ b/pkg/statistics/handle/storage/BUILD.bazel @@ -28,7 +28,6 @@ go_library( "//pkg/statistics/handle/cache", "//pkg/statistics/handle/lockstats", "//pkg/statistics/handle/logutil", - "//pkg/statistics/handle/metrics", "//pkg/statistics/handle/types", "//pkg/statistics/handle/usage/predicatecolumn", "//pkg/statistics/handle/util", @@ -36,13 +35,11 @@ go_library( "//pkg/table", "//pkg/types", "//pkg/util/chunk", - "//pkg/util/compress", "//pkg/util/intest", "//pkg/util/logutil", "//pkg/util/memory", "//pkg/util/sqlescape", "//pkg/util/sqlexec", - "@com_github_klauspost_compress//gzip", "@com_github_pingcap_errors//:errors", "@com_github_pingcap_failpoint//:failpoint", "@com_github_tikv_client_go_v2//oracle", @@ -60,7 +57,7 @@ go_test( "stats_read_writer_test.go", ], flaky = True, - shard_count = 28, + shard_count = 29, deps = [ ":storage", "//pkg/domain", diff --git a/pkg/statistics/handle/storage/dump_test.go b/pkg/statistics/handle/storage/dump_test.go index cbfb3f574b752..d33c73aa5dcc8 100644 --- a/pkg/statistics/handle/storage/dump_test.go +++ b/pkg/statistics/handle/storage/dump_test.go @@ -15,14 +15,12 @@ package storage_test import ( - "cmp" "context" "encoding/json" "errors" "fmt" "math" "runtime" - "slices" "strings" "testing" @@ -536,49 +534,6 @@ func TestLoadStatsForNewCollation(t *testing.T) { requireTableEqual(t, statsCacheTbl, loadTbl) } -func TestJSONTableToBlocks(t *testing.T) { - store, dom := testkit.CreateMockStoreAndDomain(t) - tk := testkit.NewTestKit(t, store) - tk.MustExec("set @@tidb_analyze_version = 2") - tk.MustExec("use test") - tk.MustExec("drop table if exists t") - tk.MustExec("create table t(a int, b varchar(10))") - tk.MustExec("insert into t value(1, 'aaa'), (3, 'aab'), (5, 'bba'), (2, 'bbb'), (4, 'cca'), (6, 'ccc')") - // mark column stats as needed - tk.MustExec("select * from t where a = 3") - tk.MustExec("select * from t where b = 'bbb'") - tk.MustExec("alter table t add index single(a)") - tk.MustExec("alter table t add index multi(a, b)") - tk.MustExec("analyze table t with 2 topn") - h := dom.StatsHandle() - is := dom.InfoSchema() - tableInfo, err := is.TableByName(context.Background(), ast.NewCIStr("test"), ast.NewCIStr("t")) - require.NoError(t, err) - - dumpJSONTable, err := h.DumpStatsToJSON("test", tableInfo.Meta(), nil, true) - require.NoError(t, err) - // the slice is generated from a map loop, which is randomly - slices.SortFunc(dumpJSONTable.PredicateColumns, func(a, b *statsutil.JSONPredicateColumn) int { - return cmp.Compare(a.ID, b.ID) - }) - jsOrigin, _ := json.Marshal(dumpJSONTable) - - blockSize := 30 - js, err := h.DumpStatsToJSON("test", tableInfo.Meta(), nil, true) - require.NoError(t, err) - dumpJSONBlocks, err := storage.JSONTableToBlocks(js, blockSize) - require.NoError(t, err) - jsConverted, err := storage.BlocksToJSONTable(dumpJSONBlocks) - // the slice is generated from a map loop, which is randomly - slices.SortFunc(jsConverted.PredicateColumns, func(a, b *statsutil.JSONPredicateColumn) int { - return cmp.Compare(a.ID, b.ID) - }) - require.NoError(t, err) - jsonStr, err := json.Marshal(jsConverted) - require.NoError(t, err) - require.JSONEq(t, string(jsOrigin), string(jsonStr)) -} - func TestLoadStatsFromOldVersion(t *testing.T) { store, dom := testkit.CreateMockStoreAndDomain(t) tk := testkit.NewTestKit(t, store) diff --git a/pkg/statistics/handle/storage/gc.go b/pkg/statistics/handle/storage/gc.go index 5409d0517ee04..5208f322f0d82 100644 --- a/pkg/statistics/handle/storage/gc.go +++ b/pkg/statistics/handle/storage/gc.go @@ -28,7 +28,6 @@ import ( "github.com/pingcap/tidb/pkg/statistics/handle/lockstats" "github.com/pingcap/tidb/pkg/statistics/handle/types" "github.com/pingcap/tidb/pkg/statistics/handle/util" - "github.com/pingcap/tidb/pkg/util/chunk" "github.com/pingcap/tidb/pkg/util/logutil" "github.com/pingcap/tidb/pkg/util/sqlexec" "github.com/tikv/client-go/v2/oracle" @@ -188,41 +187,63 @@ func forCount(total int64, batch int64) int64 { return result } -// ClearOutdatedHistoryStats clear outdated historical stats +// ClearOutdatedHistoryStats clears outdated historical stats. +// Nothing writes to mysql.stats_meta_history/mysql.stats_history anymore since +// the historical stats feature was removed, but rows written before an upgrade +// may remain, so we keep draining them once they fall out of the retention +// window. The two tables are counted and deleted independently: their row +// counts are unrelated, and either table may still hold rows after the other +// has been fully drained. func ClearOutdatedHistoryStats(sctx sessionctx.Context) error { - sql := "select count(*) from mysql.stats_meta_history use index (idx_create_time) where create_time <= NOW() - INTERVAL %? SECOND" - rs, err := util.Exec(sctx, sql, vardef.HistoricalStatsDuration.Load().Seconds()) + metaCount, err := countOutdatedHistoryRows(sctx, "mysql.stats_meta_history") if err != nil { return err } - if rs == nil { - return nil + historyCount, err := countOutdatedHistoryRows(sctx, "mysql.stats_history") + if err != nil { + return err } - var rows []chunk.Row - defer terror.Call(rs.Close) - if rows, err = sqlexec.DrainRecordSet(context.Background(), rs, 8); err != nil { - return errors.Trace(err) + if metaCount == 0 && historyCount == 0 { + return nil } - count := rows[0].GetInt64(0) - if count > 0 { - for range forCount(count, int64(1000)) { - sql = "delete from mysql.stats_meta_history use index (idx_create_time) where create_time <= NOW() - INTERVAL %? SECOND limit 1000 " - _, err = util.Exec(sctx, sql, vardef.HistoricalStatsDuration.Load().Seconds()) - if err != nil { - return err - } + for range forCount(metaCount, int64(1000)) { + sql := "delete from mysql.stats_meta_history use index (idx_create_time) where create_time <= NOW() - INTERVAL %? SECOND limit 1000 " + if _, err = util.Exec(sctx, sql, vardef.HistoricalStatsDuration.Load().Seconds()); err != nil { + return err } - for range forCount(count, int64(50)) { - sql = "delete from mysql.stats_history use index (idx_create_time) where create_time <= NOW() - INTERVAL %? SECOND limit 50 " - _, err = util.Exec(sctx, sql, vardef.HistoricalStatsDuration.Load().Seconds()) + } + for range forCount(historyCount, int64(50)) { + sql := "delete from mysql.stats_history use index (idx_create_time) where create_time <= NOW() - INTERVAL %? SECOND limit 50 " + if _, err = util.Exec(sctx, sql, vardef.HistoricalStatsDuration.Load().Seconds()); err != nil { return err } - logutil.BgLogger().Info("clear outdated historical stats") } + logutil.BgLogger().Info("clear outdated historical stats", + zap.Int64("stats-meta-history-rows", metaCount), + zap.Int64("stats-history-rows", historyCount)) return nil } -// gcHistoryStatsFromKV delete history stats from kv. +// countOutdatedHistoryRows counts the rows of the given historical stats table +// that have fallen out of the retention window. +func countOutdatedHistoryRows(sctx sessionctx.Context, table string) (int64, error) { + sql := "select count(*) from " + table + " use index (idx_create_time) where create_time <= NOW() - INTERVAL %? SECOND" + rs, err := util.Exec(sctx, sql, vardef.HistoricalStatsDuration.Load().Seconds()) + if err != nil { + return 0, err + } + if rs == nil { + return 0, nil + } + defer terror.Call(rs.Close) + rows, err := sqlexec.DrainRecordSet(context.Background(), rs, 8) + if err != nil { + return 0, errors.Trace(err) + } + return rows[0].GetInt64(0), nil +} + +// gcHistoryStatsFromKV deletes historical stats from kv. func gcHistoryStatsFromKV(sctx sessionctx.Context, physicalID int64) (err error) { sql := "delete from mysql.stats_history where table_id = %?" _, err = util.Exec(sctx, sql, physicalID) diff --git a/pkg/statistics/handle/storage/gc_test.go b/pkg/statistics/handle/storage/gc_test.go index 5c09443c9e385..891d60d3670b2 100644 --- a/pkg/statistics/handle/storage/gc_test.go +++ b/pkg/statistics/handle/storage/gc_test.go @@ -16,6 +16,8 @@ package storage_test import ( "context" + "fmt" + "strings" "testing" "time" @@ -134,6 +136,61 @@ func TestDeleteAnalyzeJobs(t *testing.T) { require.Equal(t, 0, len(rows)) } +func TestClearOutdatedHistoryStats(t *testing.T) { + store, dom := testkit.CreateMockStoreAndDomain(t) + tk := testkit.NewTestKit(t, store) + // The historical stats feature was removed and nothing writes to + // mysql.stats_history/mysql.stats_meta_history anymore, but rows written + // before an upgrade may remain, so seed legacy rows directly. Use more + // than one delete batch (50 rows) of stats_history rows to make sure the + // drain loops until completion instead of stopping after the first batch. + var sb strings.Builder + sb.WriteString("insert into mysql.stats_history (table_id, stats_data, seq_no, version, create_time) values ") + for i := range 120 { + if i > 0 { + sb.WriteString(",") + } + fmt.Fprintf(&sb, "(1, 'x', %d, %d, '2020-01-01 00:00:00')", i, i+1) + } + tk.MustExec(sb.String()) + tk.MustExec("insert into mysql.stats_meta_history (table_id, modify_count, count, version, source, create_time) values (1, 0, 0, 1, 'test', '2020-01-01 00:00:00')") + // Rows within the retention window must survive the drain. + tk.MustExec("insert into mysql.stats_history (table_id, stats_data, seq_no, version, create_time) values (2, 'x', 0, 1, NOW())") + tk.MustExec("insert into mysql.stats_meta_history (table_id, modify_count, count, version, source, create_time) values (2, 0, 0, 1, 'test', NOW())") + + require.NoError(t, dom.StatsHandle().ClearOutdatedHistoryStats()) + tk.MustQuery("select count(*) from mysql.stats_meta_history").Check(testkit.Rows("1")) + tk.MustQuery("select count(*) from mysql.stats_history").Check(testkit.Rows("1")) + + // Legacy rows may remain in mysql.stats_history even after + // mysql.stats_meta_history is fully drained; they must still be cleared. + tk.MustExec("insert into mysql.stats_history (table_id, stats_data, seq_no, version, create_time) values (3, 'x', 0, 1, '2020-01-01 00:00:00')") + require.NoError(t, dom.StatsHandle().ClearOutdatedHistoryStats()) + tk.MustQuery("select count(*) from mysql.stats_history where table_id = 3").Check(testkit.Rows("0")) +} + +func TestGCHistoryStatsAfterDropTable(t *testing.T) { + store, dom := testkit.CreateMockStoreAndDomain(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("create table t(a int, b int)") + tk.MustExec("insert into t values (1,2),(3,4)") + tk.MustExec("analyze table t") + tbl, err := dom.InfoSchema().TableByName(context.Background(), ast.NewCIStr("test"), ast.NewCIStr("t")) + require.NoError(t, err) + tid := tbl.Meta().ID + // Seed legacy historical stats rows for the table. Use a recent + // create_time so only the drop-table GC path (not the retention-based + // drain) can remove them. + tk.MustExec(fmt.Sprintf("insert into mysql.stats_history (table_id, stats_data, seq_no, version, create_time) values (%d, 'x', 0, 1, NOW())", tid)) + tk.MustExec(fmt.Sprintf("insert into mysql.stats_meta_history (table_id, modify_count, count, version, source, create_time) values (%d, 0, 0, 1, 'test', NOW())", tid)) + + tk.MustExec("drop table t") + require.NoError(t, dom.StatsHandle().GCStats(dom.InfoSchema(), 0)) + tk.MustQuery(fmt.Sprintf("select count(*) from mysql.stats_history where table_id = %d", tid)).Check(testkit.Rows("0")) + tk.MustQuery(fmt.Sprintf("select count(*) from mysql.stats_meta_history where table_id = %d", tid)).Check(testkit.Rows("0")) +} + func TestExtremCaseOfGC(t *testing.T) { // This case tests that there's no records in mysql.stats_histograms but this table is not deleted in fact. // We should not delete the record in mysql.stats_meta. diff --git a/pkg/statistics/handle/storage/json.go b/pkg/statistics/handle/storage/json.go index 2e13988ad8198..e07d335463de4 100644 --- a/pkg/statistics/handle/storage/json.go +++ b/pkg/statistics/handle/storage/json.go @@ -15,24 +15,15 @@ package storage import ( - "bytes" - "encoding/json" - "io" - - "github.com/klauspost/compress/gzip" "github.com/pingcap/errors" "github.com/pingcap/tidb/pkg/meta/model" "github.com/pingcap/tidb/pkg/parser/mysql" "github.com/pingcap/tidb/pkg/sessionctx" "github.com/pingcap/tidb/pkg/statistics" statstypes "github.com/pingcap/tidb/pkg/statistics/handle/types" - "github.com/pingcap/tidb/pkg/statistics/handle/util" statsutil "github.com/pingcap/tidb/pkg/statistics/util" "github.com/pingcap/tidb/pkg/types" - compressutil "github.com/pingcap/tidb/pkg/util/compress" - "github.com/pingcap/tidb/pkg/util/logutil" "github.com/pingcap/tidb/pkg/util/memory" - "go.uber.org/zap" ) func dumpJSONCol(hist *statistics.Histogram, cmsketch *statistics.CMSketch, topn *statistics.TopN, fmsketch *statistics.FMSketch, statsVer *int64) *statsutil.JSONColumn { @@ -225,117 +216,3 @@ func TableStatsFromJSON(tableInfo *model.TableInfo, physicalID int64, jsonTbl *s } return tbl, nil } - -// JSONTableToBlocks convert JSONTable to json, then compresses it to blocks by gzip. -func JSONTableToBlocks(jsTable *statsutil.JSONTable, blockSize int) ([][]byte, error) { - data, err := json.Marshal(jsTable) - if err != nil { - return nil, errors.Trace(err) - } - var gzippedData bytes.Buffer - gzipWriter := compressutil.GzipWriterPool.Get().(*gzip.Writer) - defer compressutil.GzipWriterPool.Put(gzipWriter) - gzipWriter.Reset(&gzippedData) - if _, err := gzipWriter.Write(data); err != nil { - return nil, errors.Trace(err) - } - if err := gzipWriter.Close(); err != nil { - return nil, errors.Trace(err) - } - blocksNum := gzippedData.Len() / blockSize - if gzippedData.Len()%blockSize != 0 { - blocksNum = blocksNum + 1 - } - blocks := make([][]byte, blocksNum) - for i := range blocksNum - 1 { - blocks[i] = gzippedData.Bytes()[blockSize*i : blockSize*(i+1)] - } - blocks[blocksNum-1] = gzippedData.Bytes()[blockSize*(blocksNum-1):] - return blocks, nil -} - -// BlocksToJSONTable convert gzip-compressed blocks to JSONTable -func BlocksToJSONTable(blocks [][]byte) (*statsutil.JSONTable, error) { - if len(blocks) == 0 { - return nil, errors.New("Block empty error") - } - data := blocks[0] - for i := 1; i < len(blocks); i++ { - data = append(data, blocks[i]...) - } - gzippedData := bytes.NewReader(data) - gzipReader := compressutil.GzipReaderPool.Get().(*gzip.Reader) - if err := gzipReader.Reset(gzippedData); err != nil { - compressutil.GzipReaderPool.Put(gzipReader) - return nil, err - } - defer func() { - compressutil.GzipReaderPool.Put(gzipReader) - }() - if err := gzipReader.Close(); err != nil { - return nil, err - } - jsonStr, err := io.ReadAll(gzipReader) - if err != nil { - return nil, errors.Trace(err) - } - jsonTbl := statsutil.JSONTable{} - err = json.Unmarshal(jsonStr, &jsonTbl) - if err != nil { - return nil, errors.Trace(err) - } - return &jsonTbl, nil -} - -// TableHistoricalStatsToJSON converts the historical stats of a table to JSONTable. -func TableHistoricalStatsToJSON(sctx sessionctx.Context, physicalID int64, snapshot uint64) (jt *statsutil.JSONTable, exist bool, err error) { - // get meta version - rows, _, err := util.ExecRows(sctx, "select distinct version from mysql.stats_meta_history where table_id = %? and version <= %? order by version desc limit 1", physicalID, snapshot) - if err != nil { - return nil, false, errors.AddStack(err) - } - if len(rows) < 1 { - logutil.BgLogger().Warn("failed to get records of stats_meta_history", - zap.Int64("table-id", physicalID), - zap.Uint64("snapshotTS", snapshot)) - return nil, false, nil - } - statsMetaVersion := rows[0].GetInt64(0) - // get stats meta - rows, _, err = util.ExecRows(sctx, "select modify_count, count from mysql.stats_meta_history where table_id = %? and version = %?", physicalID, statsMetaVersion) - if err != nil { - return nil, false, errors.AddStack(err) - } - modifyCount, count := rows[0].GetInt64(0), rows[0].GetInt64(1) - - // get stats version - rows, _, err = util.ExecRows(sctx, "select distinct version from mysql.stats_history where table_id = %? and version <= %? order by version desc limit 1", physicalID, snapshot) - if err != nil { - return nil, false, errors.AddStack(err) - } - if len(rows) < 1 { - logutil.BgLogger().Warn("failed to get record of stats_history", - zap.Int64("table-id", physicalID), - zap.Uint64("snapshotTS", snapshot)) - return nil, false, nil - } - statsVersion := rows[0].GetInt64(0) - - // get stats - rows, _, err = util.ExecRows(sctx, "select stats_data from mysql.stats_history where table_id = %? and version = %? order by seq_no", physicalID, statsVersion) - if err != nil { - return nil, false, errors.AddStack(err) - } - blocks := make([][]byte, 0) - for _, row := range rows { - blocks = append(blocks, row.GetBytes(0)) - } - jsonTbl, err := BlocksToJSONTable(blocks) - if err != nil { - return nil, false, errors.AddStack(err) - } - jsonTbl.Count = count - jsonTbl.ModifyCount = modifyCount - jsonTbl.IsHistoricalStats = true - return jsonTbl, true, nil -} diff --git a/pkg/statistics/handle/storage/save.go b/pkg/statistics/handle/storage/save.go index f671304687b40..8e56145ad0e46 100644 --- a/pkg/statistics/handle/storage/save.go +++ b/pkg/statistics/handle/storage/save.go @@ -378,10 +378,10 @@ func SaveMetaToStorage( sctx sessionctx.Context, refreshLastHistVer bool, metaUpdates []statstypes.MetaUpdate, -) (uint64, error) { +) error { version, err := util.GetStartTS(sctx) if err != nil { - return 0, errors.Trace(err) + return errors.Trace(err) } var sql string values := make([]string, 0, len(metaUpdates)) @@ -399,7 +399,7 @@ func SaveMetaToStorage( "on duplicate key update version = values(version), modify_count = values(modify_count), count = values(count)", strings.Join(values, ",")) } _, err = util.Exec(sctx, sql) - return version, errors.Trace(err) + return errors.Trace(err) } // InsertColStats2KV insert a record to stats_histograms with distinct_count 1 @@ -410,10 +410,10 @@ func InsertColStats2KV( sctx sessionctx.Context, physicalID int64, colInfos []*model.ColumnInfo, -) (uint64, error) { +) error { startTS, err := util.GetStartTS(sctx) if err != nil { - return 0, errors.Trace(err) + return errors.Trace(err) } // By this step we can get the count of this table, then we can sure the count and repeats of bucket. @@ -424,17 +424,17 @@ func InsertColStats2KV( physicalID, ) if err != nil { - return 0, errors.Trace(err) + return errors.Trace(err) } defer terror.Call(rs.Close) req := rs.NewChunk(nil) err = rs.Next(ctx, req) if err != nil { - return 0, errors.Trace(err) + return errors.Trace(err) } // If no row is returned, it means the stats of this table does not exist. if req.NumRows() == 0 { - return startTS, nil + return nil } count := req.GetRow(0).GetInt64(0) hasStatsUpdate := false @@ -452,7 +452,7 @@ func InsertColStats2KV( values (%?, %?, 0, %?, 0, 0)`, startTS, physicalID, colInfo.ID, ); err != nil { - return 0, errors.Trace(err) + return errors.Trace(err) } if sctx.GetSessionVars().StmtCtx.AffectedRows() > 0 { hasStatsUpdate = true @@ -462,7 +462,7 @@ func InsertColStats2KV( value, err := table.GetColOriginDefaultValue(sctx.GetExprCtx(), colInfo) if err != nil { - return 0, errors.Trace(err) + return errors.Trace(err) } if value.IsNull() { // If the adding column has default value null, all the existing rows have null value on the newly added column. @@ -473,7 +473,7 @@ func InsertColStats2KV( values (%?, %?, 0, %?, 0, %?)`, startTS, physicalID, colInfo.ID, count, ); err != nil { - return 0, errors.Trace(err) + return errors.Trace(err) } if sctx.GetSessionVars().StmtCtx.AffectedRows() > 0 { hasStatsUpdate = true @@ -489,7 +489,7 @@ func InsertColStats2KV( values (%?, %?, 0, %?, 1, GREATEST(%?, 0))`, startTS, physicalID, colInfo.ID, int64(len(value.GetBytes()))*count, ); err != nil { - return 0, errors.Trace(err) + return errors.Trace(err) } // The histogram may have been created by ANALYZE TABLE ... ALL COLUMNS before the add-column DDL event is handled. // Skip inserting the default-value bucket in that case. @@ -499,7 +499,7 @@ func InsertColStats2KV( hasStatsUpdate = true value, err = value.ConvertTo(sctx.GetSessionVars().StmtCtx.TypeCtx(), types.NewFieldType(mysql.TypeBlob)) if err != nil { - return 0, errors.Trace(err) + return errors.Trace(err) } // There must be only one bucket for this new column and the value is the default value. if _, err = util.ExecWithCtx( @@ -509,11 +509,11 @@ func InsertColStats2KV( values (%?, 0, %?, 0, %?, %?, %?, %?)`, physicalID, colInfo.ID, count, count, value.GetBytes(), value.GetBytes(), ); err != nil { - return 0, errors.Trace(err) + return errors.Trace(err) } } if !hasStatsUpdate { - return startTS, nil + return nil } _, err = util.ExecWithCtx( ctx, sctx, @@ -521,9 +521,9 @@ func InsertColStats2KV( startTS, startTS, physicalID, ) if err != nil { - return 0, errors.Trace(err) + return errors.Trace(err) } - return startTS, nil + return nil } // InsertTableStats2KV inserts a record standing for a new table to stats_meta @@ -534,17 +534,17 @@ func InsertTableStats2KV( sctx sessionctx.Context, info *model.TableInfo, physicalID int64, -) (uint64, error) { +) error { startTS, err := util.GetStartTS(sctx) if err != nil { - return 0, errors.Trace(err) + return errors.Trace(err) } if _, err = util.ExecWithCtx( ctx, sctx, "insert ignore into mysql.stats_meta (version, table_id, last_stats_histograms_version) values(%?, %?, %?)", startTS, physicalID, startTS, ); err != nil { - return 0, errors.Trace(err) + return errors.Trace(err) } for _, col := range info.Columns { if _, err = util.ExecWithCtx( @@ -554,7 +554,7 @@ func InsertTableStats2KV( values (%?, 0, %?, 0, %?)`, physicalID, col.ID, startTS, ); err != nil { - return 0, errors.Trace(err) + return errors.Trace(err) } } for _, idx := range info.Indices { @@ -565,10 +565,10 @@ func InsertTableStats2KV( values(%?, 1, %?, 0, %?)`, physicalID, idx.ID, startTS, ); err != nil { - return 0, errors.Trace(err) + return errors.Trace(err) } } - return startTS, nil + return nil } // convertBoundToBlob converts the bound to blob. The `blob` will be used to store in the `mysql.stats_buckets` table. diff --git a/pkg/statistics/handle/storage/stats_read_writer.go b/pkg/statistics/handle/storage/stats_read_writer.go index bd9bc95640797..84c1ea3904da6 100644 --- a/pkg/statistics/handle/storage/stats_read_writer.go +++ b/pkg/statistics/handle/storage/stats_read_writer.go @@ -29,12 +29,10 @@ import ( "github.com/pingcap/tidb/pkg/parser/ast" "github.com/pingcap/tidb/pkg/parser/mysql" "github.com/pingcap/tidb/pkg/sessionctx" - "github.com/pingcap/tidb/pkg/sessionctx/vardef" "github.com/pingcap/tidb/pkg/sessionctx/variable" "github.com/pingcap/tidb/pkg/statistics" "github.com/pingcap/tidb/pkg/statistics/handle/cache" statslogutil "github.com/pingcap/tidb/pkg/statistics/handle/logutil" - handle_metrics "github.com/pingcap/tidb/pkg/statistics/handle/metrics" statstypes "github.com/pingcap/tidb/pkg/statistics/handle/types" "github.com/pingcap/tidb/pkg/statistics/handle/usage/predicatecolumn" "github.com/pingcap/tidb/pkg/statistics/handle/util" @@ -65,20 +63,9 @@ func (s *statsReadWriter) ChangeGlobalStatsID(from, to int64) (err error) { // UpdateStatsMetaVersionForGC update the version of mysql.stats_meta. See more // details in the interface definition. func (s *statsReadWriter) UpdateStatsMetaVersionForGC(physicalID int64) (err error) { - statsVer := uint64(0) - defer func() { - if err == nil && statsVer != 0 { - s.statsHandler.RecordHistoricalStatsMeta(statsVer, util.StatsMetaHistorySourceSchemaChange, false, physicalID) - } - }() - return util.CallWithSCtx(s.statsHandler.SPool(), func(sctx sessionctx.Context) error { - startTS, err := UpdateStatsMetaVerAndLastHistUpdateVer(util.StatsCtx, sctx, physicalID) - if err != nil { - return errors.Trace(err) - } - statsVer = startTS - return nil + _, err := UpdateStatsMetaVerAndLastHistUpdateVer(util.StatsCtx, sctx, physicalID) + return errors.Trace(err) }, util.FlagWrapTxn) } @@ -87,7 +74,7 @@ func (s *statsReadWriter) UpdateStatsMetaVersionForGC(physicalID int64) (err err // Also, we need to update the stats meta with a more recent version to avoid other nodes missing the delta update. // Combined with the stats meta version update, we can ensure the stats cache on other TiDB nodes is consistent. // See more at stats cache's Update function. -func (s *statsReadWriter) handleSlowStatsSaving(tableID int64, start time.Time) (uint64, error) { +func (s *statsReadWriter) handleSlowStatsSaving(tableID int64, start time.Time) error { dur := time.Since(start) // Note: In unit tests, the lease is set to a value less than 0, which means the lease is disabled. // This is why we need to explicitly check the lease here. Without this check, @@ -101,7 +88,7 @@ func (s *statsReadWriter) handleSlowStatsSaving(tableID int64, start time.Time) }) if !isLoadIntervalExceeded { - return 0, nil + return nil } statslogutil.StatsLogger().Warn("Update stats cache is too slow", @@ -129,18 +116,18 @@ func (s *statsReadWriter) handleSlowStatsSaving(tableID int64, start time.Time) zap.Int64("physicalID", tableID), zap.Error(err), ) - return 0, errors.Errorf("failed to update stats meta version during analyze result save. The system may be too busy. Please retry the operation later") + return errors.Annotate(err, "failed to update stats meta version while saving statistics; the system may be too busy; please retry the operation later") } statslogutil.StatsLogger().Info("Successfully updated stats meta version for slow saving", zap.Uint64("statsVer", statsVer), zap.Int64("physicalID", tableID), ) - return statsVer, nil + return nil } // SaveAnalyzeResultToStorage saves the stats of a table to storage. -func (s *statsReadWriter) SaveAnalyzeResultToStorage(results *statistics.AnalyzeResults, analyzeSnapshot bool, source string) (err error) { +func (s *statsReadWriter) SaveAnalyzeResultToStorage(results *statistics.AnalyzeResults, analyzeSnapshot bool) (err error) { var statsVer uint64 start := time.Now() err = util.CallWithSCtx(s.statsHandler.SPool(), func(sctx sessionctx.Context) error { @@ -150,7 +137,7 @@ func (s *statsReadWriter) SaveAnalyzeResultToStorage(results *statistics.Analyze if err == nil && statsVer != 0 { tableID := results.TableID.GetStatisticsID() // Check if saving was slow and update stats version if needed - version, err2 := s.handleSlowStatsSaving(tableID, start) + err2 := s.handleSlowStatsSaving(tableID, start) if err2 != nil { statslogutil.StatsLogger().Error("Failed to update stats meta version for slow saving during analyze job execution", zap.Int64("physicalID", tableID), @@ -162,10 +149,6 @@ func (s *statsReadWriter) SaveAnalyzeResultToStorage(results *statistics.Analyze ) return err2 } - if version != 0 { - statsVer = version - } - s.statsHandler.RecordHistoricalStatsMeta(statsVer, source, true, tableID) } return err } @@ -206,7 +189,6 @@ func (s *statsReadWriter) SaveColOrIdxStatsToStorage( topN *statistics.TopN, statsVersion int, updateAnalyzeTime bool, - source string, ) (err error) { var statsVer uint64 start := time.Now() @@ -217,14 +199,10 @@ func (s *statsReadWriter) SaveColOrIdxStatsToStorage( }, util.FlagWrapTxn) if err == nil && statsVer != 0 { // Check if saving was slow and update stats version if needed - version, err2 := s.handleSlowStatsSaving(tableID, start) + err2 := s.handleSlowStatsSaving(tableID, start) if err2 != nil { return err2 } - if version != 0 { - statsVer = version - } - s.statsHandler.RecordHistoricalStatsMeta(statsVer, source, false, tableID) } return } @@ -232,23 +210,13 @@ func (s *statsReadWriter) SaveColOrIdxStatsToStorage( // SaveMetaToStorage saves stats meta to the storage. // Use the param `refreshLastHistVer` to indicate whether we need to update the last_histograms_versions in stats_meta table. func (s *statsReadWriter) SaveMetaToStorage( - source string, refreshLastHistVer bool, metaUpdates ...statstypes.MetaUpdate, ) (err error) { intest.Assert(len(metaUpdates) > 0, "meta updates is empty") - var statsVer uint64 err = util.CallWithSCtx(s.statsHandler.SPool(), func(sctx sessionctx.Context) error { - statsVer, err = SaveMetaToStorage(sctx, refreshLastHistVer, metaUpdates) - return err + return SaveMetaToStorage(sctx, refreshLastHistVer, metaUpdates) }, util.FlagWrapTxn) - if err == nil && statsVer != 0 { - tableIDs := make([]int64, 0, len(metaUpdates)) - for i := range metaUpdates { - tableIDs = append(tableIDs, metaUpdates[i].PhysicalID) - } - s.statsHandler.RecordHistoricalStatsMeta(statsVer, source, false, tableIDs...) - } return } @@ -286,70 +254,6 @@ func (s *statsReadWriter) DumpStatsToJSON(dbName string, tableInfo *model.TableI return s.DumpStatsToJSONBySnapshot(dbName, tableInfo, snapshot, dumpPartitionStats) } -// DumpHistoricalStatsBySnapshot dumped json tables from mysql.stats_meta_history and mysql.stats_history. -// As implemented in getTableHistoricalStatsToJSONWithFallback, if historical stats are nonexistent, it will fall back -// to the latest stats, and these table names (and partition names) will be returned in fallbackTbls. -func (s *statsReadWriter) DumpHistoricalStatsBySnapshot( - dbName string, - tableInfo *model.TableInfo, - snapshot uint64, -) ( - jt *statsutil.JSONTable, - fallbackTbls []string, - err error, -) { - historicalStatsEnabled, err := s.statsHandler.CheckHistoricalStatsEnable() - if err != nil { - return nil, nil, errors.Errorf("check %v failed: %v", vardef.TiDBEnableHistoricalStats, err) - } - if !historicalStatsEnabled { - return nil, nil, errors.Errorf("%v should be enabled", vardef.TiDBEnableHistoricalStats) - } - - defer func() { - if err == nil { - handle_metrics.DumpHistoricalStatsSuccessCounter.Inc() - } else { - handle_metrics.DumpHistoricalStatsFailedCounter.Inc() - } - }() - pi := tableInfo.GetPartitionInfo() - if pi == nil { - jt, fallback, err := s.getTableHistoricalStatsToJSONWithFallback(dbName, tableInfo, tableInfo.ID, snapshot) - if fallback { - fallbackTbls = append(fallbackTbls, fmt.Sprintf("%s.%s", dbName, tableInfo.Name.O)) - } - return jt, fallbackTbls, err - } - jsonTbl := &statsutil.JSONTable{ - DatabaseName: dbName, - TableName: tableInfo.Name.L, - Partitions: make(map[string]*statsutil.JSONTable, len(pi.Definitions)), - } - for _, def := range pi.Definitions { - tbl, fallback, err := s.getTableHistoricalStatsToJSONWithFallback(dbName, tableInfo, def.ID, snapshot) - if err != nil { - return nil, nil, errors.Trace(err) - } - if fallback { - fallbackTbls = append(fallbackTbls, fmt.Sprintf("%s.%s %s", dbName, tableInfo.Name.O, def.Name.O)) - } - jsonTbl.Partitions[def.Name.L] = tbl - } - tbl, fallback, err := s.getTableHistoricalStatsToJSONWithFallback(dbName, tableInfo, tableInfo.ID, snapshot) - if err != nil { - return nil, nil, err - } - if fallback { - fallbackTbls = append(fallbackTbls, fmt.Sprintf("%s.%s global", dbName, tableInfo.Name.O)) - } - // dump its global-stats if existed - if tbl != nil { - jsonTbl.Partitions[statsutil.TiDBGlobalStats] = tbl - } - return jsonTbl, fallbackTbls, nil -} - // PersistStatsBySnapshot dumps statistic to json and call the function for each partition statistic to persist. // Notice: // 1. It might call the function `persist` with nil jsontable. @@ -435,41 +339,6 @@ func (s *statsReadWriter) DumpStatsToJSONBySnapshot(dbName string, tableInfo *mo return jsonTbl, nil } -// getTableHistoricalStatsToJSONWithFallback try to get table historical stats, if not exist, directly fallback to the -// latest stats, and the second return value would be true. -func (s *statsReadWriter) getTableHistoricalStatsToJSONWithFallback( - dbName string, - tableInfo *model.TableInfo, - physicalID int64, - snapshot uint64, -) ( - *statsutil.JSONTable, - bool, - error, -) { - jt, exist, err := s.tableHistoricalStatsToJSON(physicalID, snapshot) - if err != nil { - return nil, false, err - } - if !exist { - jt, err = s.TableStatsToJSON(dbName, tableInfo, physicalID, 0) - fallback := true - if snapshot == 0 { - fallback = false - } - return jt, fallback, err - } - return jt, false, nil -} - -func (s *statsReadWriter) tableHistoricalStatsToJSON(physicalID int64, snapshot uint64) (jt *statsutil.JSONTable, exist bool, err error) { - err = util.CallWithSCtx(s.statsHandler.SPool(), func(sctx sessionctx.Context) error { - jt, exist, err = TableHistoricalStatsToJSON(sctx, physicalID, snapshot) - return err - }, util.FlagWrapTxn) - return -} - // TableStatsToJSON dumps statistic to json. func (s *statsReadWriter) TableStatsToJSON(dbName string, tableInfo *model.TableInfo, physicalID int64, snapshot uint64) (*statsutil.JSONTable, error) { tbl, err := s.TableStatsFromStorage(tableInfo, physicalID, true, snapshot) @@ -617,7 +486,7 @@ func (s *statsReadWriter) loadStatsFromJSON(tableInfo *model.TableInfo, physical // loadStatsFromJSON doesn't support partition table now. // The table level count and modify_count would be overridden by the SaveMetaToStorage below, so we don't need // to care about them here. - if err := s.SaveColOrIdxStatsToStorage(tbl.PhysicalID, tbl.RealtimeCount, 0, 0, &col.Histogram, col.CMSketch, col.TopN, int(col.GetStatsVer()), false, util.StatsMetaHistorySourceLoadStats); err != nil { + if err := s.SaveColOrIdxStatsToStorage(tbl.PhysicalID, tbl.RealtimeCount, 0, 0, &col.Histogram, col.CMSketch, col.TopN, int(col.GetStatsVer()), false); err != nil { outerErr = err return true } @@ -630,7 +499,7 @@ func (s *statsReadWriter) loadStatsFromJSON(tableInfo *model.TableInfo, physical // loadStatsFromJSON doesn't support partition table now. // The table level count and modify_count would be overridden by the SaveMetaToStorage below, so we don't need // to care about them here. - if err := s.SaveColOrIdxStatsToStorage(tbl.PhysicalID, tbl.RealtimeCount, 0, 1, &idx.Histogram, idx.CMSketch, idx.TopN, int(idx.GetStatsVer()), false, util.StatsMetaHistorySourceLoadStats); err != nil { + if err := s.SaveColOrIdxStatsToStorage(tbl.PhysicalID, tbl.RealtimeCount, 0, 1, &idx.Histogram, idx.CMSketch, idx.TopN, int(idx.GetStatsVer()), false); err != nil { outerErr = err return true } @@ -643,7 +512,7 @@ func (s *statsReadWriter) loadStatsFromJSON(tableInfo *model.TableInfo, physical if err != nil { return errors.Trace(err) } - return s.SaveMetaToStorage(util.StatsMetaHistorySourceLoadStats, true, statstypes.MetaUpdate{ + return s.SaveMetaToStorage(true, statstypes.MetaUpdate{ PhysicalID: tbl.PhysicalID, Count: tbl.RealtimeCount, ModifyCount: tbl.ModifyCount, diff --git a/pkg/statistics/handle/storage/stats_read_writer_test.go b/pkg/statistics/handle/storage/stats_read_writer_test.go index fa755c995519b..33abf7cd6ee53 100644 --- a/pkg/statistics/handle/storage/stats_read_writer_test.go +++ b/pkg/statistics/handle/storage/stats_read_writer_test.go @@ -75,13 +75,12 @@ func TestUpdateStatsMetaVersionForGC(t *testing.T) { // Parse version from stats_meta. version := rows[0][0].(string) - // Check stats_meta_history again. The version should be the same. - rows = testKit.MustQuery( + // Historical stats tables remain for compatibility, but stats updates should no longer append records. + testKit.MustQuery( "select count(*) from mysql.stats_meta_history where table_id = ? and version = ?", p0.ID, version, - ).Rows() - require.Equal(t, 1, len(rows)) + ).Check(testkit.Rows("0")) } func TestSlowStatsSaving(t *testing.T) { @@ -222,5 +221,5 @@ func TestFailedToHandleSlowStatsSaving(t *testing.T) { ) `) testKit.MustExec("insert into t values (1,2),(2,2),(6,2),(11,2),(16,2)") - testKit.MustGetErrMsg("analyze table t with 0 topn", "failed to update stats meta version during analyze result save. The system may be too busy. Please retry the operation later") + testKit.MustGetErrMsg("analyze table t with 0 topn", "failed to update stats meta version while saving statistics; the system may be too busy; please retry the operation later: mock update stats meta version failed") } diff --git a/pkg/statistics/handle/types/interfaces.go b/pkg/statistics/handle/types/interfaces.go index 6ad61fdcb4c10..68cfa01955468 100644 --- a/pkg/statistics/handle/types/interfaces.go +++ b/pkg/statistics/handle/types/interfaces.go @@ -103,18 +103,6 @@ type IndexUsage interface { GetIndexUsage(tableID int64, indexID int64) indexusage.Sample } -// StatsHistory is used to manage historical stats. -type StatsHistory interface { - // RecordHistoricalStatsMeta records the historical stats meta in mysql.stats_meta_history one by one. - RecordHistoricalStatsMeta(version uint64, source string, enforce bool, tableIDs ...int64) - - // CheckHistoricalStatsEnable check whether historical stats is enabled. - CheckHistoricalStatsEnable() (enable bool, err error) - - // RecordHistoricalStatsToStorage records the given table's stats data to mysql.stats_history - RecordHistoricalStatsToStorage(dbName string, tableInfo *model.TableInfo, physicalID int64, isPartition bool) (uint64, error) -} - // PriorityQueueSnapshot is the snapshot of the stats priority queue. type PriorityQueueSnapshot struct { CurrentJobs []AnalysisJobJSON `json:"current_jobs"` @@ -354,15 +342,15 @@ type StatsReadWriter interface { // SaveColOrIdxStatsToStorage save the column or index stats to storage. SaveColOrIdxStatsToStorage(tableID int64, count, modifyCount int64, isIndex int, hg *statistics.Histogram, - cms *statistics.CMSketch, topN *statistics.TopN, statsVersion int, updateAnalyzeTime bool, source string) (err error) + cms *statistics.CMSketch, topN *statistics.TopN, statsVersion int, updateAnalyzeTime bool) (err error) // SaveAnalyzeResultToStorage saves the analyze result to the storage. - SaveAnalyzeResultToStorage(results *statistics.AnalyzeResults, analyzeSnapshot bool, source string) (err error) + SaveAnalyzeResultToStorage(results *statistics.AnalyzeResults, analyzeSnapshot bool) (err error) // SaveMetaToStorage saves the stats meta of a table to storage. // Use the param `refreshLastHistVer` to indicate whether we need to update the last_histograms_versions in stats_meta table. // Set it to true if the column/index stats is updated. - SaveMetaToStorage(source string, needRefreshLastHistVer bool, metaUpdates ...MetaUpdate) (err error) + SaveMetaToStorage(needRefreshLastHistVer bool, metaUpdates ...MetaUpdate) (err error) // UpdateStatsMetaVersionForGC updates the version of mysql.stats_meta, // ensuring it is greater than the last garbage collection (GC) time. @@ -399,19 +387,6 @@ type StatsReadWriter interface { DumpStatsToJSON(dbName string, tableInfo *model.TableInfo, historyStatsExec sqlexec.RestrictedSQLExecutor, dumpPartitionStats bool) (*statsutil.JSONTable, error) - // DumpHistoricalStatsBySnapshot dumped json tables from mysql.stats_meta_history and mysql.stats_history. - // As implemented in getTableHistoricalStatsToJSONWithFallback, if historical stats are nonexistent, it will fall back - // to the latest stats, and these table names (and partition names) will be returned in fallbackTbls. - DumpHistoricalStatsBySnapshot( - dbName string, - tableInfo *model.TableInfo, - snapshot uint64, - ) ( - jt *statsutil.JSONTable, - fallbackTbls []string, - err error, - ) - // DumpStatsToJSONBySnapshot dumps statistic to json. DumpStatsToJSONBySnapshot(dbName string, tableInfo *model.TableInfo, snapshot uint64, dumpPartitionStats bool) (*statsutil.JSONTable, error) @@ -514,9 +489,6 @@ type StatsHandle interface { // StatsUsage is used to handle table delta and stats usage. StatsUsage - // StatsHistory is used to manage historical stats. - StatsHistory - // StatsAnalyze is used to handle auto-analyze and manage analyze jobs. StatsAnalyze diff --git a/pkg/statistics/handle/updatetest/update_test.go b/pkg/statistics/handle/updatetest/update_test.go index 2e8c5cebf3b17..6d50e2f171ed2 100644 --- a/pkg/statistics/handle/updatetest/update_test.go +++ b/pkg/statistics/handle/updatetest/update_test.go @@ -750,14 +750,12 @@ func TestStatsVariables(t *testing.T) { err = util.UpdateSCtxVarsForStats(sctx) require.NoError(t, err) require.Equal(t, 2, sctx.GetSessionVars().AnalyzeVersion) - require.Equal(t, false, sctx.GetSessionVars().EnableHistoricalStats) require.Equal(t, string(variable.Dynamic), sctx.GetSessionVars().PartitionPruneMode.Load()) require.Equal(t, false, sctx.GetSessionVars().EnableAnalyzeSnapshot) require.Equal(t, true, sctx.GetSessionVars().SkipMissingPartitionStats) tk.MustExec(`set global tidb_analyze_version=2`) tk.MustExec(`set global tidb_partition_prune_mode='static'`) - tk.MustExec(`set global tidb_enable_historical_stats=1`) tk.MustExec(`set global tidb_enable_analyze_snapshot=1`) tk.MustExec(`set global tidb_skip_missing_partition_stats=0`) @@ -767,7 +765,6 @@ func TestStatsVariables(t *testing.T) { err = util.UpdateSCtxVarsForStats(sctx) require.NoError(t, err) require.Equal(t, 2, sctx.GetSessionVars().AnalyzeVersion) - require.Equal(t, true, sctx.GetSessionVars().EnableHistoricalStats) require.Equal(t, string(variable.Static), sctx.GetSessionVars().PartitionPruneMode.Load()) require.Equal(t, true, sctx.GetSessionVars().EnableAnalyzeSnapshot) require.Equal(t, false, sctx.GetSessionVars().SkipMissingPartitionStats) diff --git a/pkg/statistics/handle/usage/BUILD.bazel b/pkg/statistics/handle/usage/BUILD.bazel index 48fc84076eff3..6b2c1ded92699 100644 --- a/pkg/statistics/handle/usage/BUILD.bazel +++ b/pkg/statistics/handle/usage/BUILD.bazel @@ -28,7 +28,6 @@ go_library( "//pkg/util/intest", "//pkg/util/sqlescape", "@com_github_pingcap_errors//:errors", - "@com_github_pingcap_failpoint//:failpoint", "@org_uber_go_zap//:zap", ], ) diff --git a/pkg/statistics/handle/usage/session_stats_collect.go b/pkg/statistics/handle/usage/session_stats_collect.go index 855d4269d6541..d8cefe21a191d 100644 --- a/pkg/statistics/handle/usage/session_stats_collect.go +++ b/pkg/statistics/handle/usage/session_stats_collect.go @@ -23,7 +23,6 @@ import ( "time" "github.com/pingcap/errors" - "github.com/pingcap/failpoint" "github.com/pingcap/tidb/pkg/infoschema" "github.com/pingcap/tidb/pkg/meta/metadef" "github.com/pingcap/tidb/pkg/meta/model" @@ -127,10 +126,7 @@ func (s *statsUsageImpl) DumpStatsDeltaToKV(forceDump bool, tableIDs ...int64) e end := min(i+dumpDeltaBatchSize, len(tableIDs)) batchTableIDs := tableIDs[i:end] - var ( - statsVersion uint64 - batchUpdates []*storage.DeltaUpdate - ) + var batchUpdates []*storage.DeltaUpdate batchStart := time.Now() err := utilstats.CallWithSCtx(s.statsHandle.SPool(), func(sctx sessionctx.Context) error { is := sctx.GetLatestInfoSchema().(infoschema.InfoSchema) @@ -161,11 +157,10 @@ func (s *statsUsageImpl) DumpStatsDeltaToKV(forceDump bool, tableIDs ...int64) e // Process all updates in the batch with a single transaction. // Note: batchUpdates may be modified in dumpStatsDeltaToKV. (e.g. sorting, updating IsLocked) - startTs, updated, err := s.dumpStatsDeltaToKV(is, sctx, batchUpdates) + updated, err := s.dumpStatsDeltaToKV(is, sctx, batchUpdates) if err != nil { return errors.Trace(err) } - statsVersion = startTs // Note: Ensure we use the updated slice after dumpStatsDeltaToKV, // because dumpStatsDeltaToKV may modify the underlying array of batchUpdates. // For example, dumpStatsDeltaToKV may sort the array. @@ -195,23 +190,6 @@ func (s *statsUsageImpl) DumpStatsDeltaToKV(forceDump bool, tableIDs ...int64) e if err != nil { return errors.Trace(err) } - startRecordHistoricalStatsMeta := time.Now() - unlockedTableIDs := make([]int64, 0, len(batchUpdates)) - for _, update := range batchUpdates { - if !update.IsLocked { - failpoint.Inject("panic-when-record-historical-stats-meta", func() { - panic("panic when record historical stats meta") - }) - unlockedTableIDs = append(unlockedTableIDs, update.TableID) - } - } - s.statsHandle.RecordHistoricalStatsMeta(statsVersion, "flush stats", false, unlockedTableIDs...) - // Log a warning if recording historical stats meta takes too long, as it can be slow for large table counts - if time.Since(startRecordHistoricalStatsMeta) > time.Minute*15 { - statslogutil.StatsSampleLogger().Warn("Recording historical stats meta is too slow", - zap.Int("tableCount", len(batchUpdates)), - zap.Duration("duration", time.Since(startRecordHistoricalStatsMeta))) - } } return nil @@ -257,14 +235,14 @@ func (s *statsUsageImpl) dumpStatsDeltaToKV( is infoschema.InfoSchema, sctx sessionctx.Context, updates []*storage.DeltaUpdate, -) (statsVersion uint64, updated []*storage.DeltaUpdate, err error) { +) (updated []*storage.DeltaUpdate, err error) { if len(updates) == 0 { - return 0, nil, nil + return nil, nil } beforeLen := len(updates) - statsVersion, err = utilstats.GetStartTS(sctx) + statsVersion, err := utilstats.GetStartTS(sctx) if err != nil { - return 0, nil, errors.Trace(err) + return nil, errors.Trace(err) } // Collect all table IDs that need lock checking. @@ -285,7 +263,7 @@ func (s *statsUsageImpl) dumpStatsDeltaToKV( // Batch get lock status for all tables. lockedTables, err := s.statsHandle.GetLockedTables(allTableIDs...) if err != nil { - return 0, nil, errors.Trace(err) + return nil, errors.Trace(err) } // Prepare batch updates @@ -342,12 +320,12 @@ func (s *statsUsageImpl) dumpStatsDeltaToKV( // Batch update stats meta. if err = storage.UpdateStatsMeta(utilstats.StatsCtx, sctx, statsVersion, updates...); err != nil { - return 0, nil, errors.Trace(err) + return nil, errors.Trace(err) } // Because we may sort the updates, we need to return the updated slice. // Otherwise the caller may use the original slice and get wrong results. - return statsVersion, updates, nil + return updates, nil } // DumpColStatsUsageToKV sweeps the whole list, updates the column stats usage map and dumps it to KV. diff --git a/pkg/statistics/handle/util/util.go b/pkg/statistics/handle/util/util.go index e8a2ff06fc50c..f0ba2d062f071 100644 --- a/pkg/statistics/handle/util/util.go +++ b/pkg/statistics/handle/util/util.go @@ -39,19 +39,6 @@ import ( "github.com/tikv/client-go/v2/oracle" ) -const ( - // StatsMetaHistorySourceAnalyze indicates stats history meta source from analyze - StatsMetaHistorySourceAnalyze = "analyze" - // StatsMetaHistorySourceLoadStats indicates stats history meta source from load stats - StatsMetaHistorySourceLoadStats = "load stats" - // StatsMetaHistorySourceFlushStats indicates stats history meta source from flush stats - StatsMetaHistorySourceFlushStats = "flush stats" - // StatsMetaHistorySourceSchemaChange indicates stats history meta source from schema change - StatsMetaHistorySourceSchemaChange = "schema change" - // StatsMetaHistorySourceExtendedStats indicates stats history meta source from extended stats - StatsMetaHistorySourceExtendedStats = "extended stats" -) - var ( // UseCurrentSessionOpt to make sure the sql is executed in current session. UseCurrentSessionOpt = []sqlexec.OptionFuncAlias{sqlexec.ExecOptionUseCurSession} @@ -130,13 +117,6 @@ func UpdateSCtxVarsForStats(sctx sessionctx.Context) error { } sctx.GetSessionVars().AnalyzeVersion = int(ver) - // enable historical stats - val, err := sctx.GetSessionVars().GlobalVarsAccessor.GetGlobalSysVar(vardef.TiDBEnableHistoricalStats) - if err != nil { - return err - } - sctx.GetSessionVars().EnableHistoricalStats = variable.TiDBOptOn(val) - // partition mode pruneMode, err := sctx.GetSessionVars().GlobalVarsAccessor.GetGlobalSysVar(vardef.TiDBPartitionPruneMode) if err != nil { @@ -152,7 +132,7 @@ func UpdateSCtxVarsForStats(sctx sessionctx.Context) error { sctx.GetSessionVars().EnableAnalyzeSnapshot = variable.TiDBOptOn(analyzeSnapshot) // enable skip column types - val, err = sctx.GetSessionVars().GlobalVarsAccessor.GetGlobalSysVar(vardef.TiDBAnalyzeSkipColumnTypes) + val, err := sctx.GetSessionVars().GlobalVarsAccessor.GetGlobalSysVar(vardef.TiDBAnalyzeSkipColumnTypes) if err != nil { return err } diff --git a/pkg/statistics/integration_test.go b/pkg/statistics/integration_test.go index e73269f436901..9c98411d717eb 100644 --- a/pkg/statistics/integration_test.go +++ b/pkg/statistics/integration_test.go @@ -389,33 +389,6 @@ func TestTableLastAnalyzeVersion(t *testing.T) { require.NotEqual(t, uint64(0), statsTbl.LastAnalyzeVersion) } -func TestGlobalIndexWithHistoricalStats(t *testing.T) { - store, dom := testkit.CreateMockStoreAndDomain(t) - tk := testkit.NewTestKit(t, store) - - tk.MustExec("set tidb_analyze_version = 2") - tk.MustExec("set global tidb_enable_historical_stats = true") - defer tk.MustExec("set global tidb_enable_historical_stats = default") - - tk.MustExec("use test") - tk.MustExec(`CREATE TABLE t ( a int, b int, c int default 0) - PARTITION BY RANGE (a) ( - PARTITION p0 VALUES LESS THAN (10), - PARTITION p1 VALUES LESS THAN (20), - PARTITION p2 VALUES LESS THAN (30), - PARTITION p3 VALUES LESS THAN (40))`) - tk.MustExec("ALTER TABLE t ADD UNIQUE INDEX idx(b) GLOBAL") - tk.MustExec("INSERT INTO t(a, b) values(1, 1), (2, 2), (3, 3), (15, 15), (25, 25), (35, 35)") - - tblID := dom.MustGetTableID(t, "test", "t") - - for range 10 { - tk.MustExec("analyze table t") - } - // Each analyze will only generate one record - tk.MustQuery(fmt.Sprintf("select count(*) from mysql.stats_history where table_id=%d", tblID)).Equal(testkit.Rows("10")) -} - func TestLastAnalyzeVersionNotChangedWithAsyncStatsLoad(t *testing.T) { store, dom := testkit.CreateMockStoreAndDomain(t) tk := testkit.NewTestKit(t, store) @@ -458,7 +431,7 @@ func TestSaveMetaToStorage(t *testing.T) { tableIDs = append(tableIDs, fmt.Sprintf("%d", tableInfo.ID)) } statsHandler := dom.StatsHandle() - err := statsHandler.SaveMetaToStorage("test", false, metaUpdates...) + err := statsHandler.SaveMetaToStorage(false, metaUpdates...) require.NoError(t, err) rows := tk.MustQuery( fmt.Sprintf( @@ -492,7 +465,7 @@ func TestSaveMetaToStorage(t *testing.T) { metaUpdates[i].ModifyCount = metaUpdates[i].Count metaUpdates[i].Count += metaUpdates[i].ModifyCount } - err = statsHandler.SaveMetaToStorage("test", true, metaUpdates...) + err = statsHandler.SaveMetaToStorage(true, metaUpdates...) require.NoError(t, err) rows = tk.MustQuery( fmt.Sprintf( diff --git a/pkg/statistics/util/json_objects.go b/pkg/statistics/util/json_objects.go index 497ebd60e85ba..45c2d73aa4e1c 100644 --- a/pkg/statistics/util/json_objects.go +++ b/pkg/statistics/util/json_objects.go @@ -28,16 +28,17 @@ const ( // JSONTable is used for dumping statistics. type JSONTable struct { - Columns map[string]*JSONColumn `json:"columns"` - Indices map[string]*JSONColumn `json:"indices"` - Partitions map[string]*JSONTable `json:"partitions"` - DatabaseName string `json:"database_name"` - TableName string `json:"table_name"` - PredicateColumns []*JSONPredicateColumn `json:"predicate_columns"` - Count int64 `json:"count"` - ModifyCount int64 `json:"modify_count"` - Version uint64 `json:"version"` - IsHistoricalStats bool `json:"is_historical_stats"` + Columns map[string]*JSONColumn `json:"columns"` + Indices map[string]*JSONColumn `json:"indices"` + Partitions map[string]*JSONTable `json:"partitions"` + DatabaseName string `json:"database_name"` + TableName string `json:"table_name"` + PredicateColumns []*JSONPredicateColumn `json:"predicate_columns"` + Count int64 `json:"count"` + ModifyCount int64 `json:"modify_count"` + Version uint64 `json:"version"` + // IsHistoricalStats is kept for stats JSON compatibility. Historical stats dumps have been removed. + IsHistoricalStats bool `json:"is_historical_stats"` } // Sort is used to sort the object in the JSONTable. it is used for testing to avoid flaky test. diff --git a/pkg/testkit/mockstore.go b/pkg/testkit/mockstore.go index 38e838c4740f1..accdf765e6866 100644 --- a/pkg/testkit/mockstore.go +++ b/pkg/testkit/mockstore.go @@ -230,7 +230,6 @@ func tryMakeImageOnce(t testing.TB) (retry bool, err error) { vardef.SetSchemaLease(500 * time.Millisecond) session.DisableStats4Test() domain.DisablePlanReplayerBackgroundJob4Test() - domain.DisableDumpHistoricalStats4Test() dom, err := session.BootstrapSession(store) if err != nil { return false, err @@ -387,7 +386,6 @@ func bootstrap4DistExecution(t testing.TB, store kv.Storage, lease time.Duration vardef.SetSchemaLease(lease) session.DisableStats4Test() domain.DisablePlanReplayerBackgroundJob4Test() - domain.DisableDumpHistoricalStats4Test() // Multi-domain tests intentionally keep multiple live domains for one mock // store. Skip the temporary global-init domain so it cannot reuse and close @@ -410,7 +408,6 @@ func bootstrap(t testing.TB, store kv.Storage, lease time.Duration) *domain.Doma vardef.SetSchemaLease(lease) session.DisableStats4Test() domain.DisablePlanReplayerBackgroundJob4Test() - domain.DisableDumpHistoricalStats4Test() dom, err := session.BootstrapSession(store) require.NoError(t, err) diff --git a/pkg/util/replayer/replayer.go b/pkg/util/replayer/replayer.go index 1cabcf770d719..4a5e4d5328465 100644 --- a/pkg/util/replayer/replayer.go +++ b/pkg/util/replayer/replayer.go @@ -41,9 +41,9 @@ type PlanReplayerTaskKey struct { } // GeneratePlanReplayerFile generates plan replayer file -func GeneratePlanReplayerFile(ctx context.Context, storage storeapi.Storage, isCapture, isContinuesCapture, enableHistoricalStatsForCapture bool) (io.WriteCloser, string, error) { +func GeneratePlanReplayerFile(ctx context.Context, storage storeapi.Storage, isCapture, isContinuesCapture bool) (io.WriteCloser, string, error) { path := GetPlanReplayerDirName() - fileName, err := generatePlanReplayerFileName(isCapture, isContinuesCapture, enableHistoricalStatsForCapture) + fileName, err := generatePlanReplayerFileName(isCapture, isContinuesCapture) if err != nil { return nil, "", errors.AddStack(err) } @@ -74,11 +74,11 @@ func (w *fileWriter) Close() error { } // GeneratePlanReplayerFileName generates plan replayer capture task name -func GeneratePlanReplayerFileName(isCapture, isContinuesCapture, enableHistoricalStatsForCapture bool) (string, error) { - return generatePlanReplayerFileName(isCapture, isContinuesCapture, enableHistoricalStatsForCapture) +func GeneratePlanReplayerFileName(isCapture, isContinuesCapture bool) (string, error) { + return generatePlanReplayerFileName(isCapture, isContinuesCapture) } -func generatePlanReplayerFileName(isCapture, isContinuesCapture, enableHistoricalStatsForCapture bool) (string, error) { +func generatePlanReplayerFileName(isCapture, isContinuesCapture bool) (string, error) { // Generate key and create zip file time := time.Now().UnixNano() failpoint.Inject("InjectPlanReplayerFileNameTimeField", func(val failpoint.Value) { @@ -91,11 +91,7 @@ func generatePlanReplayerFileName(isCapture, isContinuesCapture, enableHistorica return "", err } key := base64.URLEncoding.EncodeToString(b) - // "capture_replayer" in filename has special meaning for the /plan_replayer/dump/ HTTP handler - if isContinuesCapture || isCapture && enableHistoricalStatsForCapture { - return fmt.Sprintf("capture_replayer_%v_%v.zip", key, time), nil - } - if isCapture && !enableHistoricalStatsForCapture { + if isCapture || isContinuesCapture { return fmt.Sprintf("capture_normal_replayer_%v_%v.zip", key, time), nil } return fmt.Sprintf("replayer_%v_%v.zip", key, time), nil