From 2ad76a2dea097815a58c11d24ab6da8d4ec9575f Mon Sep 17 00:00:00 2001 From: Brian Ip Date: Tue, 9 Feb 2016 19:26:27 -0800 Subject: [PATCH 1/3] initialy add autoinc checl --- mysql/dbstat/dbstat.go | 36 +++++++++++++++++++ mysql/tablestat/tablestat.go | 68 ++++++++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+) diff --git a/mysql/dbstat/dbstat.go b/mysql/dbstat/dbstat.go index 53e3a87..a0dc71d 100644 --- a/mysql/dbstat/dbstat.go +++ b/mysql/dbstat/dbstat.go @@ -222,6 +222,42 @@ SELECT COUNT(*) as count defaultMaxConns = 5 readOnlyQuery = "SELECT @@read_only;" superReadOnlyQuery = "SELECT @@super_read_only;" + autoincQuery = `SELECT * FROM (select + table_schema, + table_name, + column_name, + proper_type, + auto_increment, + max_size, + (((max_size - auto_increment) / max_size ) * 100) AS pct_diff + from + INFORMATION_SCHEMA.columns + natural join INFORMATION_SCHEMA.tables + join ( + select 'tinyint' as proper_type, 127 as max_size + union all + select 'tinyint unsigned' as proper_type, 255 as max_size + union all + select 'smallint' as proper_type, 32767 as max_size + union all + select 'smallint unsigned' as proper_type, 65535 as max_size + union all + select 'mediumint' as proper_type, 8388607 as max_size + union all + select 'mediumint unsigned' as proper_type, 16777215 as max_size + union all + select 'int' as proper_type, 2147483647 as max_size + union all + select 'int unsigned' as proper_type, 4294967295 as max_size + union all + select 'bigint' as proper_type, 9223372036854775807 as max_size + union all + select 'bigint unsigned' as proper_type, 18446744073709551615 as max_size + ) maxes ON (proper_type = CONCAT(LEFT(column_type, GREATEST(0, LOCATE('(', column_type)-1)), RIGHT(column_type, LENGTH(column_type)-LOCATE(')', column_type)))) + where + table_schema NOT IN ('common_schema', 'mysql', '_pending_drops') + AND extra like '%auto_increment%') AS a + WHERE pct_diff < 40;` ) // New initializes mysqlstat diff --git a/mysql/tablestat/tablestat.go b/mysql/tablestat/tablestat.go index 4d07083..86119fd 100644 --- a/mysql/tablestat/tablestat.go +++ b/mysql/tablestat/tablestat.go @@ -36,6 +36,41 @@ SELECT table_schema AS db, table_name AS tbl, FROM INFORMATION_SCHEMA.TABLE_STATISTICS WHERE rows_read > 0;` defaultMaxConns = 5 + autoincQuery = `SELECT * FROM (select + table_schema, + table_name, + column_name, + proper_type, + auto_increment, + max_size, + (((max_size - auto_increment) / max_size ) * 100) AS pct_diff + from + INFORMATION_SCHEMA.columns + natural join INFORMATION_SCHEMA.tables + join ( + select 'tinyint' as proper_type, 127 as max_size + union all + select 'tinyint unsigned' as proper_type, 255 as max_size + union all + select 'smallint' as proper_type, 32767 as max_size + union all + select 'smallint unsigned' as proper_type, 65535 as max_size + union all + select 'mediumint' as proper_type, 8388607 as max_size + union all + select 'mediumint unsigned' as proper_type, 16777215 as max_size + union all + select 'int' as proper_type, 2147483647 as max_size + union all + select 'int unsigned' as proper_type, 4294967295 as max_size + union all + select 'bigint' as proper_type, 9223372036854775807 as max_size + union all + select 'bigint unsigned' as proper_type, 18446744073709551615 as max_size + ) maxes ON (proper_type = CONCAT(LEFT(column_type, GREATEST(0, LOCATE('(', column_type)-1)), RIGHT(column_type, LENGTH(column_type)-LOCATE(')', column_type)))) + where + table_schema NOT IN ('common_schema', 'mysql', '_pending_drops') + AND extra like '%auto_increment%') AS a;` ) // MysqlStatTables - main struct that contains connection to database, metric context, and map to database stats struct @@ -57,6 +92,8 @@ type MysqlStatPerTable struct { RowsRead *metrics.Counter RowsChanged *metrics.Counter RowsChangedXIndexes *metrics.Counter + Autoincrement *metrics.Counter + AutoincPercentFull *metrics.Gauge } // MysqlStatPerDB - metrics for each database @@ -261,6 +298,31 @@ func (s *MysqlStatTables) GetTableStatistics() { return } +func (s *MysqlStatTables) GetColumnStats() { + res, err := s.Db.QueryReturnColumnDict(autoincQuery) + if len(res) == 0 || err != nil { + s.Db.Log(err) + return + } + for i, tblname := range res["table_name"] { + dbname := res["table_schema"][i] + autoincrement, err := strconv.ParseInt(res["auto_increment"][i], 10, 64) + if err != nil { + s.Db.Log(err) + } + pctdiff, err := strconv.ParseFloat(res["pct_diff"][i], 64) + if err != nil { + s.Db.Log(err) + } + s.checkDB(dbname) + s.checkTable(dbname, tblname) + s.nLock.Lock() + s.DBs[dbname].Tables[tblname].Autoincrement.Set(uint64(autoincrement)) + s.DBs[dbname].Tables[tblname].AutoincPercentFull.Set(float64(pctdiff)) + s.nLock.Unlock() + } +} + // FormatGraphite writes metrics in the form "metric_name metric_value" // to the input writer func (s *MysqlStatTables) FormatGraphite(w io.Writer) error { @@ -280,6 +342,12 @@ func (s *MysqlStatTables) FormatGraphite(w io.Writer) error { strconv.FormatUint(tbl.RowsChanged.Get(), 10)) fmt.Fprintln(w, dbname+"."+tblname+".RowsChangedXIndexes "+ strconv.FormatUint(tbl.RowsChangedXIndexes.Get(), 10)) + fmt.Fprintln(w, dbname+"."+tblname+".Autoincrement.Value "+ + strconv.FormatUint(tbl.Autoincrement.Get(), 10)) + fmt.Fprintln(w, dbname+"."+tblname+".Autoincrement.Rate "+ + strconv.FormatFloat(tbl.Autoincrement.ComputeRate(), 'f', 5, 64)) + fmt.Fprintln(w, dbname+"."+tblname+".AutoincPercentFull "+ + strconv.FormatFloat(tbl.AutoincPercentFull.Get(), 'f', 5, 64)) } } return nil From f0d1497b0aa1c8b6f1df250e46c3c8a6f0eee47f Mon Sep 17 00:00:00 2001 From: Brian Ip Date: Tue, 9 Feb 2016 21:45:29 -0800 Subject: [PATCH 2/3] rename to autoincpercentleft --- mysql/dbstat/dbstat.go | 36 ------------------------------------ mysql/tablestat/tablestat.go | 11 +++++------ 2 files changed, 5 insertions(+), 42 deletions(-) diff --git a/mysql/dbstat/dbstat.go b/mysql/dbstat/dbstat.go index a0dc71d..53e3a87 100644 --- a/mysql/dbstat/dbstat.go +++ b/mysql/dbstat/dbstat.go @@ -222,42 +222,6 @@ SELECT COUNT(*) as count defaultMaxConns = 5 readOnlyQuery = "SELECT @@read_only;" superReadOnlyQuery = "SELECT @@super_read_only;" - autoincQuery = `SELECT * FROM (select - table_schema, - table_name, - column_name, - proper_type, - auto_increment, - max_size, - (((max_size - auto_increment) / max_size ) * 100) AS pct_diff - from - INFORMATION_SCHEMA.columns - natural join INFORMATION_SCHEMA.tables - join ( - select 'tinyint' as proper_type, 127 as max_size - union all - select 'tinyint unsigned' as proper_type, 255 as max_size - union all - select 'smallint' as proper_type, 32767 as max_size - union all - select 'smallint unsigned' as proper_type, 65535 as max_size - union all - select 'mediumint' as proper_type, 8388607 as max_size - union all - select 'mediumint unsigned' as proper_type, 16777215 as max_size - union all - select 'int' as proper_type, 2147483647 as max_size - union all - select 'int unsigned' as proper_type, 4294967295 as max_size - union all - select 'bigint' as proper_type, 9223372036854775807 as max_size - union all - select 'bigint unsigned' as proper_type, 18446744073709551615 as max_size - ) maxes ON (proper_type = CONCAT(LEFT(column_type, GREATEST(0, LOCATE('(', column_type)-1)), RIGHT(column_type, LENGTH(column_type)-LOCATE(')', column_type)))) - where - table_schema NOT IN ('common_schema', 'mysql', '_pending_drops') - AND extra like '%auto_increment%') AS a - WHERE pct_diff < 40;` ) // New initializes mysqlstat diff --git a/mysql/tablestat/tablestat.go b/mysql/tablestat/tablestat.go index 86119fd..a7816d9 100644 --- a/mysql/tablestat/tablestat.go +++ b/mysql/tablestat/tablestat.go @@ -93,7 +93,7 @@ type MysqlStatPerTable struct { RowsChanged *metrics.Counter RowsChangedXIndexes *metrics.Counter Autoincrement *metrics.Counter - AutoincPercentFull *metrics.Gauge + AutoincPercentLeft *metrics.Gauge } // MysqlStatPerDB - metrics for each database @@ -142,6 +142,7 @@ func (s *MysqlStatTables) Collect() { s.GetDBSizes, s.GetTableSizes, s.GetTableStatistics, + s.GetColumnStats, } util.CollectInParallel(queryFuncList) } @@ -318,7 +319,7 @@ func (s *MysqlStatTables) GetColumnStats() { s.checkTable(dbname, tblname) s.nLock.Lock() s.DBs[dbname].Tables[tblname].Autoincrement.Set(uint64(autoincrement)) - s.DBs[dbname].Tables[tblname].AutoincPercentFull.Set(float64(pctdiff)) + s.DBs[dbname].Tables[tblname].AutoincPercentLeft.Set(float64(pctdiff)) s.nLock.Unlock() } } @@ -344,10 +345,8 @@ func (s *MysqlStatTables) FormatGraphite(w io.Writer) error { strconv.FormatUint(tbl.RowsChangedXIndexes.Get(), 10)) fmt.Fprintln(w, dbname+"."+tblname+".Autoincrement.Value "+ strconv.FormatUint(tbl.Autoincrement.Get(), 10)) - fmt.Fprintln(w, dbname+"."+tblname+".Autoincrement.Rate "+ - strconv.FormatFloat(tbl.Autoincrement.ComputeRate(), 'f', 5, 64)) - fmt.Fprintln(w, dbname+"."+tblname+".AutoincPercentFull "+ - strconv.FormatFloat(tbl.AutoincPercentFull.Get(), 'f', 5, 64)) + fmt.Fprintln(w, dbname+"."+tblname+".AutoincPercentLeft "+ + strconv.FormatFloat(tbl.AutoincPercentLeft.Get(), 'f', 5, 64)) } } return nil From 00dbd70e09a2817f5458fb0f6dee3ea14202388a Mon Sep 17 00:00:00 2001 From: Brian Ip Date: Tue, 9 Feb 2016 21:51:50 -0800 Subject: [PATCH 3/3] graphite formatting --- mysql/tablestat/tablestat.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql/tablestat/tablestat.go b/mysql/tablestat/tablestat.go index a7816d9..b8fc0a6 100644 --- a/mysql/tablestat/tablestat.go +++ b/mysql/tablestat/tablestat.go @@ -343,7 +343,7 @@ func (s *MysqlStatTables) FormatGraphite(w io.Writer) error { strconv.FormatUint(tbl.RowsChanged.Get(), 10)) fmt.Fprintln(w, dbname+"."+tblname+".RowsChangedXIndexes "+ strconv.FormatUint(tbl.RowsChangedXIndexes.Get(), 10)) - fmt.Fprintln(w, dbname+"."+tblname+".Autoincrement.Value "+ + fmt.Fprintln(w, dbname+"."+tblname+".Autoincrement "+ strconv.FormatUint(tbl.Autoincrement.Get(), 10)) fmt.Fprintln(w, dbname+"."+tblname+".AutoincPercentLeft "+ strconv.FormatFloat(tbl.AutoincPercentLeft.Get(), 'f', 5, 64))