From 751333420032f2b2d7aa77680cac52e3325d3440 Mon Sep 17 00:00:00 2001 From: "marthjod@gmail.com" Date: Thu, 9 Jan 2020 16:17:31 +0100 Subject: [PATCH 1/6] fix broken example yaml --- config.yml.dist | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/config.yml.dist b/config.yml.dist index b692dd8f..b7bccc58 100644 --- a/config.yml.dist +++ b/config.yml.dist @@ -109,16 +109,6 @@ jobs: , idx_blks_read::float , idx_blks_hit::float FROM pg_statio_user_tables; -queries: - pg_statio_user_tables: | - SELECT - schemaname::text - , relname::text - , heap_blks_read::float - , heap_blks_hit::float - , idx_blks_read::float - , idx_blks_hit::float - FROM pg_statio_user_tables; - name: "athena" interval: '5m' connections: From f364d425927e76023d8f75dac07e671e6a6d9119 Mon Sep 17 00:00:00 2001 From: Dominik Schulz Date: Fri, 28 Dec 2018 07:12:22 +0100 Subject: [PATCH 2/6] Report failed scrapes Fixes #19 Signed-off-by: Dominik Schulz --- config.go | 15 +++++++++++++++ job.go | 8 ++++++++ main.go | 8 ++++---- query.go | 3 +++ 4 files changed, 30 insertions(+), 4 deletions(-) diff --git a/config.go b/config.go index d85e3679..253f9466 100644 --- a/config.go +++ b/config.go @@ -13,6 +13,20 @@ import ( "gopkg.in/yaml.v2" ) +var ( + failedScrapes = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "sql_exporter_last_scrape_failed", + Help: "Failed scrapes", + }, + []string{"driver", "host", "database", "user", "sql_job", "query"}, + ) +) + +func init() { + prometheus.MustRegister(failedScrapes) +} + // Read attempts to parse the given config and return a file // object func Read(path string) (File, error) { @@ -68,6 +82,7 @@ type Query struct { log log.Logger desc *prometheus.Desc metrics map[*connection][]prometheus.Metric + jobName string Name string `yaml:"name"` // the prometheus metric name Help string `yaml:"help"` // the prometheus metric help text Labels []string `yaml:"labels"` // expose these columns as labels per gauge diff --git a/job.go b/job.go index a4ecf02e..5f3ff5e1 100644 --- a/job.go +++ b/job.go @@ -35,6 +35,7 @@ func (j *Job) Init(logger log.Logger, queries map[string]string) error { continue } q.log = log.With(j.log, "query", q.Name) + q.jobName = j.Name if q.Query == "" && q.QueryRef != "" { if qry, found := queries[q.QueryRef]; found { q.Query = qry @@ -142,6 +143,7 @@ func (j *Job) runOnceConnection(conn *connection, done chan int) { // connect to DB if not connected already if err := conn.connect(j); err != nil { level.Warn(j.log).Log("msg", "Failed to connect", "err", err) + j.markFailed(conn) return } @@ -165,6 +167,12 @@ func (j *Job) runOnceConnection(conn *connection, done chan int) { } } +func (j *Job) markFailed(conn *connection) { + for _, q := range j.Queries { + failedScrapes.WithLabelValues(conn.driver, conn.host, conn.database, conn.user, q.jobName, q.Name).Set(1.0) + } +} + func (j *Job) runOnce() error { doneChan := make(chan int, len(j.conns)) diff --git a/main.go b/main.go index a7656af9..98d17828 100644 --- a/main.go +++ b/main.go @@ -35,10 +35,6 @@ func main() { // init logger logger := log.NewJSONLogger(os.Stdout) - logger = log.With(logger, - "ts", log.DefaultTimestampUTC, - "caller", log.DefaultCaller, - ) // set the allowed log level filter switch strings.ToLower(os.Getenv("LOGLEVEL")) { case "debug": @@ -52,6 +48,10 @@ func main() { default: logger = level.NewFilter(logger, level.AllowAll()) } + logger = log.With(logger, + "ts", log.DefaultTimestampUTC, + "caller", log.DefaultCaller, + ) logger.Log("msg", "Starting sql_exporter", "version_info", version.Info(), "build_context", version.BuildContext()) diff --git a/query.go b/query.go index 29aa8055..0de39e67 100644 --- a/query.go +++ b/query.go @@ -37,15 +37,18 @@ func (q *Query) Run(conn *connection) error { err := rows.MapScan(res) if err != nil { level.Error(q.log).Log("msg", "Failed to scan", "err", err, "host", conn.host, "db", conn.database) + failedScrapes.WithLabelValues(conn.driver, conn.host, conn.database, conn.user, q.jobName, q.Name).Set(1.0) continue } m, err := q.updateMetrics(conn, res) if err != nil { level.Error(q.log).Log("msg", "Failed to update metrics", "err", err, "host", conn.host, "db", conn.database) + failedScrapes.WithLabelValues(conn.driver, conn.host, conn.database, conn.user, q.jobName, q.Name).Set(1.0) continue } metrics = append(metrics, m...) updated++ + failedScrapes.WithLabelValues(conn.driver, conn.host, conn.database, conn.user, q.jobName, q.Name).Set(0.0) } if updated < 1 { From 93b13597e8c7394f6537343c3fbc39e888e711db Mon Sep 17 00:00:00 2001 From: "marthjod@gmail.com" Date: Thu, 9 Jan 2020 16:31:10 +0100 Subject: [PATCH 3/6] failed query = failed scrape (athena) - athena driver-based queries may break late (ie, not during connect) --- query.go | 1 + 1 file changed, 1 insertion(+) diff --git a/query.go b/query.go index 0de39e67..437890a8 100644 --- a/query.go +++ b/query.go @@ -26,6 +26,7 @@ func (q *Query) Run(conn *connection) error { // execute query rows, err := conn.conn.Queryx(q.Query) if err != nil { + failedScrapes.WithLabelValues(conn.driver, conn.host, conn.database, conn.user, q.jobName, q.Name).Set(1.0) return err } defer rows.Close() From ce4896f10c9367478d2c22b1a932b047d26152e9 Mon Sep 17 00:00:00 2001 From: Matthias Rampke Date: Thu, 26 Sep 2019 16:05:51 +0000 Subject: [PATCH 4/6] Do not attempt to parse MySQL DSNs as URLs they do not conform to the RFCs for web URLs, and Go 1.12.8+ enforce this. Instead of the parsed URL, keep the original connection string. This could probably be generalized for other databases, but I don't know if they have the same problem. Signed-off-by: Matthias Rampke --- config.go | 3 +-- job.go | 27 ++++++++++++++++++++++----- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/config.go b/config.go index 253f9466..9896cc1a 100644 --- a/config.go +++ b/config.go @@ -2,7 +2,6 @@ package main import ( "io/ioutil" - "net/url" "os" "sync" "time" @@ -69,7 +68,7 @@ type Job struct { type connection struct { conn *sqlx.DB - url *url.URL + url string driver string host string database string diff --git a/job.go b/job.go index 5f3ff5e1..eadedd60 100644 --- a/job.go +++ b/job.go @@ -12,7 +12,7 @@ import ( _ "github.com/denisenkom/go-mssqldb" // register the MS-SQL driver "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" - _ "github.com/go-sql-driver/mysql" // register the MySQL driver + "github.com/go-sql-driver/mysql" // register the MySQL driver "github.com/jmoiron/sqlx" _ "github.com/lib/pq" // register the PostgreSQL driver "github.com/prometheus/client_golang/prometheus" @@ -87,6 +87,23 @@ func (j *Job) Run() { // parse the connection URLs and create an connection object for each if len(j.conns) < len(j.Connections) { for _, conn := range j.Connections { + // MySQL DSNs do not parse cleanly as URLs as of Go 1.12.8+ + if strings.HasPrefix(conn, "mysql://") { + config, err := mysql.ParseDSN(strings.TrimPrefix(conn, "mysql://")) + if err != nil { + level.Error(j.log).Log("msg", "Failed to parse MySQL DSN", "url", conn, "err", err) + } + + j.conns = append(j.conns, &connection{ + conn: nil, + url: conn, + driver: "mysql", + host: config.Addr, + database: config.DBName, + user: config.User, + }) + continue + } u, err := url.Parse(conn) if err != nil { level.Error(j.log).Log("msg", "Failed to parse URL", "url", conn, "err", err) @@ -100,7 +117,7 @@ func (j *Job) Run() { // remember them newConn := &connection{ conn: nil, - url: u, + url: conn, driver: u.Scheme, host: u.Host, database: strings.TrimPrefix(u.Path, "/"), @@ -198,14 +215,14 @@ func (c *connection) connect(job *Job) error { if c.conn != nil { return nil } - dsn := c.url.String() - switch c.url.Scheme { + dsn := c.url + switch c.driver { case "mysql": dsn = strings.TrimPrefix(dsn, "mysql://") case "clickhouse": dsn = "tcp://" + strings.TrimPrefix(dsn, "clickhouse://") } - conn, err := sqlx.Connect(c.url.Scheme, dsn) + conn, err := sqlx.Connect(c.driver, dsn) if err != nil { return err } From 3598703046f91a706b2978770f89f30f3780d348 Mon Sep 17 00:00:00 2001 From: Matthias Rampke Date: Thu, 26 Sep 2019 16:45:50 +0000 Subject: [PATCH 5/6] Test with Go 1.13 Signed-off-by: Matthias Rampke --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 5681b53a..9e92b88b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,7 @@ language: go go: - 1.12 + - 1.13 - tip env: From b22dbd06778b654f77db2c47d986805d80428b16 Mon Sep 17 00:00:00 2001 From: "marthjod@gmail.com" Date: Fri, 11 Dec 2020 11:42:45 +0100 Subject: [PATCH 6/6] bump version to 0.4.0 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 0d91a54c..1d0ba9ea 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.0 +0.4.0