From ba1a766aeb30695bbef4fce3da723a4638deab03 Mon Sep 17 00:00:00 2001 From: jenunes Date: Thu, 11 Jun 2026 11:42:52 -0300 Subject: [PATCH 1/9] Fix MongoDB 8.x FTDC metrics and WiredTiger dashboard panels. Add queues.execution ticket fields, checkpoint/journal metrics, flatten nested FTDC documents, and update WiredTiger panel queries, units, and legends. --- ftdc_exporter/ftdc/iterator.go | 2 +- ftdc_exporter/ftdc/normalize.go | 94 +- ftdc_exporter/ftdc/normalize_test.go | 101 + .../ftdc/testdata/metrics_to_get.txt | 6 + grafana/dashboards/dashboard.json | 2233 ++++++++++++----- metrics_to_get.txt | 12 + 6 files changed, 1810 insertions(+), 638 deletions(-) create mode 100644 ftdc_exporter/ftdc/normalize_test.go diff --git a/ftdc_exporter/ftdc/iterator.go b/ftdc_exporter/ftdc/iterator.go index 9a475eb..222b535 100644 --- a/ftdc_exporter/ftdc/iterator.go +++ b/ftdc_exporter/ftdc/iterator.go @@ -1,7 +1,7 @@ package ftdc func (it *FTDCDataIterator) NormalisedDocument(includedPatterns map[string]struct{}) map[string]interface{} { - return normalizeDocument(it.doc, includedPatterns) + return normalizeMetricsDocument(it.doc, includedPatterns) } func (it *FTDCDataIterator) Close() { diff --git a/ftdc_exporter/ftdc/normalize.go b/ftdc_exporter/ftdc/normalize.go index c05339e..06d6991 100644 --- a/ftdc_exporter/ftdc/normalize.go +++ b/ftdc_exporter/ftdc/normalize.go @@ -1,6 +1,8 @@ package ftdc import ( + "strconv" + "github.com/evergreen-ci/birch" "github.com/evergreen-ci/birch/bsontype" "strings" @@ -60,6 +62,14 @@ func isIncluded(key string, includedPatterns map[string]struct{}) bool { return false } +func joinMetricKey(prefix, key string) string { + if prefix == "" { + return key + } + return prefix + "." + key +} + +// normalizeDocument preserves nested maps (used for FTDC metadata). func normalizeDocument(document *birch.Document, includedPatterns map[string]struct{}) map[string]interface{} { normalized := make(map[string]interface{}) iter := document.Iterator() @@ -67,8 +77,6 @@ func normalizeDocument(document *birch.Document, includedPatterns map[string]str for iter.Next() { elem := iter.Element() key := elem.Key() - // on some versions, the metrics are starting with a common. prefix - // I decided to get rid of it so that we do not have to change anything in the grafana dashboards. key = strings.TrimPrefix(key, "common.") val := elem.Value() if isIncluded(key, includedPatterns) { @@ -78,6 +86,83 @@ func normalizeDocument(document *birch.Document, includedPatterns map[string]str return normalized } +// normalizeMetricsDocument flattens nested FTDC samples into dotted field names for InfluxDB. +func normalizeMetricsDocument(document *birch.Document, includedPatterns map[string]struct{}) map[string]interface{} { + return flattenDocument(document, "", includedPatterns) +} + +func flattenDocument(document *birch.Document, prefix string, includedPatterns map[string]struct{}) map[string]interface{} { + normalized := make(map[string]interface{}) + iter := document.Iterator() + + for iter.Next() { + elem := iter.Element() + key := elem.Key() + key = strings.TrimPrefix(key, "common.") + fullKey := joinMetricKey(prefix, key) + val := elem.Value() + + switch val.Type() { + case bsontype.EmbeddedDocument: + for k, v := range flattenDocument(val.MutableDocument(), fullKey, includedPatterns) { + normalized[k] = v + } + case bsontype.Array: + flattenArray(val.MutableArray(), fullKey, includedPatterns, normalized) + default: + if isIncluded(fullKey, includedPatterns) { + normalized[fullKey] = normalizeScalar(val) + } + } + } + return normalized +} + +func flattenArray(arr *birch.Array, prefix string, includedPatterns map[string]struct{}, out map[string]interface{}) { + it := arr.Iterator() + i := 0 + for it.Next() { + indexKey := joinMetricKey(prefix, strconv.Itoa(i)) + val := it.Value() + switch val.Type() { + case bsontype.EmbeddedDocument: + for k, v := range flattenDocument(val.MutableDocument(), indexKey, includedPatterns) { + out[k] = v + } + case bsontype.Array: + flattenArray(val.MutableArray(), indexKey, includedPatterns, out) + default: + if isIncluded(indexKey, includedPatterns) { + out[indexKey] = normalizeScalar(val) + } + } + i++ + } +} + +func normalizeScalar(val *birch.Value) interface{} { + switch val.Type() { + case bsontype.Double: + return val.Double() + case bsontype.String: + return val.StringValue() + case bsontype.Boolean: + return val.Boolean() + case bsontype.Int32: + return val.Int32() + case bsontype.Int64: + return val.Int64() + case bsontype.Null: + return -1 + case bsontype.ObjectID: + return val.ObjectID().Hex() + case bsontype.DateTime: + return time.UnixMilli(val.DateTime()).UnixMilli() + default: + return val.Interface() + } +} + func normalizeValue(val *birch.Value, includedPatterns map[string]struct{}) interface{} { switch val.Type() { case bsontype.Double: @@ -99,16 +184,13 @@ func normalizeValue(val *birch.Value, includedPatterns map[string]struct{}) inte case bsontype.Array: out := []interface{}{} it := val.MutableArray().Iterator() - i := 0 for it.Next() { out = append(out, normalizeValue(it.Value(), includedPatterns)) - i++ } return out case bsontype.DateTime: return time.UnixMilli(val.DateTime()).UnixMilli() default: - // Handle unsupported types as raw or string, or skip - return val.Interface() // fallback + return val.Interface() } } diff --git a/ftdc_exporter/ftdc/normalize_test.go b/ftdc_exporter/ftdc/normalize_test.go new file mode 100644 index 0000000..8a631f5 --- /dev/null +++ b/ftdc_exporter/ftdc/normalize_test.go @@ -0,0 +1,101 @@ +package ftdc + +import ( + "testing" + + "github.com/evergreen-ci/birch" +) + +func TestNormalizeMetricsDocument_flatAndNestedKeys(t *testing.T) { + doc := birch.NewDocument( + birch.EC.Int64("start", 1700000000000), + birch.EC.Int64("serverStatus.wiredTiger.cache.bytes currently in the cache", 4096), + birch.EC.SubDocument("serverStatus", birch.NewDocument( + birch.EC.SubDocument("wiredTiger", birch.NewDocument( + birch.EC.SubDocument("concurrentTransactions", birch.NewDocument( + birch.EC.SubDocument("monitor", birch.NewDocument( + birch.EC.Int32("available", 100), + birch.EC.Int32("totalTickets", 128), + birch.EC.Int32("queueLength", 0), + )), + )), + )), + )), + ) + + include := map[string]struct{}{ + "start": {}, + "serverStatus.wiredTiger.cache.bytes currently in the cache": {}, + "serverStatus.wiredTiger.concurrentTransactions.monitor.available": {}, + "serverStatus.wiredTiger.concurrentTransactions.monitor.totalTickets": {}, + "serverStatus.wiredTiger.concurrentTransactions.monitor.queueLength": {}, + } + + got := normalizeMetricsDocument(doc, include) + + want := map[string]interface{}{ + "start": int64(1700000000000), + "serverStatus.wiredTiger.cache.bytes currently in the cache": int64(4096), + "serverStatus.wiredTiger.concurrentTransactions.monitor.available": int32(100), + "serverStatus.wiredTiger.concurrentTransactions.monitor.totalTickets": int32(128), + "serverStatus.wiredTiger.concurrentTransactions.monitor.queueLength": int32(0), + } + + for key, wantVal := range want { + gotVal, ok := got[key] + if !ok { + t.Fatalf("missing key %q; got %#v", key, got) + } + if gotVal != wantVal { + t.Fatalf("key %q: got %v (%T), want %v (%T)", key, gotVal, gotVal, wantVal, wantVal) + } + } + + if len(got) != len(want) { + t.Fatalf("unexpected extra keys: %#v", got) + } +} + +func TestNormalizeMetricsDocument_flatQueueExecutionKeys(t *testing.T) { + doc := birch.NewDocument( + birch.EC.Int32("serverStatus.queues.execution.read.available", 100), + birch.EC.Int32("serverStatus.queues.execution.write.available", 95), + birch.EC.Int32("serverStatus.queues.execution.read.totalTickets", 128), + ) + + include := map[string]struct{}{ + "serverStatus.queues.execution.read.available": {}, + "serverStatus.queues.execution.write.available": {}, + "serverStatus.queues.execution.read.totalTickets": {}, + } + + got := normalizeMetricsDocument(doc, include) + if got["serverStatus.queues.execution.read.available"] != int32(100) { + t.Fatalf("read.available = %#v", got) + } +} + +func TestNormalizeDocument_preservesNestedMetadata(t *testing.T) { + doc := birch.NewDocument( + birch.EC.SubDocument("doc", birch.NewDocument( + birch.EC.SubDocument("hostInfo", birch.NewDocument( + birch.EC.SubDocument("system", birch.NewDocument( + birch.EC.String("hostname", "mongod1.example:27017"), + )), + )), + birch.EC.SubDocument("buildInfo", birch.NewDocument( + birch.EC.String("version", "8.0.17-6"), + )), + )), + ) + + got := normalizeDocument(doc, map[string]struct{}{}) + hostname := getNestedString(got, "doc.hostInfo.system.hostname") + if hostname != "mongod1.example:27017" { + t.Fatalf("hostname = %q, want mongod1.example:27017", hostname) + } + version := getNestedString(got, "doc.buildInfo.version") + if version != "8.0.17-6" { + t.Fatalf("version = %q, want 8.0.17-6", version) + } +} diff --git a/ftdc_exporter/ftdc/testdata/metrics_to_get.txt b/ftdc_exporter/ftdc/testdata/metrics_to_get.txt index 6265f78..066bbe3 100644 --- a/ftdc_exporter/ftdc/testdata/metrics_to_get.txt +++ b/ftdc_exporter/ftdc/testdata/metrics_to_get.txt @@ -35,6 +35,9 @@ serverStatus.flowControl.targetRateLimit serverStatus.wiredTiger.block-manager.bytes read serverStatus.wiredTiger.block-manager.bytes written serverStatus.wiredTiger.block-manager.bytes written for checkpoint +serverStatus.wiredTiger.log.log bytes written +serverStatus.wiredTiger.transaction.transaction checkpoint most recent time (msecs) +serverStatus.wiredTiger.checkpoint.most recent time (msecs) serverStatus.wiredTiger.cache.bytes currently in the cache serverStatus.wiredTiger.cache.maximum bytes configured serverStatus.wiredTiger.cache.modified pages evicted @@ -45,6 +48,9 @@ serverStatus.wiredTiger.cache.unmodified pages evicted serverStatus.wiredTiger.data-handle.connection data handles currently active serverStatus.wiredTiger.concurrentTransactions.read.available serverStatus.wiredTiger.concurrentTransactions.write.available +serverStatus.wiredTiger.concurrentTransactions.monitor.available +serverStatus.wiredTiger.concurrentTransactions.monitor.totalTickets +serverStatus.wiredTiger.concurrentTransactions.monitor.queueLength serverStatus.flowControl.enabled serverStatus.flowControl.isLagged serverStatus.flowControl.isLaggedCount diff --git a/grafana/dashboards/dashboard.json b/grafana/dashboards/dashboard.json index 57b7944..e9a2049 100644 --- a/grafana/dashboards/dashboard.json +++ b/grafana/dashboards/dashboard.json @@ -18,7 +18,6 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, - "id": 1, "links": [], "panels": [ { @@ -101,19 +100,49 @@ }, "unit": "none" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*\\.read\\.available$" + }, + "properties": [ + { + "id": "displayName", + "value": "Tickets Available Read" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*\\.write\\.available$" + }, + "properties": [ + { + "id": "displayName", + "value": "Tickets Available Write" + } + ] + } + ] }, "gridPos": { - "h": 6, - "w": 14, + "h": 8, + "w": 13, "x": 0, "y": 1 }, "id": 1, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, @@ -123,14 +152,14 @@ "sort": "none" } }, - "pluginVersion": "12.2.0", + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", - "uid": "fdnvtvpv64tfkc" + "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.concurrentTransactions.write.available\" or r[\"_field\"] == \"serverStatus.wiredTiger.concurrentTransactions.read.available\")\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.concurrentTransactions.read.available\" or r[\"_field\"] == \"serverStatus.wiredTiger.concurrentTransactions.write.available\" or r[\"_field\"] == \"serverStatus.queues.execution.read.available\" or r[\"_field\"] == \"serverStatus.queues.execution.write.available\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n", "refId": "A" } ], @@ -142,7 +171,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "used displays the total bytes cached in memory for serving reads and writes.\n\nThese metrics include both indexes and data from the working set.\n\nSustained high cache usage indicates the RAM is too small for your workloads. Optimize your queries to avoid frequent disk reads. If write operations make cache usage high, throttle them.", + "description": "Shows wiredTiger.cache.maximum bytes configured (maximum cache size) vs wiredTiger.cache.bytes currently in the cache (data currently cached). Per MongoDB docs, bytes currently in the cache should not be greater than the maximum bytes configured. To adjust the WiredTiger internal cache size, see --wiredTigerCacheSizeGB (serverStatus.wiredTiger.cache).\n\nHow to read: two lines -- configured max and current usage, displayed in GB.\n\nTroubleshooting: when current cache approaches the configured maximum, WiredTiger must evict aggressively. Correlate with 'Pages Evicted' and 'Dirty Cache Bytes'. If cache is consistently near max, either increase wiredTigerCacheSizeGB or optimize queries to reduce the working set.", "fieldConfig": { "defaults": { "color": { @@ -200,19 +229,49 @@ }, "unit": "decgbytes" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*maximum bytes configured$" + }, + "properties": [ + { + "id": "displayName", + "value": "Maximum cache configured" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*bytes currently in the cache$" + }, + "properties": [ + { + "id": "displayName", + "value": "Current cache usage" + } + ] + } + ] }, "gridPos": { - "h": 6, - "w": 10, - "x": 14, + "h": 8, + "w": 11, + "x": 13, "y": 1 }, "id": 6, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, @@ -222,7 +281,7 @@ "sort": "none" } }, - "pluginVersion": "12.2.0", + "pluginVersion": "12.3.2", "targets": [ { "datasource": { @@ -230,11 +289,11 @@ "uid": "P3C6603E967DC8568" }, "hide": false, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.cache.maximum bytes configured\" or r[\"_field\"] == \"serverStatus.wiredTiger.cache.bytes currently in the cache\" )\r\n |> map(fn: (r) => ({ r with _value: r._value / 1024 / 1024 / 1024}))\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.cache.maximum bytes configured\" or r[\"_field\"] == \"serverStatus.wiredTiger.cache.bytes currently in the cache\" )\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> map(fn: (r) => ({ r with _value: r._value / 1024 / 1024 / 1024}))\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n\r\n", "refId": "A" } ], - "title": "WiredTiger cache", + "title": "WiredTiger Cache", "type": "timeseries" }, { @@ -242,7 +301,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "read Into displays the rate in bytes per second of data read from disk into memory to service queries.\n\nwritten From displays the rate in bytes per second of data written from memory into disk to service writes.", + "description": "Rate of wiredTiger.cache.modified pages evicted (dirty) and wiredTiger.cache.unmodified pages evicted (clean) per second. Per MongoDB docs, unmodified pages evicted is the main statistic for page eviction (serverStatus.wiredTiger.cache).\n\nHow to read: two lines showing pages/sec evicted, split by clean vs dirty. Clean eviction discards unmodified pages; dirty eviction requires writing modified data to disk first (reconciliation).\n\nTroubleshooting: spikes in dirty eviction indicate cache pressure and correlate with write latency increases. If eviction cannot keep up, application threads may be forced to do eviction themselves, causing a performance cliff. Check 'WiredTiger Cache' and 'Dirty Cache Bytes'.", "fieldConfig": { "defaults": { "color": { @@ -282,6 +341,7 @@ "mode": "off" } }, + "decimals": 0, "mappings": [], "thresholds": { "mode": "absolute", @@ -296,21 +356,51 @@ } ] }, - "unit": "decbytes" + "unit": "pages/s" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*modified pages evicted$" + }, + "properties": [ + { + "id": "displayName", + "value": "Modified pages evicted (dirty)" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*unmodified pages evicted$" + }, + "properties": [ + { + "id": "displayName", + "value": "Unmodified pages evicted (clean)" + } + ] + } + ] }, "gridPos": { - "h": 6, - "w": 6, + "h": 7, + "w": 13, "x": 0, - "y": 7 + "y": 9 }, - "id": 8, + "id": 12, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, @@ -320,18 +410,18 @@ "sort": "none" } }, - "pluginVersion": "12.2.0", + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.cache.bytes written from cache\" or r[\"_field\"] == \"serverStatus.wiredTiger.cache.bytes read into cache\")\r\n |> derivative(unit: 1s)\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.cache.modified pages evicted\" or r[\"_field\"] == \"serverStatus.wiredTiger.cache.unmodified pages evicted\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s)\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n", "refId": "A" } ], - "title": "Mb read into cache / written from cache", + "title": "Pages Evicted", "type": "timeseries" }, { @@ -339,7 +429,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Clean vs dirty data\n\nClean refers to data in the cache that is identical to the version stored on disk.\nDirty refers to data in the cache that has been modified and needs to be reconciled with the data stored on the disk.\n\nClean vs dirty eviction\n\nIn the case of clean eviction, there is no dirty content on the page and it is directly removed from the memory. After that, the page remains solely on the disk unchanged.\nIn the case of dirty eviction, the dirty page goes through reconciliation. The obsolete content on the page is thrown out, the latest value goes to the data store and older values go to the history store.", + "description": "Shows wiredTiger.data-handle.connection data handles currently active. Per MongoDB docs, a dhandle is a generic representation of any named data source (collection, index). WiredTiger maintains all dhandles in a global list protected by a R/W lock; each session has a dhandle cache (serverStatus.wiredTiger.data-handle).\n\nHow to read: single line showing the count of active dhandles over time.\n\nTroubleshooting: a very high count of active dhandles indicates many collections/indexes accessed concurrently, increasing memory overhead and lock contention on the global dhandle list. A sudden drop may indicate collections being closed or a mongod restart.", "fieldConfig": { "defaults": { "color": { @@ -379,7 +469,6 @@ "mode": "off" } }, - "decimals": 0, "mappings": [], "thresholds": { "mode": "absolute", @@ -393,22 +482,528 @@ "value": 80 } ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*data handles currently active$" + }, + "properties": [ + { + "id": "displayName", + "value": "Active data handles" + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 11, + "x": 13, + "y": 9 + }, + "id": 26, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.data-handle.connection data handles currently active\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> yield(name: \"last\")", + "refId": "A" + } + ], + "title": "WiredTiger Data Handle", + "type": "timeseries" + }, + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "description": "Shows wiredTiger.cache.tracked dirty bytes in the cache -- the size in bytes of dirty data in the cache. Per MongoDB docs, this value should be less than bytes currently in the cache (serverStatus.wiredTiger.cache).\n\nHow to read: single line showing dirty bytes over time. Dirty data = modified data not yet written to disk via checkpoint or eviction.\n\nTroubleshooting: if dirty bytes approach the eviction target (~5% of cache triggers eviction, ~20% triggers aggressive eviction), WiredTiger stalls application threads to force eviction. Correlate with checkpoint writes and eviction rate.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*tracked dirty bytes in the cache$" + }, + "properties": [ + { + "id": "displayName", + "value": "Dirty cache bytes" + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 11, + "x": 13, + "y": 14 + }, + "id": 7, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.cache.tracked dirty bytes in the cache\" )\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> yield(name: \"last\")", + "refId": "A" + } + ], + "title": "Dirty Cache Bytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "description": "Rate of wiredTiger.block-manager.bytes read and wiredTiger.block-manager.bytes written per second. Per MongoDB docs, the block manager subsystem manages the reading and writing of data from disk (serverStatus.wiredTiger.block-manager).\n\nHow to read: two lines showing read and write throughput in MB/s through the block manager.\n\nTroubleshooting: high block manager reads indicate cache misses forcing disk access. High writes correlate with checkpoints and eviction flushes. Compare with 'Disk I/O' to determine if WiredTiger is the dominant disk consumer or if other processes contribute.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "MBs" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*block-manager\\.bytes read$" + }, + "properties": [ + { + "id": "displayName", + "value": "Bytes read" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*block-manager\\.bytes written$" + }, + "properties": [ + { + "id": "displayName", + "value": "Bytes written" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 13, + "x": 0, + "y": 16 + }, + "id": 23, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.block-manager.bytes read\" or r[\"_field\"] == \"serverStatus.wiredTiger.block-manager.bytes written\" )\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> map(fn: (r) => ({ r with _value: r._value / 1024 /1024 }))\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> derivative(unit: 1s)\r\n", + "refId": "A" + } + ], + "title": "WiredTiger Block Manager", + "type": "timeseries" + }, + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "description": "Rate of wiredTiger.block-manager.bytes written for checkpoint per second. Per MongoDB docs, a checkpoint is a known point in time from which WiredTiger can recover in the event of a crash or unexpected shutdown. Checkpoints typically run every 60 seconds (serverStatus.wiredTiger.block-manager).\n\nHow to read: derivative of cumulative checkpoint bytes, showing write throughput during checkpoints.\n\nTroubleshooting: large checkpoint writes indicate heavy write workloads. If checkpoint write spikes coincide with latency increases, the checkpoint is competing for disk I/O. Correlate with 'Disk I/O', 'Disk Writes', and 'Average Operation Latency' panels.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*bytes written for checkpoint$" + }, + "properties": [ + { + "id": "displayName", + "value": "Checkpoint write throughput" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 13, + "x": 0, + "y": 23 + }, + "id": 19, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.block-manager.bytes written for checkpoint\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n|> derivative(unit: 1s, nonNegative: true)", + "refId": "A" + } + ], + "title": "Checkpoint Write Throughput", + "type": "timeseries" + }, + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "description": "Rate of wiredTiger.cache.bytes read into cache and wiredTiger.cache.bytes written from cache per second. Per MongoDB docs, pages read into cache together with pages written from cache provide an overview of I/O activity (serverStatus.wiredTiger.cache).\n\nHow to read: derivative of cumulative byte counters, showing bytes/sec. 'Read into cache' = cold data pulled from disk; 'Written from cache' = dirty data flushed to disk.\n\nTroubleshooting: high read-into-cache means frequent cache misses -- check if the working set or indexes exceed cache size. High write-from-cache correlates with write-heavy workloads or eviction pressure. Compare with 'Disk I/O' and 'WiredTiger Block Manager'.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*bytes read into cache$" + }, + "properties": [ + { + "id": "displayName", + "value": "Read into cache" + } + ] }, - "unit": "short" - }, - "overrides": [] + { + "matcher": { + "id": "byRegexp", + "options": ".*bytes written from cache$" + }, + "properties": [ + { + "id": "displayName", + "value": "Written from cache" + } + ] + } + ] }, "gridPos": { - "h": 6, - "w": 8, - "x": 6, - "y": 7 + "h": 7, + "w": 11, + "x": 13, + "y": 23 }, - "id": 12, + "id": 8, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, @@ -418,18 +1013,18 @@ "sort": "none" } }, - "pluginVersion": "12.2.0", + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.cache.modified pages evicted\" or r[\"_field\"] == \"serverStatus.wiredTiger.cache.unmodified pages evicted\")\r\n |> map(fn: (r) => ({ r with _value: r._value }))\r\n |> derivative(unit: 1s)\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.cache.bytes written from cache\" or r[\"_field\"] == \"serverStatus.wiredTiger.cache.bytes read into cache\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", "refId": "A" } ], - "title": "Pages Evicted", + "title": "Cache Read/Write Throughput", "type": "timeseries" }, { @@ -437,7 +1032,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "indicates modified data that needs to be written to the data files via a checkpoint.", + "description": "Shows the duration in milliseconds of the most recent WiredTiger checkpoint. Pre-8.0: wiredTiger.transaction.transaction checkpoint most recent time (msecs). MongoDB 8.x: wiredTiger.checkpoint.most recent time (msecs). Per MongoDB docs, an increase under steady write load may indicate I/O subsystem saturation.\n\nHow to read: single line showing the most recent checkpoint duration. Checkpoints typically run every 60 seconds.\n\nTroubleshooting: increasing checkpoint duration under steady write load indicates I/O subsystem saturation. Correlate with 'Checkpoint Write Throughput', 'Disk I/O', and 'Average Operation Latency'. Long checkpoints can cause write stalls.", "fieldConfig": { "defaults": { "color": { @@ -477,6 +1072,7 @@ "mode": "off" } }, + "decimals": 1, "mappings": [], "min": 0, "thresholds": { @@ -492,67 +1088,62 @@ } ] }, - "unit": "decbytes" + "unit": "ms" }, "overrides": [ { - "__systemRef": "hideSeriesFrom", "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "serverStatus.wiredTiger.cache.tracked dirty bytes in the cache" - ], - "prefix": "All except:", - "readOnly": true - } + "id": "byRegexp", + "options": ".*checkpoint.most recent time \\(msecs\\)$" }, "properties": [ { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": true, - "viz": true - } + "id": "displayName", + "value": "Checkpoint duration" } ] } ] }, "gridPos": { - "h": 7, - "w": 10, - "x": 14, - "y": 7 + "h": 6, + "w": 13, + "x": 0, + "y": 30 }, - "id": 7, + "id": 48, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", - "showLegend": true + "showLegend": true, + "sortBy": "Name", + "sortDesc": true }, "tooltip": { "hideZeros": false, - "mode": "single", + "mode": "multi", "sort": "none" } }, - "pluginVersion": "12.2.0", + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.cache.tracked dirty bytes in the cache\" )\r\n |> map(fn: (r) => ({ r with _value: r._value}))\r\n\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> yield(name: \"last\")", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.transaction.transaction checkpoint most recent time (msecs)\" or r[\"_field\"] == \"serverStatus.wiredTiger.checkpoint.most recent time (msecs)\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n", "refId": "A" } ], - "title": "Dirty mb in the cache", + "title": "Checkpoint Duration", "type": "timeseries" }, { @@ -560,7 +1151,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "A checkpoint is a known point in time from which WiredTiger can recover in the event of a crash or unexpected shutdown.", + "description": "Rate of wiredTiger.log.log bytes written per second. Per MongoDB docs, wiredTiger.log returns statistics on WiredTiger's write ahead log (i.e. the journal) (serverStatus.wiredTiger.log).\n\nHow to read: single line showing journal write throughput in bytes/sec.\n\nTroubleshooting: high journal write rates correlate with heavy write workloads. Compare with 'Checkpoint Write Throughput' to understand the balance between journal writes and checkpoint writes. Sustained high values may indicate the need for faster storage.", "fieldConfig": { "defaults": { "color": { @@ -572,7 +1163,6 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", - "axisSoftMax": 10, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -601,6 +1191,7 @@ "mode": "off" } }, + "decimals": 2, "mappings": [], "thresholds": { "mode": "absolute", @@ -615,50 +1206,81 @@ } ] }, - "unit": "decbytes" + "unit": "Bps" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "serverStatus.wiredTiger.log.log bytes written" + }, + "properties": [ + { + "id": "displayName", + "value": "Journal bytes written" + } + ] + } + ] }, "gridPos": { "h": 6, - "w": 6, - "x": 0, - "y": 13 + "w": 11, + "x": 13, + "y": 30 }, - "id": 19, + "id": 49, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, - "mode": "single", + "mode": "multi", "sort": "none" } }, - "pluginVersion": "12.2.0", + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.block-manager.bytes written for checkpoint\")\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> derivative(unit: v.windowPeriod)\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.log.log bytes written\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n", "refId": "A" } ], - "title": "Mb written for checkpoint", + "title": "Journal Bytes Written", "type": "timeseries" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 22, + "panels": [], + "title": "Server Status", + "type": "row" + }, { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "The WiredTiger block manager subsystem manages the reading and writing of data from the disk. It is designed to facilitate high performance, economic use of disk space and customizability.", + "description": "Shows serverStatus.mem.resident and serverStatus.mem.virtual. Per MongoDB docs, mem.resident is roughly equivalent to the amount of RAM in MiB currently used by the database process; during normal use this value tends to grow and on dedicated servers approaches total system memory. mem.virtual is the quantity in MiB of virtual memory used by the mongod process (serverStatus.mem).\n\nHow to read: two lines -- resident (physical RAM) and virtual (total address space), displayed in GB.\n\nTroubleshooting: if resident memory approaches total system RAM, the OS may start swapping -- check 'Swap Memory'. Virtual memory much larger than resident is normal (memory-mapped files). Sudden drops in resident indicate a restart; compare with 'Server Uptime'.", "fieldConfig": { "defaults": { "color": { @@ -670,6 +1292,7 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -683,7 +1306,7 @@ "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, - "pointSize": 5, + "pointSize": 1, "scaleDistribution": { "type": "linear" }, @@ -712,21 +1335,26 @@ } ] }, - "unit": "MBs" + "unit": "decgbytes" }, "overrides": [] }, "gridPos": { - "h": 6, + "h": 5, "w": 8, - "x": 6, - "y": 13 + "x": 0, + "y": 37 }, - "id": 23, + "id": 3, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, @@ -736,18 +1364,18 @@ "sort": "none" } }, - "pluginVersion": "12.2.0", + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.block-manager.bytes read\" or r[\"_field\"] == \"serverStatus.wiredTiger.block-manager.bytes written\" )\r\n |> map(fn: (r) => ({ r with _value: r._value}))\r\n |> map(fn: (r) => ({ r with _value: r._value / 1024 /1024 }))\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> derivative(unit: 1s)\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.mem.virtual\" or r[\"_field\"] == \"serverStatus.mem.resident\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: 1m, fn: median, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: r._value / 1000.0 }))\r\n |> yield(name: \"median\")\r\n", "refId": "A" } ], - "title": "wiredTiger Block Manager", + "title": "mongod Memory (RSS / Virtual)", "type": "timeseries" }, { @@ -755,7 +1383,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "A WiredTiger Data Handle (dhandle) is a generic representation of any named data source. This representation contains information such as its name, how many references there are to it, individual data source statistics and what type of underlying data object it is.\n\nWiredTiger maintains all dhandles in a global dhandle list accessed from the connection. Multiple sessions access this list, which is protected by a R/W lock. Each session also maintains a session dhandle cache which is a cache of dhandles a session has operated upon. The entries in the cache are references into the global dhandle list.", + "description": "Rate of serverStatus.extra_info.page_faults per second. Per MongoDB docs, the page_faults counter may increase dramatically during moments of poor performance and may correlate with limited memory environments and larger data sets. Limited and sporadic page faults do not necessarily indicate an issue (serverStatus.extra_info).\n\nHow to read: single line showing page faults/sec (derivative of cumulative counter).\n\nTroubleshooting: sustained high page fault rates mean the working set exceeds available physical RAM, forcing expensive disk reads. Correlate with 'WiredTiger Cache' utilization, 'Swap Memory', and 'mongod Memory (RSS / Virtual)'. If page faults are high and swap is active, the server needs more RAM.", "fieldConfig": { "defaults": { "color": { @@ -814,56 +1442,49 @@ }, "gridPos": { "h": 5, - "w": 10, - "x": 14, - "y": 14 + "w": 7, + "x": 8, + "y": 37 }, - "id": 26, + "id": 27, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, - "mode": "single", + "mode": "multi", "sort": "none" } }, - "pluginVersion": "12.2.0", + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.data-handle.connection data handles currently active\")\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> yield(name: \"last\")", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.extra_info.page_faults\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: v.windowPeriod)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", "refId": "A" } ], - "title": "WiredTiger Data Handle", + "title": "Page Faults", "type": "timeseries" }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 19 - }, - "id": 22, - "panels": [], - "title": "Server Status", - "type": "row" - }, { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "Shows serverStatus.connections.current, connections.active, and the rate of connections.totalCreated. Per MongoDB docs, connections.current is the number of incoming connections from clients including shell sessions; connections.available shows unused connections available; connections.active is client connections with operations in progress (serverStatus.connections).\n\nHow to read: active and current as gauge lines, plus totalCreated as a rate (derivative) showing new connection velocity.\n\nTroubleshooting: current connections climbing toward maxIncomingConnections (default 65536) indicates pool exhaustion. If active is much lower than current, many idle connections are consuming resources -- check application connection pool settings. A spike in totalCreated rate may indicate connection churn (short-lived connections).", "fieldConfig": { "defaults": { "color": { @@ -875,7 +1496,6 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", - "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -889,7 +1509,7 @@ "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, - "pointSize": 1, + "pointSize": 5, "scaleDistribution": { "type": "linear" }, @@ -917,22 +1537,26 @@ "value": 80 } ] - }, - "unit": "decgbytes" + } }, "overrides": [] }, "gridPos": { - "h": 5, - "w": 8, - "x": 0, - "y": 20 + "h": 7, + "w": 9, + "x": 15, + "y": 37 }, - "id": 3, + "id": 2, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, @@ -942,18 +1566,27 @@ "sort": "none" } }, - "pluginVersion": "12.2.0", + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", - "uid": "influxdb" + "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.mem.virtual\" or r[\"_field\"] == \"serverStatus.mem.resident\")\r\n |> aggregateWindow(every: 1m, fn: median, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: r._value / 1000.0 }))\r\n |> yield(name: \"median\")\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.connections.active\" or r[\"_field\"] == \"serverStatus.connections.current\" )\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> yield(name: \"last\")\r\n\r\n", "refId": "A" + }, + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "hide": false, + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.connections.totalCreated\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> derivative(unit: v.windowPeriod)", + "refId": "B" } ], - "title": "Memory", + "title": "Connections", "type": "timeseries" }, { @@ -961,6 +1594,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "Shows average operation latency in microseconds for reads, writes, and commands. Per MongoDB docs, opLatencies contains operation latencies for the instance as a whole; opLatencies.reads for read requests, opLatencies.writes for write operations, opLatencies.commands for database commands (serverStatus.opLatencies).\n\nHow to read: three lines showing average latency per operation type in microseconds, computed as delta(opLatencies.*.latency) / delta(opLatencies.*.ops).\n\nTroubleshooting: latency spikes that correlate with ticket exhaustion indicate concurrency contention. Spikes correlating with disk I/O indicate storage bottlenecks. Compare with 'Operations' to distinguish 'fewer but slower ops' from 'more ops at same speed'.", "fieldConfig": { "defaults": { "color": { @@ -1000,6 +1634,7 @@ "mode": "off" } }, + "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", @@ -1013,42 +1648,48 @@ "value": 80 } ] - } + }, + "unit": "µs" }, "overrides": [] }, "gridPos": { - "h": 5, - "w": 7, - "x": 8, - "y": 20 + "h": 7, + "w": 8, + "x": 0, + "y": 42 }, - "id": 27, + "id": 5, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, - "mode": "single", + "mode": "multi", "sort": "none" } }, - "pluginVersion": "12.2.0", + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.extra_info.page_faults\")\r\n |> derivative(unit: v.windowPeriod)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.opLatencies.reads.latency\"\r\n or r[\"_field\"] == \"serverStatus.opLatencies.reads.ops\"\r\n or r[\"_field\"] == \"serverStatus.opLatencies.writes.latency\"\r\n or r[\"_field\"] == \"serverStatus.opLatencies.writes.ops\"\r\n or r[\"_field\"] == \"serverStatus.opLatencies.commands.latency\"\r\n or r[\"_field\"] == \"serverStatus.opLatencies.commands.ops\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> pivot(rowKey: [\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n reads_avg_us: if r[\"serverStatus.opLatencies.reads.ops\"] > 0.0\r\n then r[\"serverStatus.opLatencies.reads.latency\"] / r[\"serverStatus.opLatencies.reads.ops\"]\r\n else 0.0,\r\n writes_avg_us: if r[\"serverStatus.opLatencies.writes.ops\"] > 0.0\r\n then r[\"serverStatus.opLatencies.writes.latency\"] / r[\"serverStatus.opLatencies.writes.ops\"]\r\n else 0.0,\r\n commands_avg_us: if r[\"serverStatus.opLatencies.commands.ops\"] > 0.0\r\n then r[\"serverStatus.opLatencies.commands.latency\"] / r[\"serverStatus.opLatencies.commands.ops\"]\r\n else 0.0,\r\n }))\r\n", "refId": "A" } ], - "title": "Page faults", + "title": "Average Operation Latency", "type": "timeseries" }, { @@ -1056,6 +1697,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "Rate of serverStatus.opcounters per second by type: insert, query, update, delete, getmore, command. Per MongoDB docs, opcounters treats operations that affect multiple documents (bulk insert, multi-update) as a single operation. These values reflect received operations and increment even when operations are not successful. opcounters.command counts all commands except insert, update, delete, and aggregate (serverStatus.opcounters).\n\nHow to read: multiple lines showing ops/sec per operation type (derivative of cumulative counters).\n\nTroubleshooting: sudden drops may indicate blocking -- check 'Current Queue writers / readers' and 'WiredTiger Tickets'. Sudden spikes may cause resource contention. Use alongside 'Average Operation Latency' to distinguish throughput-driven from latency-driven performance issues.", "fieldConfig": { "defaults": { "color": { @@ -1078,7 +1720,7 @@ "viz": false }, "insertNulls": false, - "lineInterpolation": "linear", + "lineInterpolation": "stepAfter", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -1114,15 +1756,20 @@ }, "gridPos": { "h": 7, - "w": 9, - "x": 15, - "y": 20 + "w": 7, + "x": 8, + "y": 42 }, - "id": 2, + "id": 9, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, @@ -1132,27 +1779,18 @@ "sort": "none" } }, - "pluginVersion": "12.2.0", + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", - "uid": "influxdb" + "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.connections.active\" or r[\"_field\"] == \"serverStatus.connections.current\" )\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> yield(name: \"last\")\r\n\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => \r\n r[\"_field\"] == \"serverStatus.opcounters.update\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.query\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.insert\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.getmore\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.delete\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.command\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)", "refId": "A" - }, - { - "datasource": { - "type": "influxdb", - "uid": "influxdb" - }, - "hide": false, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.connections.totalCreated\")\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> derivative(unit: v.windowPeriod)", - "refId": "B" } ], - "title": "Connections", + "title": "Operations", "type": "timeseries" }, { @@ -1160,6 +1798,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "Rate of serverStatus.network.numRequests per second. Per MongoDB docs, numRequests is the total number of distinct requests the server has received. Use this value to provide context for network.bytesIn and network.bytesOut to ensure network utilization is consistent with expectations (serverStatus.network).\n\nHow to read: single line showing requests/sec (derivative of cumulative counter).\n\nTroubleshooting: a sudden drop may indicate network issues or client-side failures. A spike correlates with increased load -- compare with 'Operations' to see if requests translate to actual database operations.", "fieldConfig": { "defaults": { "color": { @@ -1171,7 +1810,8 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", - "barAlignment": 0, + "barAlignment": -1, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1188,6 +1828,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -1197,13 +1838,14 @@ "mode": "off" } }, - "decimals": 1, + "fieldMinMax": true, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -1211,40 +1853,47 @@ } ] }, - "unit": "ns" + "unit": "reqps" }, "overrides": [] }, "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 25 + "h": 5, + "w": 9, + "x": 15, + "y": 44 }, - "id": 5, + "id": 28, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.opLatencies.reads.latency\" or r[\"_field\"] == \"serverStatus.opLatencies.writes.latency\" or r[\"_field\"] == \"serverStatus.opLatencies.commands.latency\" )\r\n |> derivative(unit: v.windowPeriod)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.network.numRequests\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: v.windowPeriod)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", "refId": "A" } ], - "title": "Latency", + "title": "Network Requests", "type": "timeseries" }, { @@ -1252,7 +1901,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "", + "description": "Shows serverStatus.globalLock.currentQueue.readers and currentQueue.writers. Per MongoDB docs, currentQueue.readers is the number of operations currently queued waiting for the read lock; currentQueue.writers is the same for the write lock. A consistently small queue, particularly of shorter operations, should cause no concern (serverStatus.globalLock).\n\nHow to read: two lines showing queued reader and writer counts over time.\n\nTroubleshooting: a persistent non-zero queue indicates lock contention -- this is one of the first panels to check during 'slow database' investigations. Write queue buildup often correlates with WiredTiger ticket exhaustion. Correlate with 'WiredTiger Tickets' and 'Average Operation Latency'.", "fieldConfig": { "defaults": { "color": { @@ -1265,6 +1914,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1274,13 +1924,14 @@ "viz": false }, "insertNulls": false, - "lineInterpolation": "stepAfter", + "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -1295,7 +1946,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -1307,35 +1959,42 @@ "overrides": [] }, "gridPos": { - "h": 7, - "w": 7, - "x": 8, - "y": 25 + "h": 6, + "w": 8, + "x": 0, + "y": 49 }, - "id": 9, + "id": 29, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => \r\n r[\"_field\"] == \"serverStatus.opcounters.update\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.query\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.insert\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.getmore\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.delete\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.command\")\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.globalLock.currentQueue.writers\" or r[\"_field\"] == \"serverStatus.globalLock.currentQueue.readers\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> yield(name: \"last\")", "refId": "A" } ], - "title": "Operations", + "title": "Current Queue writers / readers", "type": "timeseries" }, { @@ -1343,6 +2002,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "Shows replSetGetStatus.members.N.health for each replica set member. Health = 1 means the member is reachable and healthy; health = 0 means the member is unreachable or down.\n\nHow to read: one line per replica member showing binary health state over time.\n\nTroubleshooting: a transition from 1 to 0 means a member became unavailable -- check for network partitions, disk failures, or OOM kills. Correlate the exact timestamp of the health drop with other panels (CPU, memory, disk) to identify root cause.", "fieldConfig": { "defaults": { "color": { @@ -1355,8 +2015,9 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 25, "gradientMode": "none", "hideFrom": { "legend": false, @@ -1364,68 +2025,79 @@ "viz": false }, "insertNulls": false, - "lineInterpolation": "linear", + "lineInterpolation": "stepAfter", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", - "mode": "none" + "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", "value": 80 } ] - } + }, + "unit": "none" }, "overrides": [] }, "gridPos": { - "h": 5, - "w": 9, - "x": 15, - "y": 27 + "h": 6, + "w": 7, + "x": 8, + "y": 49 }, - "id": 28, + "id": 16, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.network.numRequests\")\r\n |> derivative(unit: v.windowPeriod)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] =~ /replSetGetStatus\\.members\\.\\d+\\.health/)\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> yield(name: \"last\")", "refId": "A" } ], - "title": "Network requests", + "title": "Replica Members Health", "type": "timeseries" }, { @@ -1433,6 +2105,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "Shows replication lag per member, computed as the difference between serverStatus.localTime and replSetGetStatus.members.N.lastAppliedWallTime in seconds.\n\nHow to read: one line per member showing lag in seconds. Zero means the member is fully caught up.\n\nTroubleshooting: increasing lag on a secondary indicates it cannot keep up with the primary's write rate. Check the secondary's disk I/O, CPU, and network bandwidth. Lag spikes during checkpoint intervals suggest I/O contention. Correlate with 'FlowControl isLagged' to see if flow control is throttling the primary.", "fieldConfig": { "defaults": { "color": { @@ -1444,7 +2117,9 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1461,6 +2136,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -1475,47 +2151,56 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", "value": 80 } ] - } + }, + "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 6, - "w": 8, - "x": 0, - "y": 32 + "w": 9, + "x": 15, + "y": 49 }, - "id": 29, + "id": 15, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.globalLock.currentQueue.writers\" or r[\"_field\"] == \"serverStatus.globalLock.currentQueue.readers\")\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> yield(name: \"last\")", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] =~ /replSetGetStatus\\.members\\.\\d+\\.lastAppliedWallTime|serverStatus\\.localTime/)\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> pivot(rowKey:[\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n member_0: if exists r[\"replSetGetStatus.members.0.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.0.lastAppliedWallTime\"])) / 1000.0 else 0.0,\r\n member_1: if exists r[\"replSetGetStatus.members.1.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.1.lastAppliedWallTime\"])) / 1000.0 else 0.0,\r\n member_2: if exists r[\"replSetGetStatus.members.2.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.2.lastAppliedWallTime\"])) / 1000.0 else 0.0,\r\n member_3: if exists r[\"replSetGetStatus.members.3.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.3.lastAppliedWallTime\"])) / 1000.0 else 0.0, // Add more members as needed\r\n }))\r\n |> yield(name: \"differences\")\r\n", "refId": "A" } ], - "title": "Current Queue writers / readers", + "title": "Replica Members Lag", "type": "timeseries" }, { @@ -1523,6 +2208,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "Rate of serverStatus.opLatencies.reads.ops, opLatencies.writes.ops, and opLatencies.commands.ops per second. Per MongoDB docs, opLatencies contains operation latencies for the instance as a whole (serverStatus.opLatencies).\n\nHow to read: three lines showing the operation rate (ops/sec) per latency class -- this is the denominator used to compute average latency.\n\nTroubleshooting: compare with 'Average Operation Latency' -- if ops/sec drops while latency rises, individual operations are getting slower. If both rise together, the system is under increased load. A divergence between reads.ops and writes.ops can indicate workload imbalance.", "fieldConfig": { "defaults": { "color": { @@ -1535,8 +2221,9 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 25, + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, @@ -1544,29 +2231,30 @@ "viz": false }, "insertNulls": false, - "lineInterpolation": "stepAfter", + "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", - "mode": "normal" + "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -1574,41 +2262,47 @@ } ] }, - "unit": "none" + "unit": "µs" }, "overrides": [] }, "gridPos": { - "h": 6, - "w": 7, - "x": 8, - "y": 32 + "h": 5, + "w": 8, + "x": 0, + "y": 55 }, - "id": 16, + "id": 10, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, - "pluginVersion": "10.4.3", + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"replSetGetStatus.members.0.health\" or r[\"_field\"] == \"replSetGetStatus.members.1.health\" or r[\"_field\"] == \"replSetGetStatus.members.2.health\")\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> yield(name: \"last\")", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.opLatencies.reads.ops\" or r[\"_field\"] == \"serverStatus.opLatencies.writes.ops\" or r[\"_field\"] == \"serverStatus.opLatencies.commands.ops\" )\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s)\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n", "refId": "A" } ], - "title": "Replica members health", + "title": "Operations Latencies Op", "type": "timeseries" }, { @@ -1616,6 +2310,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "Shows serverStatus.uptime -- the number of seconds the current MongoDB process has been active (serverStatus.uptime).\n\nHow to read: a monotonically increasing line. A drop to zero or a sudden decrease means the mongod process restarted.\n\nTroubleshooting: identify restart timestamps and correlate with other panels to determine cause -- OOM kill (check 'mongod Memory (RSS / Virtual)' and 'Swap Memory'), crash (check 'Asserts'), or manual intervention.", "fieldConfig": { "defaults": { "color": { @@ -1629,6 +2324,7 @@ "axisPlacement": "auto", "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1645,6 +2341,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -1659,7 +2356,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -1667,40 +2365,48 @@ } ] }, - "unit": "ms" + "unit": "none" }, "overrides": [] }, "gridPos": { - "h": 6, - "w": 9, - "x": 15, - "y": 32 + "h": 5, + "w": 7, + "x": 8, + "y": 55 }, - "id": 15, + "id": 20, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "none", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] =~ /replSetGetStatus\\.members\\.\\d+\\.lastAppliedWallTime|serverStatus\\.localTime/)\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> pivot(rowKey:[\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n member_0: if exists r[\"replSetGetStatus.members.0.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.0.lastAppliedWallTime\"])) / 1000.0 else 0.0,\r\n member_1: if exists r[\"replSetGetStatus.members.1.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.1.lastAppliedWallTime\"])) / 1000.0 else 0.0,\r\n member_2: if exists r[\"replSetGetStatus.members.2.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.2.lastAppliedWallTime\"])) / 1000.0 else 0.0,\r\n member_3: if exists r[\"replSetGetStatus.members.3.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.3.lastAppliedWallTime\"])) / 1000.0 else 0.0, // Add more members as needed\r\n }))\r\n |> yield(name: \"differences\")\r\n", + "hide": false, + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.uptime\" )\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n\r\n", "refId": "A" } ], - "title": "Replica members lag", + "title": "Server Uptime", "type": "timeseries" }, { @@ -1708,7 +2414,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Operation latencies for the instance as a whole.", + "description": "Shows replSetGetStatus.members.N.pingMs -- the round-trip time in milliseconds from this member to other replica set members.\n\nHow to read: one line per member showing ping latency over time.\n\nTroubleshooting: elevated ping times indicate network latency or congestion between members. Ping spikes that correlate with replication lag suggest the network is a bottleneck for oplog replication. Compare with 'TCP' panels for underlying transport issues.", "fieldConfig": { "defaults": { "color": { @@ -1721,6 +2427,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1737,6 +2444,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -1751,7 +2459,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -1759,40 +2468,47 @@ } ] }, - "unit": "µs" + "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 5, - "w": 8, - "x": 0, - "y": 38 + "w": 9, + "x": 15, + "y": 55 }, - "id": 10, + "id": 25, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.opLatencies.reads.ops\" or r[\"_field\"] == \"serverStatus.opLatencies.writes.ops\" or r[\"_field\"] == \"serverStatus.opLatencies.commands.ops\" )\r\n |> derivative(unit: 1s)\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] =~ /replSetGetStatus\\.members\\.\\d+\\.pingMs/)\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n", "refId": "A" } ], - "title": "Operations latencies op", + "title": "Replica Members Ping", "type": "timeseries" }, { @@ -1800,7 +2516,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "", + "description": "Shows the derivative of serverStatus.flowControl.isLaggedTimeMicros -- the cumulative time flow control has been engaged. Per MongoDB docs, flow control engages when the majority committed lag grows close to flowControlTargetLagSeconds; writes on the primary must then obtain tickets before taking locks (serverStatus.flowControl).\n\nHow to read: a non-zero derivative means flow control was actively throttling the primary during that interval.\n\nTroubleshooting: sustained non-zero values mean the primary is being throttled to let secondaries catch up. Check 'Replica Members Lag', secondary disk I/O, and 'Opcounters Replica' to understand why secondaries are falling behind.", "fieldConfig": { "defaults": { "color": { @@ -1812,8 +2528,8 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", - "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1824,12 +2540,13 @@ }, "insertNulls": false, "lineInterpolation": "linear", - "lineWidth": 1, + "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -1844,7 +2561,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -1852,41 +2570,47 @@ } ] }, - "unit": "none" + "unit": "µs" }, "overrides": [] }, "gridPos": { - "h": 5, - "w": 7, - "x": 8, - "y": 38 + "h": 6, + "w": 8, + "x": 0, + "y": 60 }, - "id": 20, + "id": 14, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "hide": false, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.uptime\" )\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.flowControl.isLaggedTimeMicros\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> derivative(unit: v.windowPeriod)\r\n |> yield(name: \"last\")", "refId": "A" } ], - "title": "Server Uptime", + "title": "FlowControl isLagged", "type": "timeseries" }, { @@ -1894,6 +2618,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "Rate of serverStatus.asserts by type: regular, warning, msg, user, rollovers. Per MongoDB docs, assertions are internal checks for errors that occur while the database is operating. Non-zero values are uncommon. asserts.user are errors clients may generate (e.g., out of disk space, duplicate key). asserts.rollovers counts how many times assert counters have rolled over after 2^30 assertions (serverStatus.asserts).\n\nHow to read: multiple lines showing assert rate/sec per type.\n\nTroubleshooting: any sustained increase in regular or msg asserts may indicate bugs or data corruption -- examine the MongoDB log. High user asserts indicate application-level errors (duplicate keys, validation failures). Check application logs for corresponding errors.", "fieldConfig": { "defaults": { "color": { @@ -1906,6 +2631,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1922,6 +2648,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -1936,48 +2663,55 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", "value": 80 } ] - }, - "unit": "ms" + } }, "overrides": [] }, "gridPos": { - "h": 5, - "w": 9, - "x": 15, - "y": 38 + "h": 7, + "w": 7, + "x": 8, + "y": 60 }, - "id": 25, + "id": 30, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] =~ /replSetGetStatus\\.members\\.\\d+\\.pingMs/)\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.asserts.warning\" or r[\"_field\"] == \"serverStatus.asserts.user\" or r[\"_field\"] == \"serverStatus.asserts.rollovers\" or r[\"_field\"] == \"serverStatus.asserts.regular\" or r[\"_field\"] == \"serverStatus.asserts.msg\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: v.windowPeriod)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", "refId": "A" } ], - "title": "Replica members ping", + "title": "Asserts", "type": "timeseries" }, { @@ -1985,6 +2719,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "Rate of serverStatus.metrics.queryExecutor metrics: scanned (total index items scanned, equivalent to totalKeysExamined in explain()), scannedObjects (total documents scanned, equivalent to totalDocsExamined), and collectionScans.total (queries that performed a collection scan) per second (serverStatus.metrics.queryExecutor).\n\nHow to read: multiple lines showing rates. High scannedObjects means many documents examined; high collectionScans.total means queries are not using indexes.\n\nTroubleshooting: high scannedObjects relative to documents returned indicates inefficient queries -- run explain() to identify missing indexes. High collectionScans.total is a red flag for production workloads; identify the queries via the profiler.", "fieldConfig": { "defaults": { "color": { @@ -1997,6 +2732,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2007,12 +2743,13 @@ }, "insertNulls": false, "lineInterpolation": "linear", - "lineWidth": 2, + "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -2027,7 +2764,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -2035,65 +2773,47 @@ } ] }, - "unit": "µs" + "unit": "short" }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "serverStatus.flowControl.isLaggedTimeMicros" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": true, - "viz": true - } - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 6, - "w": 8, - "x": 0, - "y": 43 + "w": 9, + "x": 15, + "y": 60 }, - "id": 14, + "id": 31, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.flowControl.isLaggedTimeMicros\")\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> derivative(unit: v.windowPeriod)\r\n |> yield(name: \"last\")", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.metrics.queryExecutor.scannedObjects\" or r[\"_field\"] == \"serverStatus.metrics.queryExecutor.scanned\" or r[\"_field\"] == \"serverStatus.metrics.queryExecutor.collectionScans.total\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: v.windowPeriod)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", "refId": "A" } ], - "title": "FlowControl isLagged", + "title": "Query Executor", "type": "timeseries" }, { @@ -2101,6 +2821,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "Rate of serverStatus.metrics.commands per second. Per MongoDB docs, metrics.commands reports on the use of database commands. opcounters.command counts all commands except insert, update, delete, and aggregate (serverStatus.metrics.commands, serverStatus.opcounters).\n\nHow to read: single line showing command execution rate.\n\nTroubleshooting: a spike in command rate may indicate runaway aggregation pipelines, administrative operations (index builds, compact), or driver heartbeat storms. Correlate with 'Average Operation Latency' and 'CPU Usage' to assess impact.", "fieldConfig": { "defaults": { "color": { @@ -2113,6 +2834,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2129,6 +2851,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -2143,7 +2866,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -2155,35 +2879,42 @@ "overrides": [] }, "gridPos": { - "h": 7, - "w": 7, - "x": 8, - "y": 43 + "h": 6, + "w": 8, + "x": 0, + "y": 66 }, - "id": 30, + "id": 32, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.asserts.warning\" or r[\"_field\"] == \"serverStatus.asserts.user\" or r[\"_field\"] == \"serverStatus.asserts.rollovers\" or r[\"_field\"] == \"serverStatus.asserts.regular\" or r[\"_field\"] == \"serverStatus.asserts.msg\")\r\n |> derivative(unit: v.windowPeriod)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.metrics.commands.createIndexes.total\" or r[\"_field\"] == \"serverStatus.metrics.commands.dataSize.total\" or r[\"_field\"] == \"serverStatus.metrics.commands.dropIndexes.total\" or r[\"_field\"] == \"serverStatus.metrics.commands.listCollections.total\" or r[\"_field\"] == \"serverStatus.metrics.commands.listDatabases.total\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n|> derivative(unit: v.windowPeriod)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", "refId": "A" } ], - "title": "Asserts", + "title": "Commands", "type": "timeseries" }, { @@ -2191,6 +2922,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "Shows dbStats.avgObjSize -- the average size of each document in bytes, computed as dataSize divided by the number of documents. Per MongoDB docs, the scale argument does not affect avgObjSize (dbStats.avgObjSize).\n\nHow to read: single line showing average document size over time.\n\nTroubleshooting: a sudden increase may indicate documents growing from array pushes or embedded document growth, which affects WiredTiger cache efficiency, network bandwidth, and replication lag. A decrease after TTL cleanup or data migration is expected.", "fieldConfig": { "defaults": { "color": { @@ -2203,6 +2935,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2219,6 +2952,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -2229,19 +2963,20 @@ } }, "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", "value": 80 } ] - }, - "unit": "short" + } }, "overrides": [] }, @@ -2249,32 +2984,39 @@ "h": 6, "w": 9, "x": 15, - "y": 43 + "y": 66 }, - "id": 31, + "id": 34, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.metrics.queryExecutor.scannedObjects\" or r[\"_field\"] == \"serverStatus.metrics.queryExecutor.scanned\" or r[\"_field\"] == \"serverStatus.metrics.queryExecutor.collectionScans.total\")\r\n |> derivative(unit: v.windowPeriod)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"local.oplog.rs.stats.storageStats.avgObjSize\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", "refId": "A" } ], - "title": "Query Executor", + "title": "avg Obj Size", "type": "timeseries" }, { @@ -2282,6 +3024,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "Shows serverStatus.metrics.cursor.open.total -- the number of cursors MongoDB is maintaining for clients. Per MongoDB docs, typically this value is small or zero; however, if there is a queue, stale tailable cursors, or a large number of operations, this value may increase. metrics.cursor.timedOut tracks cursors that timed out (serverStatus.metrics.cursor).\n\nHow to read: single line showing open cursor count over time.\n\nTroubleshooting: a growing cursor count may indicate cursor leaks in the application (not calling close()). Cursor timeouts (default 10 min) normally clean up idle cursors. If timedOut is also growing, applications are abandoning cursors without exhausting or closing them.", "fieldConfig": { "defaults": { "color": { @@ -2294,6 +3037,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2310,6 +3054,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -2320,11 +3065,13 @@ } }, "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -2337,34 +3084,41 @@ }, "gridPos": { "h": 6, - "w": 8, - "x": 0, - "y": 49 + "w": 7, + "x": 8, + "y": 67 }, - "id": 32, + "id": 33, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.metrics.commands.createIndexes.total\" or r[\"_field\"] == \"serverStatus.metrics.commands.dataSize.total\" or r[\"_field\"] == \"serverStatus.metrics.commands.dropIndexes.total\" or r[\"_field\"] == \"serverStatus.metrics.commands.listCollections.total\" or r[\"_field\"] == \"serverStatus.metrics.commands.listDatabases.total\")\r\n|> derivative(unit: v.windowPeriod)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.metrics.cursor.open.noTimeout\" or r[\"_field\"] == \"serverStatus.metrics.cursor.open.pinned\" or r[\"_field\"] == \"serverStatus.metrics.cursor.open.total\" or r[\"_field\"] == \"serverStatus.metrics.cursor.timedOut\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)", "refId": "A" } ], - "title": "Commands", + "title": "Cursors", "type": "timeseries" }, { @@ -2372,6 +3126,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "Rate of serverStatus.metrics.document counters per second: deleted, inserted, returned, updated. Per MongoDB docs, metrics.document reflects document access and modification patterns. Compare with opcounters, which tracks total number of operations -- since one query can return many documents (serverStatus.metrics.document).\n\nHow to read: four lines showing document-level operation rates.\n\nTroubleshooting: high 'returned' with low 'scannedObjects' (from 'Query Executor') indicates efficient queries. High 'returned' with high 'scannedObjects' means poor query targeting. High 'deleted' may indicate TTL index activity or bulk cleanup operations.", "fieldConfig": { "defaults": { "color": { @@ -2384,6 +3139,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2400,6 +3156,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -2410,12 +3167,12 @@ } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -2427,35 +3184,42 @@ "overrides": [] }, "gridPos": { - "h": 6, - "w": 9, - "x": 15, - "y": 49 + "h": 7, + "w": 8, + "x": 0, + "y": 72 }, - "id": 34, + "id": 36, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"local.oplog.rs.stats.storageStats.avgObjSize\")\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.metrics.document.updated\" or r[\"_field\"] == \"serverStatus.metrics.document.returned\" or r[\"_field\"] == \"serverStatus.metrics.document.inserted\" or r[\"_field\"] == \"serverStatus.metrics.document.deleted\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: v.windowPeriod)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", "refId": "A" } ], - "title": "avg Obj Size", + "title": "Documents (deleted / inserted / returned / updated)", "type": "timeseries" }, { @@ -2463,7 +3227,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "", + "description": "Rate of serverStatus.opcountersRepl per second: insert, query, update, delete, getmore, command. Per MongoDB docs, opcountersRepl reports on database replication operations by type since the mongod instance last started. These numbers grow over time until the next restart (serverStatus.opcountersRepl).\n\nHow to read: multiple lines showing replication operation rates.\n\nTroubleshooting: on a secondary, this shows how fast replication operations are being applied. If this rate drops while the primary's 'Operations' panel shows high throughput, the secondary is falling behind. Correlate with 'Replica Members Lag' and secondary disk I/O.", "fieldConfig": { "defaults": { "color": { @@ -2476,6 +3240,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2492,6 +3257,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -2502,52 +3268,60 @@ } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", "value": 80 } ] - } + }, + "unit": "ops" }, "overrides": [] }, "gridPos": { - "h": 6, - "w": 7, - "x": 8, - "y": 50 + "h": 7, + "w": 9, + "x": 15, + "y": 72 }, - "id": 33, + "id": 38, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.metrics.cursor.open.noTimeout\" or r[\"_field\"] == \"serverStatus.metrics.cursor.open.pinned\" or r[\"_field\"] == \"serverStatus.metrics.cursor.open.total\" or r[\"_field\"] == \"serverStatus.metrics.cursor.timedOut\")\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.opcountersRepl.command\" or r[\"_field\"] == \"serverStatus.opcountersRepl.delete\" or r[\"_field\"] == \"serverStatus.opcountersRepl.getmore\" or r[\"_field\"] == \"serverStatus.opcountersRepl.insert\" or r[\"_field\"] == \"serverStatus.opcountersRepl.query\" or r[\"_field\"] == \"serverStatus.opcountersRepl.update\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: v.windowPeriod)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", "refId": "A" } ], - "title": "Cursors", + "title": "Opcounters Replica", "type": "timeseries" }, { @@ -2555,7 +3329,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "", + "description": "Ratio of serverStatus.metrics.queryExecutor.scannedObjects to metrics.document.returned, computed as a per-second rate ratio. Per MongoDB docs, scannedObjects is equivalent to totalDocsExamined in explain(); document.returned is the total number of documents returned by queries (serverStatus.metrics.queryExecutor, serverStatus.metrics.document).\n\nHow to read: single line showing the scanned-to-returned ratio. A ratio of 1.0 means every scanned document was returned (perfect targeting).\n\nTroubleshooting: ratios significantly above 1.0 indicate queries scanning many documents to return few results -- the classic sign of missing or suboptimal indexes. This is often the number one cause of unnecessary disk I/O and CPU load. Run explain() on slow queries to identify the problem.", "fieldConfig": { "defaults": { "color": { @@ -2568,6 +3342,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2578,12 +3353,13 @@ }, "insertNulls": false, "lineInterpolation": "linear", - "lineWidth": 1, + "lineWidth": 4, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -2593,12 +3369,15 @@ "mode": "off" } }, + "decimals": 2, "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -2610,42 +3389,63 @@ "overrides": [] }, "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 55 + "h": 6, + "w": 7, + "x": 8, + "y": 73 }, - "id": 36, + "id": 37, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.metrics.document.updated\" or r[\"_field\"] == \"serverStatus.metrics.document.returned\" or r[\"_field\"] == \"serverStatus.metrics.document.inserted\" or r[\"_field\"] == \"serverStatus.metrics.document.deleted\")\r\n |> derivative(unit: v.windowPeriod)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.metrics.document.returned\" or \r\n r[\"_field\"] == \"serverStatus.metrics.queryExecutor.scannedObjects\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> pivot(\r\n rowKey: [\"_time\"],\r\n columnKey: [\"_field\"],\r\n valueColumn: \"_value\"\r\n )\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n scanned_returned_ratio:\r\n if r[\"serverStatus.metrics.document.returned\"] > 0.0\r\n then r[\"serverStatus.metrics.queryExecutor.scannedObjects\"]\r\n / r[\"serverStatus.metrics.document.returned\"]\r\n else 0.0\r\n }))\r\n |> yield(name: \"scanned_returned_ratio\")\r\n", "refId": "A" } ], - "title": "Documents (deleted / inserted / returned / updated)", + "title": "Query Targeting: Scanned Objects / Returned", "type": "timeseries" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 79 + }, + "id": 21, + "panels": [], + "title": "System", + "type": "row" + }, { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "CPU utilization from systemMetrics.cpu, broken down by user, system, and iowait as a percentage of total CPU time. These are derived from the OS-level CPU counters exposed through FTDC (systemMetrics).\n\nHow to read: three lines showing percentage of CPU time spent in user mode (application), system/kernel mode, and iowait (waiting on disk).\n\nTroubleshooting: high 'user' CPU typically means heavy query processing (sorts, aggregations, index builds). High 'iowait' means the CPU is blocked waiting on disk -- correlate with 'Disk I/O' and 'WiredTiger Block Manager'. High 'system' may indicate excessive context switching from too many threads/connections.", "fieldConfig": { "defaults": { "color": { @@ -2658,6 +3458,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2674,6 +3475,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -2688,47 +3490,56 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", "value": 80 } ] - } + }, + "unit": "percent" }, "overrides": [] }, "gridPos": { - "h": 7, - "w": 9, - "x": 15, - "y": 55 + "h": 9, + "w": 24, + "x": 0, + "y": 80 }, - "id": 38, + "id": 13, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.opcountersRepl.command\" or r[\"_field\"] == \"serverStatus.opcountersRepl.delete\" or r[\"_field\"] == \"serverStatus.opcountersRepl.getmore\" or r[\"_field\"] == \"serverStatus.opcountersRepl.insert\" or r[\"_field\"] == \"serverStatus.opcountersRepl.query\" or r[\"_field\"] == \"serverStatus.opcountersRepl.update\")\r\n |> derivative(unit: v.windowPeriod)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field == \"systemMetrics.cpu.idle_ms\" or r._field == \"systemMetrics.cpu.user_ms\" or r._field == \"systemMetrics.cpu.iowait_ms\" or r._field == \"systemMetrics.cpu.nice_ms\" or r._field == \"systemMetrics.cpu.softirq_ms\" or r._field == \"systemMetrics.cpu.steal_ms\" or r._field == \"systemMetrics.cpu.system_ms\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> pivot(rowKey:[\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\r\n |> derivative(unit: 1s, nonNegative: true, columns: [\"systemMetrics.cpu.idle_ms\", \"systemMetrics.cpu.user_ms\", \"systemMetrics.cpu.iowait_ms\", \"systemMetrics.cpu.nice_ms\", \"systemMetrics.cpu.softirq_ms\", \"systemMetrics.cpu.steal_ms\", \"systemMetrics.cpu.system_ms\"])\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n idle_ms: r[\"systemMetrics.cpu.idle_ms\"],\r\n user_ms: r[\"systemMetrics.cpu.user_ms\"],\r\n iowait_ms: r[\"systemMetrics.cpu.iowait_ms\"],\r\n nice_ms: r[\"systemMetrics.cpu.nice_ms\"],\r\n softirq_ms: r[\"systemMetrics.cpu.softirq_ms\"],\r\n steal_ms: r[\"systemMetrics.cpu.steal_ms\"],\r\n system_ms: r[\"systemMetrics.cpu.system_ms\"],\r\n total_ms: r[\"systemMetrics.cpu.idle_ms\"] + r[\"systemMetrics.cpu.user_ms\"] + r[\"systemMetrics.cpu.iowait_ms\"] + r[\"systemMetrics.cpu.nice_ms\"] + r[\"systemMetrics.cpu.softirq_ms\"] + r[\"systemMetrics.cpu.steal_ms\"] + r[\"systemMetrics.cpu.system_ms\"]\r\n }))\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n user: 100.0 * float(v: r.user_ms) / float(v: r.total_ms),\r\n system: 100.0 * float(v: r.system_ms) / float(v: r.total_ms),\r\n iowait: 100.0 * float(v: r.iowait_ms) / float(v: r.total_ms),\r\n }))\r\n", "refId": "A" } ], - "title": "Opcounters Replica", + "title": "CPU Usage", "type": "timeseries" }, { @@ -2736,6 +3547,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "System-level memory metrics from systemMetrics (derived from /proc/meminfo). Shows available and free memory at the OS level (systemMetrics).\n\nHow to read: multiple lines showing memory categories over time.\n\nTroubleshooting: if available memory is low and swap is active, the system is under memory pressure affecting all processes including mongod. Compare with 'mongod Memory (RSS / Virtual)' to determine how much system RAM is consumed by the database vs other services.", "fieldConfig": { "defaults": { "color": { @@ -2748,6 +3560,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2758,12 +3571,13 @@ }, "insertNulls": false, "lineInterpolation": "linear", - "lineWidth": 4, + "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -2773,74 +3587,69 @@ "mode": "off" } }, - "decimals": 2, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", "value": 80 } ] - } + }, + "unit": "deckbytes" }, "overrides": [] }, "gridPos": { - "h": 6, - "w": 7, - "x": 8, - "y": 56 + "h": 8, + "w": 12, + "x": 0, + "y": 89 }, - "id": 37, + "id": 40, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.metrics.document.returned\" or \r\n r[\"_field\"] == \"serverStatus.metrics.queryExecutor.scannedObjects\")\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> pivot(\r\n rowKey: [\"_time\"],\r\n columnKey: [\"_field\"],\r\n valueColumn: \"_value\"\r\n )\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n scanned_returned_ratio: r[\"serverStatus.metrics.queryExecutor.scannedObjects\"] / r[\"serverStatus.metrics.document.returned\"]\r\n }))\r\n |> yield(name: \"scanned_returned_ratio\")\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"systemMetrics.memory.MemAvailable_kb\" or r[\"_field\"] == \"systemMetrics.memory.MemFree_kb\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> yield(name: \"mean\")", "refId": "A" } ], - "title": "Query Targeting: Scanned Objects / Returned ", + "title": "System Memory", "type": "timeseries" }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 62 - }, - "id": 21, - "panels": [], - "title": "System", - "type": "row" - }, { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "Swap space usage from systemMetrics (systemMetrics).\n\nHow to read: lines showing swap used and swap total over time.\n\nTroubleshooting: ANY swap usage on a MongoDB server is a critical concern. Swapping replaces nanosecond RAM access with millisecond disk access, causing order-of-magnitude latency increases. If swap is in use, check 'System Memory' for overall pressure and consider increasing RAM or reducing WiredTiger cache size (--wiredTigerCacheSizeGB) to leave more memory for the OS page cache.", "fieldConfig": { "defaults": { "color": { @@ -2852,8 +3661,8 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", - "axisSoftMax": 10, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2870,6 +3679,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -2884,7 +3694,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -2892,40 +3703,47 @@ } ] }, - "unit": "decbytes" + "unit": "deckbytes" }, "overrides": [] }, "gridPos": { - "h": 5, - "w": 8, - "x": 0, - "y": 63 + "h": 8, + "w": 12, + "x": 12, + "y": 89 }, - "id": 11, + "id": 39, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.network.physicalBytesIn\" or r[\"_field\"] == \"serverStatus.network.physicalBytesOut\")\r\n |> derivative(unit: 1s)\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"systemMetrics.memory.SwapFree_kb\" or r[\"_field\"] == \"systemMetrics.memory.SwapTotal_kb\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> yield(name: \"mean\")", "refId": "A" } ], - "title": "Network In / Out physical", + "title": "Swap Memory", "type": "timeseries" }, { @@ -2933,6 +3751,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "Shows dbStats.storageSize -- the sum of disk space allocated to all collections for document storage, including free space. Per MongoDB docs, storageSize does not decrease when documents are removed or shrunk; it may be smaller than dataSize on WiredTiger with compression enabled. storageSize does not include indexes -- see indexSize (dbStats).\n\nHow to read: single line showing storage allocation over time.\n\nTroubleshooting: sudden growth may indicate unexpected data ingestion or large index creation. Compare with 'avg Obj Size' and 'Documents' panel rates. Slow steady growth is normal.", "fieldConfig": { "defaults": { "color": { @@ -2944,8 +3763,8 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", - "axisSoftMax": 10, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2962,6 +3781,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -2976,47 +3796,56 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", "value": 80 } ] - } + }, + "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 6, "w": 8, - "x": 8, - "y": 63 + "x": 0, + "y": 97 }, - "id": 4, + "id": 35, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.network.bytesOut\" or r[\"_field\"] == \"serverStatus.network.bytesIn\")\r\n |> map(fn: (r) => ({ r with _value: r._value / 1024 /1024 }))\r\n |> derivative(unit: 1s)\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"local.oplog.rs.stats.storageStats.freeStorageSize\" or r[\"_field\"] == \"local.oplog.rs.stats.storageStats.storageSize\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> yield(name: \"mean\")", "refId": "A" } ], - "title": "Network In / Out", + "title": "Storage Size", "type": "timeseries" }, { @@ -3024,7 +3853,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "", + "description": "Rate of disk read and write operations per second (IOPS) per device from systemMetrics.disks.*.reads and systemMetrics.disks.*.writes (systemMetrics).\n\nHow to read: lines showing IOPS per device.\n\nTroubleshooting: high IOPS approaching device limits causes increased latency. Compare with 'Disk I/O' (bytes) to determine if the workload is many small I/O operations vs fewer large ones. SSD and HDD have very different IOPS limits (hundreds of thousands vs hundreds).", "fieldConfig": { "defaults": { "color": { @@ -3037,6 +3866,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3053,6 +3883,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -3067,7 +3898,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -3075,40 +3907,47 @@ } ] }, - "unit": "bytes" + "unit": "iops" }, "overrides": [] }, "gridPos": { - "h": 6, + "h": 7, "w": 8, - "x": 16, - "y": 63 + "x": 8, + "y": 97 }, - "id": 35, + "id": 17, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"local.oplog.rs.stats.storageStats.freeStorageSize\" or r[\"_field\"] == \"local.oplog.rs.stats.storageStats.storageSize\")\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> yield(name: \"mean\")", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..*\\.writes$/ or r._field =~ /^systemMetrics\\.disks\\..*\\.reads$/)\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> derivative(unit: 1s)\r\n", "refId": "A" } ], - "title": "Storage Size", + "title": "Disk Writes and Reads", "type": "timeseries" }, { @@ -3116,6 +3955,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "Rate of TCP segments received (InSegs) and sent (OutSegs) per second from systemMetrics.netstat.Tcp (systemMetrics.netstat).\n\nHow to read: two lines showing TCP segment rates.\n\nTroubleshooting: shows raw TCP-level traffic volume. A high InSegs/OutSegs ratio on a secondary indicates it receives more replication data than it sends. Sudden drops indicate network outages. Compare with 'Network In / Out' for application-level context.", "fieldConfig": { "defaults": { "color": { @@ -3128,6 +3968,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3144,6 +3985,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -3158,7 +4000,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -3166,40 +4009,47 @@ } ] }, - "unit": "percent" + "unit": "none" }, "overrides": [] }, "gridPos": { - "h": 6, + "h": 5, "w": 8, - "x": 0, - "y": 68 + "x": 16, + "y": 97 }, - "id": 13, + "id": 43, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field == \"systemMetrics.cpu.idle_ms\" or r._field == \"systemMetrics.cpu.user_ms\" or r._field == \"systemMetrics.cpu.iowait_ms\" or r._field == \"systemMetrics.cpu.nice_ms\" or r._field == \"systemMetrics.cpu.softirq_ms\" or r._field == \"systemMetrics.cpu.steal_ms\" or r._field == \"systemMetrics.cpu.system_ms\")\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> pivot(rowKey:[\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\r\n |> derivative(unit: 1s, nonNegative: true, columns: [\"systemMetrics.cpu.idle_ms\", \"systemMetrics.cpu.user_ms\", \"systemMetrics.cpu.iowait_ms\", \"systemMetrics.cpu.nice_ms\", \"systemMetrics.cpu.softirq_ms\", \"systemMetrics.cpu.steal_ms\", \"systemMetrics.cpu.system_ms\"])\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n idle_ms: r[\"systemMetrics.cpu.idle_ms\"],\r\n user_ms: r[\"systemMetrics.cpu.user_ms\"],\r\n iowait_ms: r[\"systemMetrics.cpu.iowait_ms\"],\r\n nice_ms: r[\"systemMetrics.cpu.nice_ms\"],\r\n softirq_ms: r[\"systemMetrics.cpu.softirq_ms\"],\r\n steal_ms: r[\"systemMetrics.cpu.steal_ms\"],\r\n system_ms: r[\"systemMetrics.cpu.system_ms\"],\r\n total_ms: r[\"systemMetrics.cpu.idle_ms\"] + r[\"systemMetrics.cpu.user_ms\"] + r[\"systemMetrics.cpu.iowait_ms\"] + r[\"systemMetrics.cpu.nice_ms\"] + r[\"systemMetrics.cpu.softirq_ms\"] + r[\"systemMetrics.cpu.steal_ms\"] + r[\"systemMetrics.cpu.system_ms\"]\r\n }))\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n user: 100.0 * float(v: r.user_ms) / float(v: r.total_ms),\r\n system: 100.0 * float(v: r.system_ms) / float(v: r.total_ms),\r\n iowait: 100.0 * float(v: r.iowait_ms) / float(v: r.total_ms),\r\n }))\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"systemMetrics.netstat.Tcp:InSegs\" or r[\"_field\"] == \"systemMetrics.netstat.Tcp:OutSegs\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> derivative(unit: v.windowPeriod)", "refId": "A" } ], - "title": "CPU Usage", + "title": "TCP: InSegs / OutSegs", "type": "timeseries" }, { @@ -3207,6 +4057,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "Rate of serverStatus.network.bytesIn and network.bytesOut per second. Per MongoDB docs, bytesIn is the total number of logical bytes received; bytesOut is the total logical bytes sent. Logical bytes are the exact number of bytes in the data (serverStatus.network).\n\nHow to read: two lines showing logical network throughput in bytes/sec.\n\nTroubleshooting: large bytesOut may indicate queries returning too much data -- check query projections and limit clauses. Compare logical vs physical bytes to assess compression ratio.", "fieldConfig": { "defaults": { "color": { @@ -3219,6 +4070,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3235,6 +4087,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -3249,47 +4102,56 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", "value": 80 } ] - } + }, + "unit": "decbytes" }, "overrides": [] }, "gridPos": { - "h": 7, + "h": 6, "w": 8, - "x": 8, - "y": 69 + "x": 0, + "y": 103 }, - "id": 17, + "id": 4, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..*\\.writes$/ or r._field =~ /^systemMetrics\\.disks\\..*\\.reads$/)\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> derivative(unit: 1s)\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.network.bytesOut\" or r[\"_field\"] == \"serverStatus.network.bytesIn\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> map(fn: (r) => ({ r with _value: r._value / 1024 /1024 }))\r\n |> derivative(unit: 1s)\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n", "refId": "A" } ], - "title": "Disk writes and reads", + "title": "Network In / Out", "type": "timeseries" }, { @@ -3297,6 +4159,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "Disk write throughput per device from systemMetrics.disks (systemMetrics).\n\nHow to read: lines showing write bytes/sec per device.\n\nTroubleshooting: periodic write spikes every ~60 seconds correlate with WiredTiger checkpoints and are normal. Sustained high writes between checkpoints indicate heavy write workloads or eviction pressure. Compare with 'Dirty Cache Bytes' and 'Pages Evicted'.", "fieldConfig": { "defaults": { "color": { @@ -3308,7 +4171,9 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3325,6 +4190,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -3339,48 +4205,64 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", "value": 80 } ] - }, - "unit": "deckbytes" + } }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, - "x": 16, - "y": 69 + "x": 8, + "y": 104 }, - "id": 40, + "id": 46, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"systemMetrics.memory.MemAvailable_kb\" or r[\"_field\"] == \"systemMetrics.memory.MemFree_kb\")\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> yield(name: \"mean\")", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..*\\.write_sectors$/ or r._field =~ /^systemMetrics\\.disks\\..*\\.write_time_ms$/ or r._field =~ /^systemMetrics\\.disks\\..*\\.writes_merged$/)\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> derivative(unit: v.windowPeriod)", "refId": "A" + }, + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "hide": false, + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..*\\.io_in_progress$/ )\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n\r\n\r\n\r\n", + "refId": "B" } ], - "title": "Memory", + "title": "Disk Writes", "type": "timeseries" }, { @@ -3388,7 +4270,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "", + "description": "Rate of serverStatus.network.physicalBytesIn and network.physicalBytesOut per second. Per MongoDB docs, physicalBytesIn is the total number of physical bytes received over network connections; physicalBytesOut is the total physical bytes sent. Physical bytes are the bytes that actually reside on disk (serverStatus.network).\n\nHow to read: two lines showing physical network throughput in bytes/sec.\n\nTroubleshooting: compare with 'Network In / Out' (logical bytes) to assess compression effectiveness. Saturation of network bandwidth causes connection timeouts and replication lag. Spikes without corresponding operation increases may indicate replication catch-up or background traffic.", "fieldConfig": { "defaults": { "color": { @@ -3400,8 +4282,8 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", - "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3418,6 +4300,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -3432,56 +4315,56 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", "value": 80 } ] - } + }, + "unit": "decbytes" }, "overrides": [] }, "gridPos": { - "h": 7, + "h": 5, "w": 8, "x": 0, - "y": 74 + "y": 109 }, - "id": 47, + "id": 11, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..*\\.io_in_progress$/ or r._field =~ /^systemMetrics\\.disks\\..*\\.io_queued_ms$/ or r._field =~ /^systemMetrics\\.disks\\..*\\.io_time_ms$/)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> derivative(unit: v.windowPeriod)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.network.physicalBytesIn\" or r[\"_field\"] == \"serverStatus.network.physicalBytesOut\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s)\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n", "refId": "A" - }, - { - "datasource": { - "type": "influxdb", - "uid": "P3C6603E967DC8568" - }, - "hide": false, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..*\\.io_in_progress$/ )\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n\r\n\r\n\r\n", - "refId": "B" } ], - "title": "Disk I/O", + "title": "Network In / Out Physical", "type": "timeseries" }, { @@ -3489,6 +4372,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "TCP connection lifecycle counters from systemMetrics.netstat: Tcp.ActiveOpens (new outgoing connections initiated), Tcp.AttemptFails (connection attempts that failed), Tcp.EstabResets (established connections that were reset) (systemMetrics.netstat).\n\nHow to read: three lines showing rates per second.\n\nTroubleshooting: high AttemptFails indicates the server or peers are refusing connections. High EstabResets indicates established connections being unexpectedly terminated -- check for replica member crashes, network partitions, or firewall issues. Correlate with 'Replica Members Health'.", "fieldConfig": { "defaults": { "color": { @@ -3500,8 +4384,8 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", - "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3518,6 +4402,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -3532,14 +4417,16 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", "value": 80 } ] - } + }, + "unit": "none" }, "overrides": [] }, @@ -3547,41 +4434,39 @@ "h": 7, "w": 8, "x": 8, - "y": 76 + "y": 111 }, - "id": 46, + "id": 41, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..*\\.write_sectors$/ or r._field =~ /^systemMetrics\\.disks\\..*\\.write_time_ms$/ or r._field =~ /^systemMetrics\\.disks\\..*\\.writes_merged$/)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> derivative(unit: v.windowPeriod)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"systemMetrics.netstat.Tcp:ActiveOpens\" or r[\"_field\"] == \"systemMetrics.netstat.Tcp:AttemptFails\" or r[\"_field\"] == \"systemMetrics.netstat.Tcp:EstabResets\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> derivative(unit: v.windowPeriod)", "refId": "A" - }, - { - "datasource": { - "type": "influxdb", - "uid": "P3C6603E967DC8568" - }, - "hide": false, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..*\\.io_in_progress$/ )\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n\r\n\r\n\r\n", - "refId": "B" } ], - "title": "Disk writes", + "title": "TCP ActiveOpens / AttemptFails / EstabResets", "type": "timeseries" }, { @@ -3589,6 +4474,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "Disk throughput in bytes per second per device from systemMetrics.disks (systemMetrics).\n\nHow to read: lines showing read and write throughput per device.\n\nTroubleshooting: high throughput during WiredTiger checkpoint intervals (default 60s) is expected. Sustained high throughput outside checkpoints indicates cache misses or heavy write workloads. Compare with 'Disk Writes and Reads' (IOPS) for a complete picture of disk saturation.", "fieldConfig": { "defaults": { "color": { @@ -3600,7 +4486,9 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3617,6 +4505,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -3631,7 +4520,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -3645,33 +4535,49 @@ "gridPos": { "h": 7, "w": 8, - "x": 16, - "y": 76 + "x": 0, + "y": 114 }, - "id": 39, + "id": 47, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"systemMetrics.memory.SwapFree_kb\" or r[\"_field\"] == \"systemMetrics.memory.SwapTotal_kb\")\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> yield(name: \"mean\")", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..*\\.io_in_progress$/ or r._field =~ /^systemMetrics\\.disks\\..*\\.io_queued_ms$/ or r._field =~ /^systemMetrics\\.disks\\..*\\.io_time_ms$/)\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> derivative(unit: v.windowPeriod)", "refId": "A" + }, + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "hide": false, + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..*\\.io_in_progress$/ )\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n\r\n\r\n\r\n", + "refId": "B" } ], - "title": "Swap Memory", + "title": "Disk I/O", "type": "timeseries" }, { @@ -3679,6 +4585,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "TCP connection error metrics from systemMetrics.netstat (systemMetrics.netstat).\n\nHow to read: lines showing error rates per second.\n\nTroubleshooting: increasing values indicate problems establishing or maintaining TCP connections between MongoDB nodes or clients. Correlate with 'Replica Members Health', 'Replica Members Ping', and application connection error logs.", "fieldConfig": { "defaults": { "color": { @@ -3690,8 +4597,8 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", - "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3708,6 +4615,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -3718,60 +4626,61 @@ } }, "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", "value": 80 } ] - } + }, + "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, - "x": 0, - "y": 81 + "x": 8, + "y": 118 }, - "id": 18, + "id": 45, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..*\\.read_sectors$/ or r._field =~ /^systemMetrics\\.disks\\..*\\.read_time_ms$/ or r._field =~ /^systemMetrics\\.disks\\..*\\.reads_merged$/)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> derivative(unit: v.windowPeriod)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"systemMetrics.netstat.Tcp:CurrEstab\" or r[\"_field\"] == \"systemMetrics.netstat.Tcp:EstabResets\" or r[\"_field\"] == \"systemMetrics.netstat.Tcp:OutRsts\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> derivative(unit: v.windowPeriod)", "refId": "A" - }, - { - "datasource": { - "type": "influxdb", - "uid": "P3C6603E967DC8568" - }, - "hide": false, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..*\\.io_in_progress$/ )\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n\r\n\r\n\r\n", - "refId": "B" } ], - "title": "Disk reads", + "title": "TCP Connection Issues", "type": "timeseries" }, { @@ -3779,6 +4688,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "Disk read throughput per device from systemMetrics.disks (systemMetrics).\n\nHow to read: lines showing read bytes/sec per device.\n\nTroubleshooting: high disk reads indicate data not served from WiredTiger cache. Correlate with 'Cache Read/Write Throughput' (bytes read into cache) to confirm. Persistent high reads suggest the working set exceeds the configured cache size.", "fieldConfig": { "defaults": { "color": { @@ -3790,7 +4700,9 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3807,6 +4719,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -3821,48 +4734,64 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", "value": 80 } ] - }, - "unit": "none" + } }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, - "x": 8, - "y": 83 + "x": 0, + "y": 121 }, - "id": 41, + "id": 18, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"systemMetrics.netstat.Tcp:ActiveOpens\" or r[\"_field\"] == \"systemMetrics.netstat.Tcp:AttemptFails\" or r[\"_field\"] == \"systemMetrics.netstat.Tcp:EstabResets\")\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> derivative(unit: v.windowPeriod)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..*\\.read_sectors$/ or r._field =~ /^systemMetrics\\.disks\\..*\\.read_time_ms$/ or r._field =~ /^systemMetrics\\.disks\\..*\\.reads_merged$/)\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> derivative(unit: v.windowPeriod)", "refId": "A" + }, + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "hide": false, + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..*\\.io_in_progress$/ )\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n\r\n\r\n\r\n", + "refId": "B" } ], - "title": "TCP ActiveOpens / AttemptFails / EstabResets", + "title": "Disk Reads", "type": "timeseries" }, { @@ -3870,6 +4799,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "Disk I/O queue time per device from systemMetrics.disks.*.io_queued_ms. This counter represents cumulative milliseconds that I/O requests have spent waiting in the queue (systemMetrics).\n\nHow to read: lines showing queued time per device as a rate (ms of queue time per second). Values approaching 1000 ms/s indicate persistent queueing.\n\nTroubleshooting: high queue depth is a direct saturation signal -- it means I/O requests are waiting because the device cannot keep up. This is often the root cause of latency spikes. Correlate with 'Disk Latency' and 'Disk I/O'.", "fieldConfig": { "defaults": { "color": { @@ -3882,6 +4812,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3898,6 +4829,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -3912,7 +4844,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -3920,40 +4853,47 @@ } ] }, - "unit": "none" + "unit": "ms" }, "overrides": [] }, "gridPos": { - "h": 5, + "h": 6, "w": 8, - "x": 16, - "y": 83 + "x": 8, + "y": 125 }, - "id": 43, + "id": 51, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"systemMetrics.netstat.Tcp:InSegs\" or r[\"_field\"] == \"systemMetrics.netstat.Tcp:OutSegs\")\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> derivative(unit: v.windowPeriod)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..*\\.io_queued_ms$/)\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> derivative(unit: 1s)\r\n", "refId": "A" } ], - "title": "TCP: InSegs / OutSegs", + "title": "Disk Queue Depth", "type": "timeseries" }, { @@ -3961,6 +4901,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "TCP/IP routing error counters from systemMetrics.netstat: InNoRoutes, InTruncatedPkts, and related fields (systemMetrics.netstat).\n\nHow to read: lines showing error rates per second (derivatives of cumulative counters).\n\nTroubleshooting: any non-zero derivative indicates network configuration problems -- misrouted packets, MTU issues, or firewall interference. These are rare but can cause intermittent connectivity failures that are very difficult to diagnose at the application level.", "fieldConfig": { "defaults": { "color": { @@ -3973,6 +4914,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3989,6 +4931,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -4003,7 +4946,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -4019,28 +4963,35 @@ "h": 5, "w": 8, "x": 0, - "y": 88 + "y": 128 }, "id": 44, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"systemMetrics.netstat.Ip:OutNoRoutes\" )\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> derivative(unit: v.windowPeriod)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"systemMetrics.netstat.Ip:OutNoRoutes\" )\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> derivative(unit: v.windowPeriod)", "refId": "A" } ], @@ -4052,6 +5003,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "TCP packet integrity counters from systemMetrics.netstat: Tcp.InCsumErrors, Tcp.InErrs, Tcp.RetransSegs, Ip.InDiscards, Ip.OutDiscards (systemMetrics.netstat).\n\nHow to read: lines showing error rates per second.\n\nTroubleshooting: RetransSegs (retransmissions) indicate packet loss, directly causing latency spikes visible in 'Average Operation Latency' and 'Replica Members Ping'. InCsumErrors indicate data corruption on the wire. InDiscards/OutDiscards indicate kernel buffer overflows. Any sustained increase points to network hardware issues, congestion, or NIC driver/firmware bugs.", "fieldConfig": { "defaults": { "color": { @@ -4064,6 +5016,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -4080,6 +5033,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -4090,12 +5044,12 @@ } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -4108,35 +5062,42 @@ "overrides": [] }, "gridPos": { - "h": 7, + "h": 5, "w": 8, - "x": 8, - "y": 90 + "x": 0, + "y": 133 }, - "id": 45, + "id": 42, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"systemMetrics.netstat.Tcp:CurrEstab\" or r[\"_field\"] == \"systemMetrics.netstat.Tcp:EstabResets\" or r[\"_field\"] == \"systemMetrics.netstat.Tcp:OutRsts\")\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> derivative(unit: v.windowPeriod)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"systemMetrics.netstat.Tcp:InCsumErrors\" \r\n or r[\"_field\"] == \"systemMetrics.netstat.Tcp:InErrs\"\r\n or r[\"_field\"] == \"systemMetrics.netstat.Tcp:RetransSegs\"\r\n or r[\"_field\"] == \"systemMetrics.netstat.Ip:InDiscards\"\r\n or r[\"_field\"] == \"systemMetrics.netstat.Ip:OutDiscards\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> derivative(unit: v.windowPeriod)", "refId": "A" } ], - "title": "TCP Connection Issues", + "title": "TCP Packet Loss or Corruption", "type": "timeseries" }, { @@ -4144,6 +5105,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "Disk I/O latency per device from systemMetrics.disks.*.read_time_ms and write_time_ms. These counters represent cumulative milliseconds spent on I/O operations (systemMetrics).\n\nHow to read: lines showing time spent on reads and writes per device, as a rate (ms of I/O time per second). Values approaching 1000 ms/s indicate the device is saturated.\n\nTroubleshooting: high disk latency directly causes operation latency increases in MongoDB. Correlate with 'Disk Writes and Reads' (IOPS) and 'Disk I/O' (throughput) to understand if latency is from volume or device saturation.", "fieldConfig": { "defaults": { "color": { @@ -4156,6 +5118,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -4172,6 +5135,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -4186,7 +5150,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -4194,40 +5159,47 @@ } ] }, - "unit": "none" + "unit": "ms" }, "overrides": [] }, "gridPos": { - "h": 5, + "h": 6, "w": 8, "x": 0, - "y": 93 + "y": 138 }, - "id": 42, + "id": 50, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"systemMetrics.netstat.Tcp:InCsumErrors\" \r\n or r[\"_field\"] == \"systemMetrics.netstat.Tcp:InErrs\"\r\n or r[\"_field\"] == \"systemMetrics.netstat.Tcp:RetransSegs\"\r\n or r[\"_field\"] == \"systemMetrics.netstat.Ip:InDiscards\"\r\n or r[\"_field\"] == \"systemMetrics.netstat.Ip:OutDiscards\")\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> derivative(unit: v.windowPeriod)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..*\\.read_time_ms$/ or r._field =~ /^systemMetrics\\.disks\\..*\\.write_time_ms$/)\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> derivative(unit: 1s)\r\n", "refId": "A" } ], - "title": "TCP Packet Loss or Corruption", + "title": "Disk Latency", "type": "timeseries" } ], @@ -4238,13 +5210,13 @@ "templating": { "list": [ { + "allValue": ".*", "allowCustomValue": false, "definition": "import \"influxdata/influxdb/schema\"\n\nschema.tagValues(\n bucket: v.bucket,\n tag: \"hostname\"\n)", "description": "", + "includeAll": true, "label": "hostname", "multi": true, - "includeAll": true, - "allValue": ".*", "name": "hostname", "options": [], "query": { @@ -4255,13 +5227,13 @@ "type": "query" }, { + "allValue": ".*", "allowCustomValue": false, "definition": "import \"influxdata/influxdb/schema\"\n\nschema.tagValues(\n bucket: v.bucket,\n tag: \"version\"\n)", "description": "version", + "includeAll": true, "label": "version", "multi": true, - "includeAll": true, - "allValue": ".*", "name": "version", "options": [], "query": { @@ -4278,8 +5250,7 @@ "to": "2031-05-26T00:01:28.240Z" }, "timepicker": {}, - "timezone": "browser", + "timezone": "UTC", "title": "ftdc_dashboard", - "uid": "ddnw277huiv40ae", - "version": 1 -} \ No newline at end of file + "uid": "ddnw277huiv40ae" +} diff --git a/metrics_to_get.txt b/metrics_to_get.txt index 6265f78..c755c48 100644 --- a/metrics_to_get.txt +++ b/metrics_to_get.txt @@ -35,6 +35,9 @@ serverStatus.flowControl.targetRateLimit serverStatus.wiredTiger.block-manager.bytes read serverStatus.wiredTiger.block-manager.bytes written serverStatus.wiredTiger.block-manager.bytes written for checkpoint +serverStatus.wiredTiger.log.log bytes written +serverStatus.wiredTiger.transaction.transaction checkpoint most recent time (msecs) +serverStatus.wiredTiger.checkpoint.most recent time (msecs) serverStatus.wiredTiger.cache.bytes currently in the cache serverStatus.wiredTiger.cache.maximum bytes configured serverStatus.wiredTiger.cache.modified pages evicted @@ -45,6 +48,15 @@ serverStatus.wiredTiger.cache.unmodified pages evicted serverStatus.wiredTiger.data-handle.connection data handles currently active serverStatus.wiredTiger.concurrentTransactions.read.available serverStatus.wiredTiger.concurrentTransactions.write.available +serverStatus.wiredTiger.concurrentTransactions.monitor.available +serverStatus.wiredTiger.concurrentTransactions.monitor.totalTickets +serverStatus.wiredTiger.concurrentTransactions.monitor.queueLength +serverStatus.queues.execution.read.available +serverStatus.queues.execution.read.totalTickets +serverStatus.queues.execution.read.normalPriority.queueLength +serverStatus.queues.execution.write.available +serverStatus.queues.execution.write.totalTickets +serverStatus.queues.execution.write.normalPriority.queueLength serverStatus.flowControl.enabled serverStatus.flowControl.isLagged serverStatus.flowControl.isLaggedCount From c0c573018da4976ec933675ce2035ba696a6f0cc Mon Sep 17 00:00:00 2001 From: jenunes Date: Thu, 11 Jun 2026 22:15:27 -0300 Subject: [PATCH 2/9] Update FTDC dashboard and replica metrics for Grafana review. Add replSet member panels, structured queries and legends, sync metrics_to_get with testdata, docker compose in run.sh, and local gitignore/provisioning tweaks. --- .gitignore | 8 +- .../ftdc/testdata/metrics_to_get.txt | 11 + grafana/dashboards/dashboard.json | 1685 ++++++++++++----- grafana/main.yaml | 4 + metrics_to_get.txt | 5 + run.sh | 8 +- 6 files changed, 1256 insertions(+), 465 deletions(-) diff --git a/.gitignore b/.gitignore index 220f221..c70ef18 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,10 @@ diagnostic.data/ .idea .DS_Store influxdb_data/ -./ftdc_exporter/vendor \ No newline at end of file +./ftdc_exporter/vendor +.cursor/ +mongodb_ftdc_decoder/ +grafana/backups/ +grafana/DASHBOARD.md +grafana/dashboards/bkp.json +grafana/dashboards/my.json \ No newline at end of file diff --git a/ftdc_exporter/ftdc/testdata/metrics_to_get.txt b/ftdc_exporter/ftdc/testdata/metrics_to_get.txt index 066bbe3..2a0ce40 100644 --- a/ftdc_exporter/ftdc/testdata/metrics_to_get.txt +++ b/ftdc_exporter/ftdc/testdata/metrics_to_get.txt @@ -51,6 +51,12 @@ serverStatus.wiredTiger.concurrentTransactions.write.available serverStatus.wiredTiger.concurrentTransactions.monitor.available serverStatus.wiredTiger.concurrentTransactions.monitor.totalTickets serverStatus.wiredTiger.concurrentTransactions.monitor.queueLength +serverStatus.queues.execution.read.available +serverStatus.queues.execution.read.totalTickets +serverStatus.queues.execution.read.normalPriority.queueLength +serverStatus.queues.execution.write.available +serverStatus.queues.execution.write.totalTickets +serverStatus.queues.execution.write.normalPriority.queueLength serverStatus.flowControl.enabled serverStatus.flowControl.isLagged serverStatus.flowControl.isLaggedCount @@ -144,3 +150,8 @@ replSetGetStatus.members.1.lastAppliedWallTime replSetGetStatus.members.2.lastAppliedWallTime replSetGetStatus.members.3.lastAppliedWallTime replSetGetStatus.members.4.lastAppliedWallTime +replSetGetStatus.members.0.state +replSetGetStatus.members.1.state +replSetGetStatus.members.2.state +replSetGetStatus.members.3.state +replSetGetStatus.members.4.state diff --git a/grafana/dashboards/dashboard.json b/grafana/dashboards/dashboard.json index e9a2049..7fe7b9f 100644 --- a/grafana/dashboards/dashboard.json +++ b/grafana/dashboards/dashboard.json @@ -18,15 +18,498 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, + "id": 1, "links": [], "panels": [ + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "description": "### Server uptime\n\n**`serverStatus.uptime`** — seconds the current **mongod** process has been running.\n\n**Unit:** seconds (displayed as duration)\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.uptime`\n\n---\n\n### How to read\n\n- Monotonic increase while the process is up\n- **Drop** or reset → **restart** (mark the timestamp)\n- May not start at zero if the dashboard time range begins mid-run\n\n---\n\n### Troubleshooting\n\n- Restart + memory drop → OOM · **mongod Memory (RSS / Virtual)**, **Swap Memory**\n- Restart + assert spike → crash · **Asserts**\n- Planned maintenance → correlate with deploy window\n- After restart, counters reset · **Operations**, **Page Faults** rates may look odd briefly\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "dtdurations" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*\\.uptime$" + }, + "properties": [ + { + "id": "displayName", + "value": "Server uptime" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 20, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.uptime\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", + "refId": "A" + } + ], + "title": "Server Uptime", + "type": "timeseries" + }, + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "description": "### Replica members health\n\n**`replSetGetStatus.members.N.health`** — per-member reachability (1 = healthy, 0 = down).\n\n**Unit:** 0/1 (gauge)\n\n**Note:** **N** is the member array index, not hostname order.\n\n---\n\n### Metrics\n\n**Fields:** `replSetGetStatus.members.0.health` … `replSetGetStatus.members.4.health`\n\n---\n\n### How to read\n\n- One line per member index\n- **stepAfter** · flat at **1** = healthy · drop to **0** = member lost\n- Do **not** stack — each line is independent 0/1\n\n---\n\n### Troubleshooting\n\n- **1 → 0** → network partition, crash, OOM · **Server Uptime**, **Asserts**, system **CPU/Memory/Disk** on that host\n- Healthy but high lag → **Replica Members Lag**, **Replica Members Ping**\n\n---\n\n### Docs\n\n- [replSetGetStatus](https://www.mongodb.com/docs/manual/reference/command/replSetGetStatus/)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [ + { + "options": { + "0": { + "index": 0, + "text": "Down" + }, + "1": { + "index": 1, + "text": "Up" + } + }, + "type": "value" + } + ], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*members\\.0\\.health$" + }, + "properties": [ + { + "id": "displayName", + "value": "Member 0 health" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*members\\.1\\.health$" + }, + "properties": [ + { + "id": "displayName", + "value": "Member 1 health" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*members\\.2\\.health$" + }, + "properties": [ + { + "id": "displayName", + "value": "Member 2 health" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*members\\.3\\.health$" + }, + "properties": [ + { + "id": "displayName", + "value": "Member 3 health" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*members\\.4\\.health$" + }, + "properties": [ + { + "id": "displayName", + "value": "Member 4 health" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 7, + "x": 6, + "y": 0 + }, + "id": 16, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] =~ /replSetGetStatus\\.members\\.\\d+\\.health/)\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", + "refId": "A" + } + ], + "title": "Replica Members Health", + "type": "timeseries" + }, + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "description": "### ReplSet member states\n\n**`replSetGetStatus.members.N.state`** over the selected time range (same codes as [replica states](https://www.mongodb.com/docs/manual/reference/replica-states/)).\n\n**Note:** Row labels use member **array index** unless `members.N.name` is ingested. Failover shows PRIMARY moving between rows. Primary-only FTDC filter via **hostname**.\n\n---\n\n### Metrics\n\n**Fields:** `replSetGetStatus.members.0.state` … (through your max index)\n\n---\n\n### How to read\n\n- One row per member index\n- Colored band = role · **PRIMARY** (1), **SECONDARY** (2), etc.\n- Red bands = DOWN, RECOVERING, ROLLBACK, …\n\n---\n\n### Docs\n\n- [replSetGetStatus](https://www.mongodb.com/docs/manual/reference/command/replSetGetStatus/)", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "custom": { + "axisPlacement": "auto", + "fillOpacity": 100, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineWidth": 0, + "spanNulls": false + }, + "mappings": [ + { + "options": { + "0": { + "color": "semi-dark-red", + "index": 11, + "text": "STARTUP" + }, + "1": { + "color": "rgb(107, 152, 102)", + "index": 4, + "text": "PRIMARY" + }, + "2": { + "color": "rgb(193, 159, 20)", + "index": 3, + "text": "SECONDARY" + }, + "3": { + "color": "semi-dark-red", + "index": 10, + "text": "RECOVERING" + }, + "5": { + "color": "semi-dark-red", + "index": 9, + "text": "STARTUP2" + }, + "6": { + "color": "semi-dark-red", + "index": 8, + "text": "UNKNOWN" + }, + "7": { + "color": "#8B8000", + "index": 1, + "text": "ARBITER" + }, + "8": { + "color": "semi-dark-red", + "index": 7, + "text": "DOWN" + }, + "9": { + "color": "semi-dark-red", + "index": 6, + "text": "ROLLBACK" + }, + "10": { + "color": "semi-dark-red", + "index": 5, + "text": "REMOVED" + }, + "null": { + "color": "#FF7383", + "index": 2, + "text": "No data" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*members\\.0\\.state$" + }, + "properties": [ + { + "id": "displayName", + "value": "Member 0" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*members\\.1\\.state$" + }, + "properties": [ + { + "id": "displayName", + "value": "Member 1" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*members\\.2\\.state$" + }, + "properties": [ + { + "id": "displayName", + "value": "Member 2" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*members\\.3\\.state$" + }, + "properties": [ + { + "id": "displayName", + "value": "Member 3" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*members\\.4\\.state$" + }, + "properties": [ + { + "id": "displayName", + "value": "Member 4" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 11, + "x": 13, + "y": 0 + }, + "id": 54, + "options": { + "alignValue": "right", + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "mergeValues": true, + "rowHeight": 0.9, + "showValue": "auto", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] =~ /replSetGetStatus\\.members\\.\\d+\\.state/)\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", + "refId": "A" + } + ], + "title": "Member States", + "type": "state-timeline" + }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 0 + "y": 6 }, "id": 24, "panels": [], @@ -38,7 +521,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Tickets Available Read displays the number of read tickets available to the WiredTiger storage engine. Read tickets represent the number of concurrent read operations allowed into the storage engine. When this value reaches zero, new read requests might queue until a read ticket becomes available.\n\nTickets Available Write displays the number of write tickets available to the WiredTiger storage engine. Write tickets represent the number of concurrent write operations allowed into the storage engine. When this value reaches zero, new write requests might queue until a write ticket becomes available.", + "description": "### WiredTiger ticket availability\n\nShows how many **read** and **write** execution tickets are still available in the storage engine. Each ticket allows one concurrent operation; when **available** reaches **0**, new operations wait until a ticket is freed.\n\n**Unit:** ticket count (not bytes)\n\n---\n\n### Metrics (same concept, two paths)\n\n**MongoDB 8.x+** \n`serverStatus.queues.execution.read.available` \n`serverStatus.queues.execution.write.available`\n\n**Legacy** \n`serverStatus.wiredTiger.concurrentTransactions.read.available` \n`serverStatus.wiredTiger.concurrentTransactions.write.available`\n\nBoth paths queried for compatibility. Usually only one has data per version.\n\n---\n\n### How to read\n\n- Two lines: read and write **available** tickets \n- **stepAfter** reflects discrete ticket changes \n- Typical max ~**128** (dynamic adjustment on 7.0+)\n\n---\n\n### Troubleshooting\n\n- Sustained **low or zero** → concurrency saturation \n- Correlate with **Current Queue writers / readers** and **Average Operation Latency**\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)\n- [Performance tuning — WiredTiger tickets](https://www.mongodb.com/docs/manual/administration/performance-tuning/)", "fieldConfig": { "defaults": { "color": { @@ -91,14 +574,10 @@ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "none" + "unit": "short" }, "overrides": [ { @@ -131,7 +610,7 @@ "h": 8, "w": 13, "x": 0, - "y": 1 + "y": 7 }, "id": 1, "options": { @@ -171,7 +650,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Shows wiredTiger.cache.maximum bytes configured (maximum cache size) vs wiredTiger.cache.bytes currently in the cache (data currently cached). Per MongoDB docs, bytes currently in the cache should not be greater than the maximum bytes configured. To adjust the WiredTiger internal cache size, see --wiredTigerCacheSizeGB (serverStatus.wiredTiger.cache).\n\nHow to read: two lines -- configured max and current usage, displayed in GB.\n\nTroubleshooting: when current cache approaches the configured maximum, WiredTiger must evict aggressively. Correlate with 'Pages Evicted' and 'Dirty Cache Bytes'. If cache is consistently near max, either increase wiredTigerCacheSizeGB or optimize queries to reduce the working set.", + "description": "### WiredTiger cache size\n\n**maximum bytes configured** vs **bytes currently in the cache** — WiredTiger internal cache limit and current usage. **Unit:** bytes (per MongoDB docs). Current usage should not exceed the configured maximum.\n\n**Metrics:** `serverStatus.wiredTiger.cache.maximum bytes configured` · `serverStatus.wiredTiger.cache.bytes currently in the cache`\n\nTo change the limit, see `wiredTigerCacheSizeGB` / `storage.wiredTiger.engineConfig.cacheSizeGB` (config in GB; these stats are still bytes).\n\n---\n\n### How to read\n\n- Two lines: configured max and current usage\n- WiredTiger often targets ~80% of max under load; sustained saturation → eviction pressure\n\n---\n\n### Troubleshooting\n\n- Usage near max → **Pages Evicted**, **Dirty Cache Bytes**\n- Consistently at max → raise `cacheSizeGB` or reduce working set\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)\n- [WiredTiger storage engine](https://www.mongodb.com/docs/manual/core/wiredtiger/)", "fieldConfig": { "defaults": { "color": { @@ -212,22 +691,20 @@ "mode": "off" } }, + "decimals": 2, "fieldMinMax": true, "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "decgbytes" + "unit": "bytes" }, "overrides": [ { @@ -260,7 +737,7 @@ "h": 8, "w": 11, "x": 13, - "y": 1 + "y": 7 }, "id": 6, "options": { @@ -289,7 +766,7 @@ "uid": "P3C6603E967DC8568" }, "hide": false, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.cache.maximum bytes configured\" or r[\"_field\"] == \"serverStatus.wiredTiger.cache.bytes currently in the cache\" )\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> map(fn: (r) => ({ r with _value: r._value / 1024 / 1024 / 1024}))\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.cache.maximum bytes configured\" or r[\"_field\"] == \"serverStatus.wiredTiger.cache.bytes currently in the cache\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", "refId": "A" } ], @@ -301,7 +778,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Rate of wiredTiger.cache.modified pages evicted (dirty) and wiredTiger.cache.unmodified pages evicted (clean) per second. Per MongoDB docs, unmodified pages evicted is the main statistic for page eviction (serverStatus.wiredTiger.cache).\n\nHow to read: two lines showing pages/sec evicted, split by clean vs dirty. Clean eviction discards unmodified pages; dirty eviction requires writing modified data to disk first (reconciliation).\n\nTroubleshooting: spikes in dirty eviction indicate cache pressure and correlate with write latency increases. If eviction cannot keep up, application threads may be forced to do eviction themselves, causing a performance cliff. Check 'WiredTiger Cache' and 'Dirty Cache Bytes'.", + "description": "### Pages evicted\n\nRate of **modified** (dirty) and **unmodified** (clean) cache pages evicted per second. MongoDB docs note unmodified evictions as the main eviction statistic.\n\n**Unit:** pages/s (derivative of cumulative counters)\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.wiredTiger.cache.modified pages evicted` · `serverStatus.wiredTiger.cache.unmodified pages evicted`\n\n---\n\n### How to read\n\n- **Modified pages evicted (dirty)** — evicted after write/reconciliation\n- **Unmodified pages evicted (clean)** — discarded without write\n\n---\n\n### Troubleshooting\n\n- Dirty eviction spikes → cache pressure, write latency · **WiredTiger Cache**, **Dirty Cache Bytes**\n- Eviction can't keep up → app threads may evict (performance cliff)\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)\n- [WiredTiger storage engine](https://www.mongodb.com/docs/manual/core/wiredtiger/)", "fieldConfig": { "defaults": { "color": { @@ -389,7 +866,7 @@ "h": 7, "w": 13, "x": 0, - "y": 9 + "y": 15 }, "id": 12, "options": { @@ -417,7 +894,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.cache.modified pages evicted\" or r[\"_field\"] == \"serverStatus.wiredTiger.cache.unmodified pages evicted\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s)\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.cache.modified pages evicted\" or r[\"_field\"] == \"serverStatus.wiredTiger.cache.unmodified pages evicted\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n", "refId": "A" } ], @@ -429,7 +906,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Shows wiredTiger.data-handle.connection data handles currently active. Per MongoDB docs, a dhandle is a generic representation of any named data source (collection, index). WiredTiger maintains all dhandles in a global list protected by a R/W lock; each session has a dhandle cache (serverStatus.wiredTiger.data-handle).\n\nHow to read: single line showing the count of active dhandles over time.\n\nTroubleshooting: a very high count of active dhandles indicates many collections/indexes accessed concurrently, increasing memory overhead and lock contention on the global dhandle list. A sudden drop may indicate collections being closed or a mongod restart.", + "description": "### WiredTiger data handles\n\nCount of **connection data handles currently active** — WiredTiger dhandles for open collections, indexes, and other named data sources.\n\n**Unit:** count (gauge)\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.wiredTiger.data-handle.connection data handles currently active`\n\n---\n\n### How to read\n\n- Single line: active dhandles over time\n- Sudden drop → collections closed or **mongod** restart (**Server Uptime**)\n\n---\n\n### Troubleshooting\n\n- Very high count → many collections/indexes in use, memory overhead, dhandle list lock contention\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)\n- [WiredTiger storage engine](https://www.mongodb.com/docs/manual/core/wiredtiger/)", "fieldConfig": { "defaults": { "color": { @@ -441,6 +918,7 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -469,6 +947,7 @@ "mode": "off" } }, + "decimals": 0, "mappings": [], "thresholds": { "mode": "absolute", @@ -476,13 +955,10 @@ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] - } + }, + "unit": "short" }, "overrides": [ { @@ -503,7 +979,7 @@ "h": 5, "w": 11, "x": 13, - "y": 9 + "y": 15 }, "id": 26, "options": { @@ -531,7 +1007,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.data-handle.connection data handles currently active\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> yield(name: \"last\")", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.data-handle.connection data handles currently active\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)", "refId": "A" } ], @@ -543,7 +1019,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Shows wiredTiger.cache.tracked dirty bytes in the cache -- the size in bytes of dirty data in the cache. Per MongoDB docs, this value should be less than bytes currently in the cache (serverStatus.wiredTiger.cache).\n\nHow to read: single line showing dirty bytes over time. Dirty data = modified data not yet written to disk via checkpoint or eviction.\n\nTroubleshooting: if dirty bytes approach the eviction target (~5% of cache triggers eviction, ~20% triggers aggressive eviction), WiredTiger stalls application threads to force eviction. Correlate with checkpoint writes and eviction rate.", + "description": "### Dirty cache bytes\n\n**tracked dirty bytes in the cache** — modified data in the WiredTiger cache not yet flushed via checkpoint or eviction. Should stay below **bytes currently in the cache**.\n\n**Unit:** bytes (gauge)\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.wiredTiger.cache.tracked dirty bytes in the cache`\n\n---\n\n### How to read\n\n- Single line: dirty bytes over time\n- ~5% of cache → eviction pressure · ~20% → aggressive eviction / stalls\n\n---\n\n### Troubleshooting\n\n- Dirty bytes high vs **WiredTiger Cache** max → eviction pressure\n- Correlate with **Pages Evicted**, **Checkpoint Write Throughput**\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)\n- [WiredTiger storage engine](https://www.mongodb.com/docs/manual/core/wiredtiger/)", "fieldConfig": { "defaults": { "color": { @@ -620,7 +1096,7 @@ "h": 9, "w": 11, "x": 13, - "y": 14 + "y": 20 }, "id": 7, "options": { @@ -648,7 +1124,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.cache.tracked dirty bytes in the cache\" )\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> yield(name: \"last\")", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.cache.tracked dirty bytes in the cache\" )\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)", "refId": "A" } ], @@ -660,7 +1136,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Rate of wiredTiger.block-manager.bytes read and wiredTiger.block-manager.bytes written per second. Per MongoDB docs, the block manager subsystem manages the reading and writing of data from disk (serverStatus.wiredTiger.block-manager).\n\nHow to read: two lines showing read and write throughput in MB/s through the block manager.\n\nTroubleshooting: high block manager reads indicate cache misses forcing disk access. High writes correlate with checkpoints and eviction flushes. Compare with 'Disk I/O' to determine if WiredTiger is the dominant disk consumer or if other processes contribute.", + "description": "### WiredTiger block manager I/O\n\nRead/write throughput through WiredTiger's block manager (disk I/O for data files). **Unit:** MB/s (from bytes/s ÷ 1024²) — cumulative byte counters differentiated.\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.wiredTiger.block-manager.bytes read` · `serverStatus.wiredTiger.block-manager.bytes written`\n\n---\n\n### How to read\n\n- **Bytes read** — data read from disk (cache misses)\n- **Bytes written** — data written via block manager (eviction, checkpoints)\n\n---\n\n### Troubleshooting\n\n- High reads → cache pressure · **WiredTiger Cache**, **Pages Evicted**\n- High writes → checkpoints/eviction · **Checkpoint Write Throughput**, **Disk I/O**\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)\n- [WiredTiger storage engine](https://www.mongodb.com/docs/manual/core/wiredtiger/)", "fieldConfig": { "defaults": { "color": { @@ -702,16 +1178,13 @@ }, "decimals": 2, "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] }, @@ -748,7 +1221,7 @@ "h": 7, "w": 13, "x": 0, - "y": 16 + "y": 22 }, "id": 23, "options": { @@ -776,7 +1249,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.block-manager.bytes read\" or r[\"_field\"] == \"serverStatus.wiredTiger.block-manager.bytes written\" )\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> map(fn: (r) => ({ r with _value: r._value / 1024 /1024 }))\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> derivative(unit: 1s)\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.block-manager.bytes read\" or r[\"_field\"] == \"serverStatus.wiredTiger.block-manager.bytes written\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) / 1024.0 / 1024.0 }))", "refId": "A" } ], @@ -788,7 +1261,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Rate of wiredTiger.block-manager.bytes written for checkpoint per second. Per MongoDB docs, a checkpoint is a known point in time from which WiredTiger can recover in the event of a crash or unexpected shutdown. Checkpoints typically run every 60 seconds (serverStatus.wiredTiger.block-manager).\n\nHow to read: derivative of cumulative checkpoint bytes, showing write throughput during checkpoints.\n\nTroubleshooting: large checkpoint writes indicate heavy write workloads. If checkpoint write spikes coincide with latency increases, the checkpoint is competing for disk I/O. Correlate with 'Disk I/O', 'Disk Writes', and 'Average Operation Latency' panels.", + "description": "### Checkpoint write throughput\n\nRate of **bytes written for checkpoint** — disk write throughput during WiredTiger checkpoints (~every 60s). Recovery point after crash/shutdown.\n\n**Unit:** B/s (`Bps`) — derivative of cumulative bytes\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.wiredTiger.block-manager.bytes written for checkpoint`\n\n---\n\n### How to read\n\n- Spikes during checkpoint runs · baseline low between checkpoints\n\n---\n\n### Troubleshooting\n\n- Large spikes + latency → checkpoint competing for disk I/O\n- Correlate with **Disk I/O**, **Disk Writes**, **Average Operation Latency**, **Checkpoint Duration**\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)\n- [WiredTiger storage engine](https://www.mongodb.com/docs/manual/core/wiredtiger/)", "fieldConfig": { "defaults": { "color": { @@ -864,7 +1337,7 @@ "h": 7, "w": 13, "x": 0, - "y": 23 + "y": 29 }, "id": 19, "options": { @@ -892,7 +1365,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.block-manager.bytes written for checkpoint\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n|> derivative(unit: 1s, nonNegative: true)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.block-manager.bytes written for checkpoint\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", "refId": "A" } ], @@ -904,7 +1377,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Rate of wiredTiger.cache.bytes read into cache and wiredTiger.cache.bytes written from cache per second. Per MongoDB docs, pages read into cache together with pages written from cache provide an overview of I/O activity (serverStatus.wiredTiger.cache).\n\nHow to read: derivative of cumulative byte counters, showing bytes/sec. 'Read into cache' = cold data pulled from disk; 'Written from cache' = dirty data flushed to disk.\n\nTroubleshooting: high read-into-cache means frequent cache misses -- check if the working set or indexes exceed cache size. High write-from-cache correlates with write-heavy workloads or eviction pressure. Compare with 'Disk I/O' and 'WiredTiger Block Manager'.", + "description": "### Cache read/write throughput\n\nRate of bytes **read into cache** (cold data from disk) and **written from cache** (flush to disk). Overview of WiredTiger cache I/O activity.\n\n**Unit:** B/s (`Bps`) — derivative of cumulative byte counters\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.wiredTiger.cache.bytes read into cache` · `serverStatus.wiredTiger.cache.bytes written from cache`\n\n---\n\n### How to read\n\n- **Read into cache** — cache misses / data pulled from disk\n- **Written from cache** — dirty data flushed (eviction, checkpoints)\n\n---\n\n### Troubleshooting\n\n- High reads → working set may exceed cache · **WiredTiger Cache**, **Pages Evicted**\n- High writes → write load or eviction pressure · **Dirty Cache Bytes**, **WiredTiger Block Manager**\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)\n- [WiredTiger storage engine](https://www.mongodb.com/docs/manual/core/wiredtiger/)", "fieldConfig": { "defaults": { "color": { @@ -992,7 +1465,7 @@ "h": 7, "w": 11, "x": 13, - "y": 23 + "y": 29 }, "id": 8, "options": { @@ -1032,7 +1505,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Shows the duration in milliseconds of the most recent WiredTiger checkpoint. Pre-8.0: wiredTiger.transaction.transaction checkpoint most recent time (msecs). MongoDB 8.x: wiredTiger.checkpoint.most recent time (msecs). Per MongoDB docs, an increase under steady write load may indicate I/O subsystem saturation.\n\nHow to read: single line showing the most recent checkpoint duration. Checkpoints typically run every 60 seconds.\n\nTroubleshooting: increasing checkpoint duration under steady write load indicates I/O subsystem saturation. Correlate with 'Checkpoint Write Throughput', 'Disk I/O', and 'Average Operation Latency'. Long checkpoints can cause write stalls.", + "description": "### Checkpoint duration\n\nDuration in **milliseconds** of the **most recent** WiredTiger checkpoint. Value updates when a checkpoint completes (~every 60s), then holds until the next.\n\n**Unit:** ms (gauge)\n\n---\n\n### Metrics (same concept, two paths)\n\n**Legacy:** `serverStatus.wiredTiger.transaction.transaction checkpoint most recent time (msecs)` \n**MongoDB 8.x+:** `serverStatus.wiredTiger.checkpoint.most recent time (msecs)`\n\nBoth queried; usually one path has data per version.\n\n---\n\n### How to read\n\n- Single line · rising trend under steady write load → I/O pressure\n\n---\n\n### Troubleshooting\n\n- Increasing duration → I/O saturation · **Checkpoint Write Throughput**, **Disk I/O**, **Average Operation Latency**\n- Long checkpoints → possible write stalls\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)\n- [WiredTiger storage engine](https://www.mongodb.com/docs/manual/core/wiredtiger/)", "fieldConfig": { "defaults": { "color": { @@ -1055,7 +1528,7 @@ "viz": false }, "insertNulls": false, - "lineInterpolation": "linear", + "lineInterpolation": "stepAfter", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -1081,10 +1554,6 @@ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] }, @@ -1109,7 +1578,7 @@ "h": 6, "w": 13, "x": 0, - "y": 30 + "y": 36 }, "id": 48, "options": { @@ -1151,7 +1620,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Rate of wiredTiger.log.log bytes written per second. Per MongoDB docs, wiredTiger.log returns statistics on WiredTiger's write ahead log (i.e. the journal) (serverStatus.wiredTiger.log).\n\nHow to read: single line showing journal write throughput in bytes/sec.\n\nTroubleshooting: high journal write rates correlate with heavy write workloads. Compare with 'Checkpoint Write Throughput' to understand the balance between journal writes and checkpoint writes. Sustained high values may indicate the need for faster storage.", + "description": "### Journal bytes written\n\nRate of WAL/journal bytes written by WiredTiger (`wiredTiger.log`). **Unit:** B/s (`Bps`) — derivative of cumulative bytes.\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.wiredTiger.log.log bytes written`\n\n---\n\n### How to read\n\n- Single line · sustained high rate → heavy write workload\n\n---\n\n### Troubleshooting\n\n- Compare with **Checkpoint Write Throughput** (journal vs checkpoint balance)\n- Sustained high values → storage may be a bottleneck\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)\n- [WiredTiger storage engine](https://www.mongodb.com/docs/manual/core/wiredtiger/)", "fieldConfig": { "defaults": { "color": { @@ -1217,135 +1686,19 @@ "properties": [ { "id": "displayName", - "value": "Journal bytes written" - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 11, - "x": 13, - "y": 30 - }, - "id": 49, - "options": { - "legend": { - "calcs": [ - "min", - "max", - "mean", - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "12.3.2", - "targets": [ - { - "datasource": { - "type": "influxdb", - "uid": "P3C6603E967DC8568" - }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.log.log bytes written\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n", - "refId": "A" - } - ], - "title": "Journal Bytes Written", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 36 - }, - "id": 22, - "panels": [], - "title": "Server Status", - "type": "row" - }, - { - "datasource": { - "type": "influxdb", - "uid": "P3C6603E967DC8568" - }, - "description": "Shows serverStatus.mem.resident and serverStatus.mem.virtual. Per MongoDB docs, mem.resident is roughly equivalent to the amount of RAM in MiB currently used by the database process; during normal use this value tends to grow and on dedicated servers approaches total system memory. mem.virtual is the quantity in MiB of virtual memory used by the mongod process (serverStatus.mem).\n\nHow to read: two lines -- resident (physical RAM) and virtual (total address space), displayed in GB.\n\nTroubleshooting: if resident memory approaches total system RAM, the OS may start swapping -- check 'Swap Memory'. Virtual memory much larger than resident is normal (memory-mapped files). Sudden drops in resident indicate a restart; compare with 'Server Uptime'.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 1, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 + "value": "Journal bytes written" } ] - }, - "unit": "decgbytes" - }, - "overrides": [] + } + ] }, "gridPos": { - "h": 5, - "w": 8, - "x": 0, - "y": 37 + "h": 6, + "w": 11, + "x": 13, + "y": 36 }, - "id": 3, + "id": 49, "options": { "legend": { "calcs": [ @@ -1371,19 +1724,32 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.mem.virtual\" or r[\"_field\"] == \"serverStatus.mem.resident\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: 1m, fn: median, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: r._value / 1000.0 }))\r\n |> yield(name: \"median\")\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.log.log bytes written\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n", "refId": "A" } ], - "title": "mongod Memory (RSS / Virtual)", + "title": "Journal Bytes Written", "type": "timeseries" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 42 + }, + "id": 22, + "panels": [], + "title": "Server Status", + "type": "row" + }, { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Rate of serverStatus.extra_info.page_faults per second. Per MongoDB docs, the page_faults counter may increase dramatically during moments of poor performance and may correlate with limited memory environments and larger data sets. Limited and sporadic page faults do not necessarily indicate an issue (serverStatus.extra_info).\n\nHow to read: single line showing page faults/sec (derivative of cumulative counter).\n\nTroubleshooting: sustained high page fault rates mean the working set exceeds available physical RAM, forcing expensive disk reads. Correlate with 'WiredTiger Cache' utilization, 'Swap Memory', and 'mongod Memory (RSS / Virtual)'. If page faults are high and swap is active, the server needs more RAM.", + "description": "### mongod memory (RSS / virtual)\n\n**mem.resident** (physical RAM) and **mem.virtual** (virtual address space) used by the mongod process.\n\n**Unit:** IEC bytes on chart (auto-scaled GiB/MiB). Source fields are **MiB** in `serverStatus.mem` (`db.serverStatus().mem`); query multiplies by 1024² for display.\n\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.mem.resident` · `serverStatus.mem.virtual`\n\n---\n\n### How to read\n\n- Two lines: resident vs virtual\n- Virtual ≫ resident is normal (mapped files)\n- Sudden resident drop → restart · **Server Uptime**\n- Tooltip/axis values are auto-scaled; divide by 1024² or compare trends, not raw shell integers\n\n---\n\n### Troubleshooting\n\n- Resident near system RAM → swap risk · **Swap Memory**, **Page Faults**\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)", "fieldConfig": { "defaults": { "color": { @@ -1395,6 +1761,7 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -1408,7 +1775,7 @@ "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, - "pointSize": 5, + "pointSize": 1, "scaleDistribution": { "type": "linear" }, @@ -1423,6 +1790,7 @@ "mode": "off" } }, + "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", @@ -1430,23 +1798,45 @@ { "color": "green", "value": 0 - }, + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*mem\\.resident$" + }, + "properties": [ { - "color": "red", - "value": 80 + "id": "displayName", + "value": "Resident (RSS)" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*mem\\.virtual$" + }, + "properties": [ + { + "id": "displayName", + "value": "Virtual memory" } ] } - }, - "overrides": [] + ] }, "gridPos": { - "h": 5, - "w": 7, - "x": 8, - "y": 37 + "h": 7, + "w": 10, + "x": 0, + "y": 43 }, - "id": 27, + "id": 3, "options": { "legend": { "calcs": [ @@ -1472,11 +1862,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.extra_info.page_faults\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: v.windowPeriod)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.mem.virtual\" or r[\"_field\"] == \"serverStatus.mem.resident\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) * 1024.0 * 1024.0 }))\r\n", "refId": "A" } ], - "title": "Page Faults", + "title": "mongod Memory (RSS / Virtual)", "type": "timeseries" }, { @@ -1484,7 +1874,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Shows serverStatus.connections.current, connections.active, and the rate of connections.totalCreated. Per MongoDB docs, connections.current is the number of incoming connections from clients including shell sessions; connections.available shows unused connections available; connections.active is client connections with operations in progress (serverStatus.connections).\n\nHow to read: active and current as gauge lines, plus totalCreated as a rate (derivative) showing new connection velocity.\n\nTroubleshooting: current connections climbing toward maxIncomingConnections (default 65536) indicates pool exhaustion. If active is much lower than current, many idle connections are consuming resources -- check application connection pool settings. A spike in totalCreated rate may indicate connection churn (short-lived connections).", + "description": "### Connections\n\n**current**, **active**, and **available** connection counts.\n\n**Unit:** count (gauge)\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.connections.current` · `serverStatus.connections.active` · `serverStatus.connections.available`\n\n---\n\n### How to read\n\n- **Current** — all open connections (incl. idle)\n- **Active** — connections running operations\n- **Available** — unused connection slots\n\n---\n\n### Troubleshooting\n\n- **Current** near `maxIncomingConnections` (65536) → pool exhaustion\n- **Active** ≪ **Current** → idle connections · check app pool settings\n- For connection **churn**, see **Connections Created**\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)", "fieldConfig": { "defaults": { "color": { @@ -1496,6 +1886,7 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -1524,6 +1915,7 @@ "mode": "off" } }, + "decimals": 0, "mappings": [], "thresholds": { "mode": "absolute", @@ -1531,21 +1923,55 @@ { "color": "green", "value": 0 - }, + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*connections\\.current$" + }, + "properties": [ { - "color": "red", - "value": 80 + "id": "displayName", + "value": "Current connections" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*connections\\.active$" + }, + "properties": [ + { + "id": "displayName", + "value": "Active connections" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*connections\\.available$" + }, + "properties": [ + { + "id": "displayName", + "value": "Available connections" } ] } - }, - "overrides": [] + ] }, "gridPos": { - "h": 7, - "w": 9, - "x": 15, - "y": 37 + "h": 9, + "w": 14, + "x": 10, + "y": 43 }, "id": 2, "options": { @@ -1573,17 +1999,8 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.connections.active\" or r[\"_field\"] == \"serverStatus.connections.current\" )\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> yield(name: \"last\")\r\n\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.connections.current\" or\r\n r[\"_field\"] == \"serverStatus.connections.active\" or\r\n r[\"_field\"] == \"serverStatus.connections.available\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", "refId": "A" - }, - { - "datasource": { - "type": "influxdb", - "uid": "P3C6603E967DC8568" - }, - "hide": false, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.connections.totalCreated\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> derivative(unit: v.windowPeriod)", - "refId": "B" } ], "title": "Connections", @@ -1594,7 +2011,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Shows average operation latency in microseconds for reads, writes, and commands. Per MongoDB docs, opLatencies contains operation latencies for the instance as a whole; opLatencies.reads for read requests, opLatencies.writes for write operations, opLatencies.commands for database commands (serverStatus.opLatencies).\n\nHow to read: three lines showing average latency per operation type in microseconds, computed as delta(opLatencies.*.latency) / delta(opLatencies.*.ops).\n\nTroubleshooting: latency spikes that correlate with ticket exhaustion indicate concurrency contention. Spikes correlating with disk I/O indicate storage bottlenecks. Compare with 'Operations' to distinguish 'fewer but slower ops' from 'more ops at same speed'.", + "description": "### Page faults\n\nRate of `extra_info.page_faults` (cumulative counter since process start). Page faults occur when MongoDB must access data not in physical memory. Per MongoDB docs, limited sporadic faults are normal; sustained high rates often mean the working set exceeds RAM.\n\n**Unit:** page faults/sec (Grafana `ops`)\n\n**Note:** On Windows, hard and soft page faults are both counted; fields in `extra_info` vary by platform.\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.extra_info.page_faults`\n\n---\n\n### How to read\n\n- Single line · spikes → memory pressure or cold data access\n- Flat near zero → working set mostly in memory\n\n---\n\n### Troubleshooting\n\n- Sustained high rate → working set > RAM · correlate with **WiredTiger Cache**, **mongod Memory (RSS / Virtual)**, **Swap Memory**\n- Spikes during scans/index builds → expected · check query patterns if persistent\n- High faults + active swap → add RAM or reduce working set\n\n---\n\n### Docs\n\n- [serverStatus — extra_info](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#extra_info)", "fieldConfig": { "defaults": { "color": { @@ -1606,6 +2023,7 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -1642,24 +2060,33 @@ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "µs" + "unit": "ops" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*page_faults$" + }, + "properties": [ + { + "id": "displayName", + "value": "Page faults (/sec)" + } + ] + } + ] }, "gridPos": { - "h": 7, - "w": 8, + "h": 6, + "w": 10, "x": 0, - "y": 42 + "y": 50 }, - "id": 5, + "id": 27, "options": { "legend": { "calcs": [ @@ -1674,7 +2101,7 @@ }, "tooltip": { "hideZeros": false, - "mode": "multi", + "mode": "single", "sort": "none" } }, @@ -1685,11 +2112,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.opLatencies.reads.latency\"\r\n or r[\"_field\"] == \"serverStatus.opLatencies.reads.ops\"\r\n or r[\"_field\"] == \"serverStatus.opLatencies.writes.latency\"\r\n or r[\"_field\"] == \"serverStatus.opLatencies.writes.ops\"\r\n or r[\"_field\"] == \"serverStatus.opLatencies.commands.latency\"\r\n or r[\"_field\"] == \"serverStatus.opLatencies.commands.ops\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> pivot(rowKey: [\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n reads_avg_us: if r[\"serverStatus.opLatencies.reads.ops\"] > 0.0\r\n then r[\"serverStatus.opLatencies.reads.latency\"] / r[\"serverStatus.opLatencies.reads.ops\"]\r\n else 0.0,\r\n writes_avg_us: if r[\"serverStatus.opLatencies.writes.ops\"] > 0.0\r\n then r[\"serverStatus.opLatencies.writes.latency\"] / r[\"serverStatus.opLatencies.writes.ops\"]\r\n else 0.0,\r\n commands_avg_us: if r[\"serverStatus.opLatencies.commands.ops\"] > 0.0\r\n then r[\"serverStatus.opLatencies.commands.latency\"] / r[\"serverStatus.opLatencies.commands.ops\"]\r\n else 0.0,\r\n }))\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.extra_info.page_faults\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", "refId": "A" } ], - "title": "Average Operation Latency", + "title": "Page Faults", "type": "timeseries" }, { @@ -1697,7 +2124,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Rate of serverStatus.opcounters per second by type: insert, query, update, delete, getmore, command. Per MongoDB docs, opcounters treats operations that affect multiple documents (bulk insert, multi-update) as a single operation. These values reflect received operations and increment even when operations are not successful. opcounters.command counts all commands except insert, update, delete, and aggregate (serverStatus.opcounters).\n\nHow to read: multiple lines showing ops/sec per operation type (derivative of cumulative counters).\n\nTroubleshooting: sudden drops may indicate blocking -- check 'Current Queue writers / readers' and 'WiredTiger Tickets'. Sudden spikes may cause resource contention. Use alongside 'Average Operation Latency' to distinguish throughput-driven from latency-driven performance issues.", + "description": "### Network requests\n\nRate of **network.numRequests** — distinct client requests received by mongod.\n\nUse with **Network In / Out** to relate request volume to logical byte throughput.\n\n**Unit:** requests/sec (`reqps`)\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.network.numRequests`\n\n---\n\n### How to read\n\n- Single line · request arrival rate\n- Not the same as **Operations** (opcounters) — one request can trigger multiple ops\n\n---\n\n### Troubleshooting\n\n- Sudden drop → client/network issue or load shift\n- Spike without matching **Operations** → lightweight commands (heartbeats, metadata)\n- High rate + high **Network In / Out** bytesOut → large responses · check projections/limits\n\n---\n\n### Docs\n\n- [serverStatus — network](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.network)", "fieldConfig": { "defaults": { "color": { @@ -1709,6 +2136,7 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -1720,7 +2148,7 @@ "viz": false }, "insertNulls": false, - "lineInterpolation": "stepAfter", + "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -1737,6 +2165,7 @@ "mode": "off" } }, + "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", @@ -1744,23 +2173,33 @@ { "color": "green", "value": 0 - }, + } + ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*numRequests$" + }, + "properties": [ { - "color": "red", - "value": 80 + "id": "displayName", + "value": "Network requests (/sec)" } ] } - }, - "overrides": [] + ] }, "gridPos": { - "h": 7, - "w": 7, - "x": 8, - "y": 42 + "h": 5, + "w": 14, + "x": 10, + "y": 52 }, - "id": 9, + "id": 28, "options": { "legend": { "calcs": [ @@ -1775,7 +2214,7 @@ }, "tooltip": { "hideZeros": false, - "mode": "multi", + "mode": "single", "sort": "none" } }, @@ -1786,11 +2225,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => \r\n r[\"_field\"] == \"serverStatus.opcounters.update\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.query\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.insert\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.getmore\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.delete\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.command\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.network.numRequests\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", "refId": "A" } ], - "title": "Operations", + "title": "Network Requests", "type": "timeseries" }, { @@ -1798,7 +2237,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Rate of serverStatus.network.numRequests per second. Per MongoDB docs, numRequests is the total number of distinct requests the server has received. Use this value to provide context for network.bytesIn and network.bytesOut to ensure network utilization is consistent with expectations (serverStatus.network).\n\nHow to read: single line showing requests/sec (derivative of cumulative counter).\n\nTroubleshooting: a sudden drop may indicate network issues or client-side failures. A spike correlates with increased load -- compare with 'Operations' to see if requests translate to actual database operations.", + "description": "### Average operation latency\n\nRolling average latency per operation type, derived from cumulative `opLatencies` counters.\n\n**Formula:** `derivative(*.latency) / derivative(*.ops)` → **µs per operation** (same as Δlatency / Δops over each window).\n\n**Unit:** microseconds (`µs`)\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.opLatencies.reads.latency` · `serverStatus.opLatencies.reads.ops`\n- `serverStatus.opLatencies.writes.latency` · `serverStatus.opLatencies.writes.ops`\n- `serverStatus.opLatencies.commands.latency` · `serverStatus.opLatencies.commands.ops`\n\n---\n\n### How to read\n\n- Three lines: read, write, command average latency\n- Spikes → slower operations in that class\n- Flat at **0** → no ops in window (idle), not zero latency\n\n---\n\n### Troubleshooting\n\n- Spikes + low **WiredTiger Tickets** → concurrency contention\n- Spikes + disk I/O → storage bottleneck · **Page Faults**, **Disk I/O**\n- Latency up, ops down → **Operations Latencies Op**, **Operations**\n- Commands line often highest → includes admin/aggregate work\n\n---\n\n### Docs\n\n- [serverStatus — opLatencies](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.opLatencies)", "fieldConfig": { "defaults": { "color": { @@ -1810,7 +2249,8 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", - "barAlignment": -1, + "axisSoftMin": 0, + "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, @@ -1838,7 +2278,7 @@ "mode": "off" } }, - "fieldMinMax": true, + "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", @@ -1846,24 +2286,57 @@ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "reqps" + "unit": "µs" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "reads_avg_us" + }, + "properties": [ + { + "id": "displayName", + "value": "Read latency (avg)" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "writes_avg_us" + }, + "properties": [ + { + "id": "displayName", + "value": "Write latency (avg)" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "commands_avg_us" + }, + "properties": [ + { + "id": "displayName", + "value": "Command latency (avg)" + } + ] + } + ] }, "gridPos": { - "h": 5, - "w": 9, - "x": 15, - "y": 44 + "h": 9, + "w": 10, + "x": 0, + "y": 56 }, - "id": 28, + "id": 5, "options": { "legend": { "calcs": [ @@ -1889,11 +2362,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.network.numRequests\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: v.windowPeriod)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.opLatencies.reads.latency\" or\r\n r[\"_field\"] == \"serverStatus.opLatencies.reads.ops\" or\r\n r[\"_field\"] == \"serverStatus.opLatencies.writes.latency\" or\r\n r[\"_field\"] == \"serverStatus.opLatencies.writes.ops\" or\r\n r[\"_field\"] == \"serverStatus.opLatencies.commands.latency\" or\r\n r[\"_field\"] == \"serverStatus.opLatencies.commands.ops\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> pivot(rowKey: [\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n reads_avg_us: if float(v: r[\"serverStatus.opLatencies.reads.ops\"]) > 0.0\r\n then float(v: r[\"serverStatus.opLatencies.reads.latency\"]) / float(v: r[\"serverStatus.opLatencies.reads.ops\"])\r\n else 0.0,\r\n writes_avg_us: if float(v: r[\"serverStatus.opLatencies.writes.ops\"]) > 0.0\r\n then float(v: r[\"serverStatus.opLatencies.writes.latency\"]) / float(v: r[\"serverStatus.opLatencies.writes.ops\"])\r\n else 0.0,\r\n commands_avg_us: if float(v: r[\"serverStatus.opLatencies.commands.ops\"]) > 0.0\r\n then float(v: r[\"serverStatus.opLatencies.commands.latency\"]) / float(v: r[\"serverStatus.opLatencies.commands.ops\"])\r\n else 0.0,\r\n }))", "refId": "A" } ], - "title": "Network Requests", + "title": "Average Operation Latency", "type": "timeseries" }, { @@ -1901,7 +2374,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Shows serverStatus.globalLock.currentQueue.readers and currentQueue.writers. Per MongoDB docs, currentQueue.readers is the number of operations currently queued waiting for the read lock; currentQueue.writers is the same for the write lock. A consistently small queue, particularly of shorter operations, should cause no concern (serverStatus.globalLock).\n\nHow to read: two lines showing queued reader and writer counts over time.\n\nTroubleshooting: a persistent non-zero queue indicates lock contention -- this is one of the first panels to check during 'slow database' investigations. Write queue buildup often correlates with WiredTiger ticket exhaustion. Correlate with 'WiredTiger Tickets' and 'Average Operation Latency'.", + "description": "### Connections created\n\nRate of new connections (`connections.totalCreated` derivative). Indicates connection **churn** — apps opening/closing connections frequently.\n\n**Unit:** connections/sec (`ops`)\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.connections.totalCreated`\n\n---\n\n### How to read\n\n- Single line · spikes → bursts of new connections\n- Sustained high rate → pool churn · check app connection pool settings\n\n---\n\n### Troubleshooting\n\n- High rate + high **Current** on **Connections** → many short-lived connections\n- Correlate with **Connections** (counts)\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)", "fieldConfig": { "defaults": { "color": { @@ -1941,6 +2414,7 @@ "mode": "off" } }, + "decimals": 2, "mappings": [], "thresholds": { "mode": "absolute", @@ -1948,23 +2422,33 @@ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] - } + }, + "unit": "ops" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*connections\\.totalCreated$" + }, + "properties": [ + { + "id": "displayName", + "value": "Connections created (/sec)" + } + ] + } + ] }, "gridPos": { "h": 6, - "w": 8, - "x": 0, - "y": 49 + "w": 14, + "x": 10, + "y": 57 }, - "id": 29, + "id": 52, "options": { "legend": { "calcs": [ @@ -1979,7 +2463,7 @@ }, "tooltip": { "hideZeros": false, - "mode": "multi", + "mode": "single", "sort": "none" } }, @@ -1990,11 +2474,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.globalLock.currentQueue.writers\" or r[\"_field\"] == \"serverStatus.globalLock.currentQueue.readers\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> yield(name: \"last\")", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.connections.totalCreated\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", "refId": "A" } ], - "title": "Current Queue writers / readers", + "title": "Connections Created", "type": "timeseries" }, { @@ -2002,7 +2486,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Shows replSetGetStatus.members.N.health for each replica set member. Health = 1 means the member is reachable and healthy; health = 0 means the member is unreachable or down.\n\nHow to read: one line per replica member showing binary health state over time.\n\nTroubleshooting: a transition from 1 to 0 means a member became unavailable -- check for network partitions, disk failures, or OOM kills. Correlate the exact timestamp of the health drop with other panels (CPU, memory, disk) to identify root cause.", + "description": "### Replica members lag\n\nWall-clock replication lag per member:\n\n**Formula:** `(serverStatus.localTime − members.N.lastAppliedWallTime) / 1000` → **seconds**\n\nBoth timestamps are **milliseconds** in FTDC; the query converts the delta to seconds.\n\n**Unit:** seconds (`s`)\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.localTime`\n- `replSetGetStatus.members.0.lastAppliedWallTime` … `replSetGetStatus.members.4.lastAppliedWallTime`\n\nMember **N** is the array index in `replSetGetStatus.members` (not necessarily hostname order).\n\n---\n\n### How to read\n\n- One line per member index (0–4)\n- **~0** → caught up (primary or healthy secondary)\n- Rising trend on a secondary → replication falling behind\n- Flat **0** for missing index → no member in that slot\n\n---\n\n### Troubleshooting\n\n- Secondary lag rising → disk I/O, CPU, network on that node · **Opcounters Replica**, **Disk I/O**\n- Lag spikes during checkpoints → I/O contention\n- Primary throttling → **FlowControl isLagged** · **Replica Members Ping**\n- Member down → **Replica Members Health**\n\n---\n\n### Docs\n\n- [replSetGetStatus](https://www.mongodb.com/docs/manual/reference/command/replSetGetStatus/)\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)", "fieldConfig": { "defaults": { "color": { @@ -2014,10 +2498,11 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 25, + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, @@ -2025,7 +2510,7 @@ "viz": false }, "insertNulls": false, - "lineInterpolation": "stepAfter", + "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -2036,38 +2521,95 @@ "spanNulls": false, "stacking": { "group": "A", - "mode": "normal" + "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, + "decimals": 1, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "none" + "unit": "s" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "member_0" + }, + "properties": [ + { + "id": "displayName", + "value": "Member 0 lag" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "member_1" + }, + "properties": [ + { + "id": "displayName", + "value": "Member 1 lag" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "member_2" + }, + "properties": [ + { + "id": "displayName", + "value": "Member 2 lag" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "member_3" + }, + "properties": [ + { + "id": "displayName", + "value": "Member 3 lag" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "member_4" + }, + "properties": [ + { + "id": "displayName", + "value": "Member 4 lag" + } + ] + } + ] }, "gridPos": { - "h": 6, - "w": 7, - "x": 8, - "y": 49 + "h": 7, + "w": 14, + "x": 10, + "y": 63 }, - "id": 16, + "id": 15, "options": { "legend": { "calcs": [ @@ -2093,11 +2635,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] =~ /replSetGetStatus\\.members\\.\\d+\\.health/)\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> yield(name: \"last\")", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] =~ /replSetGetStatus\\.members\\.\\d+\\.lastAppliedWallTime|serverStatus\\.localTime/)\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> pivot(rowKey: [\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n member_0: if exists r[\"replSetGetStatus.members.0.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.0.lastAppliedWallTime\"])) / 1000.0 else 0.0,\r\n member_1: if exists r[\"replSetGetStatus.members.1.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.1.lastAppliedWallTime\"])) / 1000.0 else 0.0,\r\n member_2: if exists r[\"replSetGetStatus.members.2.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.2.lastAppliedWallTime\"])) / 1000.0 else 0.0,\r\n member_3: if exists r[\"replSetGetStatus.members.3.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.3.lastAppliedWallTime\"])) / 1000.0 else 0.0,\r\n member_4: if exists r[\"replSetGetStatus.members.4.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.4.lastAppliedWallTime\"])) / 1000.0 else 0.0,\r\n }))", "refId": "A" } ], - "title": "Replica Members Health", + "title": "Replica Members Lag", "type": "timeseries" }, { @@ -2105,7 +2647,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Shows replication lag per member, computed as the difference between serverStatus.localTime and replSetGetStatus.members.N.lastAppliedWallTime in seconds.\n\nHow to read: one line per member showing lag in seconds. Zero means the member is fully caught up.\n\nTroubleshooting: increasing lag on a secondary indicates it cannot keep up with the primary's write rate. Check the secondary's disk I/O, CPU, and network bandwidth. Lag spikes during checkpoint intervals suggest I/O contention. Correlate with 'FlowControl isLagged' to see if flow control is throttling the primary.", + "description": "### Flow control is lagged\n\nBoolean **`flowControl.isLagged`** — whether flow control is **currently** throttling writes on the **primary**.\n\n**Unit:** 0 = off · 1 = on\n\n**Note:** Primary only; secondaries return placeholders. For **time spent** throttled, see **Flow control lagged time**.\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.flowControl.isLagged`\n\n---\n\n### How to read\n\n- **0** → not engaging (even if secondaries lag)\n- **1** → primary write rate limited so secondaries can catch up\n\n---\n\n### Troubleshooting\n\n- **1** sustained → **Replica Members Lag**, **Opcounters Replica**, secondary disk/network\n- Lag without **1** → lag below `flowControlTargetLagSeconds` threshold\n\n---\n\n### Docs\n\n- [serverStatus — flowControl](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.flowControl)", "fieldConfig": { "defaults": { "color": { @@ -2117,6 +2659,7 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMax": 1, "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, @@ -2129,8 +2672,8 @@ "viz": false }, "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, + "lineInterpolation": "stepAfter", + "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" @@ -2146,31 +2689,57 @@ "mode": "off" } }, - "mappings": [], + "decimals": 0, + "mappings": [ + { + "options": { + "0": { + "index": 0, + "text": "Off" + }, + "1": { + "index": 1, + "text": "On" + } + }, + "type": "value" + } + ], + "max": 1, + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "ms" + "unit": "short" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*isLagged$" + }, + "properties": [ + { + "id": "displayName", + "value": "Flow control engaged" + } + ] + } + ] }, "gridPos": { "h": 6, - "w": 9, - "x": 15, - "y": 49 + "w": 10, + "x": 0, + "y": 65 }, - "id": 15, + "id": 14, "options": { "legend": { "calcs": [ @@ -2185,7 +2754,7 @@ }, "tooltip": { "hideZeros": false, - "mode": "multi", + "mode": "single", "sort": "none" } }, @@ -2196,11 +2765,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] =~ /replSetGetStatus\\.members\\.\\d+\\.lastAppliedWallTime|serverStatus\\.localTime/)\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> pivot(rowKey:[\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n member_0: if exists r[\"replSetGetStatus.members.0.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.0.lastAppliedWallTime\"])) / 1000.0 else 0.0,\r\n member_1: if exists r[\"replSetGetStatus.members.1.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.1.lastAppliedWallTime\"])) / 1000.0 else 0.0,\r\n member_2: if exists r[\"replSetGetStatus.members.2.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.2.lastAppliedWallTime\"])) / 1000.0 else 0.0,\r\n member_3: if exists r[\"replSetGetStatus.members.3.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.3.lastAppliedWallTime\"])) / 1000.0 else 0.0, // Add more members as needed\r\n }))\r\n |> yield(name: \"differences\")\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.flowControl.isLagged\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", "refId": "A" } ], - "title": "Replica Members Lag", + "title": "FlowControl isLagged", "type": "timeseries" }, { @@ -2208,7 +2777,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Rate of serverStatus.opLatencies.reads.ops, opLatencies.writes.ops, and opLatencies.commands.ops per second. Per MongoDB docs, opLatencies contains operation latencies for the instance as a whole (serverStatus.opLatencies).\n\nHow to read: three lines showing the operation rate (ops/sec) per latency class -- this is the denominator used to compute average latency.\n\nTroubleshooting: compare with 'Average Operation Latency' -- if ops/sec drops while latency rises, individual operations are getting slower. If both rise together, the system is under increased load. A divergence between reads.ops and writes.ops can indicate workload imbalance.", + "description": "### Replica members ping\n\n**`replSetGetStatus.members.N.pingMs`** — round-trip time in **milliseconds** from **this FTDC host** to each replica set member (repl heartbeat).\n\n**Unit:** milliseconds (`ms`)\n\n**Note:** **N** is the member array index in `replSetGetStatus.members`, not hostname order. Use the **hostname** template variable to pick which mongod’s view of the set you are viewing.\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `replSetGetStatus.members.0.pingMs`\n- `replSetGetStatus.members.1.pingMs`\n- `replSetGetStatus.members.2.pingMs`\n- `replSetGetStatus.members.3.pingMs`\n- `replSetGetStatus.members.4.pingMs`\n\n---\n\n### How to read\n\n- One line per member index (**Member 0 ping** … **Member 4 ping**)\n- **stepAfter** reflects discrete heartbeat samples\n- Low stable values → healthy paths between members\n- Spikes → latency or congestion between nodes\n\n---\n\n### Troubleshooting\n\n- Ping rising with **Replica Members Lag** → network may limit replication\n- Ping OK but lag high → disk/CPU on secondaries, not network\n- Correlate with **TCP** panels (RetransSegs, InSegs/OutSegs) and **Replica Members Health**\n\n---\n\n### Docs\n\n- [replSetGetStatus](https://www.mongodb.com/docs/manual/reference/command/replSetGetStatus/)", "fieldConfig": { "defaults": { "color": { @@ -2220,6 +2789,7 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -2231,7 +2801,7 @@ "viz": false }, "insertNulls": false, - "lineInterpolation": "linear", + "lineInterpolation": "stepAfter", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -2248,6 +2818,7 @@ "mode": "off" } }, + "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", @@ -2255,24 +2826,81 @@ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "µs" + "unit": "ms" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*members\\.0\\.pingMs$" + }, + "properties": [ + { + "id": "displayName", + "value": "Member 0 ping" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*members\\.1\\.pingMs$" + }, + "properties": [ + { + "id": "displayName", + "value": "Member 1 ping" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*members\\.2\\.pingMs$" + }, + "properties": [ + { + "id": "displayName", + "value": "Member 2 ping" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*members\\.3\\.pingMs$" + }, + "properties": [ + { + "id": "displayName", + "value": "Member 3 ping" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*members\\.4\\.pingMs$" + }, + "properties": [ + { + "id": "displayName", + "value": "Member 4 ping" + } + ] + } + ] }, "gridPos": { "h": 5, - "w": 8, - "x": 0, - "y": 55 + "w": 9, + "x": 10, + "y": 70 }, - "id": 10, + "id": 25, "options": { "legend": { "calcs": [ @@ -2298,11 +2926,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.opLatencies.reads.ops\" or r[\"_field\"] == \"serverStatus.opLatencies.writes.ops\" or r[\"_field\"] == \"serverStatus.opLatencies.commands.ops\" )\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s)\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"replSetGetStatus.members.0.pingMs\" or\r\n r[\"_field\"] == \"replSetGetStatus.members.1.pingMs\" or\r\n r[\"_field\"] == \"replSetGetStatus.members.2.pingMs\" or\r\n r[\"_field\"] == \"replSetGetStatus.members.3.pingMs\" or\r\n r[\"_field\"] == \"replSetGetStatus.members.4.pingMs\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", "refId": "A" } ], - "title": "Operations Latencies Op", + "title": "Replica Members Ping", "type": "timeseries" }, { @@ -2310,7 +2938,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Shows serverStatus.uptime -- the number of seconds the current MongoDB process has been active (serverStatus.uptime).\n\nHow to read: a monotonically increasing line. A drop to zero or a sudden decrease means the mongod process restarted.\n\nTroubleshooting: identify restart timestamps and correlate with other panels to determine cause -- OOM kill (check 'mongod Memory (RSS / Virtual)' and 'Swap Memory'), crash (check 'Asserts'), or manual intervention.", + "description": "### Flow control lagged time\n\n**Fraction of each window** the primary spent with flow control engaged, from **`flowControl.isLaggedTimeMicros`**.\n\n**Formula:** `derivative(isLaggedTimeMicros, 1s) / 1e6` → 0–100%\n\n**Note:** Primary only. For **on/off now**, see **FlowControl isLagged**.\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.flowControl.isLaggedTimeMicros`\n\n---\n\n### How to read\n\n- **0%** → no throttling in that window\n- **100%** → flow control engaged the entire window\n- Spikes → brief throttling bursts\n\n---\n\n### Troubleshooting\n\n- Sustained **>0%** → correlate with **Replica Members Lag**, **FlowControl isLagged**\n- High % + rising lag → secondaries cannot keep up · disk/CPU/network\n\n---\n\n### Docs\n\n- [serverStatus — flowControl](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.flowControl)", "fieldConfig": { "defaults": { "color": { @@ -2322,6 +2950,7 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMax": 1, "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, @@ -2335,7 +2964,7 @@ }, "insertNulls": false, "lineInterpolation": "linear", - "lineWidth": 1, + "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" @@ -2351,31 +2980,43 @@ "mode": "off" } }, + "decimals": 1, "mappings": [], + "max": 1, + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "none" + "unit": "percentunit" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*isLaggedTimeMicros$" + }, + "properties": [ + { + "id": "displayName", + "value": "Flow control engaged (% of time)" + } + ] + } + ] }, "gridPos": { - "h": 5, - "w": 7, - "x": 8, - "y": 55 + "h": 6, + "w": 10, + "x": 0, + "y": 71 }, - "id": 20, + "id": 53, "options": { "legend": { "calcs": [ @@ -2390,7 +3031,7 @@ }, "tooltip": { "hideZeros": false, - "mode": "multi", + "mode": "single", "sort": "none" } }, @@ -2401,12 +3042,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "hide": false, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.uptime\" )\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.flowControl.isLaggedTimeMicros\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) / 1000000.0 }))", "refId": "A" } ], - "title": "Server Uptime", + "title": "Flow control lagged time", "type": "timeseries" }, { @@ -2414,7 +3054,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Shows replSetGetStatus.members.N.pingMs -- the round-trip time in milliseconds from this member to other replica set members.\n\nHow to read: one line per member showing ping latency over time.\n\nTroubleshooting: elevated ping times indicate network latency or congestion between members. Ping spikes that correlate with replication lag suggest the network is a bottleneck for oplog replication. Compare with 'TCP' panels for underlying transport issues.", + "description": "Rate of serverStatus.opcounters per second by type: insert, query, update, delete, getmore, command. Per MongoDB docs, opcounters treats operations that affect multiple documents (bulk insert, multi-update) as a single operation. These values reflect received operations and increment even when operations are not successful. opcounters.command counts all commands except insert, update, delete, and aggregate (serverStatus.opcounters).\n\nHow to read: multiple lines showing ops/sec per operation type (derivative of cumulative counters).\n\nTroubleshooting: sudden drops may indicate blocking -- check 'Current Queue writers / readers' and 'WiredTiger Tickets'. Sudden spikes may cause resource contention. Use alongside 'Average Operation Latency' to distinguish throughput-driven from latency-driven performance issues.", "fieldConfig": { "defaults": { "color": { @@ -2437,7 +3077,7 @@ "viz": false }, "insertNulls": false, - "lineInterpolation": "linear", + "lineInterpolation": "stepAfter", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -2467,18 +3107,17 @@ "value": 80 } ] - }, - "unit": "ms" + } }, "overrides": [] }, "gridPos": { - "h": 5, - "w": 9, - "x": 15, - "y": 55 + "h": 7, + "w": 7, + "x": 10, + "y": 75 }, - "id": 25, + "id": 9, "options": { "legend": { "calcs": [ @@ -2504,11 +3143,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] =~ /replSetGetStatus\\.members\\.\\d+\\.pingMs/)\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => \r\n r[\"_field\"] == \"serverStatus.opcounters.update\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.query\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.insert\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.getmore\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.delete\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.command\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)", "refId": "A" } ], - "title": "Replica Members Ping", + "title": "Operations", "type": "timeseries" }, { @@ -2516,7 +3155,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Shows the derivative of serverStatus.flowControl.isLaggedTimeMicros -- the cumulative time flow control has been engaged. Per MongoDB docs, flow control engages when the majority committed lag grows close to flowControlTargetLagSeconds; writes on the primary must then obtain tickets before taking locks (serverStatus.flowControl).\n\nHow to read: a non-zero derivative means flow control was actively throttling the primary during that interval.\n\nTroubleshooting: sustained non-zero values mean the primary is being throttled to let secondaries catch up. Check 'Replica Members Lag', secondary disk I/O, and 'Opcounters Replica' to understand why secondaries are falling behind.", + "description": "Rate of serverStatus.metrics.document counters per second: deleted, inserted, returned, updated. Per MongoDB docs, metrics.document reflects document access and modification patterns. Compare with opcounters, which tracks total number of operations -- since one query can return many documents (serverStatus.metrics.document).\n\nHow to read: four lines showing document-level operation rates.\n\nTroubleshooting: high 'returned' with low 'scannedObjects' (from 'Query Executor') indicates efficient queries. High 'returned' with high 'scannedObjects' means poor query targeting. High 'deleted' may indicate TTL index activity or bulk cleanup operations.", "fieldConfig": { "defaults": { "color": { @@ -2540,7 +3179,7 @@ }, "insertNulls": false, "lineInterpolation": "linear", - "lineWidth": 2, + "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" @@ -2569,18 +3208,17 @@ "value": 80 } ] - }, - "unit": "µs" + } }, "overrides": [] }, "gridPos": { - "h": 6, + "h": 7, "w": 8, "x": 0, - "y": 60 + "y": 77 }, - "id": 14, + "id": 36, "options": { "legend": { "calcs": [ @@ -2606,11 +3244,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.flowControl.isLaggedTimeMicros\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> derivative(unit: v.windowPeriod)\r\n |> yield(name: \"last\")", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.metrics.document.updated\" or r[\"_field\"] == \"serverStatus.metrics.document.returned\" or r[\"_field\"] == \"serverStatus.metrics.document.inserted\" or r[\"_field\"] == \"serverStatus.metrics.document.deleted\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: v.windowPeriod)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", "refId": "A" } ], - "title": "FlowControl isLagged", + "title": "Documents (deleted / inserted / returned / updated)", "type": "timeseries" }, { @@ -2618,7 +3256,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Rate of serverStatus.asserts by type: regular, warning, msg, user, rollovers. Per MongoDB docs, assertions are internal checks for errors that occur while the database is operating. Non-zero values are uncommon. asserts.user are errors clients may generate (e.g., out of disk space, duplicate key). asserts.rollovers counts how many times assert counters have rolled over after 2^30 assertions (serverStatus.asserts).\n\nHow to read: multiple lines showing assert rate/sec per type.\n\nTroubleshooting: any sustained increase in regular or msg asserts may indicate bugs or data corruption -- examine the MongoDB log. High user asserts indicate application-level errors (duplicate keys, validation failures). Check application logs for corresponding errors.", + "description": "Ratio of serverStatus.metrics.queryExecutor.scannedObjects to metrics.document.returned, computed as a per-second rate ratio. Per MongoDB docs, scannedObjects is equivalent to totalDocsExamined in explain(); document.returned is the total number of documents returned by queries (serverStatus.metrics.queryExecutor, serverStatus.metrics.document).\n\nHow to read: single line showing the scanned-to-returned ratio. A ratio of 1.0 means every scanned document was returned (perfect targeting).\n\nTroubleshooting: ratios significantly above 1.0 indicate queries scanning many documents to return few results -- the classic sign of missing or suboptimal indexes. This is often the number one cause of unnecessary disk I/O and CPU load. Run explain() on slow queries to identify the problem.", "fieldConfig": { "defaults": { "color": { @@ -2642,7 +3280,7 @@ }, "insertNulls": false, "lineInterpolation": "linear", - "lineWidth": 1, + "lineWidth": 4, "pointSize": 5, "scaleDistribution": { "type": "linear" @@ -2658,7 +3296,9 @@ "mode": "off" } }, + "decimals": 2, "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -2676,12 +3316,12 @@ "overrides": [] }, "gridPos": { - "h": 7, + "h": 6, "w": 7, - "x": 8, - "y": 60 + "x": 10, + "y": 82 }, - "id": 30, + "id": 37, "options": { "legend": { "calcs": [ @@ -2707,11 +3347,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.asserts.warning\" or r[\"_field\"] == \"serverStatus.asserts.user\" or r[\"_field\"] == \"serverStatus.asserts.rollovers\" or r[\"_field\"] == \"serverStatus.asserts.regular\" or r[\"_field\"] == \"serverStatus.asserts.msg\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: v.windowPeriod)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.metrics.document.returned\" or \r\n r[\"_field\"] == \"serverStatus.metrics.queryExecutor.scannedObjects\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> pivot(\r\n rowKey: [\"_time\"],\r\n columnKey: [\"_field\"],\r\n valueColumn: \"_value\"\r\n )\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n scanned_returned_ratio:\r\n if r[\"serverStatus.metrics.document.returned\"] > 0.0\r\n then r[\"serverStatus.metrics.queryExecutor.scannedObjects\"]\r\n / r[\"serverStatus.metrics.document.returned\"]\r\n else 0.0\r\n }))\r\n |> yield(name: \"scanned_returned_ratio\")\r\n", "refId": "A" } ], - "title": "Asserts", + "title": "Query Targeting: Scanned Objects / Returned", "type": "timeseries" }, { @@ -2719,7 +3359,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Rate of serverStatus.metrics.queryExecutor metrics: scanned (total index items scanned, equivalent to totalKeysExamined in explain()), scannedObjects (total documents scanned, equivalent to totalDocsExamined), and collectionScans.total (queries that performed a collection scan) per second (serverStatus.metrics.queryExecutor).\n\nHow to read: multiple lines showing rates. High scannedObjects means many documents examined; high collectionScans.total means queries are not using indexes.\n\nTroubleshooting: high scannedObjects relative to documents returned indicates inefficient queries -- run explain() to identify missing indexes. High collectionScans.total is a red flag for production workloads; identify the queries via the profiler.", + "description": "Rate of serverStatus.metrics.commands per second. Per MongoDB docs, metrics.commands reports on the use of database commands. opcounters.command counts all commands except insert, update, delete, and aggregate (serverStatus.metrics.commands, serverStatus.opcounters).\n\nHow to read: single line showing command execution rate.\n\nTroubleshooting: a spike in command rate may indicate runaway aggregation pipelines, administrative operations (index builds, compact), or driver heartbeat storms. Correlate with 'Average Operation Latency' and 'CPU Usage' to assess impact.", "fieldConfig": { "defaults": { "color": { @@ -2772,18 +3412,17 @@ "value": 80 } ] - }, - "unit": "short" + } }, "overrides": [] }, "gridPos": { "h": 6, - "w": 9, - "x": 15, - "y": 60 + "w": 8, + "x": 0, + "y": 84 }, - "id": 31, + "id": 32, "options": { "legend": { "calcs": [ @@ -2809,11 +3448,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.metrics.queryExecutor.scannedObjects\" or r[\"_field\"] == \"serverStatus.metrics.queryExecutor.scanned\" or r[\"_field\"] == \"serverStatus.metrics.queryExecutor.collectionScans.total\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: v.windowPeriod)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.metrics.commands.createIndexes.total\" or r[\"_field\"] == \"serverStatus.metrics.commands.dataSize.total\" or r[\"_field\"] == \"serverStatus.metrics.commands.dropIndexes.total\" or r[\"_field\"] == \"serverStatus.metrics.commands.listCollections.total\" or r[\"_field\"] == \"serverStatus.metrics.commands.listDatabases.total\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n|> derivative(unit: v.windowPeriod)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", "refId": "A" } ], - "title": "Query Executor", + "title": "Commands", "type": "timeseries" }, { @@ -2821,7 +3460,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Rate of serverStatus.metrics.commands per second. Per MongoDB docs, metrics.commands reports on the use of database commands. opcounters.command counts all commands except insert, update, delete, and aggregate (serverStatus.metrics.commands, serverStatus.opcounters).\n\nHow to read: single line showing command execution rate.\n\nTroubleshooting: a spike in command rate may indicate runaway aggregation pipelines, administrative operations (index builds, compact), or driver heartbeat storms. Correlate with 'Average Operation Latency' and 'CPU Usage' to assess impact.", + "description": "Rate of serverStatus.metrics.queryExecutor metrics: scanned (total index items scanned, equivalent to totalKeysExamined in explain()), scannedObjects (total documents scanned, equivalent to totalDocsExamined), and collectionScans.total (queries that performed a collection scan) per second (serverStatus.metrics.queryExecutor).\n\nHow to read: multiple lines showing rates. High scannedObjects means many documents examined; high collectionScans.total means queries are not using indexes.\n\nTroubleshooting: high scannedObjects relative to documents returned indicates inefficient queries -- run explain() to identify missing indexes. High collectionScans.total is a red flag for production workloads; identify the queries via the profiler.", "fieldConfig": { "defaults": { "color": { @@ -2874,17 +3513,18 @@ "value": 80 } ] - } + }, + "unit": "short" }, "overrides": [] }, "gridPos": { "h": 6, - "w": 8, - "x": 0, - "y": 66 + "w": 9, + "x": 10, + "y": 88 }, - "id": 32, + "id": 31, "options": { "legend": { "calcs": [ @@ -2910,11 +3550,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.metrics.commands.createIndexes.total\" or r[\"_field\"] == \"serverStatus.metrics.commands.dataSize.total\" or r[\"_field\"] == \"serverStatus.metrics.commands.dropIndexes.total\" or r[\"_field\"] == \"serverStatus.metrics.commands.listCollections.total\" or r[\"_field\"] == \"serverStatus.metrics.commands.listDatabases.total\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n|> derivative(unit: v.windowPeriod)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.metrics.queryExecutor.scannedObjects\" or r[\"_field\"] == \"serverStatus.metrics.queryExecutor.scanned\" or r[\"_field\"] == \"serverStatus.metrics.queryExecutor.collectionScans.total\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: v.windowPeriod)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", "refId": "A" } ], - "title": "Commands", + "title": "Query Executor", "type": "timeseries" }, { @@ -2922,7 +3562,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Shows dbStats.avgObjSize -- the average size of each document in bytes, computed as dataSize divided by the number of documents. Per MongoDB docs, the scale argument does not affect avgObjSize (dbStats.avgObjSize).\n\nHow to read: single line showing average document size over time.\n\nTroubleshooting: a sudden increase may indicate documents growing from array pushes or embedded document growth, which affects WiredTiger cache efficiency, network bandwidth, and replication lag. A decrease after TTL cleanup or data migration is expected.", + "description": "Shows serverStatus.metrics.cursor.open.total -- the number of cursors MongoDB is maintaining for clients. Per MongoDB docs, typically this value is small or zero; however, if there is a queue, stale tailable cursors, or a large number of operations, this value may increase. metrics.cursor.timedOut tracks cursors that timed out (serverStatus.metrics.cursor).\n\nHow to read: single line showing open cursor count over time.\n\nTroubleshooting: a growing cursor count may indicate cursor leaks in the application (not calling close()). Cursor timeouts (default 10 min) normally clean up idle cursors. If timedOut is also growing, applications are abandoning cursors without exhausting or closing them.", "fieldConfig": { "defaults": { "color": { @@ -2982,11 +3622,11 @@ }, "gridPos": { "h": 6, - "w": 9, - "x": 15, - "y": 66 + "w": 7, + "x": 0, + "y": 90 }, - "id": 34, + "id": 33, "options": { "legend": { "calcs": [ @@ -3012,11 +3652,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"local.oplog.rs.stats.storageStats.avgObjSize\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.metrics.cursor.open.noTimeout\" or r[\"_field\"] == \"serverStatus.metrics.cursor.open.pinned\" or r[\"_field\"] == \"serverStatus.metrics.cursor.open.total\" or r[\"_field\"] == \"serverStatus.metrics.cursor.timedOut\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)", "refId": "A" } ], - "title": "avg Obj Size", + "title": "Cursors", "type": "timeseries" }, { @@ -3024,7 +3664,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Shows serverStatus.metrics.cursor.open.total -- the number of cursors MongoDB is maintaining for clients. Per MongoDB docs, typically this value is small or zero; however, if there is a queue, stale tailable cursors, or a large number of operations, this value may increase. metrics.cursor.timedOut tracks cursors that timed out (serverStatus.metrics.cursor).\n\nHow to read: single line showing open cursor count over time.\n\nTroubleshooting: a growing cursor count may indicate cursor leaks in the application (not calling close()). Cursor timeouts (default 10 min) normally clean up idle cursors. If timedOut is also growing, applications are abandoning cursors without exhausting or closing them.", + "description": "Rate of serverStatus.opLatencies.reads.ops, opLatencies.writes.ops, and opLatencies.commands.ops per second. Per MongoDB docs, opLatencies contains operation latencies for the instance as a whole (serverStatus.opLatencies).\n\nHow to read: three lines showing the operation rate (ops/sec) per latency class -- this is the denominator used to compute average latency.\n\nTroubleshooting: compare with 'Average Operation Latency' -- if ops/sec drops while latency rises, individual operations are getting slower. If both rise together, the system is under increased load. A divergence between reads.ops and writes.ops can indicate workload imbalance.", "fieldConfig": { "defaults": { "color": { @@ -3065,7 +3705,6 @@ } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -3078,17 +3717,18 @@ "value": 80 } ] - } + }, + "unit": "µs" }, "overrides": [] }, "gridPos": { - "h": 6, - "w": 7, + "h": 5, + "w": 8, "x": 8, - "y": 67 + "y": 94 }, - "id": 33, + "id": 10, "options": { "legend": { "calcs": [ @@ -3114,11 +3754,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.metrics.cursor.open.noTimeout\" or r[\"_field\"] == \"serverStatus.metrics.cursor.open.pinned\" or r[\"_field\"] == \"serverStatus.metrics.cursor.open.total\" or r[\"_field\"] == \"serverStatus.metrics.cursor.timedOut\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.opLatencies.reads.ops\" or r[\"_field\"] == \"serverStatus.opLatencies.writes.ops\" or r[\"_field\"] == \"serverStatus.opLatencies.commands.ops\" )\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s)\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n", "refId": "A" } ], - "title": "Cursors", + "title": "Operations Latencies Op", "type": "timeseries" }, { @@ -3126,7 +3766,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Rate of serverStatus.metrics.document counters per second: deleted, inserted, returned, updated. Per MongoDB docs, metrics.document reflects document access and modification patterns. Compare with opcounters, which tracks total number of operations -- since one query can return many documents (serverStatus.metrics.document).\n\nHow to read: four lines showing document-level operation rates.\n\nTroubleshooting: high 'returned' with low 'scannedObjects' (from 'Query Executor') indicates efficient queries. High 'returned' with high 'scannedObjects' means poor query targeting. High 'deleted' may indicate TTL index activity or bulk cleanup operations.", + "description": "Shows dbStats.avgObjSize -- the average size of each document in bytes, computed as dataSize divided by the number of documents. Per MongoDB docs, the scale argument does not affect avgObjSize (dbStats.avgObjSize).\n\nHow to read: single line showing average document size over time.\n\nTroubleshooting: a sudden increase may indicate documents growing from array pushes or embedded document growth, which affects WiredTiger cache efficiency, network bandwidth, and replication lag. A decrease after TTL cleanup or data migration is expected.", "fieldConfig": { "defaults": { "color": { @@ -3167,6 +3807,7 @@ } }, "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -3184,12 +3825,12 @@ "overrides": [] }, "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 72 + "h": 6, + "w": 9, + "x": 7, + "y": 99 }, - "id": 36, + "id": 34, "options": { "legend": { "calcs": [ @@ -3215,11 +3856,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.metrics.document.updated\" or r[\"_field\"] == \"serverStatus.metrics.document.returned\" or r[\"_field\"] == \"serverStatus.metrics.document.inserted\" or r[\"_field\"] == \"serverStatus.metrics.document.deleted\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: v.windowPeriod)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"local.oplog.rs.stats.storageStats.avgObjSize\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", "refId": "A" } ], - "title": "Documents (deleted / inserted / returned / updated)", + "title": "avg Obj Size", "type": "timeseries" }, { @@ -3227,7 +3868,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Rate of serverStatus.opcountersRepl per second: insert, query, update, delete, getmore, command. Per MongoDB docs, opcountersRepl reports on database replication operations by type since the mongod instance last started. These numbers grow over time until the next restart (serverStatus.opcountersRepl).\n\nHow to read: multiple lines showing replication operation rates.\n\nTroubleshooting: on a secondary, this shows how fast replication operations are being applied. If this rate drops while the primary's 'Operations' panel shows high throughput, the secondary is falling behind. Correlate with 'Replica Members Lag' and secondary disk I/O.", + "description": "### Current queue (writers / readers)\n\nOperations **waiting** for the global read or write lock (`globalLock.currentQueue`). Snapshot gauge — not a rate.\n\n**Unit:** count (`short`)\n\n**Note:** On MongoDB 4.2+, these queues are often **0**; use **WiredTiger Tickets** and **Average Operation Latency** for concurrency pressure on modern releases.\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.globalLock.currentQueue.readers` · `serverStatus.globalLock.currentQueue.writers`\n\n---\n\n### How to read\n\n- Two lines: queued readers vs writers\n- Brief spikes can be normal · sustained non-zero → contention\n\n---\n\n### Troubleshooting\n\n- Persistent write queue → **WiredTiger Tickets**, **Average Operation Latency**, **Operations**\n- Persistent read queue → long-running reads blocking others\n- Always zero but slow DB → focus on tickets/latency, not this panel alone\n\n---\n\n### Docs\n\n- [serverStatus — globalLock](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.globalLock)", "fieldConfig": { "defaults": { "color": { @@ -3239,6 +3880,7 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -3250,7 +3892,7 @@ "viz": false }, "insertNulls": false, - "lineInterpolation": "linear", + "lineInterpolation": "stepAfter", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -3267,6 +3909,7 @@ "mode": "off" } }, + "decimals": 0, "mappings": [], "thresholds": { "mode": "absolute", @@ -3274,24 +3917,45 @@ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "ops" + "unit": "short" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*currentQueue\\.readers$" + }, + "properties": [ + { + "id": "displayName", + "value": "Queued readers" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*currentQueue\\.writers$" + }, + "properties": [ + { + "id": "displayName", + "value": "Queued writers" + } + ] + } + ] }, "gridPos": { "h": 7, - "w": 9, - "x": 15, - "y": 72 + "w": 10, + "x": 0, + "y": 105 }, - "id": 38, + "id": 29, "options": { "legend": { "calcs": [ @@ -3317,11 +3981,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.opcountersRepl.command\" or r[\"_field\"] == \"serverStatus.opcountersRepl.delete\" or r[\"_field\"] == \"serverStatus.opcountersRepl.getmore\" or r[\"_field\"] == \"serverStatus.opcountersRepl.insert\" or r[\"_field\"] == \"serverStatus.opcountersRepl.query\" or r[\"_field\"] == \"serverStatus.opcountersRepl.update\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: v.windowPeriod)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.globalLock.currentQueue.writers\" or\r\n r[\"_field\"] == \"serverStatus.globalLock.currentQueue.readers\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", "refId": "A" } ], - "title": "Opcounters Replica", + "title": "Current Queue writers / readers", "type": "timeseries" }, { @@ -3329,7 +3993,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Ratio of serverStatus.metrics.queryExecutor.scannedObjects to metrics.document.returned, computed as a per-second rate ratio. Per MongoDB docs, scannedObjects is equivalent to totalDocsExamined in explain(); document.returned is the total number of documents returned by queries (serverStatus.metrics.queryExecutor, serverStatus.metrics.document).\n\nHow to read: single line showing the scanned-to-returned ratio. A ratio of 1.0 means every scanned document was returned (perfect targeting).\n\nTroubleshooting: ratios significantly above 1.0 indicate queries scanning many documents to return few results -- the classic sign of missing or suboptimal indexes. This is often the number one cause of unnecessary disk I/O and CPU load. Run explain() on slow queries to identify the problem.", + "description": "Rate of serverStatus.opcountersRepl per second: insert, query, update, delete, getmore, command. Per MongoDB docs, opcountersRepl reports on database replication operations by type since the mongod instance last started. These numbers grow over time until the next restart (serverStatus.opcountersRepl).\n\nHow to read: multiple lines showing replication operation rates.\n\nTroubleshooting: on a secondary, this shows how fast replication operations are being applied. If this rate drops while the primary's 'Operations' panel shows high throughput, the secondary is falling behind. Correlate with 'Replica Members Lag' and secondary disk I/O.", "fieldConfig": { "defaults": { "color": { @@ -3353,7 +4017,7 @@ }, "insertNulls": false, "lineInterpolation": "linear", - "lineWidth": 4, + "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" @@ -3369,9 +4033,7 @@ "mode": "off" } }, - "decimals": 2, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -3384,17 +4046,18 @@ "value": 80 } ] - } + }, + "unit": "ops" }, "overrides": [] }, "gridPos": { - "h": 6, - "w": 7, - "x": 8, - "y": 73 + "h": 7, + "w": 9, + "x": 10, + "y": 105 }, - "id": 37, + "id": 38, "options": { "legend": { "calcs": [ @@ -3420,11 +4083,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.metrics.document.returned\" or \r\n r[\"_field\"] == \"serverStatus.metrics.queryExecutor.scannedObjects\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> pivot(\r\n rowKey: [\"_time\"],\r\n columnKey: [\"_field\"],\r\n valueColumn: \"_value\"\r\n )\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n scanned_returned_ratio:\r\n if r[\"serverStatus.metrics.document.returned\"] > 0.0\r\n then r[\"serverStatus.metrics.queryExecutor.scannedObjects\"]\r\n / r[\"serverStatus.metrics.document.returned\"]\r\n else 0.0\r\n }))\r\n |> yield(name: \"scanned_returned_ratio\")\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.opcountersRepl.command\" or r[\"_field\"] == \"serverStatus.opcountersRepl.delete\" or r[\"_field\"] == \"serverStatus.opcountersRepl.getmore\" or r[\"_field\"] == \"serverStatus.opcountersRepl.insert\" or r[\"_field\"] == \"serverStatus.opcountersRepl.query\" or r[\"_field\"] == \"serverStatus.opcountersRepl.update\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: v.windowPeriod)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", "refId": "A" } ], - "title": "Query Targeting: Scanned Objects / Returned", + "title": "Opcounters Replica", "type": "timeseries" }, { @@ -3433,7 +4096,7 @@ "h": 1, "w": 24, "x": 0, - "y": 79 + "y": 112 }, "id": 21, "panels": [], @@ -3507,7 +4170,7 @@ "h": 9, "w": 24, "x": 0, - "y": 80 + "y": 113 }, "id": 13, "options": { @@ -3609,7 +4272,7 @@ "h": 8, "w": 12, "x": 0, - "y": 89 + "y": 122 }, "id": 40, "options": { @@ -3711,7 +4374,7 @@ "h": 8, "w": 12, "x": 12, - "y": 89 + "y": 122 }, "id": 39, "options": { @@ -3810,10 +4473,10 @@ "overrides": [] }, "gridPos": { - "h": 6, + "h": 7, "w": 8, "x": 0, - "y": 97 + "y": 130 }, "id": 35, "options": { @@ -3915,7 +4578,7 @@ "h": 7, "w": 8, "x": 8, - "y": 97 + "y": 130 }, "id": 17, "options": { @@ -4014,10 +4677,10 @@ "overrides": [] }, "gridPos": { - "h": 5, + "h": 7, "w": 8, "x": 16, - "y": 97 + "y": 130 }, "id": 43, "options": { @@ -4119,7 +4782,7 @@ "h": 6, "w": 8, "x": 0, - "y": 103 + "y": 137 }, "id": 4, "options": { @@ -4221,7 +4884,7 @@ "h": 7, "w": 8, "x": 8, - "y": 104 + "y": 137 }, "id": 46, "options": { @@ -4332,7 +4995,7 @@ "h": 5, "w": 8, "x": 0, - "y": 109 + "y": 143 }, "id": 11, "options": { @@ -4434,7 +5097,7 @@ "h": 7, "w": 8, "x": 8, - "y": 111 + "y": 144 }, "id": 41, "options": { @@ -4536,7 +5199,7 @@ "h": 7, "w": 8, "x": 0, - "y": 114 + "y": 148 }, "id": 47, "options": { @@ -4648,7 +5311,7 @@ "h": 7, "w": 8, "x": 8, - "y": 118 + "y": 151 }, "id": 45, "options": { @@ -4750,7 +5413,7 @@ "h": 7, "w": 8, "x": 0, - "y": 121 + "y": 155 }, "id": 18, "options": { @@ -4861,7 +5524,7 @@ "h": 6, "w": 8, "x": 8, - "y": 125 + "y": 158 }, "id": 51, "options": { @@ -4901,7 +5564,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "TCP/IP routing error counters from systemMetrics.netstat: InNoRoutes, InTruncatedPkts, and related fields (systemMetrics.netstat).\n\nHow to read: lines showing error rates per second (derivatives of cumulative counters).\n\nTroubleshooting: any non-zero derivative indicates network configuration problems -- misrouted packets, MTU issues, or firewall interference. These are rare but can cause intermittent connectivity failures that are very difficult to diagnose at the application level.", + "description": "TCP packet integrity counters from systemMetrics.netstat: Tcp.InCsumErrors, Tcp.InErrs, Tcp.RetransSegs, Ip.InDiscards, Ip.OutDiscards (systemMetrics.netstat).\n\nHow to read: lines showing error rates per second.\n\nTroubleshooting: RetransSegs (retransmissions) indicate packet loss, directly causing latency spikes visible in 'Average Operation Latency' and 'Replica Members Ping'. InCsumErrors indicate data corruption on the wire. InDiscards/OutDiscards indicate kernel buffer overflows. Any sustained increase points to network hardware issues, congestion, or NIC driver/firmware bugs.", "fieldConfig": { "defaults": { "color": { @@ -4963,9 +5626,9 @@ "h": 5, "w": 8, "x": 0, - "y": 128 + "y": 162 }, - "id": 44, + "id": 42, "options": { "legend": { "calcs": [ @@ -4991,11 +5654,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"systemMetrics.netstat.Ip:OutNoRoutes\" )\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> derivative(unit: v.windowPeriod)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"systemMetrics.netstat.Tcp:InCsumErrors\" \r\n or r[\"_field\"] == \"systemMetrics.netstat.Tcp:InErrs\"\r\n or r[\"_field\"] == \"systemMetrics.netstat.Tcp:RetransSegs\"\r\n or r[\"_field\"] == \"systemMetrics.netstat.Ip:InDiscards\"\r\n or r[\"_field\"] == \"systemMetrics.netstat.Ip:OutDiscards\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> derivative(unit: v.windowPeriod)", "refId": "A" } ], - "title": "TCP Routing Issues", + "title": "TCP Packet Loss or Corruption", "type": "timeseries" }, { @@ -5003,7 +5666,108 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "TCP packet integrity counters from systemMetrics.netstat: Tcp.InCsumErrors, Tcp.InErrs, Tcp.RetransSegs, Ip.InDiscards, Ip.OutDiscards (systemMetrics.netstat).\n\nHow to read: lines showing error rates per second.\n\nTroubleshooting: RetransSegs (retransmissions) indicate packet loss, directly causing latency spikes visible in 'Average Operation Latency' and 'Replica Members Ping'. InCsumErrors indicate data corruption on the wire. InDiscards/OutDiscards indicate kernel buffer overflows. Any sustained increase points to network hardware issues, congestion, or NIC driver/firmware bugs.", + "description": "Rate of serverStatus.asserts by type: regular, warning, msg, user, rollovers. Per MongoDB docs, assertions are internal checks for errors that occur while the database is operating. Non-zero values are uncommon. asserts.user are errors clients may generate (e.g., out of disk space, duplicate key). asserts.rollovers counts how many times assert counters have rolled over after 2^30 assertions (serverStatus.asserts).\n\nHow to read: multiple lines showing assert rate/sec per type.\n\nTroubleshooting: any sustained increase in regular or msg asserts may indicate bugs or data corruption -- examine the MongoDB log. High user asserts indicate application-level errors (duplicate keys, validation failures). Check application logs for corresponding errors.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 7, + "x": 8, + "y": 164 + }, + "id": 30, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.asserts.warning\" or r[\"_field\"] == \"serverStatus.asserts.user\" or r[\"_field\"] == \"serverStatus.asserts.rollovers\" or r[\"_field\"] == \"serverStatus.asserts.regular\" or r[\"_field\"] == \"serverStatus.asserts.msg\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: v.windowPeriod)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", + "refId": "A" + } + ], + "title": "Asserts", + "type": "timeseries" + }, + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "description": "TCP/IP routing error counters from systemMetrics.netstat: InNoRoutes, InTruncatedPkts, and related fields (systemMetrics.netstat).\n\nHow to read: lines showing error rates per second (derivatives of cumulative counters).\n\nTroubleshooting: any non-zero derivative indicates network configuration problems -- misrouted packets, MTU issues, or firewall interference. These are rare but can cause intermittent connectivity failures that are very difficult to diagnose at the application level.", "fieldConfig": { "defaults": { "color": { @@ -5065,9 +5829,9 @@ "h": 5, "w": 8, "x": 0, - "y": 133 + "y": 167 }, - "id": 42, + "id": 44, "options": { "legend": { "calcs": [ @@ -5093,11 +5857,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"systemMetrics.netstat.Tcp:InCsumErrors\" \r\n or r[\"_field\"] == \"systemMetrics.netstat.Tcp:InErrs\"\r\n or r[\"_field\"] == \"systemMetrics.netstat.Tcp:RetransSegs\"\r\n or r[\"_field\"] == \"systemMetrics.netstat.Ip:InDiscards\"\r\n or r[\"_field\"] == \"systemMetrics.netstat.Ip:OutDiscards\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> derivative(unit: v.windowPeriod)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"systemMetrics.netstat.Ip:OutNoRoutes\" )\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> derivative(unit: v.windowPeriod)", "refId": "A" } ], - "title": "TCP Packet Loss or Corruption", + "title": "TCP Routing Issues", "type": "timeseries" }, { @@ -5167,7 +5931,7 @@ "h": 6, "w": 8, "x": 0, - "y": 138 + "y": 172 }, "id": 50, "options": { @@ -5252,5 +6016,6 @@ "timepicker": {}, "timezone": "UTC", "title": "ftdc_dashboard", - "uid": "ddnw277huiv40ae" -} + "uid": "ddnw277huiv40ae", + "version": 1 +} \ No newline at end of file diff --git a/grafana/main.yaml b/grafana/main.yaml index fc2534e..590cf66 100644 --- a/grafana/main.yaml +++ b/grafana/main.yaml @@ -4,5 +4,9 @@ providers: - name: "ftdc" orgId: 1 type: file + disableDeletion: false + updateIntervalSeconds: 10 + allowUiUpdates: false options: path: /etc/grafana/provisioning/dashboards/ + foldersFromFilesStructure: false diff --git a/metrics_to_get.txt b/metrics_to_get.txt index c755c48..2a0ce40 100644 --- a/metrics_to_get.txt +++ b/metrics_to_get.txt @@ -150,3 +150,8 @@ replSetGetStatus.members.1.lastAppliedWallTime replSetGetStatus.members.2.lastAppliedWallTime replSetGetStatus.members.3.lastAppliedWallTime replSetGetStatus.members.4.lastAppliedWallTime +replSetGetStatus.members.0.state +replSetGetStatus.members.1.state +replSetGetStatus.members.2.state +replSetGetStatus.members.3.state +replSetGetStatus.members.4.state diff --git a/run.sh b/run.sh index 682d2ea..66a5821 100755 --- a/run.sh +++ b/run.sh @@ -70,7 +70,7 @@ esac # Function to handle Ctrl-C (SIGINT) cleanup() { echo "Stopping Docker containers..." - $ENVS docker-compose down -v + $ENVS docker compose down -v echo "Docker containers stopped." exit 0 } @@ -80,13 +80,13 @@ trap cleanup SIGINT echo "Checking for image changes and building if needed..." -docker-compose build +docker compose build # remove any running container -docker-compose down -v +docker compose down -v # Start Docker containers in detached mode -docker-compose up -d +docker compose up -d echo "🔐 Credentials:" echo "--------------------------" From 9e79b125682664d332845ffb2d778c038adca31a Mon Sep 17 00:00:00 2001 From: jenunes Date: Fri, 12 Jun 2026 17:22:17 -0300 Subject: [PATCH 3/9] Refine FTDC Grafana dashboard layout and add upstream publish scripts. Reorder and resize panels (network, connections, replica, flow control), add a dedicated Network Requests panel, and add scripts to stage changes and open PRs against upstream. Co-authored-by: Cursor --- grafana/dashboards/dashboard.json | 1177 ++++++++++++++++++++--------- scripts/publish-to-upstream.sh | 58 ++ scripts/stage-for-upstream.sh | 28 + 3 files changed, 917 insertions(+), 346 deletions(-) create mode 100755 scripts/publish-to-upstream.sh create mode 100755 scripts/stage-for-upstream.sh diff --git a/grafana/dashboards/dashboard.json b/grafana/dashboards/dashboard.json index 7fe7b9f..6b47caa 100644 --- a/grafana/dashboards/dashboard.json +++ b/grafana/dashboards/dashboard.json @@ -1869,6 +1869,119 @@ "title": "mongod Memory (RSS / Virtual)", "type": "timeseries" }, + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "description": "### Network requests\n\nRate of **network.numRequests** — distinct client requests received by mongod.\n\nUse with **Network In / Out** to relate request volume to logical byte throughput.\n\n**Unit:** requests/sec (`reqps`)\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.network.numRequests`\n\n---\n\n### How to read\n\n- Single line · request arrival rate\n- Not the same as **Operations** (opcounters) — one request can trigger multiple ops\n\n---\n\n### Troubleshooting\n\n- Sudden drop → client/network issue or load shift\n- Spike without matching **Operations** → lightweight commands (heartbeats, metadata)\n- High rate + high **Network In / Out** bytesOut → large responses · check projections/limits\n\n---\n\n### Docs\n\n- [serverStatus — network](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.network)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*numRequests$" + }, + "properties": [ + { + "id": "displayName", + "value": "Network requests (/sec)" + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 14, + "x": 10, + "y": 43 + }, + "id": 28, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.network.numRequests\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", + "refId": "A" + } + ], + "title": "Network Requests", + "type": "timeseries" + }, { "datasource": { "type": "influxdb", @@ -1964,14 +2077,39 @@ "value": "Available connections" } ] + }, + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Active connections", + "Current connections" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": true, + "viz": true + } + } + ] } ] }, "gridPos": { - "h": 9, + "h": 6, "w": 14, "x": 10, - "y": 43 + "y": 48 }, "id": 2, "options": { @@ -1983,7 +2121,7 @@ "lastNotNull" ], "displayMode": "table", - "placement": "bottom", + "placement": "right", "showLegend": true }, "tooltip": { @@ -2124,7 +2262,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Network requests\n\nRate of **network.numRequests** — distinct client requests received by mongod.\n\nUse with **Network In / Out** to relate request volume to logical byte throughput.\n\n**Unit:** requests/sec (`reqps`)\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.network.numRequests`\n\n---\n\n### How to read\n\n- Single line · request arrival rate\n- Not the same as **Operations** (opcounters) — one request can trigger multiple ops\n\n---\n\n### Troubleshooting\n\n- Sudden drop → client/network issue or load shift\n- Spike without matching **Operations** → lightweight commands (heartbeats, metadata)\n- High rate + high **Network In / Out** bytesOut → large responses · check projections/limits\n\n---\n\n### Docs\n\n- [serverStatus — network](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.network)", + "description": "### Connections created\n\nRate of new connections (`connections.totalCreated` derivative). Indicates connection **churn** — apps opening/closing connections frequently.\n\n**Unit:** connections/sec (`ops`)\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.connections.totalCreated`\n\n---\n\n### How to read\n\n- Single line · spikes → bursts of new connections\n- Sustained high rate → pool churn · check app connection pool settings\n\n---\n\n### Troubleshooting\n\n- High rate + high **Current** on **Connections** → many short-lived connections\n- Correlate with **Connections** (counts)\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)", "fieldConfig": { "defaults": { "color": { @@ -2136,7 +2274,6 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", - "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -2165,7 +2302,7 @@ "mode": "off" } }, - "decimals": 1, + "decimals": 2, "mappings": [], "thresholds": { "mode": "absolute", @@ -2176,30 +2313,30 @@ } ] }, - "unit": "reqps" + "unit": "ops" }, "overrides": [ { "matcher": { "id": "byRegexp", - "options": ".*numRequests$" + "options": ".*connections\\.totalCreated$" }, "properties": [ { "id": "displayName", - "value": "Network requests (/sec)" + "value": "Connections created (/sec)" } ] } ] }, "gridPos": { - "h": 5, + "h": 6, "w": 14, "x": 10, - "y": 52 + "y": 54 }, - "id": 28, + "id": 52, "options": { "legend": { "calcs": [ @@ -2209,7 +2346,7 @@ "lastNotNull" ], "displayMode": "table", - "placement": "bottom", + "placement": "right", "showLegend": true }, "tooltip": { @@ -2225,11 +2362,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.network.numRequests\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.connections.totalCreated\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", "refId": "A" } ], - "title": "Network Requests", + "title": "Connections Created", "type": "timeseries" }, { @@ -2331,7 +2468,7 @@ ] }, "gridPos": { - "h": 9, + "h": 10, "w": 10, "x": 0, "y": 56 @@ -2374,7 +2511,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Connections created\n\nRate of new connections (`connections.totalCreated` derivative). Indicates connection **churn** — apps opening/closing connections frequently.\n\n**Unit:** connections/sec (`ops`)\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.connections.totalCreated`\n\n---\n\n### How to read\n\n- Single line · spikes → bursts of new connections\n- Sustained high rate → pool churn · check app connection pool settings\n\n---\n\n### Troubleshooting\n\n- High rate + high **Current** on **Connections** → many short-lived connections\n- Correlate with **Connections** (counts)\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)", + "description": "### Cursors\n\n**Open cursors** from **`metrics.cursor.open.*`** (current count, gauge). **`metrics.cursor.timedOut`** is a **cumulative counter** — shown here as **timeouts per second** (derivative).\n\n**Unit:** count (`short`) for open · timeouts/sec (`ops`) for timed out rate\n\n---\n\n### Metrics\n\n**Fields (open — gauge, `last`):**\n\n- `serverStatus.metrics.cursor.open.total`\n- `serverStatus.metrics.cursor.open.noTimeout`\n- `serverStatus.metrics.cursor.open.pinned`\n\n**Field (timeouts — counter, `derivative(1s)`):**\n\n- `serverStatus.metrics.cursor.timedOut`\n\n---\n\n### How to read\n\n- **Open total** — cursors mongod holds for clients; usually small\n- **Open (no timeout)** / **Open (pinned)** — subsets of total\n- **Timed out (/sec)** — idle cursors killed (default cursor timeout ~10 min)\n- **stepAfter** on open lines reflects discrete count changes\n\n---\n\n### Troubleshooting\n\n- **Open total** rising → possible cursor leak (app not closing/exhausting cursors)\n- **Timed out (/sec)** rising → clients abandoning cursors · correlate with **Operations** (getmore/query)\n- Tailable / pinned cursors can keep **open** elevated without a leak\n\n---\n\n### Docs\n\n- [serverStatus — metrics.cursor](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.metrics.cursor)", "fieldConfig": { "defaults": { "color": { @@ -2386,6 +2523,7 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -2397,7 +2535,7 @@ "viz": false }, "insertNulls": false, - "lineInterpolation": "linear", + "lineInterpolation": "stepAfter", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -2414,8 +2552,9 @@ "mode": "off" } }, - "decimals": 2, + "decimals": 0, "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -2425,18 +2564,106 @@ } ] }, - "unit": "ops" + "unit": "short" }, "overrides": [ { "matcher": { "id": "byRegexp", - "options": ".*connections\\.totalCreated$" + "options": ".*\\.open\\.total$" }, "properties": [ { "id": "displayName", - "value": "Connections created (/sec)" + "value": "Open total" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 0 + }, + { + "id": "custom.lineInterpolation", + "value": "stepAfter" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*\\.open\\.noTimeout$" + }, + "properties": [ + { + "id": "displayName", + "value": "Open (no timeout)" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 0 + }, + { + "id": "custom.lineInterpolation", + "value": "stepAfter" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*\\.open\\.pinned$" + }, + "properties": [ + { + "id": "displayName", + "value": "Open (pinned)" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 0 + }, + { + "id": "custom.lineInterpolation", + "value": "stepAfter" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*\\.timedOut$" + }, + "properties": [ + { + "id": "displayName", + "value": "Timed out (/sec)" + }, + { + "id": "unit", + "value": "ops" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "custom.lineInterpolation", + "value": "linear" } ] } @@ -2446,15 +2673,14 @@ "h": 6, "w": 14, "x": 10, - "y": 57 + "y": 60 }, - "id": 52, + "id": 33, "options": { "legend": { "calcs": [ "min", "max", - "mean", "lastNotNull" ], "displayMode": "table", @@ -2463,7 +2689,7 @@ }, "tooltip": { "hideZeros": false, - "mode": "single", + "mode": "multi", "sort": "none" } }, @@ -2474,11 +2700,20 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.connections.totalCreated\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.metrics.cursor.open.total\" or\r\n r[\"_field\"] == \"serverStatus.metrics.cursor.open.noTimeout\" or\r\n r[\"_field\"] == \"serverStatus.metrics.cursor.open.pinned\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", "refId": "A" + }, + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "hide": false, + "query": "from(bucket: v.defaultBucket)\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.metrics.cursor.timedOut\")\n |> drop(columns: [\"hostname\", \"version\"])\n |> derivative(unit: 1s, nonNegative: true)\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", + "refId": "B" } ], - "title": "Connections Created", + "title": "Cursors", "type": "timeseries" }, { @@ -2486,7 +2721,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Replica members lag\n\nWall-clock replication lag per member:\n\n**Formula:** `(serverStatus.localTime − members.N.lastAppliedWallTime) / 1000` → **seconds**\n\nBoth timestamps are **milliseconds** in FTDC; the query converts the delta to seconds.\n\n**Unit:** seconds (`s`)\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.localTime`\n- `replSetGetStatus.members.0.lastAppliedWallTime` … `replSetGetStatus.members.4.lastAppliedWallTime`\n\nMember **N** is the array index in `replSetGetStatus.members` (not necessarily hostname order).\n\n---\n\n### How to read\n\n- One line per member index (0–4)\n- **~0** → caught up (primary or healthy secondary)\n- Rising trend on a secondary → replication falling behind\n- Flat **0** for missing index → no member in that slot\n\n---\n\n### Troubleshooting\n\n- Secondary lag rising → disk I/O, CPU, network on that node · **Opcounters Replica**, **Disk I/O**\n- Lag spikes during checkpoints → I/O contention\n- Primary throttling → **FlowControl isLagged** · **Replica Members Ping**\n- Member down → **Replica Members Health**\n\n---\n\n### Docs\n\n- [replSetGetStatus](https://www.mongodb.com/docs/manual/reference/command/replSetGetStatus/)\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)", + "description": "### Admin Commands (metrics.commands)\n\nPer-second rate of selected **`serverStatus.metrics.commands.*.total`** counters — admin/catalog-style commands tracked individually by MongoDB.\n\n**Not** the same as **Operations** (`opcounters.command`), which aggregates most command types except insert, update, delete, and aggregate.\n\n**Unit:** commands per second (`ops`)\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.metrics.commands.createIndexes.total`\n- `serverStatus.metrics.commands.dataSize.total`\n- `serverStatus.metrics.commands.dropIndexes.total`\n- `serverStatus.metrics.commands.listCollections.total`\n- `serverStatus.metrics.commands.listDatabases.total`\n- `serverStatus.metrics.commands.listIndexes.total`\n\n**Query:** `derivative(..., unit: 1s, nonNegative: true)` → cmds/sec\n\n---\n\n### How to read\n\n- One line per command counter above\n- Spikes on **listCollections** / **listDatabases** / **listIndexes** → metadata/catalog traffic (drivers, tools, monitoring)\n- Spikes on **createIndexes** / **dropIndexes** → schema/index maintenance\n- **dataSize** → `dataSize` command usage\n\n---\n\n### Troubleshooting\n\n- Sustained high list* rates → chatty clients or health checks · compare with **Network Requests**\n- Index build bursts → **createIndexes** + **CPU Usage** + **Disk I/O**\n- For overall op mix, use **Operations**; for document volume, **Documents**\n\n---\n\n### Docs\n\n- [serverStatus — metrics.commands](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.metrics.commands)", "fieldConfig": { "defaults": { "color": { @@ -2527,7 +2762,6 @@ "mode": "off" } }, - "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", @@ -2538,78 +2772,90 @@ } ] }, - "unit": "s" + "unit": "ops" }, "overrides": [ { "matcher": { - "id": "byName", - "options": "member_0" + "id": "byRegexp", + "options": ".*\\.createIndexes\\.total$" }, "properties": [ { "id": "displayName", - "value": "Member 0 lag" + "value": "createIndexes" } ] }, { "matcher": { - "id": "byName", - "options": "member_1" + "id": "byRegexp", + "options": ".*\\.dropIndexes\\.total$" }, "properties": [ { "id": "displayName", - "value": "Member 1 lag" + "value": "dropIndexes" } ] }, { "matcher": { - "id": "byName", - "options": "member_2" + "id": "byRegexp", + "options": ".*\\.listCollections\\.total$" }, "properties": [ { "id": "displayName", - "value": "Member 2 lag" + "value": "listCollections" } ] }, { "matcher": { - "id": "byName", - "options": "member_3" + "id": "byRegexp", + "options": ".*\\.listDatabases\\.total$" }, "properties": [ { "id": "displayName", - "value": "Member 3 lag" + "value": "listDatabases" } ] }, { "matcher": { - "id": "byName", - "options": "member_4" + "id": "byRegexp", + "options": ".*\\.listIndexes\\.total$" }, "properties": [ { "id": "displayName", - "value": "Member 4 lag" + "value": "listIndexes" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*\\.dataSize\\.total$" + }, + "properties": [ + { + "id": "displayName", + "value": "dataSize" } ] } ] }, "gridPos": { - "h": 7, - "w": 14, - "x": 10, - "y": 63 + "h": 8, + "w": 10, + "x": 0, + "y": 66 }, - "id": 15, + "id": 32, "options": { "legend": { "calcs": [ @@ -2635,11 +2881,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] =~ /replSetGetStatus\\.members\\.\\d+\\.lastAppliedWallTime|serverStatus\\.localTime/)\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> pivot(rowKey: [\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n member_0: if exists r[\"replSetGetStatus.members.0.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.0.lastAppliedWallTime\"])) / 1000.0 else 0.0,\r\n member_1: if exists r[\"replSetGetStatus.members.1.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.1.lastAppliedWallTime\"])) / 1000.0 else 0.0,\r\n member_2: if exists r[\"replSetGetStatus.members.2.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.2.lastAppliedWallTime\"])) / 1000.0 else 0.0,\r\n member_3: if exists r[\"replSetGetStatus.members.3.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.3.lastAppliedWallTime\"])) / 1000.0 else 0.0,\r\n member_4: if exists r[\"replSetGetStatus.members.4.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.4.lastAppliedWallTime\"])) / 1000.0 else 0.0,\r\n }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.metrics.commands.createIndexes.total\" or\r\n r[\"_field\"] == \"serverStatus.metrics.commands.dataSize.total\" or\r\n r[\"_field\"] == \"serverStatus.metrics.commands.dropIndexes.total\" or\r\n r[\"_field\"] == \"serverStatus.metrics.commands.listCollections.total\" or\r\n r[\"_field\"] == \"serverStatus.metrics.commands.listDatabases.total\" or\r\n r[\"_field\"] == \"serverStatus.metrics.commands.listIndexes.total\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", "refId": "A" } ], - "title": "Replica Members Lag", + "title": "Admin Commands", "type": "timeseries" }, { @@ -2647,7 +2893,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Flow control is lagged\n\nBoolean **`flowControl.isLagged`** — whether flow control is **currently** throttling writes on the **primary**.\n\n**Unit:** 0 = off · 1 = on\n\n**Note:** Primary only; secondaries return placeholders. For **time spent** throttled, see **Flow control lagged time**.\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.flowControl.isLagged`\n\n---\n\n### How to read\n\n- **0** → not engaging (even if secondaries lag)\n- **1** → primary write rate limited so secondaries can catch up\n\n---\n\n### Troubleshooting\n\n- **1** sustained → **Replica Members Lag**, **Opcounters Replica**, secondary disk/network\n- Lag without **1** → lag below `flowControlTargetLagSeconds` threshold\n\n---\n\n### Docs\n\n- [serverStatus — flowControl](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.flowControl)", + "description": "### Replica members lag\n\nWall-clock replication lag per member:\n\n**Formula:** `(serverStatus.localTime − members.N.lastAppliedWallTime) / 1000` → **seconds**\n\nBoth timestamps are **milliseconds** in FTDC; the query converts the delta to seconds.\n\n**Unit:** seconds (`s`)\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.localTime`\n- `replSetGetStatus.members.0.lastAppliedWallTime` … `replSetGetStatus.members.4.lastAppliedWallTime`\n\nMember **N** is the array index in `replSetGetStatus.members` (not necessarily hostname order).\n\n---\n\n### How to read\n\n- One line per member index (0–4)\n- **~0** → caught up (primary or healthy secondary)\n- Rising trend on a secondary → replication falling behind\n- Flat **0** for missing index → no member in that slot\n\n---\n\n### Troubleshooting\n\n- Secondary lag rising → disk I/O, CPU, network on that node · **Opcounters Replica**, **Disk I/O**\n- Lag spikes during checkpoints → I/O contention\n- Primary throttling → **FlowControl isLagged** · **Replica Members Ping**\n- Member down → **Replica Members Health**\n\n---\n\n### Docs\n\n- [replSetGetStatus](https://www.mongodb.com/docs/manual/reference/command/replSetGetStatus/)\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)", "fieldConfig": { "defaults": { "color": { @@ -2659,7 +2905,6 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", - "axisSoftMax": 1, "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, @@ -2672,8 +2917,8 @@ "viz": false }, "insertNulls": false, - "lineInterpolation": "stepAfter", - "lineWidth": 2, + "lineInterpolation": "linear", + "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" @@ -2689,24 +2934,8 @@ "mode": "off" } }, - "decimals": 0, - "mappings": [ - { - "options": { - "0": { - "index": 0, - "text": "Off" - }, - "1": { - "index": 1, - "text": "On" - } - }, - "type": "value" - } - ], - "max": 1, - "min": 0, + "decimals": 1, + "mappings": [], "thresholds": { "mode": "absolute", "steps": [ @@ -2716,30 +2945,78 @@ } ] }, - "unit": "short" + "unit": "s" }, "overrides": [ { "matcher": { - "id": "byRegexp", - "options": ".*isLagged$" + "id": "byName", + "options": "member_0" }, "properties": [ { "id": "displayName", - "value": "Flow control engaged" + "value": "Member 0 lag" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "member_1" + }, + "properties": [ + { + "id": "displayName", + "value": "Member 1 lag" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "member_2" + }, + "properties": [ + { + "id": "displayName", + "value": "Member 2 lag" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "member_3" + }, + "properties": [ + { + "id": "displayName", + "value": "Member 3 lag" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "member_4" + }, + "properties": [ + { + "id": "displayName", + "value": "Member 4 lag" } ] } ] }, "gridPos": { - "h": 6, - "w": 10, - "x": 0, - "y": 65 + "h": 7, + "w": 14, + "x": 10, + "y": 66 }, - "id": 14, + "id": 15, "options": { "legend": { "calcs": [ @@ -2749,12 +3026,12 @@ "lastNotNull" ], "displayMode": "table", - "placement": "bottom", + "placement": "right", "showLegend": true }, "tooltip": { "hideZeros": false, - "mode": "single", + "mode": "multi", "sort": "none" } }, @@ -2765,11 +3042,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.flowControl.isLagged\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] =~ /replSetGetStatus\\.members\\.\\d+\\.lastAppliedWallTime|serverStatus\\.localTime/)\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> pivot(rowKey: [\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n member_0: if exists r[\"replSetGetStatus.members.0.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.0.lastAppliedWallTime\"])) / 1000.0 else 0.0,\r\n member_1: if exists r[\"replSetGetStatus.members.1.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.1.lastAppliedWallTime\"])) / 1000.0 else 0.0,\r\n member_2: if exists r[\"replSetGetStatus.members.2.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.2.lastAppliedWallTime\"])) / 1000.0 else 0.0,\r\n member_3: if exists r[\"replSetGetStatus.members.3.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.3.lastAppliedWallTime\"])) / 1000.0 else 0.0,\r\n member_4: if exists r[\"replSetGetStatus.members.4.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.4.lastAppliedWallTime\"])) / 1000.0 else 0.0,\r\n }))", "refId": "A" } ], - "title": "FlowControl isLagged", + "title": "Replica Members Lag", "type": "timeseries" }, { @@ -2895,10 +3172,10 @@ ] }, "gridPos": { - "h": 5, - "w": 9, + "h": 6, + "w": 14, "x": 10, - "y": 70 + "y": 73 }, "id": 25, "options": { @@ -2910,7 +3187,7 @@ "lastNotNull" ], "displayMode": "table", - "placement": "bottom", + "placement": "right", "showLegend": true }, "tooltip": { @@ -2938,7 +3215,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Flow control lagged time\n\n**Fraction of each window** the primary spent with flow control engaged, from **`flowControl.isLaggedTimeMicros`**.\n\n**Formula:** `derivative(isLaggedTimeMicros, 1s) / 1e6` → 0–100%\n\n**Note:** Primary only. For **on/off now**, see **FlowControl isLagged**.\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.flowControl.isLaggedTimeMicros`\n\n---\n\n### How to read\n\n- **0%** → no throttling in that window\n- **100%** → flow control engaged the entire window\n- Spikes → brief throttling bursts\n\n---\n\n### Troubleshooting\n\n- Sustained **>0%** → correlate with **Replica Members Lag**, **FlowControl isLagged**\n- High % + rising lag → secondaries cannot keep up · disk/CPU/network\n\n---\n\n### Docs\n\n- [serverStatus — flowControl](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.flowControl)", + "description": "### Document metrics (deleted / inserted / returned / updated)\n\nPer-second rate of **`serverStatus.metrics.document`** counters — document-level access and modification (not the same as **Operations** / opcounters: one op can touch many documents).\n\n**Unit:** documents per second (`ops`)\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.metrics.document.deleted`\n- `serverStatus.metrics.document.inserted`\n- `serverStatus.metrics.document.returned`\n- `serverStatus.metrics.document.updated`\n\n**Query:** `derivative(..., unit: 1s, nonNegative: true)` → docs/sec\n\n---\n\n### How to read\n\n- Four lines: delete, insert, return, update **rates**\n- **Returned** high with low **Scanned / Returned Ratio** → efficient reads\n- **Returned** high with high ratio → scans without good targeting · **Query Executor**\n\n---\n\n### Troubleshooting\n\n- High **deleted** → TTL indexes, bulk cleanup, or churn\n- High **returned** + high **Scanned / Returned Ratio** → missing/suboptimal indexes · run `explain()`\n- Compare with **Operations** (op-level) and **Opcounters Replica** on secondaries\n\n---\n\n### Docs\n\n- [serverStatus — metrics.document](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.metrics.document)", "fieldConfig": { "defaults": { "color": { @@ -2950,7 +3227,6 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", - "axisSoftMax": 1, "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, @@ -2964,7 +3240,7 @@ }, "insertNulls": false, "lineInterpolation": "linear", - "lineWidth": 2, + "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" @@ -2980,10 +3256,7 @@ "mode": "off" } }, - "decimals": 1, "mappings": [], - "max": 1, - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -2993,131 +3266,66 @@ } ] }, - "unit": "percentunit" + "unit": "ops" }, "overrides": [ { "matcher": { "id": "byRegexp", - "options": ".*isLaggedTimeMicros$" + "options": ".*\\.document\\.deleted$" }, "properties": [ { "id": "displayName", - "value": "Flow control engaged (% of time)" + "value": "Documents deleted" } ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 10, - "x": 0, - "y": 71 - }, - "id": 53, - "options": { - "legend": { - "calcs": [ - "min", - "max", - "mean", - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.3.2", - "targets": [ - { - "datasource": { - "type": "influxdb", - "uid": "P3C6603E967DC8568" - }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.flowControl.isLaggedTimeMicros\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) / 1000000.0 }))", - "refId": "A" - } - ], - "title": "Flow control lagged time", - "type": "timeseries" - }, - { - "datasource": { - "type": "influxdb", - "uid": "P3C6603E967DC8568" - }, - "description": "Rate of serverStatus.opcounters per second by type: insert, query, update, delete, getmore, command. Per MongoDB docs, opcounters treats operations that affect multiple documents (bulk insert, multi-update) as a single operation. These values reflect received operations and increment even when operations are not successful. opcounters.command counts all commands except insert, update, delete, and aggregate (serverStatus.opcounters).\n\nHow to read: multiple lines showing ops/sec per operation type (derivative of cumulative counters).\n\nTroubleshooting: sudden drops may indicate blocking -- check 'Current Queue writers / readers' and 'WiredTiger Tickets'. Sudden spikes may cause resource contention. Use alongside 'Average Operation Latency' to distinguish throughput-driven from latency-driven performance issues.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "stepAfter", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + { + "matcher": { + "id": "byRegexp", + "options": ".*\\.document\\.inserted$" }, - "thresholdsStyle": { - "mode": "off" - } + "properties": [ + { + "id": "displayName", + "value": "Documents inserted" + } + ] }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*\\.document\\.returned$" + }, + "properties": [ { - "color": "green", - "value": 0 - }, + "id": "displayName", + "value": "Documents returned" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*\\.document\\.updated$" + }, + "properties": [ { - "color": "red", - "value": 80 + "id": "displayName", + "value": "Documents updated" } ] } - }, - "overrides": [] + ] }, "gridPos": { - "h": 7, - "w": 7, - "x": 10, - "y": 75 + "h": 12, + "w": 10, + "x": 0, + "y": 74 }, - "id": 9, + "id": 36, "options": { "legend": { "calcs": [ @@ -3143,11 +3351,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => \r\n r[\"_field\"] == \"serverStatus.opcounters.update\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.query\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.insert\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.getmore\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.delete\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.command\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.metrics.document.deleted\" or\r\n r[\"_field\"] == \"serverStatus.metrics.document.inserted\" or\r\n r[\"_field\"] == \"serverStatus.metrics.document.returned\" or\r\n r[\"_field\"] == \"serverStatus.metrics.document.updated\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", "refId": "A" } ], - "title": "Operations", + "title": "Documents (deleted / inserted / returned / updated)", "type": "timeseries" }, { @@ -3155,7 +3363,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Rate of serverStatus.metrics.document counters per second: deleted, inserted, returned, updated. Per MongoDB docs, metrics.document reflects document access and modification patterns. Compare with opcounters, which tracks total number of operations -- since one query can return many documents (serverStatus.metrics.document).\n\nHow to read: four lines showing document-level operation rates.\n\nTroubleshooting: high 'returned' with low 'scannedObjects' (from 'Query Executor') indicates efficient queries. High 'returned' with high 'scannedObjects' means poor query targeting. High 'deleted' may indicate TTL index activity or bulk cleanup operations.", + "description": "### Flow control is lagged\n\nBoolean **`flowControl.isLagged`** — whether flow control is **currently** throttling writes on the **primary**.\n\n**Unit:** 0 = off · 1 = on\n\n**Note:** Primary only; secondaries return placeholders. For **time spent** throttled, see **Flow control lagged time**.\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.flowControl.isLagged`\n\n---\n\n### How to read\n\n- **0** → not engaging (even if secondaries lag)\n- **1** → primary write rate limited so secondaries can catch up\n\n---\n\n### Troubleshooting\n\n- **1** sustained → **Replica Members Lag**, **Opcounters Replica**, secondary disk/network\n- Lag without **1** → lag below `flowControlTargetLagSeconds` threshold\n\n---\n\n### Docs\n\n- [serverStatus — flowControl](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.flowControl)", "fieldConfig": { "defaults": { "color": { @@ -3167,6 +3375,8 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMax": 1, + "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -3178,8 +3388,8 @@ "viz": false }, "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, + "lineInterpolation": "stepAfter", + "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" @@ -3195,30 +3405,57 @@ "mode": "off" } }, - "mappings": [], + "decimals": 0, + "mappings": [ + { + "options": { + "0": { + "index": 0, + "text": "Off" + }, + "1": { + "index": 1, + "text": "On" + } + }, + "type": "value" + } + ], + "max": 1, + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 - }, + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*isLagged$" + }, + "properties": [ { - "color": "red", - "value": 80 + "id": "displayName", + "value": "Flow control engaged" } ] } - }, - "overrides": [] + ] }, "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 77 + "h": 5, + "w": 14, + "x": 10, + "y": 79 }, - "id": 36, + "id": 14, "options": { "legend": { "calcs": [ @@ -3233,7 +3470,7 @@ }, "tooltip": { "hideZeros": false, - "mode": "multi", + "mode": "single", "sort": "none" } }, @@ -3244,11 +3481,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.metrics.document.updated\" or r[\"_field\"] == \"serverStatus.metrics.document.returned\" or r[\"_field\"] == \"serverStatus.metrics.document.inserted\" or r[\"_field\"] == \"serverStatus.metrics.document.deleted\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: v.windowPeriod)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.flowControl.isLagged\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", "refId": "A" } ], - "title": "Documents (deleted / inserted / returned / updated)", + "title": "FlowControl isLagged", "type": "timeseries" }, { @@ -3256,7 +3493,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Ratio of serverStatus.metrics.queryExecutor.scannedObjects to metrics.document.returned, computed as a per-second rate ratio. Per MongoDB docs, scannedObjects is equivalent to totalDocsExamined in explain(); document.returned is the total number of documents returned by queries (serverStatus.metrics.queryExecutor, serverStatus.metrics.document).\n\nHow to read: single line showing the scanned-to-returned ratio. A ratio of 1.0 means every scanned document was returned (perfect targeting).\n\nTroubleshooting: ratios significantly above 1.0 indicate queries scanning many documents to return few results -- the classic sign of missing or suboptimal indexes. This is often the number one cause of unnecessary disk I/O and CPU load. Run explain() on slow queries to identify the problem.", + "description": "### Flow control lagged time\n\n**Fraction of each window** the primary spent with flow control engaged, from **`flowControl.isLaggedTimeMicros`**.\n\n**Formula:** `derivative(isLaggedTimeMicros, 1s) / 1e6` → 0–100%\n\n**Note:** Primary only. For **on/off now**, see **FlowControl isLagged**.\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.flowControl.isLaggedTimeMicros`\n\n---\n\n### How to read\n\n- **0%** → no throttling in that window\n- **100%** → flow control engaged the entire window\n- Spikes → brief throttling bursts\n\n---\n\n### Troubleshooting\n\n- Sustained **>0%** → correlate with **Replica Members Lag**, **FlowControl isLagged**\n- High % + rising lag → secondaries cannot keep up · disk/CPU/network\n\n---\n\n### Docs\n\n- [serverStatus — flowControl](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.flowControl)", "fieldConfig": { "defaults": { "color": { @@ -3268,6 +3505,8 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMax": 1, + "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -3280,7 +3519,7 @@ }, "insertNulls": false, "lineInterpolation": "linear", - "lineWidth": 4, + "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" @@ -3296,8 +3535,9 @@ "mode": "off" } }, - "decimals": 2, + "decimals": 1, "mappings": [], + "max": 1, "min": 0, "thresholds": { "mode": "absolute", @@ -3305,23 +3545,33 @@ { "color": "green", "value": 0 - }, + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*isLaggedTimeMicros$" + }, + "properties": [ { - "color": "red", - "value": 80 + "id": "displayName", + "value": "Flow control engaged (% of time)" } ] } - }, - "overrides": [] + ] }, "gridPos": { "h": 6, - "w": 7, + "w": 14, "x": 10, - "y": 82 + "y": 84 }, - "id": 37, + "id": 53, "options": { "legend": { "calcs": [ @@ -3336,7 +3586,7 @@ }, "tooltip": { "hideZeros": false, - "mode": "multi", + "mode": "single", "sort": "none" } }, @@ -3347,11 +3597,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.metrics.document.returned\" or \r\n r[\"_field\"] == \"serverStatus.metrics.queryExecutor.scannedObjects\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> pivot(\r\n rowKey: [\"_time\"],\r\n columnKey: [\"_field\"],\r\n valueColumn: \"_value\"\r\n )\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n scanned_returned_ratio:\r\n if r[\"serverStatus.metrics.document.returned\"] > 0.0\r\n then r[\"serverStatus.metrics.queryExecutor.scannedObjects\"]\r\n / r[\"serverStatus.metrics.document.returned\"]\r\n else 0.0\r\n }))\r\n |> yield(name: \"scanned_returned_ratio\")\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.flowControl.isLaggedTimeMicros\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) / 1000000.0 }))", "refId": "A" } ], - "title": "Query Targeting: Scanned Objects / Returned", + "title": "Flow control lagged time", "type": "timeseries" }, { @@ -3359,7 +3609,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Rate of serverStatus.metrics.commands per second. Per MongoDB docs, metrics.commands reports on the use of database commands. opcounters.command counts all commands except insert, update, delete, and aggregate (serverStatus.metrics.commands, serverStatus.opcounters).\n\nHow to read: single line showing command execution rate.\n\nTroubleshooting: a spike in command rate may indicate runaway aggregation pipelines, administrative operations (index builds, compact), or driver heartbeat storms. Correlate with 'Average Operation Latency' and 'CPU Usage' to assess impact.", + "description": "### Current queue (writers / readers)\n\nOperations **waiting** for the global read or write lock (`globalLock.currentQueue`). Snapshot gauge — not a rate.\n\n**Unit:** count (`short`)\n\n**Note:** On MongoDB 4.2+, these queues are often **0**; use **WiredTiger Tickets** and **Average Operation Latency** for concurrency pressure on modern releases.\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.globalLock.currentQueue.readers` · `serverStatus.globalLock.currentQueue.writers`\n\n---\n\n### How to read\n\n- Two lines: queued readers vs writers\n- Brief spikes can be normal · sustained non-zero → contention\n\n---\n\n### Troubleshooting\n\n- Persistent write queue → **WiredTiger Tickets**, **Average Operation Latency**, **Operations**\n- Persistent read queue → long-running reads blocking others\n- Always zero but slow DB → focus on tickets/latency, not this panel alone\n\n---\n\n### Docs\n\n- [serverStatus — globalLock](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.globalLock)", "fieldConfig": { "defaults": { "color": { @@ -3371,6 +3621,7 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -3382,7 +3633,7 @@ "viz": false }, "insertNulls": false, - "lineInterpolation": "linear", + "lineInterpolation": "stepAfter", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -3399,6 +3650,7 @@ "mode": "off" } }, + "decimals": 0, "mappings": [], "thresholds": { "mode": "absolute", @@ -3406,23 +3658,45 @@ { "color": "green", "value": 0 - }, + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*currentQueue\\.readers$" + }, + "properties": [ { - "color": "red", - "value": 80 + "id": "displayName", + "value": "Queued readers" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*currentQueue\\.writers$" + }, + "properties": [ + { + "id": "displayName", + "value": "Queued writers" } ] } - }, - "overrides": [] + ] }, "gridPos": { - "h": 6, - "w": 8, + "h": 7, + "w": 10, "x": 0, - "y": 84 + "y": 86 }, - "id": 32, + "id": 29, "options": { "legend": { "calcs": [ @@ -3448,11 +3722,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.metrics.commands.createIndexes.total\" or r[\"_field\"] == \"serverStatus.metrics.commands.dataSize.total\" or r[\"_field\"] == \"serverStatus.metrics.commands.dropIndexes.total\" or r[\"_field\"] == \"serverStatus.metrics.commands.listCollections.total\" or r[\"_field\"] == \"serverStatus.metrics.commands.listDatabases.total\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n|> derivative(unit: v.windowPeriod)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.globalLock.currentQueue.writers\" or\r\n r[\"_field\"] == \"serverStatus.globalLock.currentQueue.readers\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", "refId": "A" } ], - "title": "Commands", + "title": "Current Queue writers / readers", "type": "timeseries" }, { @@ -3460,7 +3734,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Rate of serverStatus.metrics.queryExecutor metrics: scanned (total index items scanned, equivalent to totalKeysExamined in explain()), scannedObjects (total documents scanned, equivalent to totalDocsExamined), and collectionScans.total (queries that performed a collection scan) per second (serverStatus.metrics.queryExecutor).\n\nHow to read: multiple lines showing rates. High scannedObjects means many documents examined; high collectionScans.total means queries are not using indexes.\n\nTroubleshooting: high scannedObjects relative to documents returned indicates inefficient queries -- run explain() to identify missing indexes. High collectionScans.total is a red flag for production workloads; identify the queries via the profiler.", + "description": "### Query executor\n\nPer-second rates from **queryExecutor** counters — how much work the query engine does examining indexes and documents.\n\n**Unit:** operations/sec (`ops`) — keys scanned, docs scanned, and collection-scan events are all counter derivatives.\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.metrics.queryExecutor.scanned` — index keys examined (`totalKeysExamined` in explain())\n- `serverStatus.metrics.queryExecutor.scannedObjects` — documents examined (`totalDocsExamined`)\n- `serverStatus.metrics.queryExecutor.collectionScans.total` — queries that performed a **collection scan** (COLLSCAN)\n\nAll are cumulative counters; query uses **`derivative(unit: 1s, nonNegative: true)`**.\n\n---\n\n### How to read\n\n- **Index keys scanned / sec** — index traversal volume\n- **Documents scanned / sec** — document examination volume (often dominates I/O)\n- **Collection scans / sec** (right axis) — queries **not** using a useful index; should stay near **0** in production\n\n**Keys vs docs:** high docs with low keys can mean wide index scans or many docs per key; high docs with high **Collection scans** confirms COLLSCAN.\n\nCompare with **Query Targeting: Scanned Objects / Returned** (ratio) and **Documents** (`document.returned`).\n\n---\n\n### Troubleshooting\n\n- High **Documents scanned** + high targeting ratio → missing/poor indexes · run `explain()` on slow queries\n- Sustained **Collection scans / sec** → enable profiler / Performance Advisor · fix query patterns or add indexes\n- High scan rates + **Page Faults** / **Disk I/O** → storage pressure from inefficient reads\n- **Operations** `opcounters.query` rising with scan rates → read-heavy workload under index stress\n\n---\n\n### Docs\n\n- [serverStatus — metrics.queryExecutor](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.metrics.queryExecutor)", "fieldConfig": { "defaults": { "color": { @@ -3472,6 +3746,7 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -3483,7 +3758,7 @@ "viz": false }, "insertNulls": false, - "lineInterpolation": "linear", + "lineInterpolation": "stepAfter", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -3507,22 +3782,59 @@ { "color": "green", "value": 0 + } + ] + }, + "unit": "ops" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "serverStatus.metrics.queryExecutor.scanned" + }, + "properties": [ + { + "id": "displayName", + "value": "Index keys scanned / sec" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "serverStatus.metrics.queryExecutor.scannedObjects" + }, + "properties": [ + { + "id": "displayName", + "value": "Documents scanned / sec" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "serverStatus.metrics.queryExecutor.collectionScans.total" + }, + "properties": [ + { + "id": "displayName", + "value": "Collection scans / sec" }, { - "color": "red", - "value": 80 + "id": "custom.axisPlacement", + "value": "right" } ] - }, - "unit": "short" - }, - "overrides": [] + } + ] }, "gridPos": { - "h": 6, - "w": 9, + "h": 9, + "w": 14, "x": 10, - "y": 88 + "y": 90 }, "id": 31, "options": { @@ -3550,7 +3862,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.metrics.queryExecutor.scannedObjects\" or r[\"_field\"] == \"serverStatus.metrics.queryExecutor.scanned\" or r[\"_field\"] == \"serverStatus.metrics.queryExecutor.collectionScans.total\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: v.windowPeriod)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.metrics.queryExecutor.scannedObjects\" or\r\n r[\"_field\"] == \"serverStatus.metrics.queryExecutor.scanned\" or\r\n r[\"_field\"] == \"serverStatus.metrics.queryExecutor.collectionScans.total\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", "refId": "A" } ], @@ -3562,7 +3874,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Shows serverStatus.metrics.cursor.open.total -- the number of cursors MongoDB is maintaining for clients. Per MongoDB docs, typically this value is small or zero; however, if there is a queue, stale tailable cursors, or a large number of operations, this value may increase. metrics.cursor.timedOut tracks cursors that timed out (serverStatus.metrics.cursor).\n\nHow to read: single line showing open cursor count over time.\n\nTroubleshooting: a growing cursor count may indicate cursor leaks in the application (not calling close()). Cursor timeouts (default 10 min) normally clean up idle cursors. If timedOut is also growing, applications are abandoning cursors without exhausting or closing them.", + "description": "### Oplog average entry size\n\nAverage size in **bytes** of documents in the **`local.oplog.rs`** capped collection (`collStats` / `storageStats.avgObjSize`).\n\n**Unit:** bytes (`bytes`)\n\n**Not `dbStats.avgObjSize`:** FTDC exposes oplog collStats here, not per-database average user document size.\n\n---\n\n### Metrics\n\n**Field:** `local.oplog.rs.stats.storageStats.avgObjSize`\n\nSnapshot gauge — **`aggregateWindow(..., fn: last)`**, no derivative.\n\n---\n\n### How to read\n\n- Single line · **Oplog avg entry size**\n- Rises when writes carry larger payloads (big documents, wide updates, bulk ops)\n- Slow drift over days → changing write patterns or schema\n- Compare with **Storage Size** (`local.oplog.rs` `storageSize` / `freeStorageSize`)\n\n---\n\n### Troubleshooting\n\n- Step change + write workload shift → larger oplog entries · more replication network/disk\n- High avg size + lag → secondary apply pressure · check **Replica Members Lag**\n- **Arbiter / standalone** may show no or stale series (no meaningful oplog)\n- For **user collection** document sizing → `db.collection.stats().avgObjSize` (not in this FTDC export)\n\n---\n\n### Docs\n\n- [collStats — avgObjSize](https://www.mongodb.com/docs/manual/reference/command/collStats/)\n- [Change oplog size](https://www.mongodb.com/docs/manual/tutorial/change-oplog-size/)", "fieldConfig": { "defaults": { "color": { @@ -3574,6 +3886,7 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -3610,23 +3923,33 @@ { "color": "green", "value": 0 - }, + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "local.oplog.rs.stats.storageStats.avgObjSize" + }, + "properties": [ { - "color": "red", - "value": 80 + "id": "displayName", + "value": "Oplog avg entry size" } ] } - }, - "overrides": [] + ] }, "gridPos": { "h": 6, - "w": 7, + "w": 10, "x": 0, - "y": 90 + "y": 93 }, - "id": 33, + "id": 34, "options": { "legend": { "calcs": [ @@ -3652,11 +3975,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.metrics.cursor.open.noTimeout\" or r[\"_field\"] == \"serverStatus.metrics.cursor.open.pinned\" or r[\"_field\"] == \"serverStatus.metrics.cursor.open.total\" or r[\"_field\"] == \"serverStatus.metrics.cursor.timedOut\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"local.oplog.rs.stats.storageStats.avgObjSize\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", "refId": "A" } ], - "title": "Cursors", + "title": "Oplog average entry size", "type": "timeseries" }, { @@ -3664,7 +3987,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Rate of serverStatus.opLatencies.reads.ops, opLatencies.writes.ops, and opLatencies.commands.ops per second. Per MongoDB docs, opLatencies contains operation latencies for the instance as a whole (serverStatus.opLatencies).\n\nHow to read: three lines showing the operation rate (ops/sec) per latency class -- this is the denominator used to compute average latency.\n\nTroubleshooting: compare with 'Average Operation Latency' -- if ops/sec drops while latency rises, individual operations are getting slower. If both rise together, the system is under increased load. A divergence between reads.ops and writes.ops can indicate workload imbalance.", + "description": "### Operations (opcounters)\n\nPer-second rate of **`serverStatus.opcounters`** — **operation-level** counts (one bulk/multi-doc op = one op). Not the same as **Documents** (`metrics.document`), which counts documents touched.\n\n**Unit:** operations per second (`ops`)\n\n**Note:** `opcounters.command` includes most commands except insert, update, delete, and **aggregate** (see MongoDB docs).\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.opcounters.command`\n- `serverStatus.opcounters.delete`\n- `serverStatus.opcounters.getmore`\n- `serverStatus.opcounters.insert`\n- `serverStatus.opcounters.query`\n- `serverStatus.opcounters.update`\n\n**Query:** `derivative(..., unit: 1s, nonNegative: true)` → ops/sec\n\n---\n\n### How to read\n\n- Six lines: command, delete, getmore, insert, query, update **rates**\n- Sudden drop → blocking or idle period\n- Spike → load burst; check **Average Operation Latency** and **WiredTiger Tickets**\n\n---\n\n### Troubleshooting\n\n- Sustained high write ops + latency → **Current Queue writers / readers**, disk I/O\n- High **query** + **getmore** → cursor-heavy workload\n- On secondaries, use **Opcounters Replica** instead of this panel\n\n---\n\n### Docs\n\n- [serverStatus — opcounters](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.opcounters)", "fieldConfig": { "defaults": { "color": { @@ -3676,6 +3999,7 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -3687,7 +4011,7 @@ "viz": false }, "insertNulls": false, - "lineInterpolation": "linear", + "lineInterpolation": "stepAfter", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -3711,24 +4035,93 @@ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "µs" + "unit": "ops" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*\\.opcounters\\.command$" + }, + "properties": [ + { + "id": "displayName", + "value": "Command" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*\\.opcounters\\.delete$" + }, + "properties": [ + { + "id": "displayName", + "value": "Delete" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*\\.opcounters\\.getmore$" + }, + "properties": [ + { + "id": "displayName", + "value": "Getmore" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*\\.opcounters\\.insert$" + }, + "properties": [ + { + "id": "displayName", + "value": "Insert" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*\\.opcounters\\.query$" + }, + "properties": [ + { + "id": "displayName", + "value": "Query" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*\\.opcounters\\.update$" + }, + "properties": [ + { + "id": "displayName", + "value": "Update" + } + ] + } + ] }, "gridPos": { - "h": 5, - "w": 8, - "x": 8, - "y": 94 + "h": 11, + "w": 12, + "x": 0, + "y": 99 }, - "id": 10, + "id": 9, "options": { "legend": { "calcs": [ @@ -3754,11 +4147,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.opLatencies.reads.ops\" or r[\"_field\"] == \"serverStatus.opLatencies.writes.ops\" or r[\"_field\"] == \"serverStatus.opLatencies.commands.ops\" )\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s)\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.opcounters.command\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.delete\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.getmore\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.insert\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.query\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.update\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", "refId": "A" } ], - "title": "Operations Latencies Op", + "title": "Operations", "type": "timeseries" }, { @@ -3766,7 +4159,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Shows dbStats.avgObjSize -- the average size of each document in bytes, computed as dataSize divided by the number of documents. Per MongoDB docs, the scale argument does not affect avgObjSize (dbStats.avgObjSize).\n\nHow to read: single line showing average document size over time.\n\nTroubleshooting: a sudden increase may indicate documents growing from array pushes or embedded document growth, which affects WiredTiger cache efficiency, network bandwidth, and replication lag. A decrease after TTL cleanup or data migration is expected.", + "description": "### Replication opcounters (If secondary)\n\nPer-second rates from **`serverStatus.opcountersRepl`** — operations applied when this member is acting as a **secondary** (oplog apply).\n\n**Unit:** operations/sec (`ops`)\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.opcountersRepl.insert`\n- `serverStatus.opcountersRepl.query`\n- `serverStatus.opcountersRepl.update`\n- `serverStatus.opcountersRepl.delete`\n- `serverStatus.opcountersRepl.getmore`\n- `serverStatus.opcountersRepl.command`\n\nCumulative counters; query uses **`derivative(unit: 1s, nonNegative: true)`**.\n\n---\n\n### How to read\n\n- Six lines · replication apply rate by type\n- **View a secondary hostname** — answers “how fast is this node applying replication?”\n- **Primary / write source** — expect near-zero or unhelpful lines; switch host or open **Operations (write source)**\n- Fractional ops/sec → normal with sparse FTDC sampling\n- **0** → idle or no replication apply in window\n\nCompare with **Operations (write source)** on the primary, **Replica Members Lag**, **Flow control is lagged**.\n\n---\n\n### Troubleshooting\n\n- Repl rates drop + **lag rising** → secondary cannot keep up · disk I/O, CPU, **Disk I/O**\n- Primary **Operations (write source)** high, secondary repl low → bottleneck on secondary\n- Repl **update** ≫ primary **update** → expected replication amplification\n- **Flow control is lagged** = 1 + lag → primary throttling writes\n\n---\n\n### Docs\n\n- [serverStatus — opcountersRepl](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.opcountersRepl)\n- [serverStatus — opcounters](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.opcounters)", "fieldConfig": { "defaults": { "color": { @@ -3778,6 +4171,7 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -3807,30 +4201,99 @@ } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 - }, + } + ] + }, + "unit": "ops" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "serverStatus.opcountersRepl.insert" + }, + "properties": [ { - "color": "red", - "value": 80 + "id": "displayName", + "value": "Insert / sec" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "serverStatus.opcountersRepl.query" + }, + "properties": [ + { + "id": "displayName", + "value": "Query / sec" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "serverStatus.opcountersRepl.update" + }, + "properties": [ + { + "id": "displayName", + "value": "Update / sec" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "serverStatus.opcountersRepl.delete" + }, + "properties": [ + { + "id": "displayName", + "value": "Delete / sec" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "serverStatus.opcountersRepl.getmore" + }, + "properties": [ + { + "id": "displayName", + "value": "Getmore / sec" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "serverStatus.opcountersRepl.command" + }, + "properties": [ + { + "id": "displayName", + "value": "Command / sec" } ] } - }, - "overrides": [] + ] }, "gridPos": { - "h": 6, - "w": 9, - "x": 7, + "h": 11, + "w": 12, + "x": 12, "y": 99 }, - "id": 34, + "id": 38, "options": { "legend": { "calcs": [ @@ -3856,11 +4319,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"local.oplog.rs.stats.storageStats.avgObjSize\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.insert\" or\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.query\" or\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.update\" or\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.delete\" or\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.getmore\" or\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.command\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", "refId": "A" } ], - "title": "avg Obj Size", + "title": "Opcounters Replica (If secondary)", "type": "timeseries" }, { @@ -3868,7 +4331,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Current queue (writers / readers)\n\nOperations **waiting** for the global read or write lock (`globalLock.currentQueue`). Snapshot gauge — not a rate.\n\n**Unit:** count (`short`)\n\n**Note:** On MongoDB 4.2+, these queues are often **0**; use **WiredTiger Tickets** and **Average Operation Latency** for concurrency pressure on modern releases.\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.globalLock.currentQueue.readers` · `serverStatus.globalLock.currentQueue.writers`\n\n---\n\n### How to read\n\n- Two lines: queued readers vs writers\n- Brief spikes can be normal · sustained non-zero → contention\n\n---\n\n### Troubleshooting\n\n- Persistent write queue → **WiredTiger Tickets**, **Average Operation Latency**, **Operations**\n- Persistent read queue → long-running reads blocking others\n- Always zero but slow DB → focus on tickets/latency, not this panel alone\n\n---\n\n### Docs\n\n- [serverStatus — globalLock](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.globalLock)", + "description": "### Operation latency ops\n\nPer-second rates from **opLatencies.*.ops** counters — how many read, write, and command operations ran in each latency class.\n\n**Unit:** operations/sec (`ops`)\n\nThis is the **denominator** for **Average Operation Latency** (`derivative(latency) / derivative(ops)` → µs/op).\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.opLatencies.reads.ops`\n- `serverStatus.opLatencies.writes.ops`\n- `serverStatus.opLatencies.commands.ops`\n\nCumulative counters; query uses **`derivative(unit: 1s, nonNegative: true)`**.\n\n**Not the same as opcounters:** `opLatencies` groups reads / writes / commands for latency stats; **Operations** uses `opcounters.*` (query, insert, update, delete, getmore, command).\n\n---\n\n### How to read\n\n- **Read / write / command ops / sec** — throughput per latency class\n- **Commands** often includes admin, aggregate, and other command work\n- Fractional values → normal with sparse FTDC sampling\n- **0** → idle window for that class\n\nCompare with **Average Operation Latency** (numerator/denominator) and **Operations** (opcounters workload mix).\n\n---\n\n### Troubleshooting\n\n- **Ops up, avg latency up** → load + slower per-op work\n- **Ops down, avg latency up** → fewer ops but each slower (contention, I/O)\n- **Ops up, avg latency flat** → healthy scale-out of throughput\n- Read/write imbalance → workload skew · correlate with **Query Executor**, **Documents**, **Disk I/O**\n\n---\n\n### Docs\n\n- [serverStatus — opLatencies](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.opLatencies)", "fieldConfig": { "defaults": { "color": { @@ -3909,7 +4372,6 @@ "mode": "off" } }, - "decimals": 0, "mappings": [], "thresholds": { "mode": "absolute", @@ -3920,42 +4382,54 @@ } ] }, - "unit": "short" + "unit": "ops" }, "overrides": [ { "matcher": { - "id": "byRegexp", - "options": ".*currentQueue\\.readers$" + "id": "byName", + "options": "serverStatus.opLatencies.reads.ops" }, "properties": [ { "id": "displayName", - "value": "Queued readers" + "value": "Read ops / sec" } ] }, { "matcher": { - "id": "byRegexp", - "options": ".*currentQueue\\.writers$" + "id": "byName", + "options": "serverStatus.opLatencies.writes.ops" }, "properties": [ { "id": "displayName", - "value": "Queued writers" + "value": "Write ops / sec" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "serverStatus.opLatencies.commands.ops" + }, + "properties": [ + { + "id": "displayName", + "value": "Command ops / sec" } ] } ] }, "gridPos": { - "h": 7, - "w": 10, + "h": 9, + "w": 24, "x": 0, - "y": 105 + "y": 110 }, - "id": 29, + "id": 10, "options": { "legend": { "calcs": [ @@ -3981,11 +4455,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.globalLock.currentQueue.writers\" or\r\n r[\"_field\"] == \"serverStatus.globalLock.currentQueue.readers\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.opLatencies.reads.ops\" or\r\n r[\"_field\"] == \"serverStatus.opLatencies.writes.ops\" or\r\n r[\"_field\"] == \"serverStatus.opLatencies.commands.ops\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", "refId": "A" } ], - "title": "Current Queue writers / readers", + "title": "Operation Latency Ops", "type": "timeseries" }, { @@ -3993,7 +4467,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Rate of serverStatus.opcountersRepl per second: insert, query, update, delete, getmore, command. Per MongoDB docs, opcountersRepl reports on database replication operations by type since the mongod instance last started. These numbers grow over time until the next restart (serverStatus.opcountersRepl).\n\nHow to read: multiple lines showing replication operation rates.\n\nTroubleshooting: on a secondary, this shows how fast replication operations are being applied. If this rate drops while the primary's 'Operations' panel shows high throughput, the secondary is falling behind. Correlate with 'Replica Members Lag' and secondary disk I/O.", + "description": "### Query targeting: scanned objects / returned\n\nRatio of **document scan rate** to **document return rate**:\n\n**Formula:** `derivative(scannedObjects, 1s) / derivative(document.returned, 1s)`\n\nSame idea as **totalDocsExamined / docs returned** in explain() — aggregated over time.\n\n**Unit:** dimensionless ratio (`none`) — **1.0** ≈ one doc examined per doc returned\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.metrics.queryExecutor.scannedObjects`\n- `serverStatus.metrics.document.returned`\n\nBoth are cumulative counters; query uses **`derivative(unit: 1s, nonNegative: true)`** before the ratio.\n\n---\n\n### How to read\n\n- Single line · **Scanned / returned ratio**\n- **~1** → strong targeting (most examined docs were returned)\n- **> 1** (especially **10+**) → scans dominate · missing/poor indexes or large in-memory filters\n- **0** → no returned docs in window (idle) or zero denominator — **not** “perfect”\n\n---\n\n### Troubleshooting\n\n- High ratio + high **Documents returned** → **Query Executor** · run `explain()` on hot queries\n- Correlate with **Page Faults**, **Disk I/O**, **Average Operation Latency**\n- **Collection scans** panel (if present) confirms full collection scans\n\n---\n\n### Docs\n\n- [serverStatus — metrics.queryExecutor](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.metrics.queryExecutor)\n- [serverStatus — metrics.document](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.metrics.document)", "fieldConfig": { "defaults": { "color": { @@ -4005,6 +4479,7 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -4033,6 +4508,7 @@ "mode": "off" } }, + "decimals": 2, "mappings": [], "thresholds": { "mode": "absolute", @@ -4040,24 +4516,33 @@ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "ops" + "unit": "none" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "scanned_returned_ratio" + }, + "properties": [ + { + "id": "displayName", + "value": "Scanned / returned ratio" + } + ] + } + ] }, "gridPos": { - "h": 7, - "w": 9, - "x": 10, - "y": 105 + "h": 6, + "w": 24, + "x": 0, + "y": 119 }, - "id": 38, + "id": 37, "options": { "legend": { "calcs": [ @@ -4083,11 +4568,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.opcountersRepl.command\" or r[\"_field\"] == \"serverStatus.opcountersRepl.delete\" or r[\"_field\"] == \"serverStatus.opcountersRepl.getmore\" or r[\"_field\"] == \"serverStatus.opcountersRepl.insert\" or r[\"_field\"] == \"serverStatus.opcountersRepl.query\" or r[\"_field\"] == \"serverStatus.opcountersRepl.update\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: v.windowPeriod)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.metrics.document.returned\" or\r\n r[\"_field\"] == \"serverStatus.metrics.queryExecutor.scannedObjects\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> pivot(rowKey: [\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n scanned_returned_ratio:\r\n if exists r[\"serverStatus.metrics.document.returned\"]\r\n and float(v: r[\"serverStatus.metrics.document.returned\"]) > 0.0\r\n and exists r[\"serverStatus.metrics.queryExecutor.scannedObjects\"]\r\n then float(v: r[\"serverStatus.metrics.queryExecutor.scannedObjects\"])\r\n / float(v: r[\"serverStatus.metrics.document.returned\"])\r\n else 0.0\r\n }))", "refId": "A" } ], - "title": "Opcounters Replica", + "title": "Query Targeting: Scanned Objects / Returned", "type": "timeseries" }, { @@ -4096,7 +4581,7 @@ "h": 1, "w": 24, "x": 0, - "y": 112 + "y": 125 }, "id": 21, "panels": [], @@ -4170,7 +4655,7 @@ "h": 9, "w": 24, "x": 0, - "y": 113 + "y": 126 }, "id": 13, "options": { @@ -4272,7 +4757,7 @@ "h": 8, "w": 12, "x": 0, - "y": 122 + "y": 135 }, "id": 40, "options": { @@ -4374,7 +4859,7 @@ "h": 8, "w": 12, "x": 12, - "y": 122 + "y": 135 }, "id": 39, "options": { @@ -4476,7 +4961,7 @@ "h": 7, "w": 8, "x": 0, - "y": 130 + "y": 143 }, "id": 35, "options": { @@ -4578,7 +5063,7 @@ "h": 7, "w": 8, "x": 8, - "y": 130 + "y": 143 }, "id": 17, "options": { @@ -4680,7 +5165,7 @@ "h": 7, "w": 8, "x": 16, - "y": 130 + "y": 143 }, "id": 43, "options": { @@ -4782,7 +5267,7 @@ "h": 6, "w": 8, "x": 0, - "y": 137 + "y": 150 }, "id": 4, "options": { @@ -4884,7 +5369,7 @@ "h": 7, "w": 8, "x": 8, - "y": 137 + "y": 150 }, "id": 46, "options": { @@ -4995,7 +5480,7 @@ "h": 5, "w": 8, "x": 0, - "y": 143 + "y": 156 }, "id": 11, "options": { @@ -5097,7 +5582,7 @@ "h": 7, "w": 8, "x": 8, - "y": 144 + "y": 157 }, "id": 41, "options": { @@ -5199,7 +5684,7 @@ "h": 7, "w": 8, "x": 0, - "y": 148 + "y": 161 }, "id": 47, "options": { @@ -5311,7 +5796,7 @@ "h": 7, "w": 8, "x": 8, - "y": 151 + "y": 164 }, "id": 45, "options": { @@ -5413,7 +5898,7 @@ "h": 7, "w": 8, "x": 0, - "y": 155 + "y": 168 }, "id": 18, "options": { @@ -5524,7 +6009,7 @@ "h": 6, "w": 8, "x": 8, - "y": 158 + "y": 171 }, "id": 51, "options": { @@ -5626,7 +6111,7 @@ "h": 5, "w": 8, "x": 0, - "y": 162 + "y": 175 }, "id": 42, "options": { @@ -5727,7 +6212,7 @@ "h": 7, "w": 7, "x": 8, - "y": 164 + "y": 177 }, "id": 30, "options": { @@ -5829,7 +6314,7 @@ "h": 5, "w": 8, "x": 0, - "y": 167 + "y": 180 }, "id": 44, "options": { @@ -5931,7 +6416,7 @@ "h": 6, "w": 8, "x": 0, - "y": 172 + "y": 185 }, "id": 50, "options": { diff --git a/scripts/publish-to-upstream.sh b/scripts/publish-to-upstream.sh new file mode 100755 index 0000000..d0ca010 --- /dev/null +++ b/scripts/publish-to-upstream.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# Push the current branch to your fork and open a PR against devops-land/mongodb_ftdc_viewer. +# Prerequisites: gh auth login -h github.com, and at least one commit on the branch. +set -euo pipefail +cd "$(dirname "$0")/.." + +UPSTREAM="devops-land/mongodb_ftdc_viewer" +BRANCH="$(git branch --show-current)" + +if [[ "$BRANCH" == "main" ]]; then + echo "Create a feature branch first, e.g.: git checkout -b contrib/improvements" + exit 1 +fi + +if ! gh auth status -h github.com &>/dev/null; then + echo "GitHub CLI is not authenticated. Run: gh auth login -h github.com" + exit 1 +fi + +if git diff --quiet && git diff --cached --quiet; then + : +else + echo "You have unstaged or uncommitted changes. Commit first, or run ./scripts/stage-for-upstream.sh" + exit 1 +fi + +if ! git rev-list --count "upstream/main..HEAD" | grep -qv '^0$'; then + echo "No commits ahead of upstream/main. Stage and commit your changes first." + exit 1 +fi + +if ! git remote get-url origin &>/dev/null; then + GH_USER="$(gh api user --jq .login)" + echo "Creating fork and adding remote: origin -> https://github.com/${GH_USER}/mongodb_ftdc_viewer.git" + gh repo fork "$UPSTREAM" --remote=false --clone=false + git remote add origin "https://github.com/${GH_USER}/mongodb_ftdc_viewer.git" +fi + +GH_USER="$(gh api user --jq .login)" + +git push -u origin "$BRANCH" +gh pr create \ + --repo "$UPSTREAM" \ + --base main \ + --head "${GH_USER}:${BRANCH}" \ + --title "Fix MongoDB 8.x FTDC metrics and dashboard improvements" \ + --body "$(cat <<'EOF' +## Summary +- Support MongoDB 8.x ticket metrics under `serverStatus.queues.execution.*` +- Flatten nested FTDC metric documents in the exporter +- Simplify WiredTiger Tickets panel to read/write available tickets only + +## Test plan +- [ ] Re-ingest FTDC data from MongoDB 8.0.17 and confirm WiredTiger Tickets panel shows data +- [ ] `go test ./ftdc/...` in `ftdc_exporter` +- [ ] Verify legacy MongoDB versions still show ticket metrics where applicable +EOF +)" diff --git a/scripts/stage-for-upstream.sh b/scripts/stage-for-upstream.sh new file mode 100755 index 0000000..543343d --- /dev/null +++ b/scripts/stage-for-upstream.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Stage files intended for the upstream PR. Run from repo root after all improvements are done. +set -euo pipefail +cd "$(dirname "$0")/.." + +git add \ + .gitignore \ + ftdc_exporter/ftdc/normalize.go \ + ftdc_exporter/ftdc/iterator.go \ + ftdc_exporter/ftdc/normalize_test.go \ + ftdc_exporter/ftdc/testdata/metrics_to_get.txt \ + metrics_to_get.txt \ + grafana/dashboards/dashboard.json \ + run.sh \ + scripts/ + +echo "Staged for upstream PR:" +git diff --cached --stat +echo +echo "Review: git diff --cached" +echo "Commit: git commit -m \"\$(cat <<'EOF'" +echo "Fix MongoDB 8.x FTDC metrics and WiredTiger Tickets dashboard." +echo +echo "Add queues.execution ticket fields, flatten nested FTDC metric documents," +echo "and limit the WiredTiger Tickets panel to read/write available tickets." +echo "EOF" +echo ")\"" +echo "Publish: ./scripts/publish-to-upstream.sh" From e30d34d4e3ac1a6ed44c99e66e92887203d871bb Mon Sep 17 00:00:00 2001 From: jenunes Date: Fri, 12 Jun 2026 17:32:31 -0300 Subject: [PATCH 4/9] Stop tracking local scripts directory. Add scripts/ to .gitignore; publish/stage helpers are for local fork workflow only. --- .gitignore | 2 +- scripts/publish-to-upstream.sh | 58 ---------------------------------- scripts/stage-for-upstream.sh | 28 ---------------- 3 files changed, 1 insertion(+), 87 deletions(-) delete mode 100755 scripts/publish-to-upstream.sh delete mode 100755 scripts/stage-for-upstream.sh diff --git a/.gitignore b/.gitignore index c70ef18..43b1a79 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,4 @@ mongodb_ftdc_decoder/ grafana/backups/ grafana/DASHBOARD.md grafana/dashboards/bkp.json -grafana/dashboards/my.json \ No newline at end of file +grafana/dashboards/my.jsonscripts/ diff --git a/scripts/publish-to-upstream.sh b/scripts/publish-to-upstream.sh deleted file mode 100755 index d0ca010..0000000 --- a/scripts/publish-to-upstream.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env bash -# Push the current branch to your fork and open a PR against devops-land/mongodb_ftdc_viewer. -# Prerequisites: gh auth login -h github.com, and at least one commit on the branch. -set -euo pipefail -cd "$(dirname "$0")/.." - -UPSTREAM="devops-land/mongodb_ftdc_viewer" -BRANCH="$(git branch --show-current)" - -if [[ "$BRANCH" == "main" ]]; then - echo "Create a feature branch first, e.g.: git checkout -b contrib/improvements" - exit 1 -fi - -if ! gh auth status -h github.com &>/dev/null; then - echo "GitHub CLI is not authenticated. Run: gh auth login -h github.com" - exit 1 -fi - -if git diff --quiet && git diff --cached --quiet; then - : -else - echo "You have unstaged or uncommitted changes. Commit first, or run ./scripts/stage-for-upstream.sh" - exit 1 -fi - -if ! git rev-list --count "upstream/main..HEAD" | grep -qv '^0$'; then - echo "No commits ahead of upstream/main. Stage and commit your changes first." - exit 1 -fi - -if ! git remote get-url origin &>/dev/null; then - GH_USER="$(gh api user --jq .login)" - echo "Creating fork and adding remote: origin -> https://github.com/${GH_USER}/mongodb_ftdc_viewer.git" - gh repo fork "$UPSTREAM" --remote=false --clone=false - git remote add origin "https://github.com/${GH_USER}/mongodb_ftdc_viewer.git" -fi - -GH_USER="$(gh api user --jq .login)" - -git push -u origin "$BRANCH" -gh pr create \ - --repo "$UPSTREAM" \ - --base main \ - --head "${GH_USER}:${BRANCH}" \ - --title "Fix MongoDB 8.x FTDC metrics and dashboard improvements" \ - --body "$(cat <<'EOF' -## Summary -- Support MongoDB 8.x ticket metrics under `serverStatus.queues.execution.*` -- Flatten nested FTDC metric documents in the exporter -- Simplify WiredTiger Tickets panel to read/write available tickets only - -## Test plan -- [ ] Re-ingest FTDC data from MongoDB 8.0.17 and confirm WiredTiger Tickets panel shows data -- [ ] `go test ./ftdc/...` in `ftdc_exporter` -- [ ] Verify legacy MongoDB versions still show ticket metrics where applicable -EOF -)" diff --git a/scripts/stage-for-upstream.sh b/scripts/stage-for-upstream.sh deleted file mode 100755 index 543343d..0000000 --- a/scripts/stage-for-upstream.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash -# Stage files intended for the upstream PR. Run from repo root after all improvements are done. -set -euo pipefail -cd "$(dirname "$0")/.." - -git add \ - .gitignore \ - ftdc_exporter/ftdc/normalize.go \ - ftdc_exporter/ftdc/iterator.go \ - ftdc_exporter/ftdc/normalize_test.go \ - ftdc_exporter/ftdc/testdata/metrics_to_get.txt \ - metrics_to_get.txt \ - grafana/dashboards/dashboard.json \ - run.sh \ - scripts/ - -echo "Staged for upstream PR:" -git diff --cached --stat -echo -echo "Review: git diff --cached" -echo "Commit: git commit -m \"\$(cat <<'EOF'" -echo "Fix MongoDB 8.x FTDC metrics and WiredTiger Tickets dashboard." -echo -echo "Add queues.execution ticket fields, flatten nested FTDC metric documents," -echo "and limit the WiredTiger Tickets panel to read/write available tickets." -echo "EOF" -echo ")\"" -echo "Publish: ./scripts/publish-to-upstream.sh" From c98cfd3df7cc5ce36da10386b0ec36b9564f8e68 Mon Sep 17 00:00:00 2001 From: jenunes Date: Fri, 12 Jun 2026 17:37:59 -0300 Subject: [PATCH 5/9] Remove lastNotNull from dashboard legend calcs. --- grafana/dashboards/dashboard.json | 149 ++++++++++-------------------- 1 file changed, 50 insertions(+), 99 deletions(-) diff --git a/grafana/dashboards/dashboard.json b/grafana/dashboards/dashboard.json index 6b47caa..11b7f3c 100644 --- a/grafana/dashboards/dashboard.json +++ b/grafana/dashboards/dashboard.json @@ -279,8 +279,7 @@ "legend": { "calcs": [ "min", - "max", - "lastNotNull" + "max" ], "displayMode": "table", "placement": "bottom", @@ -618,8 +617,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", @@ -745,8 +743,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", @@ -874,8 +871,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", @@ -987,8 +983,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", @@ -1104,8 +1099,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", @@ -1229,8 +1223,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", @@ -1345,8 +1338,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", @@ -1473,8 +1465,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", @@ -1586,8 +1577,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", @@ -1704,8 +1694,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", @@ -1842,8 +1831,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", @@ -1955,8 +1943,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "right", @@ -2117,8 +2104,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "right", @@ -2230,8 +2216,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", @@ -2342,8 +2327,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "right", @@ -2479,8 +2463,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", @@ -2680,8 +2663,7 @@ "legend": { "calcs": [ "min", - "max", - "lastNotNull" + "max" ], "displayMode": "table", "placement": "bottom", @@ -2861,8 +2843,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", @@ -3022,8 +3003,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "right", @@ -3183,8 +3163,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "right", @@ -3331,8 +3310,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", @@ -3461,8 +3439,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", @@ -3577,8 +3554,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", @@ -3702,8 +3678,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", @@ -3842,8 +3817,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", @@ -3955,8 +3929,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", @@ -4127,8 +4100,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", @@ -4299,8 +4271,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", @@ -4435,8 +4406,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", @@ -4548,8 +4518,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", @@ -4663,8 +4632,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", @@ -4765,8 +4733,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", @@ -4867,8 +4834,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", @@ -4969,8 +4935,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", @@ -5071,8 +5036,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", @@ -5173,8 +5137,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", @@ -5275,8 +5238,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", @@ -5377,8 +5339,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", @@ -5488,8 +5449,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", @@ -5590,8 +5550,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", @@ -5692,8 +5651,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", @@ -5804,8 +5762,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", @@ -5906,8 +5863,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", @@ -6017,8 +5973,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", @@ -6119,8 +6074,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", @@ -6220,8 +6174,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", @@ -6322,8 +6275,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", @@ -6424,8 +6376,7 @@ "calcs": [ "min", "max", - "mean", - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", @@ -6503,4 +6454,4 @@ "title": "ftdc_dashboard", "uid": "ddnw277huiv40ae", "version": 1 -} \ No newline at end of file +} From aabeb1ae0454490de5ad0462df4ef4e5c535cb15 Mon Sep 17 00:00:00 2001 From: jenunes Date: Wed, 17 Jun 2026 22:14:07 -0300 Subject: [PATCH 6/9] Improve Grafana dashboard after FTDC panel review. Fix TCP lifecycle query, disk panel naming/units, descriptions, replication and query-targeting gaps; update metrics_to_get.txt. --- grafana/dashboards/dashboard.json | 1975 ++++++++++++++++------------- metrics_to_get.txt | 5 + 2 files changed, 1088 insertions(+), 892 deletions(-) diff --git a/grafana/dashboards/dashboard.json b/grafana/dashboards/dashboard.json index 11b7f3c..ee4795c 100644 --- a/grafana/dashboards/dashboard.json +++ b/grafana/dashboards/dashboard.json @@ -21,6 +21,19 @@ "id": 1, "links": [], "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 55, + "panels": [], + "title": "Mongo Health", + "type": "row" + }, { "datasource": { "type": "influxdb", @@ -96,10 +109,10 @@ ] }, "gridPos": { - "h": 6, - "w": 6, + "h": 5, + "w": 12, "x": 0, - "y": 0 + "y": 1 }, "id": 20, "options": { @@ -129,6 +142,208 @@ "title": "Server Uptime", "type": "timeseries" }, + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "description": "### ReplSet member states\n\n**`replSetGetStatus.members.N.state`** over the selected time range (same codes as [replica states](https://www.mongodb.com/docs/manual/reference/replica-states/)).\n\n**Note:** Row labels use member **array index** unless `members.N.name` is ingested. Failover shows PRIMARY moving between rows. Primary-only FTDC filter via **hostname**.\n\n---\n\n### Metrics\n\n**Fields:** `replSetGetStatus.members.0.state` … (through your max index)\n\n---\n\n### How to read\n\n- One row per member index\n- Colored band = role · **PRIMARY** (1), **SECONDARY** (2), etc.\n- Red bands = DOWN, RECOVERING, ROLLBACK, …\n\n---\n\n### Docs\n\n- [replSetGetStatus](https://www.mongodb.com/docs/manual/reference/command/replSetGetStatus/)", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "custom": { + "axisPlacement": "auto", + "fillOpacity": 100, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineWidth": 0, + "spanNulls": false + }, + "mappings": [ + { + "options": { + "0": { + "color": "semi-dark-red", + "index": 11, + "text": "STARTUP" + }, + "1": { + "color": "rgb(107, 152, 102)", + "index": 4, + "text": "PRIMARY" + }, + "2": { + "color": "rgb(193, 159, 20)", + "index": 3, + "text": "SECONDARY" + }, + "3": { + "color": "semi-dark-red", + "index": 10, + "text": "RECOVERING" + }, + "4": { + "color": "semi-dark-red", + "index": 12, + "text": "FATAL" + }, + "5": { + "color": "semi-dark-red", + "index": 9, + "text": "STARTUP2" + }, + "6": { + "color": "semi-dark-red", + "index": 8, + "text": "UNKNOWN" + }, + "7": { + "color": "#8B8000", + "index": 1, + "text": "ARBITER" + }, + "8": { + "color": "semi-dark-red", + "index": 7, + "text": "DOWN" + }, + "9": { + "color": "semi-dark-red", + "index": 6, + "text": "ROLLBACK" + }, + "10": { + "color": "semi-dark-red", + "index": 5, + "text": "REMOVED" + }, + "null": { + "color": "#FF7383", + "index": 2, + "text": "No data" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*members\\.0\\.state$" + }, + "properties": [ + { + "id": "displayName", + "value": "Member 0" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*members\\.1\\.state$" + }, + "properties": [ + { + "id": "displayName", + "value": "Member 1" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*members\\.2\\.state$" + }, + "properties": [ + { + "id": "displayName", + "value": "Member 2" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*members\\.3\\.state$" + }, + "properties": [ + { + "id": "displayName", + "value": "Member 3" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*members\\.4\\.state$" + }, + "properties": [ + { + "id": "displayName", + "value": "Member 4" + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 54, + "options": { + "alignValue": "right", + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "mergeValues": true, + "rowHeight": 0.9, + "showValue": "auto", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] =~ /replSetGetStatus\\.members\\.\\d+\\.state/)\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", + "refId": "A" + } + ], + "title": "Member States", + "type": "state-timeline" + }, { "datasource": { "type": "influxdb", @@ -269,10 +484,10 @@ ] }, "gridPos": { - "h": 6, - "w": 7, - "x": 6, - "y": 0 + "h": 5, + "w": 12, + "x": 0, + "y": 6 }, "id": 16, "options": { @@ -310,87 +525,48 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### ReplSet member states\n\n**`replSetGetStatus.members.N.state`** over the selected time range (same codes as [replica states](https://www.mongodb.com/docs/manual/reference/replica-states/)).\n\n**Note:** Row labels use member **array index** unless `members.N.name` is ingested. Failover shows PRIMARY moving between rows. Primary-only FTDC filter via **hostname**.\n\n---\n\n### Metrics\n\n**Fields:** `replSetGetStatus.members.0.state` … (through your max index)\n\n---\n\n### How to read\n\n- One row per member index\n- Colored band = role · **PRIMARY** (1), **SECONDARY** (2), etc.\n- Red bands = DOWN, RECOVERING, ROLLBACK, …\n\n---\n\n### Docs\n\n- [replSetGetStatus](https://www.mongodb.com/docs/manual/reference/command/replSetGetStatus/)", + "description": "### Asserts\n\nPer-second rate of **`serverStatus.asserts`** — mongod internal checks and user-triggered assertion counters.\n\n**Unit:** asserts/sec (`ops`)\n\n**Expectation:** normally **flat at zero**; any sustained non-zero rate warrants investigation.\n\n---\n\n### Metrics\n\n**Fields:**\n\n`serverStatus.asserts.regular`\n`serverStatus.asserts.warning`\n`serverStatus.asserts.msg`\n`serverStatus.asserts.user`\n`serverStatus.asserts.rollovers`\n\n**Query type:** cumulative counter → `derivative(unit: 1s, nonNegative: true)` → `aggregateWindow(mean)`\n\n**Rollovers:** count of times assert counters wrapped past 2³⁰ — spikes are rare and distinct from error asserts.\n\n---\n\n### How to read\n\n- **Regular** / **Msg** — internal mongod assertion failures (unusual)\n- **Warning** — non-fatal internal warnings\n- **User** — client-visible errors (duplicate key, validation, disk full, etc.)\n- **Rollovers** — counter wrap events, not application errors\n- Brief spikes on restart or deploy may appear as counter resets\n\n---\n\n### Troubleshooting\n\n- **Regular** or **Msg** sustained → possible bug or corruption · check mongod logs immediately\n- **User** spikes → application errors · correlate with app logs and **Operations**\n- Spike + **Server uptime** drop → crash/assert abort · **Replica Members Health**\n- Zero across all lines → healthy (typical)\n\nCorrelate with **Server uptime** and **Average Operation Latency**.\n\n---\n\n### Docs\n\n- [serverStatus — asserts](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.asserts)", "fieldConfig": { "defaults": { "color": { - "fixedColor": "blue", - "mode": "fixed" + "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", "axisPlacement": "auto", - "fillOpacity": 100, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, - "lineWidth": 0, - "spanNulls": false - }, - "mappings": [ - { - "options": { - "0": { - "color": "semi-dark-red", - "index": 11, - "text": "STARTUP" - }, - "1": { - "color": "rgb(107, 152, 102)", - "index": 4, - "text": "PRIMARY" - }, - "2": { - "color": "rgb(193, 159, 20)", - "index": 3, - "text": "SECONDARY" - }, - "3": { - "color": "semi-dark-red", - "index": 10, - "text": "RECOVERING" - }, - "5": { - "color": "semi-dark-red", - "index": 9, - "text": "STARTUP2" - }, - "6": { - "color": "semi-dark-red", - "index": 8, - "text": "UNKNOWN" - }, - "7": { - "color": "#8B8000", - "index": 1, - "text": "ARBITER" - }, - "8": { - "color": "semi-dark-red", - "index": 7, - "text": "DOWN" - }, - "9": { - "color": "semi-dark-red", - "index": 6, - "text": "ROLLBACK" - }, - "10": { - "color": "semi-dark-red", - "index": 5, - "text": "REMOVED" - }, - "null": { - "color": "#FF7383", - "index": 2, - "text": "No data" - } - }, - "type": "value" + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } - ], + }, + "mappings": [], "thresholds": { "mode": "absolute", "steps": [ @@ -400,91 +576,116 @@ } ] }, - "unit": "short" + "unit": "ops" }, "overrides": [ { "matcher": { "id": "byRegexp", - "options": ".*members\\.0\\.state$" + "options": ".*\\.regular$" }, "properties": [ { "id": "displayName", - "value": "Member 0" + "value": "Regular" } ] }, { "matcher": { "id": "byRegexp", - "options": ".*members\\.1\\.state$" + "options": ".*\\.warning$" }, "properties": [ { "id": "displayName", - "value": "Member 1" + "value": "Warning" } ] }, { "matcher": { "id": "byRegexp", - "options": ".*members\\.2\\.state$" + "options": ".*\\.msg$" }, "properties": [ { "id": "displayName", - "value": "Member 2" + "value": "Msg" } ] }, { "matcher": { "id": "byRegexp", - "options": ".*members\\.3\\.state$" + "options": ".*\\.user$" }, "properties": [ { "id": "displayName", - "value": "Member 3" + "value": "User" } ] }, { "matcher": { "id": "byRegexp", - "options": ".*members\\.4\\.state$" + "options": ".*\\.rollovers$" }, "properties": [ { "id": "displayName", - "value": "Member 4" + "value": "Rollovers" + } + ] + }, + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Msg" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": true, + "viz": true + } } ] } ] }, "gridPos": { - "h": 6, - "w": 11, - "x": 13, - "y": 0 + "h": 5, + "w": 12, + "x": 12, + "y": 6 }, - "id": 54, + "id": 30, "options": { - "alignValue": "right", "legend": { - "displayMode": "list", - "placement": "bottom", + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "right", "showLegend": true }, - "mergeValues": true, - "rowHeight": 0.9, - "showValue": "auto", "tooltip": { "hideZeros": false, - "mode": "single", + "mode": "multi", "sort": "none" } }, @@ -495,12 +696,12 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] =~ /replSetGetStatus\\.members\\.\\d+\\.state/)\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.asserts.regular\" or\r\n r[\"_field\"] == \"serverStatus.asserts.warning\" or\r\n r[\"_field\"] == \"serverStatus.asserts.msg\" or\r\n r[\"_field\"] == \"serverStatus.asserts.user\" or\r\n r[\"_field\"] == \"serverStatus.asserts.rollovers\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", "refId": "A" } ], - "title": "Member States", - "type": "state-timeline" + "title": "Asserts", + "type": "timeseries" }, { "collapsed": false, @@ -508,7 +709,7 @@ "h": 1, "w": 24, "x": 0, - "y": 6 + "y": 11 }, "id": 24, "panels": [], @@ -609,7 +810,7 @@ "h": 8, "w": 13, "x": 0, - "y": 7 + "y": 12 }, "id": 1, "options": { @@ -735,7 +936,7 @@ "h": 8, "w": 11, "x": 13, - "y": 7 + "y": 12 }, "id": 6, "options": { @@ -775,7 +976,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Pages evicted\n\nRate of **modified** (dirty) and **unmodified** (clean) cache pages evicted per second. MongoDB docs note unmodified evictions as the main eviction statistic.\n\n**Unit:** pages/s (derivative of cumulative counters)\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.wiredTiger.cache.modified pages evicted` · `serverStatus.wiredTiger.cache.unmodified pages evicted`\n\n---\n\n### How to read\n\n- **Modified pages evicted (dirty)** — evicted after write/reconciliation\n- **Unmodified pages evicted (clean)** — discarded without write\n\n---\n\n### Troubleshooting\n\n- Dirty eviction spikes → cache pressure, write latency · **WiredTiger Cache**, **Dirty Cache Bytes**\n- Eviction can't keep up → app threads may evict (performance cliff)\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)\n- [WiredTiger storage engine](https://www.mongodb.com/docs/manual/core/wiredtiger/)", + "description": "### WiredTiger Pages Evicted\n\nRate of **modified** (dirty) and **unmodified** (clean) cache pages evicted per second. MongoDB docs note unmodified evictions as the main eviction statistic.\n\n**Unit:** pages/s (derivative of cumulative counters)\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.wiredTiger.cache.modified pages evicted` · `serverStatus.wiredTiger.cache.unmodified pages evicted`\n\n---\n\n### How to read\n\n- **Modified pages evicted (dirty)** — evicted after write/reconciliation\n- **Unmodified pages evicted (clean)** — discarded without write\n\n---\n\n### Troubleshooting\n\n- Dirty eviction spikes → cache pressure, write latency · **WiredTiger Cache**, **Dirty Cache Bytes**\n- Eviction can't keep up → app threads may evict (performance cliff)\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)\n- [WiredTiger storage engine](https://www.mongodb.com/docs/manual/core/wiredtiger/)", "fieldConfig": { "defaults": { "color": { @@ -787,6 +988,7 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -823,10 +1025,6 @@ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] }, @@ -863,7 +1061,7 @@ "h": 7, "w": 13, "x": 0, - "y": 15 + "y": 20 }, "id": 12, "options": { @@ -890,11 +1088,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.cache.modified pages evicted\" or r[\"_field\"] == \"serverStatus.wiredTiger.cache.unmodified pages evicted\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.cache.modified pages evicted\" or r[\"_field\"] == \"serverStatus.wiredTiger.cache.unmodified pages evicted\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n", "refId": "A" } ], - "title": "Pages Evicted", + "title": "WiredTiger Pages Evicted", "type": "timeseries" }, { @@ -975,7 +1173,7 @@ "h": 5, "w": 11, "x": 13, - "y": 15 + "y": 20 }, "id": 26, "options": { @@ -1063,10 +1261,6 @@ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] }, @@ -1091,7 +1285,7 @@ "h": 9, "w": 11, "x": 13, - "y": 20 + "y": 25 }, "id": 7, "options": { @@ -1215,7 +1409,7 @@ "h": 7, "w": 13, "x": 0, - "y": 22 + "y": 27 }, "id": 23, "options": { @@ -1302,10 +1496,6 @@ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] }, @@ -1330,7 +1520,7 @@ "h": 7, "w": 13, "x": 0, - "y": 29 + "y": 34 }, "id": 19, "options": { @@ -1417,10 +1607,6 @@ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] }, @@ -1457,7 +1643,7 @@ "h": 7, "w": 11, "x": 13, - "y": 29 + "y": 34 }, "id": 8, "options": { @@ -1569,7 +1755,7 @@ "h": 6, "w": 13, "x": 0, - "y": 36 + "y": 41 }, "id": 48, "options": { @@ -1658,10 +1844,6 @@ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] }, @@ -1686,7 +1868,7 @@ "h": 6, "w": 11, "x": 13, - "y": 36 + "y": 41 }, "id": 49, "options": { @@ -1726,7 +1908,7 @@ "h": 1, "w": 24, "x": 0, - "y": 42 + "y": 47 }, "id": 22, "panels": [], @@ -1823,7 +2005,7 @@ "h": 7, "w": 10, "x": 0, - "y": 43 + "y": 48 }, "id": 3, "options": { @@ -1935,7 +2117,7 @@ "h": 5, "w": 14, "x": 10, - "y": 43 + "y": 48 }, "id": 28, "options": { @@ -2096,7 +2278,7 @@ "h": 6, "w": 14, "x": 10, - "y": 48 + "y": 53 }, "id": 2, "options": { @@ -2208,7 +2390,7 @@ "h": 6, "w": 10, "x": 0, - "y": 50 + "y": 55 }, "id": 27, "options": { @@ -2319,7 +2501,7 @@ "h": 6, "w": 14, "x": 10, - "y": 54 + "y": 59 }, "id": 52, "options": { @@ -2455,7 +2637,7 @@ "h": 10, "w": 10, "x": 0, - "y": 56 + "y": 61 }, "id": 5, "options": { @@ -2656,7 +2838,7 @@ "h": 6, "w": 14, "x": 10, - "y": 60 + "y": 65 }, "id": 33, "options": { @@ -2703,7 +2885,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Admin Commands (metrics.commands)\n\nPer-second rate of selected **`serverStatus.metrics.commands.*.total`** counters — admin/catalog-style commands tracked individually by MongoDB.\n\n**Not** the same as **Operations** (`opcounters.command`), which aggregates most command types except insert, update, delete, and aggregate.\n\n**Unit:** commands per second (`ops`)\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.metrics.commands.createIndexes.total`\n- `serverStatus.metrics.commands.dataSize.total`\n- `serverStatus.metrics.commands.dropIndexes.total`\n- `serverStatus.metrics.commands.listCollections.total`\n- `serverStatus.metrics.commands.listDatabases.total`\n- `serverStatus.metrics.commands.listIndexes.total`\n\n**Query:** `derivative(..., unit: 1s, nonNegative: true)` → cmds/sec\n\n---\n\n### How to read\n\n- One line per command counter above\n- Spikes on **listCollections** / **listDatabases** / **listIndexes** → metadata/catalog traffic (drivers, tools, monitoring)\n- Spikes on **createIndexes** / **dropIndexes** → schema/index maintenance\n- **dataSize** → `dataSize` command usage\n\n---\n\n### Troubleshooting\n\n- Sustained high list* rates → chatty clients or health checks · compare with **Network Requests**\n- Index build bursts → **createIndexes** + **CPU Usage** + **Disk I/O**\n- For overall op mix, use **Operations**; for document volume, **Documents**\n\n---\n\n### Docs\n\n- [serverStatus — metrics.commands](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.metrics.commands)", + "description": "### Admin Commands (metrics.commands)\n\nPer-second rate of selected **`serverStatus.metrics.commands.*.total`** counters — admin/catalog-style commands tracked individually by MongoDB.\n\n**Not** the same as **Operations** (`opcounters.command`), which rolls up non-opcounter command traffic (**including aggregate**); this panel tracks specific `metrics.commands.*` counters individually.\n\n**Unit:** commands per second (`ops`)\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.metrics.commands.createIndexes.total`\n- `serverStatus.metrics.commands.dataSize.total`\n- `serverStatus.metrics.commands.dropIndexes.total`\n- `serverStatus.metrics.commands.listCollections.total`\n- `serverStatus.metrics.commands.listDatabases.total`\n- `serverStatus.metrics.commands.listIndexes.total`\n\n**Query:** `derivative(..., unit: 1s, nonNegative: true)` → cmds/sec\n\n---\n\n### How to read\n\n- One line per command counter above\n- Spikes on **listCollections** / **listDatabases** / **listIndexes** → metadata/catalog traffic (drivers, tools, monitoring)\n- Spikes on **createIndexes** / **dropIndexes** → schema/index maintenance\n- **dataSize** → `dataSize` command usage\n\n---\n\n### Troubleshooting\n\n- Sustained high list* rates → chatty clients or health checks · compare with **Network Requests**\n- Index build bursts → **createIndexes** + **CPU Usage** + **Disk I/O**\n- For overall op mix, use **Operations**; for document volume, **Documents**\n\n---\n\n### Docs\n\n- [serverStatus — metrics.commands](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.metrics.commands)", "fieldConfig": { "defaults": { "color": { @@ -2835,7 +3017,7 @@ "h": 8, "w": 10, "x": 0, - "y": 66 + "y": 71 }, "id": 32, "options": { @@ -2874,7 +3056,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Replica members lag\n\nWall-clock replication lag per member:\n\n**Formula:** `(serverStatus.localTime − members.N.lastAppliedWallTime) / 1000` → **seconds**\n\nBoth timestamps are **milliseconds** in FTDC; the query converts the delta to seconds.\n\n**Unit:** seconds (`s`)\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.localTime`\n- `replSetGetStatus.members.0.lastAppliedWallTime` … `replSetGetStatus.members.4.lastAppliedWallTime`\n\nMember **N** is the array index in `replSetGetStatus.members` (not necessarily hostname order).\n\n---\n\n### How to read\n\n- One line per member index (0–4)\n- **~0** → caught up (primary or healthy secondary)\n- Rising trend on a secondary → replication falling behind\n- Flat **0** for missing index → no member in that slot\n\n---\n\n### Troubleshooting\n\n- Secondary lag rising → disk I/O, CPU, network on that node · **Opcounters Replica**, **Disk I/O**\n- Lag spikes during checkpoints → I/O contention\n- Primary throttling → **FlowControl isLagged** · **Replica Members Ping**\n- Member down → **Replica Members Health**\n\n---\n\n### Docs\n\n- [replSetGetStatus](https://www.mongodb.com/docs/manual/reference/command/replSetGetStatus/)\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)", + "description": "### Replica members lag\n\nWall-clock replication lag per member:\n\n**Formula:** `(serverStatus.localTime − members.N.lastAppliedWallTime) / 1000` → **seconds**\n\nBoth timestamps are **milliseconds** in FTDC; the query converts the delta to seconds.\n\n**Unit:** seconds (`s`)\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.localTime`\n- `replSetGetStatus.members.0.lastAppliedWallTime` … `replSetGetStatus.members.4.lastAppliedWallTime`\n\nMember **N** is the array index in `replSetGetStatus.members` (not necessarily hostname order).\n\n---\n\n### How to read\n\n- One line per member index (0–4)\n- **~0** → caught up (primary or healthy secondary)\n- Rising trend on a secondary → replication falling behind\n- **Gap** (not 0) for missing member slots — no `lastAppliedWallTime` in FTDC for that index\n\n---\n\n### Troubleshooting\n\n- Secondary lag rising → disk I/O, CPU, network on that node · **Opcounters Replica**, **Disk I/O**\n- Lag spikes during checkpoints → I/O contention\n- Primary throttling → **FlowControl isLagged** · **Replica Members Ping**\n- Member down → **Replica Members Health**\n\n---\n\n### Docs\n\n- [replSetGetStatus](https://www.mongodb.com/docs/manual/reference/command/replSetGetStatus/)\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)", "fieldConfig": { "defaults": { "color": { @@ -2995,7 +3177,7 @@ "h": 7, "w": 14, "x": 10, - "y": 66 + "y": 71 }, "id": 15, "options": { @@ -3022,7 +3204,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] =~ /replSetGetStatus\\.members\\.\\d+\\.lastAppliedWallTime|serverStatus\\.localTime/)\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> pivot(rowKey: [\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n member_0: if exists r[\"replSetGetStatus.members.0.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.0.lastAppliedWallTime\"])) / 1000.0 else 0.0,\r\n member_1: if exists r[\"replSetGetStatus.members.1.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.1.lastAppliedWallTime\"])) / 1000.0 else 0.0,\r\n member_2: if exists r[\"replSetGetStatus.members.2.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.2.lastAppliedWallTime\"])) / 1000.0 else 0.0,\r\n member_3: if exists r[\"replSetGetStatus.members.3.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.3.lastAppliedWallTime\"])) / 1000.0 else 0.0,\r\n member_4: if exists r[\"replSetGetStatus.members.4.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.4.lastAppliedWallTime\"])) / 1000.0 else 0.0,\r\n }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] =~ /replSetGetStatus\\.members\\.\\d+\\.lastAppliedWallTime|serverStatus\\.localTime/)\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> pivot(rowKey: [\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n member_0: if exists r[\"replSetGetStatus.members.0.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.0.lastAppliedWallTime\"])) / 1000.0 else float(v: \"NaN\"),\r\n member_1: if exists r[\"replSetGetStatus.members.1.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.1.lastAppliedWallTime\"])) / 1000.0 else float(v: \"NaN\"),\r\n member_2: if exists r[\"replSetGetStatus.members.2.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.2.lastAppliedWallTime\"])) / 1000.0 else float(v: \"NaN\"),\r\n member_3: if exists r[\"replSetGetStatus.members.3.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.3.lastAppliedWallTime\"])) / 1000.0 else float(v: \"NaN\"),\r\n member_4: if exists r[\"replSetGetStatus.members.4.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.4.lastAppliedWallTime\"])) / 1000.0 else float(v: \"NaN\"),\r\n }))", "refId": "A" } ], @@ -3155,7 +3337,7 @@ "h": 6, "w": 14, "x": 10, - "y": 73 + "y": 78 }, "id": 25, "options": { @@ -3302,7 +3484,7 @@ "h": 12, "w": 10, "x": 0, - "y": 74 + "y": 79 }, "id": 36, "options": { @@ -3403,276 +3585,37 @@ "min": 0, "thresholds": { "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*isLagged$" - }, - "properties": [ - { - "id": "displayName", - "value": "Flow control engaged" - } - ] - } - ] - }, - "gridPos": { - "h": 5, - "w": 14, - "x": 10, - "y": 79 - }, - "id": 14, - "options": { - "legend": { - "calcs": [ - "min", - "max", - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.3.2", - "targets": [ - { - "datasource": { - "type": "influxdb", - "uid": "P3C6603E967DC8568" - }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.flowControl.isLagged\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", - "refId": "A" - } - ], - "title": "FlowControl isLagged", - "type": "timeseries" - }, - { - "datasource": { - "type": "influxdb", - "uid": "P3C6603E967DC8568" - }, - "description": "### Flow control lagged time\n\n**Fraction of each window** the primary spent with flow control engaged, from **`flowControl.isLaggedTimeMicros`**.\n\n**Formula:** `derivative(isLaggedTimeMicros, 1s) / 1e6` → 0–100%\n\n**Note:** Primary only. For **on/off now**, see **FlowControl isLagged**.\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.flowControl.isLaggedTimeMicros`\n\n---\n\n### How to read\n\n- **0%** → no throttling in that window\n- **100%** → flow control engaged the entire window\n- Spikes → brief throttling bursts\n\n---\n\n### Troubleshooting\n\n- Sustained **>0%** → correlate with **Replica Members Lag**, **FlowControl isLagged**\n- High % + rising lag → secondaries cannot keep up · disk/CPU/network\n\n---\n\n### Docs\n\n- [serverStatus — flowControl](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.flowControl)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 1, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*isLaggedTimeMicros$" - }, - "properties": [ - { - "id": "displayName", - "value": "Flow control engaged (% of time)" - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 14, - "x": 10, - "y": 84 - }, - "id": 53, - "options": { - "legend": { - "calcs": [ - "min", - "max", - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.3.2", - "targets": [ - { - "datasource": { - "type": "influxdb", - "uid": "P3C6603E967DC8568" - }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.flowControl.isLaggedTimeMicros\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) / 1000000.0 }))", - "refId": "A" - } - ], - "title": "Flow control lagged time", - "type": "timeseries" - }, - { - "datasource": { - "type": "influxdb", - "uid": "P3C6603E967DC8568" - }, - "description": "### Current queue (writers / readers)\n\nOperations **waiting** for the global read or write lock (`globalLock.currentQueue`). Snapshot gauge — not a rate.\n\n**Unit:** count (`short`)\n\n**Note:** On MongoDB 4.2+, these queues are often **0**; use **WiredTiger Tickets** and **Average Operation Latency** for concurrency pressure on modern releases.\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.globalLock.currentQueue.readers` · `serverStatus.globalLock.currentQueue.writers`\n\n---\n\n### How to read\n\n- Two lines: queued readers vs writers\n- Brief spikes can be normal · sustained non-zero → contention\n\n---\n\n### Troubleshooting\n\n- Persistent write queue → **WiredTiger Tickets**, **Average Operation Latency**, **Operations**\n- Persistent read queue → long-running reads blocking others\n- Always zero but slow DB → focus on tickets/latency, not this panel alone\n\n---\n\n### Docs\n\n- [serverStatus — globalLock](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.globalLock)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "stepAfter", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*currentQueue\\.readers$" - }, - "properties": [ + "steps": [ { - "id": "displayName", - "value": "Queued readers" + "color": "green", + "value": 0 } ] }, + "unit": "short" + }, + "overrides": [ { "matcher": { "id": "byRegexp", - "options": ".*currentQueue\\.writers$" + "options": ".*isLagged$" }, "properties": [ { "id": "displayName", - "value": "Queued writers" + "value": "Flow control engaged" } ] } ] }, "gridPos": { - "h": 7, - "w": 10, - "x": 0, - "y": 86 + "h": 6, + "w": 7, + "x": 10, + "y": 84 }, - "id": 29, + "id": 14, "options": { "legend": { "calcs": [ @@ -3686,7 +3629,7 @@ }, "tooltip": { "hideZeros": false, - "mode": "multi", + "mode": "single", "sort": "none" } }, @@ -3697,11 +3640,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.globalLock.currentQueue.writers\" or\r\n r[\"_field\"] == \"serverStatus.globalLock.currentQueue.readers\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.flowControl.isLagged\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", "refId": "A" } ], - "title": "Current Queue writers / readers", + "title": "FlowControl isLagged", "type": "timeseries" }, { @@ -3709,7 +3652,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Query executor\n\nPer-second rates from **queryExecutor** counters — how much work the query engine does examining indexes and documents.\n\n**Unit:** operations/sec (`ops`) — keys scanned, docs scanned, and collection-scan events are all counter derivatives.\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.metrics.queryExecutor.scanned` — index keys examined (`totalKeysExamined` in explain())\n- `serverStatus.metrics.queryExecutor.scannedObjects` — documents examined (`totalDocsExamined`)\n- `serverStatus.metrics.queryExecutor.collectionScans.total` — queries that performed a **collection scan** (COLLSCAN)\n\nAll are cumulative counters; query uses **`derivative(unit: 1s, nonNegative: true)`**.\n\n---\n\n### How to read\n\n- **Index keys scanned / sec** — index traversal volume\n- **Documents scanned / sec** — document examination volume (often dominates I/O)\n- **Collection scans / sec** (right axis) — queries **not** using a useful index; should stay near **0** in production\n\n**Keys vs docs:** high docs with low keys can mean wide index scans or many docs per key; high docs with high **Collection scans** confirms COLLSCAN.\n\nCompare with **Query Targeting: Scanned Objects / Returned** (ratio) and **Documents** (`document.returned`).\n\n---\n\n### Troubleshooting\n\n- High **Documents scanned** + high targeting ratio → missing/poor indexes · run `explain()` on slow queries\n- Sustained **Collection scans / sec** → enable profiler / Performance Advisor · fix query patterns or add indexes\n- High scan rates + **Page Faults** / **Disk I/O** → storage pressure from inefficient reads\n- **Operations** `opcounters.query` rising with scan rates → read-heavy workload under index stress\n\n---\n\n### Docs\n\n- [serverStatus — metrics.queryExecutor](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.metrics.queryExecutor)", + "description": "### Flow control lagged time\n\n**Fraction of each window** the primary spent with flow control engaged, from **`flowControl.isLaggedTimeMicros`**.\n\n**Formula:** `derivative(isLaggedTimeMicros, 1s) / 1e6` → 0–100%\n\n**Note:** Primary only. For **on/off now**, see **FlowControl isLagged**.\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.flowControl.isLaggedTimeMicros`\n\n---\n\n### How to read\n\n- **0%** → no throttling in that window\n- **100%** → flow control engaged the entire window\n- Spikes → brief throttling bursts\n\n---\n\n### Troubleshooting\n\n- Sustained **>0%** → correlate with **Replica Members Lag**, **FlowControl isLagged**\n- High % + rising lag → secondaries cannot keep up · disk/CPU/network\n\n---\n\n### Docs\n\n- [serverStatus — flowControl](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.flowControl)", "fieldConfig": { "defaults": { "color": { @@ -3721,6 +3664,7 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMax": 1, "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, @@ -3733,8 +3677,8 @@ "viz": false }, "insertNulls": false, - "lineInterpolation": "stepAfter", - "lineWidth": 1, + "lineInterpolation": "linear", + "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" @@ -3750,7 +3694,10 @@ "mode": "off" } }, + "decimals": 1, "mappings": [], + "max": 1, + "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -3760,58 +3707,30 @@ } ] }, - "unit": "ops" + "unit": "percentunit" }, "overrides": [ { "matcher": { - "id": "byName", - "options": "serverStatus.metrics.queryExecutor.scanned" - }, - "properties": [ - { - "id": "displayName", - "value": "Index keys scanned / sec" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "serverStatus.metrics.queryExecutor.scannedObjects" - }, - "properties": [ - { - "id": "displayName", - "value": "Documents scanned / sec" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "serverStatus.metrics.queryExecutor.collectionScans.total" + "id": "byRegexp", + "options": ".*isLaggedTimeMicros$" }, "properties": [ { "id": "displayName", - "value": "Collection scans / sec" - }, - { - "id": "custom.axisPlacement", - "value": "right" + "value": "Flow control engaged (% of time)" } ] } ] }, "gridPos": { - "h": 9, - "w": 14, - "x": 10, - "y": 90 + "h": 6, + "w": 7, + "x": 17, + "y": 84 }, - "id": 31, + "id": 53, "options": { "legend": { "calcs": [ @@ -3825,7 +3744,7 @@ }, "tooltip": { "hideZeros": false, - "mode": "multi", + "mode": "single", "sort": "none" } }, @@ -3836,11 +3755,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.metrics.queryExecutor.scannedObjects\" or\r\n r[\"_field\"] == \"serverStatus.metrics.queryExecutor.scanned\" or\r\n r[\"_field\"] == \"serverStatus.metrics.queryExecutor.collectionScans.total\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.flowControl.isLaggedTimeMicros\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) / 1000000.0 }))", "refId": "A" } ], - "title": "Query Executor", + "title": "Flow control lagged time", "type": "timeseries" }, { @@ -3848,7 +3767,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Oplog average entry size\n\nAverage size in **bytes** of documents in the **`local.oplog.rs`** capped collection (`collStats` / `storageStats.avgObjSize`).\n\n**Unit:** bytes (`bytes`)\n\n**Not `dbStats.avgObjSize`:** FTDC exposes oplog collStats here, not per-database average user document size.\n\n---\n\n### Metrics\n\n**Field:** `local.oplog.rs.stats.storageStats.avgObjSize`\n\nSnapshot gauge — **`aggregateWindow(..., fn: last)`**, no derivative.\n\n---\n\n### How to read\n\n- Single line · **Oplog avg entry size**\n- Rises when writes carry larger payloads (big documents, wide updates, bulk ops)\n- Slow drift over days → changing write patterns or schema\n- Compare with **Storage Size** (`local.oplog.rs` `storageSize` / `freeStorageSize`)\n\n---\n\n### Troubleshooting\n\n- Step change + write workload shift → larger oplog entries · more replication network/disk\n- High avg size + lag → secondary apply pressure · check **Replica Members Lag**\n- **Arbiter / standalone** may show no or stale series (no meaningful oplog)\n- For **user collection** document sizing → `db.collection.stats().avgObjSize` (not in this FTDC export)\n\n---\n\n### Docs\n\n- [collStats — avgObjSize](https://www.mongodb.com/docs/manual/reference/command/collStats/)\n- [Change oplog size](https://www.mongodb.com/docs/manual/tutorial/change-oplog-size/)", + "description": "### Oplog storage size\n\nDisk space allocated to the **`local.oplog.rs`** capped collection (`collStats` / `storageStats`). **Oplog storage size** is total allocation on disk; **Oplog free storage** is unused space within that allocation.\n\n**Unit:** bytes (`bytes`)\n\n**Not `dbStats.storageSize`:** FTDC exposes oplog **collStats** here, not per-database storage totals.\n\n---\n\n### Metrics\n\n**Fields:**\n\n`local.oplog.rs.stats.storageStats.storageSize`\n`local.oplog.rs.stats.storageStats.freeStorageSize`\n\nSnapshot gauge — **`aggregateWindow(..., fn: last)`**, no derivative.\n\n---\n\n### How to read\n\n- **Oplog storage size** — grows as the capped oplog fills its on-disk allocation (does not shrink when entries roll off)\n- **Oplog free storage** — headroom within that allocation\n- **Used ≈ storage size − free storage** at each point\n- Flat **storage size** with falling **free storage** → oplog nearing capacity\n\n---\n\n### Troubleshooting\n\n- **Free storage** near zero → oplog cap pressure · replication lag risk on secondaries · **Oplog average entry size**, **Replica Members Lag**\n- Step growth in **storage size** → oplog resize or first-time allocation growth\n- High **storage size** + large **Oplog average entry size** → heavy write payloads · more replication I/O\n- **Arbiter / standalone** may show no or stale series\n\nCorrelate with **Operations (write source)**, **Oplog average entry size**, and **Documents**.\n\n---\n\n### Docs\n\n- [collStats — storageSize / freeStorageSize](https://www.mongodb.com/docs/manual/reference/command/collStats/)\n- [Change oplog size](https://www.mongodb.com/docs/manual/tutorial/change-oplog-size/)", "fieldConfig": { "defaults": { "color": { @@ -3889,8 +3808,8 @@ "mode": "off" } }, + "decimals": 1, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -3905,25 +3824,37 @@ "overrides": [ { "matcher": { - "id": "byName", - "options": "local.oplog.rs.stats.storageStats.avgObjSize" + "id": "byRegexp", + "options": ".*\\.storageStats\\.storageSize$" }, "properties": [ { "id": "displayName", - "value": "Oplog avg entry size" + "value": "Oplog storage size" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*freeStorageSize$" + }, + "properties": [ + { + "id": "displayName", + "value": "Oplog free storage" } ] } ] }, "gridPos": { - "h": 6, - "w": 10, - "x": 0, - "y": 93 + "h": 8, + "w": 7, + "x": 10, + "y": 90 }, - "id": 34, + "id": 35, "options": { "legend": { "calcs": [ @@ -3948,11 +3879,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"local.oplog.rs.stats.storageStats.avgObjSize\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"local.oplog.rs.stats.storageStats.storageSize\" or\r\n r[\"_field\"] == \"local.oplog.rs.stats.storageStats.freeStorageSize\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", "refId": "A" } ], - "title": "Oplog average entry size", + "title": "Oplog storage size", "type": "timeseries" }, { @@ -3960,7 +3891,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Operations (opcounters)\n\nPer-second rate of **`serverStatus.opcounters`** — **operation-level** counts (one bulk/multi-doc op = one op). Not the same as **Documents** (`metrics.document`), which counts documents touched.\n\n**Unit:** operations per second (`ops`)\n\n**Note:** `opcounters.command` includes most commands except insert, update, delete, and **aggregate** (see MongoDB docs).\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.opcounters.command`\n- `serverStatus.opcounters.delete`\n- `serverStatus.opcounters.getmore`\n- `serverStatus.opcounters.insert`\n- `serverStatus.opcounters.query`\n- `serverStatus.opcounters.update`\n\n**Query:** `derivative(..., unit: 1s, nonNegative: true)` → ops/sec\n\n---\n\n### How to read\n\n- Six lines: command, delete, getmore, insert, query, update **rates**\n- Sudden drop → blocking or idle period\n- Spike → load burst; check **Average Operation Latency** and **WiredTiger Tickets**\n\n---\n\n### Troubleshooting\n\n- Sustained high write ops + latency → **Current Queue writers / readers**, disk I/O\n- High **query** + **getmore** → cursor-heavy workload\n- On secondaries, use **Opcounters Replica** instead of this panel\n\n---\n\n### Docs\n\n- [serverStatus — opcounters](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.opcounters)", + "description": "### Oplog average entry size\n\nAverage size in **bytes** of documents in the **`local.oplog.rs`** capped collection (`collStats` / `storageStats.avgObjSize`).\n\n**Unit:** bytes (`bytes`)\n\n**Not `dbStats.avgObjSize`:** FTDC exposes oplog collStats here, not per-database average user document size.\n\n---\n\n### Metrics\n\n**Field:** `local.oplog.rs.stats.storageStats.avgObjSize`\n\nSnapshot gauge — **`aggregateWindow(..., fn: last)`**, no derivative.\n\n---\n\n### How to read\n\n- Single line · **Oplog avg entry size**\n- Rises when writes carry larger payloads (big documents, wide updates, bulk ops)\n- Slow drift over days → changing write patterns or schema\n- Compare with **Oplog storage size** (`local.oplog.rs` `storageSize` / `freeStorageSize`)\n\n---\n\n### Troubleshooting\n\n- Step change + write workload shift → larger oplog entries · more replication network/disk\n- High avg size + lag → secondary apply pressure · check **Replica Members Lag**\n- **Arbiter / standalone** may show no or stale series (no meaningful oplog)\n- For **user collection** document sizing → `db.collection.stats().avgObjSize` (not in this FTDC export)\n\n---\n\n### Docs\n\n- [collStats — avgObjSize](https://www.mongodb.com/docs/manual/reference/command/collStats/)\n- [Change oplog size](https://www.mongodb.com/docs/manual/tutorial/change-oplog-size/)", "fieldConfig": { "defaults": { "color": { @@ -3984,7 +3915,7 @@ "viz": false }, "insertNulls": false, - "lineInterpolation": "stepAfter", + "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -4002,6 +3933,7 @@ } }, "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -4011,90 +3943,30 @@ } ] }, - "unit": "ops" + "unit": "bytes" }, "overrides": [ { "matcher": { - "id": "byRegexp", - "options": ".*\\.opcounters\\.command$" - }, - "properties": [ - { - "id": "displayName", - "value": "Command" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*\\.opcounters\\.delete$" - }, - "properties": [ - { - "id": "displayName", - "value": "Delete" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*\\.opcounters\\.getmore$" - }, - "properties": [ - { - "id": "displayName", - "value": "Getmore" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*\\.opcounters\\.insert$" - }, - "properties": [ - { - "id": "displayName", - "value": "Insert" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*\\.opcounters\\.query$" - }, - "properties": [ - { - "id": "displayName", - "value": "Query" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*\\.opcounters\\.update$" + "id": "byName", + "options": "local.oplog.rs.stats.storageStats.avgObjSize" }, "properties": [ { "id": "displayName", - "value": "Update" + "value": "Oplog avg entry size" } ] } ] }, "gridPos": { - "h": 11, - "w": 12, - "x": 0, - "y": 99 + "h": 8, + "w": 7, + "x": 17, + "y": 90 }, - "id": 9, + "id": 34, "options": { "legend": { "calcs": [ @@ -4119,11 +3991,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.opcounters.command\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.delete\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.getmore\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.insert\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.query\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.update\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"local.oplog.rs.stats.storageStats.avgObjSize\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", "refId": "A" } ], - "title": "Operations", + "title": "Oplog average entry size", "type": "timeseries" }, { @@ -4131,7 +4003,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Replication opcounters (If secondary)\n\nPer-second rates from **`serverStatus.opcountersRepl`** — operations applied when this member is acting as a **secondary** (oplog apply).\n\n**Unit:** operations/sec (`ops`)\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.opcountersRepl.insert`\n- `serverStatus.opcountersRepl.query`\n- `serverStatus.opcountersRepl.update`\n- `serverStatus.opcountersRepl.delete`\n- `serverStatus.opcountersRepl.getmore`\n- `serverStatus.opcountersRepl.command`\n\nCumulative counters; query uses **`derivative(unit: 1s, nonNegative: true)`**.\n\n---\n\n### How to read\n\n- Six lines · replication apply rate by type\n- **View a secondary hostname** — answers “how fast is this node applying replication?”\n- **Primary / write source** — expect near-zero or unhelpful lines; switch host or open **Operations (write source)**\n- Fractional ops/sec → normal with sparse FTDC sampling\n- **0** → idle or no replication apply in window\n\nCompare with **Operations (write source)** on the primary, **Replica Members Lag**, **Flow control is lagged**.\n\n---\n\n### Troubleshooting\n\n- Repl rates drop + **lag rising** → secondary cannot keep up · disk I/O, CPU, **Disk I/O**\n- Primary **Operations (write source)** high, secondary repl low → bottleneck on secondary\n- Repl **update** ≫ primary **update** → expected replication amplification\n- **Flow control is lagged** = 1 + lag → primary throttling writes\n\n---\n\n### Docs\n\n- [serverStatus — opcountersRepl](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.opcountersRepl)\n- [serverStatus — opcounters](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.opcounters)", + "description": "### Current queue (writers / readers)\n\nOperations **waiting** for the global read or write lock (`globalLock.currentQueue`). Snapshot gauge — not a rate.\n\n**Unit:** count (`short`)\n\n**Note:** On MongoDB 4.2+, these queues are often **0**; use **WiredTiger Tickets** and **Average Operation Latency** for concurrency pressure on modern releases.\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.globalLock.currentQueue.readers` · `serverStatus.globalLock.currentQueue.writers`\n\n---\n\n### How to read\n\n- Two lines: queued readers vs writers\n- Brief spikes can be normal · sustained non-zero → contention\n\n---\n\n### Troubleshooting\n\n- Persistent write queue → **WiredTiger Tickets**, **Average Operation Latency**, **Operations**\n- Persistent read queue → long-running reads blocking others\n- Always zero but slow DB → focus on tickets/latency, not this panel alone\n\n---\n\n### Docs\n\n- [serverStatus — globalLock](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.globalLock)", "fieldConfig": { "defaults": { "color": { @@ -4155,7 +4027,7 @@ "viz": false }, "insertNulls": false, - "lineInterpolation": "linear", + "lineInterpolation": "stepAfter", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -4172,6 +4044,7 @@ "mode": "off" } }, + "decimals": 0, "mappings": [], "thresholds": { "mode": "absolute", @@ -4182,90 +4055,42 @@ } ] }, - "unit": "ops" + "unit": "short" }, "overrides": [ { "matcher": { - "id": "byName", - "options": "serverStatus.opcountersRepl.insert" - }, - "properties": [ - { - "id": "displayName", - "value": "Insert / sec" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "serverStatus.opcountersRepl.query" - }, - "properties": [ - { - "id": "displayName", - "value": "Query / sec" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "serverStatus.opcountersRepl.update" - }, - "properties": [ - { - "id": "displayName", - "value": "Update / sec" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "serverStatus.opcountersRepl.delete" - }, - "properties": [ - { - "id": "displayName", - "value": "Delete / sec" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "serverStatus.opcountersRepl.getmore" + "id": "byRegexp", + "options": ".*currentQueue\\.readers$" }, "properties": [ { "id": "displayName", - "value": "Getmore / sec" + "value": "Queued readers" } ] }, { "matcher": { - "id": "byName", - "options": "serverStatus.opcountersRepl.command" + "id": "byRegexp", + "options": ".*currentQueue\\.writers$" }, "properties": [ { "id": "displayName", - "value": "Command / sec" + "value": "Queued writers" } ] } ] }, "gridPos": { - "h": 11, - "w": 12, - "x": 12, - "y": 99 + "h": 7, + "w": 10, + "x": 0, + "y": 91 }, - "id": 38, + "id": 29, "options": { "legend": { "calcs": [ @@ -4290,11 +4115,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.insert\" or\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.query\" or\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.update\" or\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.delete\" or\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.getmore\" or\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.command\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.globalLock.currentQueue.writers\" or\r\n r[\"_field\"] == \"serverStatus.globalLock.currentQueue.readers\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", "refId": "A" } ], - "title": "Opcounters Replica (If secondary)", + "title": "Current Queue writers / readers", "type": "timeseries" }, { @@ -4302,7 +4127,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Operation latency ops\n\nPer-second rates from **opLatencies.*.ops** counters — how many read, write, and command operations ran in each latency class.\n\n**Unit:** operations/sec (`ops`)\n\nThis is the **denominator** for **Average Operation Latency** (`derivative(latency) / derivative(ops)` → µs/op).\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.opLatencies.reads.ops`\n- `serverStatus.opLatencies.writes.ops`\n- `serverStatus.opLatencies.commands.ops`\n\nCumulative counters; query uses **`derivative(unit: 1s, nonNegative: true)`**.\n\n**Not the same as opcounters:** `opLatencies` groups reads / writes / commands for latency stats; **Operations** uses `opcounters.*` (query, insert, update, delete, getmore, command).\n\n---\n\n### How to read\n\n- **Read / write / command ops / sec** — throughput per latency class\n- **Commands** often includes admin, aggregate, and other command work\n- Fractional values → normal with sparse FTDC sampling\n- **0** → idle window for that class\n\nCompare with **Average Operation Latency** (numerator/denominator) and **Operations** (opcounters workload mix).\n\n---\n\n### Troubleshooting\n\n- **Ops up, avg latency up** → load + slower per-op work\n- **Ops down, avg latency up** → fewer ops but each slower (contention, I/O)\n- **Ops up, avg latency flat** → healthy scale-out of throughput\n- Read/write imbalance → workload skew · correlate with **Query Executor**, **Documents**, **Disk I/O**\n\n---\n\n### Docs\n\n- [serverStatus — opLatencies](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.opLatencies)", + "description": "### Query executor\n\nPer-second rates from **queryExecutor** counters — how much work the query engine does examining indexes and documents.\n\n**Unit:** operations/sec (`ops`) — keys scanned, docs scanned, and collection-scan events are all counter derivatives.\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.metrics.queryExecutor.scanned` — index keys examined (`totalKeysExamined` in explain())\n- `serverStatus.metrics.queryExecutor.scannedObjects` — documents examined (`totalDocsExamined`)\n- `serverStatus.metrics.queryExecutor.collectionScans.total` — queries that performed a **collection scan** (COLLSCAN)\n\nAll are cumulative counters; query uses **`derivative(unit: 1s, nonNegative: true)`**.\n\n---\n\n### How to read\n\n- **Index keys scanned / sec** — index traversal volume\n- **Documents scanned / sec** — document examination volume (often dominates I/O)\n- **Collection scans / sec** (right axis) — queries **not** using a useful index; should stay near **0** in production\n\n**Keys vs docs:** high docs with low keys can mean wide index scans or many docs per key; high docs with high **Collection scans** confirms COLLSCAN.\n\nCompare with **Query Targeting: Scanned Objects / Returned** (ratio) and **Documents** (`document.returned`).\n\n---\n\n### Troubleshooting\n\n- High **Documents scanned** + high targeting ratio → missing/poor indexes · run `explain()` on slow queries\n- Sustained **Collection scans / sec** → enable profiler / Performance Advisor · fix query patterns or add indexes\n- High scan rates + **Page Faults** / **Disk I/O** → storage pressure from inefficient reads\n- **Operations** `opcounters.query` rising with scan rates → read-heavy workload under index stress\n\n---\n\n### Docs\n\n- [serverStatus — metrics.queryExecutor](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.metrics.queryExecutor)", "fieldConfig": { "defaults": { "color": { @@ -4359,48 +4184,52 @@ { "matcher": { "id": "byName", - "options": "serverStatus.opLatencies.reads.ops" + "options": "serverStatus.metrics.queryExecutor.scanned" }, "properties": [ { "id": "displayName", - "value": "Read ops / sec" + "value": "Index keys scanned / sec" } ] }, { "matcher": { "id": "byName", - "options": "serverStatus.opLatencies.writes.ops" + "options": "serverStatus.metrics.queryExecutor.scannedObjects" }, "properties": [ { "id": "displayName", - "value": "Write ops / sec" + "value": "Documents scanned / sec" } ] }, { "matcher": { "id": "byName", - "options": "serverStatus.opLatencies.commands.ops" + "options": "serverStatus.metrics.queryExecutor.collectionScans.total" }, "properties": [ { "id": "displayName", - "value": "Command ops / sec" + "value": "Collection scans / sec" + }, + { + "id": "custom.axisPlacement", + "value": "right" } ] } ] }, "gridPos": { - "h": 9, + "h": 7, "w": 24, "x": 0, - "y": 110 + "y": 98 }, - "id": 10, + "id": 31, "options": { "legend": { "calcs": [ @@ -4425,11 +4254,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.opLatencies.reads.ops\" or\r\n r[\"_field\"] == \"serverStatus.opLatencies.writes.ops\" or\r\n r[\"_field\"] == \"serverStatus.opLatencies.commands.ops\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.metrics.queryExecutor.scannedObjects\" or\r\n r[\"_field\"] == \"serverStatus.metrics.queryExecutor.scanned\" or\r\n r[\"_field\"] == \"serverStatus.metrics.queryExecutor.collectionScans.total\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", "refId": "A" } ], - "title": "Operation Latency Ops", + "title": "Query Executor", "type": "timeseries" }, { @@ -4437,7 +4266,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Query targeting: scanned objects / returned\n\nRatio of **document scan rate** to **document return rate**:\n\n**Formula:** `derivative(scannedObjects, 1s) / derivative(document.returned, 1s)`\n\nSame idea as **totalDocsExamined / docs returned** in explain() — aggregated over time.\n\n**Unit:** dimensionless ratio (`none`) — **1.0** ≈ one doc examined per doc returned\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.metrics.queryExecutor.scannedObjects`\n- `serverStatus.metrics.document.returned`\n\nBoth are cumulative counters; query uses **`derivative(unit: 1s, nonNegative: true)`** before the ratio.\n\n---\n\n### How to read\n\n- Single line · **Scanned / returned ratio**\n- **~1** → strong targeting (most examined docs were returned)\n- **> 1** (especially **10+**) → scans dominate · missing/poor indexes or large in-memory filters\n- **0** → no returned docs in window (idle) or zero denominator — **not** “perfect”\n\n---\n\n### Troubleshooting\n\n- High ratio + high **Documents returned** → **Query Executor** · run `explain()` on hot queries\n- Correlate with **Page Faults**, **Disk I/O**, **Average Operation Latency**\n- **Collection scans** panel (if present) confirms full collection scans\n\n---\n\n### Docs\n\n- [serverStatus — metrics.queryExecutor](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.metrics.queryExecutor)\n- [serverStatus — metrics.document](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.metrics.document)", + "description": "### Operations (opcounters)\n\nPer-second rate of **`serverStatus.opcounters`** — **operation-level** counts (one bulk/multi-doc op = one op). Not the same as **Documents** (`metrics.document`), which counts documents touched.\n\n**Unit:** operations per second (`ops`)\n\n**Note:** `opcounters.command` counts commands **not** already tracked as insert, update, delete, query, or getmore — **including aggregate** and other admin/catalog commands.\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.opcounters.command`\n- `serverStatus.opcounters.delete`\n- `serverStatus.opcounters.getmore`\n- `serverStatus.opcounters.insert`\n- `serverStatus.opcounters.query`\n- `serverStatus.opcounters.update`\n\n**Query:** `derivative(..., unit: 1s, nonNegative: true)` → ops/sec\n\n---\n\n### How to read\n\n- Six lines: command, delete, getmore, insert, query, update **rates**\n- Sudden drop → blocking or idle period\n- Spike → load burst; check **Average Operation Latency** and **WiredTiger Tickets**\n\n---\n\n### Troubleshooting\n\n- Sustained high write ops + latency → **Current Queue writers / readers**, disk I/O\n- High **query** + **getmore** → cursor-heavy workload\n- On secondaries, use **Opcounters Replica** instead of this panel\n\n---\n\n### Docs\n\n- [serverStatus — opcounters](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.opcounters)", "fieldConfig": { "defaults": { "color": { @@ -4461,7 +4290,7 @@ "viz": false }, "insertNulls": false, - "lineInterpolation": "linear", + "lineInterpolation": "stepAfter", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -4478,7 +4307,6 @@ "mode": "off" } }, - "decimals": 2, "mappings": [], "thresholds": { "mode": "absolute", @@ -4489,30 +4317,90 @@ } ] }, - "unit": "none" + "unit": "ops" }, "overrides": [ { "matcher": { - "id": "byName", - "options": "scanned_returned_ratio" + "id": "byRegexp", + "options": ".*\\.opcounters\\.command$" }, "properties": [ { "id": "displayName", - "value": "Scanned / returned ratio" + "value": "Command" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*\\.opcounters\\.delete$" + }, + "properties": [ + { + "id": "displayName", + "value": "Delete" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*\\.opcounters\\.getmore$" + }, + "properties": [ + { + "id": "displayName", + "value": "Getmore" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*\\.opcounters\\.insert$" + }, + "properties": [ + { + "id": "displayName", + "value": "Insert" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*\\.opcounters\\.query$" + }, + "properties": [ + { + "id": "displayName", + "value": "Query" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*\\.opcounters\\.update$" + }, + "properties": [ + { + "id": "displayName", + "value": "Update" } ] } ] }, "gridPos": { - "h": 6, - "w": 24, + "h": 11, + "w": 12, "x": 0, - "y": 119 + "y": 105 }, - "id": 37, + "id": 9, "options": { "legend": { "calcs": [ @@ -4537,32 +4425,19 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.metrics.document.returned\" or\r\n r[\"_field\"] == \"serverStatus.metrics.queryExecutor.scannedObjects\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> pivot(rowKey: [\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n scanned_returned_ratio:\r\n if exists r[\"serverStatus.metrics.document.returned\"]\r\n and float(v: r[\"serverStatus.metrics.document.returned\"]) > 0.0\r\n and exists r[\"serverStatus.metrics.queryExecutor.scannedObjects\"]\r\n then float(v: r[\"serverStatus.metrics.queryExecutor.scannedObjects\"])\r\n / float(v: r[\"serverStatus.metrics.document.returned\"])\r\n else 0.0\r\n }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.opcounters.command\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.delete\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.getmore\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.insert\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.query\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.update\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", "refId": "A" } ], - "title": "Query Targeting: Scanned Objects / Returned", + "title": "Operations", "type": "timeseries" }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 125 - }, - "id": 21, - "panels": [], - "title": "System", - "type": "row" - }, { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "CPU utilization from systemMetrics.cpu, broken down by user, system, and iowait as a percentage of total CPU time. These are derived from the OS-level CPU counters exposed through FTDC (systemMetrics).\n\nHow to read: three lines showing percentage of CPU time spent in user mode (application), system/kernel mode, and iowait (waiting on disk).\n\nTroubleshooting: high 'user' CPU typically means heavy query processing (sorts, aggregations, index builds). High 'iowait' means the CPU is blocked waiting on disk -- correlate with 'Disk I/O' and 'WiredTiger Block Manager'. High 'system' may indicate excessive context switching from too many threads/connections.", + "description": "### Replication opcounters (If secondary)\n\nPer-second rates from **`serverStatus.opcountersRepl`** — operations applied when this member is acting as a **secondary** (oplog apply).\n\n**Unit:** operations/sec (`ops`)\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.opcountersRepl.insert`\n- `serverStatus.opcountersRepl.query`\n- `serverStatus.opcountersRepl.update`\n- `serverStatus.opcountersRepl.delete`\n- `serverStatus.opcountersRepl.getmore`\n- `serverStatus.opcountersRepl.command`\n\nCumulative counters; query uses **`derivative(unit: 1s, nonNegative: true)`**.\n\n---\n\n### How to read\n\n- Six lines · replication apply rate by type\n- **View a secondary hostname** — answers “how fast is this node applying replication?”\n- **Primary / write source** — expect near-zero or unhelpful lines; switch host or open **Operations (write source)**\n- Fractional ops/sec → normal with sparse FTDC sampling\n- **0** → idle or no replication apply in window\n\nCompare with **Operations (write source)** on the primary, **Replica Members Lag**, **Flow control is lagged**.\n\n---\n\n### Troubleshooting\n\n- Repl rates drop + **lag rising** → secondary cannot keep up · disk I/O, CPU, **Disk I/O**\n- Primary **Operations (write source)** high, secondary repl low → bottleneck on secondary\n- Repl **update** ≫ primary **update** → expected replication amplification\n- **Flow control is lagged** = 1 + lag → primary throttling writes\n\n---\n\n### Docs\n\n- [serverStatus — opcountersRepl](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.opcountersRepl)\n- [serverStatus — opcounters](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.opcounters)", "fieldConfig": { "defaults": { "color": { @@ -4574,6 +4449,7 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -4609,24 +4485,93 @@ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "percent" + "unit": "ops" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "serverStatus.opcountersRepl.insert" + }, + "properties": [ + { + "id": "displayName", + "value": "Insert / sec" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "serverStatus.opcountersRepl.query" + }, + "properties": [ + { + "id": "displayName", + "value": "Query / sec" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "serverStatus.opcountersRepl.update" + }, + "properties": [ + { + "id": "displayName", + "value": "Update / sec" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "serverStatus.opcountersRepl.delete" + }, + "properties": [ + { + "id": "displayName", + "value": "Delete / sec" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "serverStatus.opcountersRepl.getmore" + }, + "properties": [ + { + "id": "displayName", + "value": "Getmore / sec" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "serverStatus.opcountersRepl.command" + }, + "properties": [ + { + "id": "displayName", + "value": "Command / sec" + } + ] + } + ] }, "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 126 + "h": 11, + "w": 12, + "x": 12, + "y": 105 }, - "id": 13, + "id": 38, "options": { "legend": { "calcs": [ @@ -4651,11 +4596,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field == \"systemMetrics.cpu.idle_ms\" or r._field == \"systemMetrics.cpu.user_ms\" or r._field == \"systemMetrics.cpu.iowait_ms\" or r._field == \"systemMetrics.cpu.nice_ms\" or r._field == \"systemMetrics.cpu.softirq_ms\" or r._field == \"systemMetrics.cpu.steal_ms\" or r._field == \"systemMetrics.cpu.system_ms\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> pivot(rowKey:[\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\r\n |> derivative(unit: 1s, nonNegative: true, columns: [\"systemMetrics.cpu.idle_ms\", \"systemMetrics.cpu.user_ms\", \"systemMetrics.cpu.iowait_ms\", \"systemMetrics.cpu.nice_ms\", \"systemMetrics.cpu.softirq_ms\", \"systemMetrics.cpu.steal_ms\", \"systemMetrics.cpu.system_ms\"])\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n idle_ms: r[\"systemMetrics.cpu.idle_ms\"],\r\n user_ms: r[\"systemMetrics.cpu.user_ms\"],\r\n iowait_ms: r[\"systemMetrics.cpu.iowait_ms\"],\r\n nice_ms: r[\"systemMetrics.cpu.nice_ms\"],\r\n softirq_ms: r[\"systemMetrics.cpu.softirq_ms\"],\r\n steal_ms: r[\"systemMetrics.cpu.steal_ms\"],\r\n system_ms: r[\"systemMetrics.cpu.system_ms\"],\r\n total_ms: r[\"systemMetrics.cpu.idle_ms\"] + r[\"systemMetrics.cpu.user_ms\"] + r[\"systemMetrics.cpu.iowait_ms\"] + r[\"systemMetrics.cpu.nice_ms\"] + r[\"systemMetrics.cpu.softirq_ms\"] + r[\"systemMetrics.cpu.steal_ms\"] + r[\"systemMetrics.cpu.system_ms\"]\r\n }))\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n user: 100.0 * float(v: r.user_ms) / float(v: r.total_ms),\r\n system: 100.0 * float(v: r.system_ms) / float(v: r.total_ms),\r\n iowait: 100.0 * float(v: r.iowait_ms) / float(v: r.total_ms),\r\n }))\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.insert\" or\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.query\" or\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.update\" or\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.delete\" or\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.getmore\" or\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.command\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", "refId": "A" } ], - "title": "CPU Usage", + "title": "Opcounters Replica (If secondary)", "type": "timeseries" }, { @@ -4663,7 +4608,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "System-level memory metrics from systemMetrics (derived from /proc/meminfo). Shows available and free memory at the OS level (systemMetrics).\n\nHow to read: multiple lines showing memory categories over time.\n\nTroubleshooting: if available memory is low and swap is active, the system is under memory pressure affecting all processes including mongod. Compare with 'mongod Memory (RSS / Virtual)' to determine how much system RAM is consumed by the database vs other services.", + "description": "### Operation latency ops\n\nPer-second rates from **opLatencies.*.ops** counters — how many read, write, and command operations ran in each latency class.\n\n**Unit:** operations/sec (`ops`)\n\nThis is the **denominator** for **Average Operation Latency** (`derivative(latency) / derivative(ops)` → µs/op).\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.opLatencies.reads.ops`\n- `serverStatus.opLatencies.writes.ops`\n- `serverStatus.opLatencies.commands.ops`\n\nCumulative counters; query uses **`derivative(unit: 1s, nonNegative: true)`**.\n\n**Not the same as opcounters:** `opLatencies` groups reads / writes / commands for latency stats; **Operations** uses `opcounters.*` (query, insert, update, delete, getmore, command).\n\n---\n\n### How to read\n\n- **Read / write / command ops / sec** — throughput per latency class\n- **Commands** often includes admin, aggregate, and other command work\n- Fractional values → normal with sparse FTDC sampling\n- **0** → idle window for that class\n\nCompare with **Average Operation Latency** (numerator/denominator) and **Operations** (opcounters workload mix).\n\n---\n\n### Troubleshooting\n\n- **Ops up, avg latency up** → load + slower per-op work\n- **Ops down, avg latency up** → fewer ops but each slower (contention, I/O)\n- **Ops up, avg latency flat** → healthy scale-out of throughput\n- Read/write imbalance → workload skew · correlate with **Query Executor**, **Documents**, **Disk I/O**\n\n---\n\n### Docs\n\n- [serverStatus — opLatencies](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.opLatencies)", "fieldConfig": { "defaults": { "color": { @@ -4675,6 +4620,7 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -4686,7 +4632,7 @@ "viz": false }, "insertNulls": false, - "lineInterpolation": "linear", + "lineInterpolation": "stepAfter", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -4710,24 +4656,57 @@ { "color": "green", "value": 0 - }, + } + ] + }, + "unit": "ops" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "serverStatus.opLatencies.reads.ops" + }, + "properties": [ + { + "id": "displayName", + "value": "Read ops / sec" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "serverStatus.opLatencies.writes.ops" + }, + "properties": [ { - "color": "red", - "value": 80 + "id": "displayName", + "value": "Write ops / sec" } ] }, - "unit": "deckbytes" - }, - "overrides": [] + { + "matcher": { + "id": "byName", + "options": "serverStatus.opLatencies.commands.ops" + }, + "properties": [ + { + "id": "displayName", + "value": "Command ops / sec" + } + ] + } + ] }, "gridPos": { - "h": 8, - "w": 12, + "h": 9, + "w": 24, "x": 0, - "y": 135 + "y": 116 }, - "id": 40, + "id": 10, "options": { "legend": { "calcs": [ @@ -4752,11 +4731,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"systemMetrics.memory.MemAvailable_kb\" or r[\"_field\"] == \"systemMetrics.memory.MemFree_kb\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> yield(name: \"mean\")", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.opLatencies.reads.ops\" or\r\n r[\"_field\"] == \"serverStatus.opLatencies.writes.ops\" or\r\n r[\"_field\"] == \"serverStatus.opLatencies.commands.ops\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", "refId": "A" } ], - "title": "System Memory", + "title": "Operation Latency Ops", "type": "timeseries" }, { @@ -4764,7 +4743,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Swap space usage from systemMetrics (systemMetrics).\n\nHow to read: lines showing swap used and swap total over time.\n\nTroubleshooting: ANY swap usage on a MongoDB server is a critical concern. Swapping replaces nanosecond RAM access with millisecond disk access, causing order-of-magnitude latency increases. If swap is in use, check 'System Memory' for overall pressure and consider increasing RAM or reducing WiredTiger cache size (--wiredTigerCacheSizeGB) to leave more memory for the OS page cache.", + "description": "### Query targeting: scanned objects / returned\n\nRatio of **document scan rate** to **document return rate**:\n\n**Formula:** `derivative(scannedObjects, 1s) / derivative(document.returned, 1s)`\n\nSame idea as **totalDocsExamined / docs returned** in explain() — aggregated over time.\n\n**Unit:** dimensionless ratio (`none`) — **1.0** ≈ one doc examined per doc returned\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.metrics.queryExecutor.scannedObjects`\n- `serverStatus.metrics.document.returned`\n\nBoth are cumulative counters; query uses **`derivative(unit: 1s, nonNegative: true)`** before the ratio.\n\n---\n\n### How to read\n\n- Single line · **Scanned / returned ratio**\n- **~1** → strong targeting (most examined docs were returned)\n- **> 1** (especially **10+**) → scans dominate · missing/poor indexes or large in-memory filters\n- **Gap** (not 0) → no returned docs in window or zero denominator — **not** “perfect”\n\n---\n\n### Troubleshooting\n\n- High ratio + high **Documents returned** → **Query Executor** · run `explain()` on hot queries\n- Correlate with **Page Faults**, **Disk I/O**, **Average Operation Latency**\n- **Collection scans** on **Query Executor** (right axis) confirms full collection scans\n\n---\n\n### Docs\n\n- [serverStatus — metrics.queryExecutor](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.metrics.queryExecutor)\n- [serverStatus — metrics.document](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.metrics.document)", "fieldConfig": { "defaults": { "color": { @@ -4776,6 +4755,7 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -4804,6 +4784,7 @@ "mode": "off" } }, + "decimals": 2, "mappings": [], "thresholds": { "mode": "absolute", @@ -4811,24 +4792,33 @@ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "deckbytes" + "unit": "none" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "scanned_returned_ratio" + }, + "properties": [ + { + "id": "displayName", + "value": "Scanned / returned ratio" + } + ] + } + ] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 135 + "h": 6, + "w": 24, + "x": 0, + "y": 125 }, - "id": 39, + "id": 37, "options": { "legend": { "calcs": [ @@ -4853,19 +4843,32 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"systemMetrics.memory.SwapFree_kb\" or r[\"_field\"] == \"systemMetrics.memory.SwapTotal_kb\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> yield(name: \"mean\")", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.metrics.document.returned\" or\r\n r[\"_field\"] == \"serverStatus.metrics.queryExecutor.scannedObjects\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> pivot(rowKey: [\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n scanned_returned_ratio:\r\n if exists r[\"serverStatus.metrics.document.returned\"]\r\n and float(v: r[\"serverStatus.metrics.document.returned\"]) > 0.0\r\n and exists r[\"serverStatus.metrics.queryExecutor.scannedObjects\"]\r\n then float(v: r[\"serverStatus.metrics.queryExecutor.scannedObjects\"])\r\n / float(v: r[\"serverStatus.metrics.document.returned\"])\r\n else float(v: \"NaN\")\r\n }))", "refId": "A" } ], - "title": "Swap Memory", + "title": "Query Targeting: Scanned Objects / Returned", "type": "timeseries" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 131 + }, + "id": 21, + "panels": [], + "title": "System", + "type": "row" + }, { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Shows dbStats.storageSize -- the sum of disk space allocated to all collections for document storage, including free space. Per MongoDB docs, storageSize does not decrease when documents are removed or shrunk; it may be smaller than dataSize on WiredTiger with compression enabled. storageSize does not include indexes -- see indexSize (dbStats).\n\nHow to read: single line showing storage allocation over time.\n\nTroubleshooting: sudden growth may indicate unexpected data ingestion or large index creation. Compare with 'avg Obj Size' and 'Documents' panel rates. Slow steady growth is normal.", + "description": "### CPU usage\n\nHost CPU time by mode, from FTDC **systemMetrics** (OS counters in milliseconds). Each line is that mode’s share of total CPU time in the window after per-mode `derivative(1s)`.\n\n**Formula:** `aggregateWindow(last)` → `pivot` → `derivative(*_ms, 1s)` → `(mode_ms / total_ms) × 100`\n\n**Unit:** percent (`percent`, 0–100)\n\n---\n\n### Metrics\n\n**Fields:**\n\n`systemMetrics.cpu.user_ms`\n`systemMetrics.cpu.system_ms`\n`systemMetrics.cpu.iowait_ms`\n`systemMetrics.cpu.idle_ms`\n`systemMetrics.cpu.nice_ms`\n`systemMetrics.cpu.softirq_ms`\n`systemMetrics.cpu.steal_ms`\n\n**Plotted:** user, system, iowait percentages only. Idle, nice, softirq, and steal are included in **total_ms** (denominator) but not shown as separate lines.\n\n**Query type:** computed ratio (not a single counter derivative). Output uses `_field` / `_value` via `union` for Grafana.\n\n---\n\n### How to read\n\n- **User %** — application time (mongod, queries, aggregations)\n- **System (kernel) %** — kernel time, syscalls, context switching\n- **I/O wait %** — CPU idle while waiting on disk\n- Three lines sum to **≤ 100%** (unplotted modes account for the rest)\n\n---\n\n### Troubleshooting\n\n- High **User %** with **Operations** or **Query Executor** → CPU-bound workload\n- High **I/O wait %** → disk bottleneck · **Disk I/O**, **WiredTiger Block Manager**, **Page Faults**\n- High **System %** with many **Connections** → thread/syscall overhead\n- VM steal not plotted — inspect raw `systemMetrics.cpu.steal_ms` in Explore if host may be oversubscribed\n\nCorrelate with **mongod Memory (RSS / Virtual)** and **System Memory**.\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)", "fieldConfig": { "defaults": { "color": { @@ -4877,6 +4880,7 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -4912,24 +4916,20 @@ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "bytes" + "unit": "percent" }, "overrides": [] }, "gridPos": { - "h": 7, - "w": 8, + "h": 9, + "w": 24, "x": 0, - "y": 143 + "y": 132 }, - "id": 35, + "id": 13, "options": { "legend": { "calcs": [ @@ -4954,11 +4954,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"local.oplog.rs.stats.storageStats.freeStorageSize\" or r[\"_field\"] == \"local.oplog.rs.stats.storageStats.storageSize\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> yield(name: \"mean\")", + "query": "base =\r\n from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r._field == \"systemMetrics.cpu.idle_ms\" or\r\n r._field == \"systemMetrics.cpu.user_ms\" or\r\n r._field == \"systemMetrics.cpu.iowait_ms\" or\r\n r._field == \"systemMetrics.cpu.nice_ms\" or\r\n r._field == \"systemMetrics.cpu.softirq_ms\" or\r\n r._field == \"systemMetrics.cpu.steal_ms\" or\r\n r._field == \"systemMetrics.cpu.system_ms\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> pivot(rowKey: [\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\r\n |> derivative(\r\n unit: 1s,\r\n nonNegative: true,\r\n columns: [\r\n \"systemMetrics.cpu.idle_ms\",\r\n \"systemMetrics.cpu.user_ms\",\r\n \"systemMetrics.cpu.iowait_ms\",\r\n \"systemMetrics.cpu.nice_ms\",\r\n \"systemMetrics.cpu.softirq_ms\",\r\n \"systemMetrics.cpu.steal_ms\",\r\n \"systemMetrics.cpu.system_ms\",\r\n ],\r\n )\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n user_ms: float(v: r[\"systemMetrics.cpu.user_ms\"]),\r\n system_ms: float(v: r[\"systemMetrics.cpu.system_ms\"]),\r\n iowait_ms: float(v: r[\"systemMetrics.cpu.iowait_ms\"]),\r\n total_ms:\r\n float(v: r[\"systemMetrics.cpu.idle_ms\"]) +\r\n float(v: r[\"systemMetrics.cpu.user_ms\"]) +\r\n float(v: r[\"systemMetrics.cpu.iowait_ms\"]) +\r\n float(v: r[\"systemMetrics.cpu.nice_ms\"]) +\r\n float(v: r[\"systemMetrics.cpu.softirq_ms\"]) +\r\n float(v: r[\"systemMetrics.cpu.steal_ms\"]) +\r\n float(v: r[\"systemMetrics.cpu.system_ms\"]),\r\n }))\r\n\r\nunion(tables: [\r\n base |> map(fn: (r) => ({\r\n _time: r._time,\r\n _field: \"User %\",\r\n _value: if r.total_ms > 0.0 then 100.0 * r.user_ms / r.total_ms else 0.0,\r\n })),\r\n base |> map(fn: (r) => ({\r\n _time: r._time,\r\n _field: \"System (kernel) %\",\r\n _value: if r.total_ms > 0.0 then 100.0 * r.system_ms / r.total_ms else 0.0,\r\n })),\r\n base |> map(fn: (r) => ({\r\n _time: r._time,\r\n _field: \"I/O wait %\",\r\n _value: if r.total_ms > 0.0 then 100.0 * r.iowait_ms / r.total_ms else 0.0,\r\n })),\r\n])\r\n |> group(columns: [\"_field\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", "refId": "A" } ], - "title": "Storage Size", + "title": "CPU Usage", "type": "timeseries" }, { @@ -4966,7 +4966,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Rate of disk read and write operations per second (IOPS) per device from systemMetrics.disks.*.reads and systemMetrics.disks.*.writes (systemMetrics).\n\nHow to read: lines showing IOPS per device.\n\nTroubleshooting: high IOPS approaching device limits causes increased latency. Compare with 'Disk I/O' (bytes) to determine if the workload is many small I/O operations vs fewer large ones. SSD and HDD have very different IOPS limits (hundreds of thousands vs hundreds).", + "description": "### System memory\n\nHost RAM from FTDC **systemMetrics** (`/proc/meminfo`, values stored as **kB**). Shows how much memory the OS reports as **available** vs **free**.\n\n**Unit:** bytes (Grafana auto-scale; FTDC fields are `*_kb`, query multiplies by 1024)\n\n---\n\n### Metrics\n\n**Fields:**\n\n`systemMetrics.memory.MemAvailable_kb`\n`systemMetrics.memory.MemFree_kb`\n\n**MemAvailable** — estimate of memory startable for new workloads without swapping (includes reclaimable cache).\n\n**MemFree** — completely unallocated pages (often much lower than available).\n\n---\n\n### How to read\n\n- **Memory available** — primary signal for OS headroom\n- **Memory free** — strictly idle RAM; low free + healthy available is normal on Linux\n- Both lines fall when mongod, page cache, or other services consume RAM\n\n---\n\n### Troubleshooting\n\n- **Available** trending toward zero → memory pressure · **Swap Memory**, **Page Faults**, **WiredTiger Cache**\n- Available low but **mongod Memory (RSS / Virtual)** moderate → other processes or OS page cache\n- Available and free both low + high **CPU Usage** I/O wait → possible thrashing\n- After mongod restart, available may jump as RSS is released\n\nCorrelate with **mongod Memory (RSS / Virtual)** and **Swap Memory**.\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)\n- Linux [meminfo](https://www.kernel.org/doc/html/latest/admin-guide/mm/memory-model.html) — `MemAvailable` vs `MemFree`", "fieldConfig": { "defaults": { "color": { @@ -4978,6 +4978,7 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -5006,6 +5007,7 @@ "mode": "off" } }, + "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", @@ -5013,24 +5015,45 @@ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "iops" + "unit": "bytes" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*MemAvailable_kb$" + }, + "properties": [ + { + "id": "displayName", + "value": "Memory available" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*MemFree_kb$" + }, + "properties": [ + { + "id": "displayName", + "value": "Memory free" + } + ] + } + ] }, "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 143 + "h": 8, + "w": 12, + "x": 0, + "y": 141 }, - "id": 17, + "id": 40, "options": { "legend": { "calcs": [ @@ -5055,11 +5078,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..*\\.writes$/ or r._field =~ /^systemMetrics\\.disks\\..*\\.reads$/)\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> derivative(unit: 1s)\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"systemMetrics.memory.MemAvailable_kb\" or\r\n r[\"_field\"] == \"systemMetrics.memory.MemFree_kb\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) * 1024.0 }))", "refId": "A" } ], - "title": "Disk Writes and Reads", + "title": "System Memory", "type": "timeseries" }, { @@ -5067,7 +5090,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Rate of TCP segments received (InSegs) and sent (OutSegs) per second from systemMetrics.netstat.Tcp (systemMetrics.netstat).\n\nHow to read: two lines showing TCP segment rates.\n\nTroubleshooting: shows raw TCP-level traffic volume. A high InSegs/OutSegs ratio on a secondary indicates it receives more replication data than it sends. Sudden drops indicate network outages. Compare with 'Network In / Out' for application-level context.", + "description": "### Swap memory\n\nHost swap from FTDC **systemMetrics** (`/proc/meminfo`, values stored as **kB**). Shows **used**, **free**, and **total** swap capacity. **Swap used** is computed as `SwapTotal − SwapFree`.\n\n**Unit:** bytes (Grafana auto-scale; FTDC fields are `*_kb`, query multiplies by 1024)\n\n---\n\n### Metrics\n\n**Fields:**\n\n`systemMetrics.memory.SwapTotal_kb`\n`systemMetrics.memory.SwapFree_kb`\n\n**Computed:** Swap used = `SwapTotal_kb − SwapFree_kb` (not a separate FTDC field)\n\n**Plotted:** Swap used (computed) · Swap free · Swap total\n\n---\n\n### How to read\n\n- **Swap used** — should stay at **0** on a MongoDB host\n- **Swap free** — unused swap capacity\n- **Swap total** — configured swap size (may be **0** if swap is disabled)\n- **Swap used + Swap free ≈ Swap total** at each point\n\n---\n\n### Troubleshooting\n\n- **Any sustained Swap used > 0** → critical · RAM pressure · latency spikes\n- Correlate with **System Memory**, **Page Faults**, **mongod Memory (RSS / Virtual)**, **WiredTiger Cache**\n- High **CPU Usage** I/O wait + swap → severe thrashing\n- Remediation: add RAM, reduce `--wiredTigerCacheSizeGB`, fix memory leaks, stop co-located heavy processes\n- **Swap total = 0** → swap disabled (common on tuned MongoDB VMs)\n\nCorrelate with **System Memory** and **Average Operation Latency**.\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)\n- [MongoDB production notes — memory](https://www.mongodb.com/docs/manual/administration/production-notes/#std-label-production-notes-memory)", "fieldConfig": { "defaults": { "color": { @@ -5079,6 +5102,7 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -5107,6 +5131,7 @@ "mode": "off" } }, + "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", @@ -5114,24 +5139,20 @@ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "none" + "unit": "bytes" }, "overrides": [] }, "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 143 + "h": 8, + "w": 12, + "x": 12, + "y": 141 }, - "id": 43, + "id": 39, "options": { "legend": { "calcs": [ @@ -5156,11 +5177,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"systemMetrics.netstat.Tcp:InSegs\" or r[\"_field\"] == \"systemMetrics.netstat.Tcp:OutSegs\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> derivative(unit: v.windowPeriod)", + "query": "base =\r\n from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"systemMetrics.memory.SwapFree_kb\" or\r\n r[\"_field\"] == \"systemMetrics.memory.SwapTotal_kb\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> pivot(rowKey: [\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n swap_total_kb: float(v: r[\"systemMetrics.memory.SwapTotal_kb\"]),\r\n swap_free_kb: float(v: r[\"systemMetrics.memory.SwapFree_kb\"]),\r\n swap_used_kb:\r\n float(v: r[\"systemMetrics.memory.SwapTotal_kb\"]) -\r\n float(v: r[\"systemMetrics.memory.SwapFree_kb\"]),\r\n }))\r\n\r\nunion(tables: [\r\n base |> map(fn: (r) => ({\r\n _time: r._time,\r\n _field: \"Swap used\",\r\n _value: r.swap_used_kb * 1024.0,\r\n })),\r\n base |> map(fn: (r) => ({\r\n _time: r._time,\r\n _field: \"Swap free\",\r\n _value: r.swap_free_kb * 1024.0,\r\n })),\r\n base |> map(fn: (r) => ({\r\n _time: r._time,\r\n _field: \"Swap total\",\r\n _value: r.swap_total_kb * 1024.0,\r\n })),\r\n])\r\n |> group(columns: [\"_field\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)", "refId": "A" } ], - "title": "TCP: InSegs / OutSegs", + "title": "Swap Memory", "type": "timeseries" }, { @@ -5168,7 +5189,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Rate of serverStatus.network.bytesIn and network.bytesOut per second. Per MongoDB docs, bytesIn is the total number of logical bytes received; bytesOut is the total logical bytes sent. Logical bytes are the exact number of bytes in the data (serverStatus.network).\n\nHow to read: two lines showing logical network throughput in bytes/sec.\n\nTroubleshooting: large bytesOut may indicate queries returning too much data -- check query projections and limit clauses. Compare logical vs physical bytes to assess compression ratio.", + "description": "### Disk writes and reads\n\nPer-device **IOPS** (read/write operations per second) from FTDC **systemMetrics** (`/proc/diskstats` cumulative `reads` / `writes` counters).\n\n**Unit:** IOPS (`iops`) — `derivative(unit: 1s)` on OS counters\n\n**Not mongod-only:** includes all processes on the host (OS view).\n\n---\n\n### Metrics\n\n**Pattern (device names vary by host):**\n\n`systemMetrics.disks..reads`\n`systemMetrics.disks..writes`\n\nExample: `systemMetrics.disks.xvda.reads` · `systemMetrics.disks.sdb.writes`\n\nAll disk stats matching the exporter’s disk allowlist are ingested automatically (no per-device entries in `metrics_to_get.txt`).\n\n**Query:** `derivative(..., unit: 1s, nonNegative: true)` → IOPS\n\n---\n\n### How to read\n\n- One line per **device · reads** and **device · writes**\n- Spikes during WiredTiger **checkpoints** (~60s) are normal on the data volume\n- Many small IOPS + low byte throughput → small random I/O · compare **Disk I/O**\n\n---\n\n### Troubleshooting\n\n- Sustained high IOPS + high **CPU Usage** I/O wait → disk saturation\n- High reads → cache misses · **WiredTiger Block Manager**, **Page Faults**\n- High writes → checkpoints/eviction · **Checkpoint Write Throughput**, **WiredTiger Pages Evicted**\n- Correlate with **Disk I/O** (bytes/sec) and **Disk I/O Busy Time / Disk Queue Time** panels\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)\n- Linux [diskstats](https://www.kernel.org/doc/html/latest/admin-guide/iostats.html)", "fieldConfig": { "defaults": { "color": { @@ -5180,6 +5201,7 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -5215,24 +5237,20 @@ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "decbytes" + "unit": "iops" }, "overrides": [] }, "gridPos": { - "h": 6, - "w": 8, + "h": 8, + "w": 12, "x": 0, - "y": 150 + "y": 149 }, - "id": 4, + "id": 17, "options": { "legend": { "calcs": [ @@ -5257,11 +5275,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.network.bytesOut\" or r[\"_field\"] == \"serverStatus.network.bytesIn\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> map(fn: (r) => ({ r with _value: r._value / 1024 /1024 }))\r\n |> derivative(unit: 1s)\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n", + "query": "import \"strings\"\r\n\r\nfrom(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r._field =~ /^systemMetrics\\.disks\\..+\\.reads$/ or\r\n r._field =~ /^systemMetrics\\.disks\\..+\\.writes$/)\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({\r\n r with\r\n _value: float(v: r._value),\r\n _field:\r\n if strings.hasSuffix(v: r._field, suffix: \".reads\") then\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".reads\",\r\n ) + \" reads\"\r\n else\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".writes\",\r\n ) + \" writes\",\r\n }))", "refId": "A" } ], - "title": "Network In / Out", + "title": "Disk Writes and Reads", "type": "timeseries" }, { @@ -5269,7 +5287,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Disk write throughput per device from systemMetrics.disks (systemMetrics).\n\nHow to read: lines showing write bytes/sec per device.\n\nTroubleshooting: periodic write spikes every ~60 seconds correlate with WiredTiger checkpoints and are normal. Sustained high writes between checkpoints indicate heavy write workloads or eviction pressure. Compare with 'Dirty Cache Bytes' and 'Pages Evicted'.", + "description": "### Disk I/O\n\nPer-device **read/write throughput** from FTDC **systemMetrics** (`/proc/diskstats` cumulative sector counters).\n\n**Formula:** `derivative(read_sectors|write_sectors, 1s) × 512` → **bytes/sec**\n\n**Unit:** B/s (`Bps`) — Linux diskstats sectors are **512 bytes** each\n\n**Not mongod-only:** OS view of all processes on the host.\n\n---\n\n### Metrics\n\n**Pattern (device names vary by host):**\n\n`systemMetrics.disks..read_sectors`\n`systemMetrics.disks..write_sectors`\n\nExample: `systemMetrics.disks.xvda.read_sectors` · `systemMetrics.disks.sdb.write_sectors`\n\nDisk metrics are ingested via the exporter disk allowlist (no per-device lines in `metrics_to_get.txt`).\n\n**Query:** `derivative(..., unit: 1s, nonNegative: true)` → sectors/s → × 512 → B/s\n\n---\n\n### How to read\n\n- One line per **device · read** and **device · write** throughput\n- Write spikes every ~**60s** → WiredTiger **checkpoints** (normal on data volume)\n- High B/s + low IOPS → large sequential I/O · compare **Disk writes and reads**\n- High B/s + high IOPS → random or mixed workload\n\n---\n\n### Troubleshooting\n\n- Sustained high throughput + **CPU Usage** I/O wait → disk bandwidth saturation\n- Read-heavy → cache pressure · **WiredTiger Block Manager**, **Page Faults**\n- Write-heavy → **Checkpoint Write Throughput**, **Pages Evicted**\n- Queueing / busy time → **Disk Queue Time**, **Disk I/O Busy Time** (not this panel)\n\nCorrelate with **Disk writes and reads** (IOPS) and **WiredTiger Block Manager**.\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)\n- Linux [diskstats](https://www.kernel.org/doc/html/latest/admin-guide/iostats.html)", "fieldConfig": { "defaults": { "color": { @@ -5310,6 +5328,7 @@ "mode": "off" } }, + "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", @@ -5317,23 +5336,20 @@ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] - } + }, + "unit": "Bps" }, "overrides": [] }, "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 150 + "h": 8, + "w": 12, + "x": 12, + "y": 149 }, - "id": 46, + "id": 47, "options": { "legend": { "calcs": [ @@ -5358,20 +5374,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..*\\.write_sectors$/ or r._field =~ /^systemMetrics\\.disks\\..*\\.write_time_ms$/ or r._field =~ /^systemMetrics\\.disks\\..*\\.writes_merged$/)\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> derivative(unit: v.windowPeriod)", + "query": "import \"strings\"\r\n\r\nfrom(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r._field =~ /^systemMetrics\\.disks\\..+\\.read_sectors$/ or\r\n r._field =~ /^systemMetrics\\.disks\\..+\\.write_sectors$/)\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({\r\n r with\r\n _value: float(v: r._value) * 512.0,\r\n _field:\r\n if strings.hasSuffix(v: r._field, suffix: \".read_sectors\") then\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".read_sectors\",\r\n ) + \" read\"\r\n else\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".write_sectors\",\r\n ) + \" write\",\r\n }))", "refId": "A" - }, - { - "datasource": { - "type": "influxdb", - "uid": "P3C6603E967DC8568" - }, - "hide": false, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..*\\.io_in_progress$/ )\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n\r\n\r\n\r\n", - "refId": "B" } ], - "title": "Disk Writes", + "title": "Disk I/O", "type": "timeseries" }, { @@ -5379,7 +5386,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Rate of serverStatus.network.physicalBytesIn and network.physicalBytesOut per second. Per MongoDB docs, physicalBytesIn is the total number of physical bytes received over network connections; physicalBytesOut is the total physical bytes sent. Physical bytes are the bytes that actually reside on disk (serverStatus.network).\n\nHow to read: two lines showing physical network throughput in bytes/sec.\n\nTroubleshooting: compare with 'Network In / Out' (logical bytes) to assess compression effectiveness. Saturation of network bandwidth causes connection timeouts and replication lag. Spikes without corresponding operation increases may indicate replication catch-up or background traffic.", + "description": "### Disk reads\n\nPer-device **read throughput** from FTDC **systemMetrics** (`/proc/diskstats` cumulative `read_sectors` counter).\n\n**Formula:** `derivative(read_sectors, 1s) × 512` → **bytes/sec**\n\n**Unit:** B/s (`Bps`) — Linux diskstats sectors are **512 bytes** each\n\n**Not mongod-only:** OS view of all processes on the host (not `serverStatus`).\n\n---\n\n### Metrics\n\n**Pattern (device names vary by host — regex filter):**\n\n`systemMetrics.disks..read_sectors`\n\nExample: `systemMetrics.disks.xvda.read_sectors`\n\nIngested via exporter disk allowlist (`isDiskMetric`); no per-device entries in `metrics_to_get.txt`.\n\n**Query type:** cumulative counter → `derivative(unit: 1s, nonNegative: true)` → `aggregateWindow(mean)` → × 512\n\n**Not this panel:** `read_time_ms` → **Disk I/O Busy Time** · `.reads` IOPS → **Disk writes and reads**\n\n---\n\n### How to read\n\n- One line per device · legend **`{device} read`**\n- Sustained high reads → cold data / cache misses · working set vs RAM\n- High B/s + low read IOPS → large sequential reads · compare **Disk writes and reads**\n\n---\n\n### Troubleshooting\n\n- High reads + **Page Faults** → memory pressure\n- High reads + **WiredTiger Cache** full → increase cache or RAM\n- High reads + **CPU Usage** I/O wait → disk bandwidth · **Disk I/O**, **Disk I/O Busy Time**\n- Mongod-specific reads → **WiredTiger Block Manager** (bytes read)\n\nCorrelate with **Disk I/O** (combined read/write B/s).\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)\n- Linux [diskstats](https://www.kernel.org/doc/html/latest/admin-guide/iostats.html)", "fieldConfig": { "defaults": { "color": { @@ -5391,6 +5398,7 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -5419,6 +5427,7 @@ "mode": "off" } }, + "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", @@ -5426,24 +5435,20 @@ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "decbytes" + "unit": "Bps" }, "overrides": [] }, "gridPos": { - "h": 5, - "w": 8, + "h": 8, + "w": 12, "x": 0, - "y": 156 + "y": 157 }, - "id": 11, + "id": 18, "options": { "legend": { "calcs": [ @@ -5468,11 +5473,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.network.physicalBytesIn\" or r[\"_field\"] == \"serverStatus.network.physicalBytesOut\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s)\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n", + "query": "import \"strings\"\r\n\r\nfrom(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..+\\.read_sectors$/)\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({\r\n r with\r\n _value: float(v: r._value) * 512.0,\r\n _field:\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".read_sectors\",\r\n ) + \" read\",\r\n }))", "refId": "A" } ], - "title": "Network In / Out Physical", + "title": "Disk Reads", "type": "timeseries" }, { @@ -5480,7 +5485,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "TCP connection lifecycle counters from systemMetrics.netstat: Tcp.ActiveOpens (new outgoing connections initiated), Tcp.AttemptFails (connection attempts that failed), Tcp.EstabResets (established connections that were reset) (systemMetrics.netstat).\n\nHow to read: three lines showing rates per second.\n\nTroubleshooting: high AttemptFails indicates the server or peers are refusing connections. High EstabResets indicates established connections being unexpectedly terminated -- check for replica member crashes, network partitions, or firewall issues. Correlate with 'Replica Members Health'.", + "description": "### Disk writes\n\nPer-device **write throughput** from FTDC **systemMetrics** (`/proc/diskstats` cumulative `write_sectors` counter).\n\n**Formula:** `derivative(write_sectors, 1s) × 512` → **bytes/sec**\n\n**Unit:** B/s (`Bps`) — Linux diskstats sectors are **512 bytes** each\n\n**Not mongod-only:** OS view of all processes on the host (not `serverStatus`).\n\n---\n\n### Metrics\n\n**Pattern (device names vary by host — regex filter):**\n\n`systemMetrics.disks..write_sectors`\n\nExample: `systemMetrics.disks.xvda.write_sectors`\n\nIngested via exporter disk allowlist (`isDiskMetric`).\n\n**Query type:** cumulative counter → `derivative(unit: 1s, nonNegative: true)` → `aggregateWindow(mean)` → × 512\n\n**Not this panel:** `write_time_ms` → **Disk I/O Busy Time** · `.writes` IOPS → **Disk writes and reads**\n\n---\n\n### How to read\n\n- One line per device · legend **`{device} write`**\n- Spikes every ~**60s** → WiredTiger **checkpoints** on data volume (normal)\n- Sustained high writes between spikes → heavy write load or eviction\n\n---\n\n### Troubleshooting\n\n- Write spikes + latency → checkpoint I/O · **Checkpoint Write Throughput**, **Checkpoint Duration**\n- Sustained high writes → **Dirty Cache Bytes**, **Pages Evicted**, **Operations**\n- High write B/s + high **CPU Usage** I/O wait → disk bandwidth · **Disk I/O**, **Disk I/O Busy Time**\n\nCorrelate with **Disk I/O** (combined read/write B/s) and **WiredTiger Block Manager**.\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)\n- Linux [diskstats](https://www.kernel.org/doc/html/latest/admin-guide/iostats.html)", "fieldConfig": { "defaults": { "color": { @@ -5492,6 +5497,7 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -5520,6 +5526,7 @@ "mode": "off" } }, + "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", @@ -5527,24 +5534,20 @@ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "none" + "unit": "Bps" }, "overrides": [] }, "gridPos": { - "h": 7, - "w": 8, - "x": 8, + "h": 8, + "w": 12, + "x": 12, "y": 157 }, - "id": 41, + "id": 46, "options": { "legend": { "calcs": [ @@ -5569,11 +5572,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"systemMetrics.netstat.Tcp:ActiveOpens\" or r[\"_field\"] == \"systemMetrics.netstat.Tcp:AttemptFails\" or r[\"_field\"] == \"systemMetrics.netstat.Tcp:EstabResets\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> derivative(unit: v.windowPeriod)", + "query": "import \"strings\"\r\n\r\nfrom(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..+\\.write_sectors$/)\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({\r\n r with\r\n _value: float(v: r._value) * 512.0,\r\n _field:\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".write_sectors\",\r\n ) + \" write\",\r\n }))", "refId": "A" } ], - "title": "TCP ActiveOpens / AttemptFails / EstabResets", + "title": "Disk Writes", "type": "timeseries" }, { @@ -5581,7 +5584,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Disk throughput in bytes per second per device from systemMetrics.disks (systemMetrics).\n\nHow to read: lines showing read and write throughput per device.\n\nTroubleshooting: high throughput during WiredTiger checkpoint intervals (default 60s) is expected. Sustained high throughput outside checkpoints indicates cache misses or heavy write workloads. Compare with 'Disk Writes and Reads' (IOPS) for a complete picture of disk saturation.", + "description": "### Disk queue time\n\nPer-device **I/O queue wait time rate** from FTDC **systemMetrics** (`/proc/diskstats` cumulative `io_queued_ms` counter).\n\n**Formula:** `derivative(io_queued_ms, 1s)` → **ms/s** (milliseconds of queue wait accrued per wall-clock second)\n\n**Unit:** ms/s — **not** in-flight I/O count · **not** `read_time_ms` / `write_time_ms` (**Disk I/O Busy Time**)\n\n**Not mongod-only:** OS view of all processes on the host.\n\n**Saturation hint:** sustained values toward **~1000 ms/s** → persistent queueing on that device.\n\n---\n\n### Metrics\n\n**Pattern (device names vary by host — regex filter):**\n\n`systemMetrics.disks..io_queued_ms`\n\nExample: `systemMetrics.disks.xvda.io_queued_ms`\n\nIngested via exporter disk allowlist (`isDiskMetric`).\n\n**Query type:** cumulative counter → `derivative(unit: 1s, nonNegative: true)` → `aggregateWindow(mean)`\n\n**In-flight I/O count:** `systemMetrics.disks..io_in_progress` (gauge) — not plotted here.\n\n---\n\n### How to read\n\n- One line per device · **`{device} queue time`** (ms/s)\n- Rising queue time + flat throughput → device falling behind\n- Spikes with checkpoint windows → normal on data volume if brief\n\n---\n\n### Troubleshooting\n\n- Sustained high ms/s → disk saturation · **Disk I/O Busy Time**, **Disk I/O**, **CPU Usage** I/O wait\n- Queue time up + **Average Operation Latency** up → storage bottleneck\n- Queue time without high B/s → random/small I/O or slow device · **Disk writes and reads**\n\nCorrelate with **Disk I/O Busy Time** and **Disk I/O**.\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)\n- Linux [diskstats](https://www.kernel.org/doc/html/latest/admin-guide/iostats.html)", "fieldConfig": { "defaults": { "color": { @@ -5591,9 +5594,8 @@ "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "", + "axisLabel": "ms/s", "axisPlacement": "auto", - "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -5622,6 +5624,7 @@ "mode": "off" } }, + "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", @@ -5629,23 +5632,20 @@ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] - } + }, + "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, - "w": 8, + "w": 12, "x": 0, - "y": 161 + "y": 165 }, - "id": 47, + "id": 51, "options": { "legend": { "calcs": [ @@ -5670,20 +5670,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..*\\.io_in_progress$/ or r._field =~ /^systemMetrics\\.disks\\..*\\.io_queued_ms$/ or r._field =~ /^systemMetrics\\.disks\\..*\\.io_time_ms$/)\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> derivative(unit: v.windowPeriod)", + "query": "import \"strings\"\r\n\r\nfrom(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..+\\.io_queued_ms$/)\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({\r\n r with\r\n _value: float(v: r._value),\r\n _field:\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".io_queued_ms\",\r\n ) + \" queue time\",\r\n }))", "refId": "A" - }, - { - "datasource": { - "type": "influxdb", - "uid": "P3C6603E967DC8568" - }, - "hide": false, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..*\\.io_in_progress$/ )\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n\r\n\r\n\r\n", - "refId": "B" } ], - "title": "Disk I/O", + "title": "Disk Queue Time", "type": "timeseries" }, { @@ -5691,7 +5682,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "TCP connection error metrics from systemMetrics.netstat (systemMetrics.netstat).\n\nHow to read: lines showing error rates per second.\n\nTroubleshooting: increasing values indicate problems establishing or maintaining TCP connections between MongoDB nodes or clients. Correlate with 'Replica Members Health', 'Replica Members Ping', and application connection error logs.", + "description": "### Disk I/O busy time\n\nPer-device **I/O service time rate** from FTDC **systemMetrics** (`/proc/diskstats` cumulative `read_time_ms` / `write_time_ms` counters).\n\n**Formula:** `derivative(read_time_ms|write_time_ms, 1s)` → **ms/s** (milliseconds of I/O time accrued per wall-clock second)\n\n**Unit:** ms/s — **not** average latency per operation · **not** `serverStatus.opLatencies`\n\n**Not mongod-only:** OS view of all processes on the host.\n\n**Saturation hint:** values approaching **~1000 ms/s** on a line → device spending ~100% of wall time in that I/O direction.\n\n---\n\n### Metrics\n\n**Pattern (device names vary by host — regex filter):**\n\n`systemMetrics.disks..read_time_ms`\n`systemMetrics.disks..write_time_ms`\n\nExample: `systemMetrics.disks.xvda.read_time_ms` · `systemMetrics.disks.sdb.write_time_ms`\n\nIngested via exporter disk allowlist (`isDiskMetric`).\n\n**Query type:** cumulative counter → `derivative(unit: 1s, nonNegative: true)` → `aggregateWindow(mean)`\n\n**Not this panel:** queue wait → **Disk Queue Time** (`io_queued_ms`) · throughput → **Disk I/O** · IOPS → **Disk writes and reads**\n\n---\n\n### How to read\n\n- One line per device · **read time** / **write time** (ms/s)\n- Write spikes ~every **60s** → checkpoint I/O on data volume (normal)\n- High ms/s + high **Disk I/O** B/s → bandwidth-heavy workload\n- High ms/s + high IOPS → random/small I/O or saturation\n\n---\n\n### Troubleshooting\n\n- Sustained high ms/s → disk busy · **Disk Queue Time**, **CPU Usage** I/O wait\n- High ms/s + rising **Average Operation Latency** → storage bottleneck\n- High read time + **Page Faults** → cache / memory pressure\n- High write time + **Checkpoint Write Throughput** → checkpoint / eviction pressure\n\nCorrelate with **Disk I/O**, **Disk writes and reads**, and **Disk Queue Time**.\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)\n- Linux [diskstats](https://www.kernel.org/doc/html/latest/admin-guide/iostats.html)", "fieldConfig": { "defaults": { "color": { @@ -5701,8 +5692,9 @@ "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "", + "axisLabel": "ms/s", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -5731,18 +5723,14 @@ "mode": "off" } }, + "decimals": 1, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] }, @@ -5752,11 +5740,11 @@ }, "gridPos": { "h": 7, - "w": 8, - "x": 8, - "y": 164 + "w": 12, + "x": 12, + "y": 165 }, - "id": 45, + "id": 50, "options": { "legend": { "calcs": [ @@ -5781,11 +5769,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"systemMetrics.netstat.Tcp:CurrEstab\" or r[\"_field\"] == \"systemMetrics.netstat.Tcp:EstabResets\" or r[\"_field\"] == \"systemMetrics.netstat.Tcp:OutRsts\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> derivative(unit: v.windowPeriod)", + "query": "import \"strings\"\r\n\r\nfrom(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r._field =~ /^systemMetrics\\.disks\\..+\\.read_time_ms$/ or\r\n r._field =~ /^systemMetrics\\.disks\\..+\\.write_time_ms$/)\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({\r\n r with\r\n _value: float(v: r._value),\r\n _field:\r\n if strings.hasSuffix(v: r._field, suffix: \".read_time_ms\") then\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".read_time_ms\",\r\n ) + \" read time\"\r\n else\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".write_time_ms\",\r\n ) + \" write time\",\r\n }))", "refId": "A" } ], - "title": "TCP Connection Issues", + "title": "Disk I/O Busy Time", "type": "timeseries" }, { @@ -5793,7 +5781,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Disk read throughput per device from systemMetrics.disks (systemMetrics).\n\nHow to read: lines showing read bytes/sec per device.\n\nTroubleshooting: high disk reads indicate data not served from WiredTiger cache. Correlate with 'Cache Read/Write Throughput' (bytes read into cache) to confirm. Persistent high reads suggest the working set exceeds the configured cache size.", + "description": "### TCP segments in / out\n\nPer-second rates of **Tcp:InSegs** and **Tcp:OutSegs** from FTDC **systemMetrics** (`/proc/net/snmp`).\n\n**Unit:** segments/s (`ops`)\n\n**Not mongod-only:** whole-host TCP stack.\n\n---\n\n### Metrics\n\n**Fields:**\n\n`systemMetrics.netstat.Tcp:InSegs`\n`systemMetrics.netstat.Tcp:OutSegs`\n\n**Query:** `derivative(unit: 1s, nonNegative: true)` → `aggregateWindow(mean)`\n\n---\n\n### How to read\n\n- **TCP segments in** / **TCP segments out** — raw TCP-level volume\n- High In/Out ratio on a secondary → receives more replication than it sends\n\n---\n\n### Troubleshooting\n\n- Sudden drops → network outage\n- Compare with **Network In / Out** (application bytes) and **Replica Members Ping**\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)", "fieldConfig": { "defaults": { "color": { @@ -5834,6 +5822,7 @@ "mode": "off" } }, + "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", @@ -5841,23 +5830,45 @@ { "color": "green", "value": 0 - }, + } + ] + }, + "unit": "ops" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*InSegs$" + }, + "properties": [ + { + "id": "displayName", + "value": "TCP segments in" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*OutSegs$" + }, + "properties": [ { - "color": "red", - "value": 80 + "id": "displayName", + "value": "TCP segments out" } ] } - }, - "overrides": [] + ] }, "gridPos": { "h": 7, "w": 8, "x": 0, - "y": 168 + "y": 172 }, - "id": 18, + "id": 43, "options": { "legend": { "calcs": [ @@ -5882,20 +5893,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..*\\.read_sectors$/ or r._field =~ /^systemMetrics\\.disks\\..*\\.read_time_ms$/ or r._field =~ /^systemMetrics\\.disks\\..*\\.reads_merged$/)\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> derivative(unit: v.windowPeriod)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:InSegs\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:OutSegs\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", "refId": "A" - }, - { - "datasource": { - "type": "influxdb", - "uid": "P3C6603E967DC8568" - }, - "hide": false, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..*\\.io_in_progress$/ )\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n\r\n\r\n\r\n", - "refId": "B" } ], - "title": "Disk Reads", + "title": "TCP segments in / out", "type": "timeseries" }, { @@ -5903,7 +5905,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Disk I/O queue time per device from systemMetrics.disks.*.io_queued_ms. This counter represents cumulative milliseconds that I/O requests have spent waiting in the queue (systemMetrics).\n\nHow to read: lines showing queued time per device as a rate (ms of queue time per second). Values approaching 1000 ms/s indicate persistent queueing.\n\nTroubleshooting: high queue depth is a direct saturation signal -- it means I/O requests are waiting because the device cannot keep up. This is often the root cause of latency spikes. Correlate with 'Disk Latency' and 'Disk I/O'.", + "description": "### Network In / Out\n\nPer-second **logical** network throughput for **mongod** — cumulative **`serverStatus.network.bytesIn`** / **`bytesOut`**.\n\n**Unit:** B/s (`Bps`) — `derivative(unit: 1s)` on byte counters\n\n**Logical bytes:** BSON/document size as counted by mongod (not necessarily on-wire size). Compare with **Network Physical In / Out** for compression/wire bytes.\n\n---\n\n### Metrics\n\n**Fields:**\n\n`serverStatus.network.bytesIn`\n`serverStatus.network.bytesOut`\n\n**Query type:** cumulative counter → `derivative(unit: 1s, nonNegative: true)` → `aggregateWindow(mean)`\n\n---\n\n### How to read\n\n- **Logical bytes in** — data received by mongod\n- **Logical bytes out** — data sent to clients / other nodes\n- Secondary: often **in ≥ out** during replication catch-up\n- Primary: **out** rises with large query results or replication fan-out\n\n---\n\n### Troubleshooting\n\n- High **bytes out** + high **Documents** returned → large result sets · check projections/`limit`\n- High bytes + low **Operations** → big payloads per op\n- Logical ≫ physical on **Network Physical In / Out** → compression effective (normal)\n- Logical ≈ physical → little compression · replication or already-compressed payloads\n- Bytes flat but lag → **Replica Members Lag**, **TCP InSegs / OutSegs**, **Replica Members Ping**\n\nCorrelate with **Network requests** and **Operations**.\n\n---\n\n### Docs\n\n- [serverStatus — network](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.network)", "fieldConfig": { "defaults": { "color": { @@ -5915,6 +5917,7 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -5943,6 +5946,7 @@ "mode": "off" } }, + "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", @@ -5950,24 +5954,45 @@ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "ms" + "unit": "Bps" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*\\.bytesIn$" + }, + "properties": [ + { + "id": "displayName", + "value": "Logical bytes in" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*\\.bytesOut$" + }, + "properties": [ + { + "id": "displayName", + "value": "Logical bytes out" + } + ] + } + ] }, "gridPos": { - "h": 6, + "h": 7, "w": 8, "x": 8, - "y": 171 + "y": 172 }, - "id": 51, + "id": 4, "options": { "legend": { "calcs": [ @@ -5992,11 +6017,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..*\\.io_queued_ms$/)\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> derivative(unit: 1s)\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.network.bytesIn\" or\r\n r[\"_field\"] == \"serverStatus.network.bytesOut\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", "refId": "A" } ], - "title": "Disk Queue Depth", + "title": "Network In / Out", "type": "timeseries" }, { @@ -6004,7 +6029,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "TCP packet integrity counters from systemMetrics.netstat: Tcp.InCsumErrors, Tcp.InErrs, Tcp.RetransSegs, Ip.InDiscards, Ip.OutDiscards (systemMetrics.netstat).\n\nHow to read: lines showing error rates per second.\n\nTroubleshooting: RetransSegs (retransmissions) indicate packet loss, directly causing latency spikes visible in 'Average Operation Latency' and 'Replica Members Ping'. InCsumErrors indicate data corruption on the wire. InDiscards/OutDiscards indicate kernel buffer overflows. Any sustained increase points to network hardware issues, congestion, or NIC driver/firmware bugs.", + "description": "### Network In / Out Physical\n\nPer-second **physical (on-the-wire)** network throughput for **mongod** — cumulative **`serverStatus.network.physicalBytesIn`** / **`physicalBytesOut`**.\n\n**Unit:** B/s (`Bps`) — `derivative(unit: 1s)` on byte counters\n\n**Physical vs logical:** **Physical** = bytes sent/received on network connections (wire size, after compression where applicable). **Logical** = document/BSON bytes as counted by mongod · compare **Network In / Out**.\n\n**Not disk I/O:** network bytes, not storage · see **Disk I/O** for block device throughput.\n\n---\n\n### Metrics\n\n**Fields:**\n\n`serverStatus.network.physicalBytesIn`\n`serverStatus.network.physicalBytesOut`\n\n**Query type:** cumulative counter → `derivative(unit: 1s, nonNegative: true)` → `aggregateWindow(mean)`\n\n---\n\n### How to read\n\n- **Physical bytes in** — wire bytes received by mongod\n- **Physical bytes out** — wire bytes sent by mongod\n- **Logical > physical** on paired panels → compression saving bandwidth (common on replication)\n- **Logical ≈ physical** → little compression benefit on the wire\n\n---\n\n### Troubleshooting\n\n- High physical **out** on primary → replication + client traffic · **Replica Members Lag**, **TCP InSegs / OutSegs**\n- High physical **in** on secondary → replication catch-up\n- Physical throughput near link capacity → timeouts, lag · **Replica Members Ping**\n- High physical + low **Operations** → large compressed payloads or oplog batches\n\nCorrelate with **Network In / Out** (logical) and **Network requests**.\n\n---\n\n### Docs\n\n- [serverStatus — network](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.network)", "fieldConfig": { "defaults": { "color": { @@ -6016,6 +6041,7 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -6044,6 +6070,7 @@ "mode": "off" } }, + "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", @@ -6051,24 +6078,45 @@ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "none" + "unit": "Bps" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*physicalBytesIn$" + }, + "properties": [ + { + "id": "displayName", + "value": "Physical bytes in" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*physicalBytesOut$" + }, + "properties": [ + { + "id": "displayName", + "value": "Physical bytes out" + } + ] + } + ] }, "gridPos": { - "h": 5, + "h": 7, "w": 8, - "x": 0, - "y": 175 + "x": 16, + "y": 172 }, - "id": 42, + "id": 11, "options": { "legend": { "calcs": [ @@ -6093,11 +6141,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"systemMetrics.netstat.Tcp:InCsumErrors\" \r\n or r[\"_field\"] == \"systemMetrics.netstat.Tcp:InErrs\"\r\n or r[\"_field\"] == \"systemMetrics.netstat.Tcp:RetransSegs\"\r\n or r[\"_field\"] == \"systemMetrics.netstat.Ip:InDiscards\"\r\n or r[\"_field\"] == \"systemMetrics.netstat.Ip:OutDiscards\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> derivative(unit: v.windowPeriod)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.network.physicalBytesIn\" or\r\n r[\"_field\"] == \"serverStatus.network.physicalBytesOut\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", "refId": "A" } ], - "title": "TCP Packet Loss or Corruption", + "title": "Network In / Out Physical", "type": "timeseries" }, { @@ -6105,7 +6153,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Rate of serverStatus.asserts by type: regular, warning, msg, user, rollovers. Per MongoDB docs, assertions are internal checks for errors that occur while the database is operating. Non-zero values are uncommon. asserts.user are errors clients may generate (e.g., out of disk space, duplicate key). asserts.rollovers counts how many times assert counters have rolled over after 2^30 assertions (serverStatus.asserts).\n\nHow to read: multiple lines showing assert rate/sec per type.\n\nTroubleshooting: any sustained increase in regular or msg asserts may indicate bugs or data corruption -- examine the MongoDB log. High user asserts indicate application-level errors (duplicate keys, validation failures). Check application logs for corresponding errors.", + "description": "### TCP connection lifecycle\n\nPer-second rates of host **TCP connection lifecycle counters** from FTDC **systemMetrics** (`/proc/net/snmp`).\n\n**Unit:** events/sec — `derivative(unit: 1s, nonNegative: true)` on cumulative counters\n\n**Not mongod-only:** whole-host TCP stack.\n\n---\n\n### Metrics\n\n**Fields:**\n\n`systemMetrics.netstat.Tcp:ActiveOpens`\n`systemMetrics.netstat.Tcp:AttemptFails`\n`systemMetrics.netstat.Tcp:EstabResets`\n`systemMetrics.netstat.Tcp:OutRsts`\n\n**Query type:** cumulative counter → `derivative(unit: 1s, nonNegative: true)` → `aggregateWindow(mean)`\n\n**Not this panel:** `Tcp:CurrEstab` (open connection **count**, gauge — use `last`, not derivative)\n\n---\n\n### How to read\n\n- **TCP active opens** — successful new connections initiated (normally **> 0** on a busy host)\n- **TCP connect failures** — failed connection attempts (expect **~0**)\n- **TCP established resets** — established connections reset (expect **~0**)\n- **TCP RST sent** — segments with RST flag (expect **~0**)\n- Brief spikes during deploys/restarts are normal\n\n---\n\n### Troubleshooting\n\n- Sustained **AttemptFails** → refused connections · firewall, `ulimit`, mongod down, wrong port\n- **EstabResets** / **RST sent** → mid-connection drops · **Replica Members Health**, node restarts\n- High **ActiveOpens** with errors → churny clients or replication mesh · compare **TCP InSegs / OutSegs**\n- With **TCP Packet Loss or Corruption** → loss before reset\n- With lag → **Replica Members Lag**, **Replica Members Ping**\n\nCorrelate with **TCP InSegs / OutSegs** and **Replica Members Health**.\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)\n- Linux [SNMP-TCP-MIB / netstat](https://www.kernel.org/doc/html/latest/networking/snmp_counter.html)", "fieldConfig": { "defaults": { "color": { @@ -6115,8 +6163,9 @@ "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "", + "axisLabel": "events/s", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -6145,30 +6194,78 @@ "mode": "off" } }, + "decimals": 2, "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 - }, + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*AttemptFails$" + }, + "properties": [ + { + "id": "displayName", + "value": "TCP connect failures" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*EstabResets$" + }, + "properties": [ + { + "id": "displayName", + "value": "TCP established resets" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*ActiveOpens$" + }, + "properties": [ + { + "id": "displayName", + "value": "TCP active opens" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*OutRsts$" + }, + "properties": [ { - "color": "red", - "value": 80 + "id": "displayName", + "value": "TCP RST sent" } ] } - }, - "overrides": [] + ] }, "gridPos": { "h": 7, - "w": 7, - "x": 8, - "y": 177 + "w": 8, + "x": 0, + "y": 179 }, - "id": 30, + "id": 45, "options": { "legend": { "calcs": [ @@ -6193,11 +6290,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.asserts.warning\" or r[\"_field\"] == \"serverStatus.asserts.user\" or r[\"_field\"] == \"serverStatus.asserts.rollovers\" or r[\"_field\"] == \"serverStatus.asserts.regular\" or r[\"_field\"] == \"serverStatus.asserts.msg\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: v.windowPeriod)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:ActiveOpens\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:AttemptFails\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:EstabResets\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:OutRsts\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", "refId": "A" } ], - "title": "Asserts", + "title": "TCP Connection lifecycle", "type": "timeseries" }, { @@ -6205,7 +6302,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "TCP/IP routing error counters from systemMetrics.netstat: InNoRoutes, InTruncatedPkts, and related fields (systemMetrics.netstat).\n\nHow to read: lines showing error rates per second (derivatives of cumulative counters).\n\nTroubleshooting: any non-zero derivative indicates network configuration problems -- misrouted packets, MTU issues, or firewall interference. These are rare but can cause intermittent connectivity failures that are very difficult to diagnose at the application level.", + "description": "### TCP routing issues\n\nPer-second rates of host **IP routing and fragmentation error counters** from FTDC **systemMetrics** (`/proc/net/snmp`).\n\n**Unit:** events/sec — `derivative(unit: 1s)` on cumulative counters\n\n**Layer note:** these are **IP** counters (title says “TCP routing” historically). **Not mongod-only.**\n\n**Expectation:** normally **flat at zero**; any sustained rate → routing, MTU, or path problems.\n\n---\n\n### Metrics\n\n**Fields:**\n\n`systemMetrics.netstat.Ip:OutNoRoutes`\n`systemMetrics.netstat.Ip:FragFails`\n`systemMetrics.netstat.Ip:ReasmFails`\n\n**Query type:** cumulative counter → `derivative(unit: 1s, nonNegative: true)` → `aggregateWindow(mean)`\n\n**Not ingested (optional FTDC `IpExt` paths):** `IpExt:InNoRoutes` · `IpExt:InTruncatedPkts` — add to `metrics_to_get.txt` if present in your `metrics.txt`\n\n**Not this panel:** `Ip:InDiscards` / `Ip:OutDiscards` → **TCP Packet Loss or Corruption**\n\n---\n\n### How to read\n\n- **IP no route (out)** — packets dropped with no route to destination\n- **IP fragmentation failures** — could not fragment (often **MTU / PMTU** issues)\n- **IP reassembly failures** — fragmented packets could not be reassembled\n\n---\n\n### Troubleshooting\n\n- **OutNoRoutes** → wrong/missing routes, asymmetric routing, firewall blackholes\n- **FragFails** / **ReasmFails** → **MTU mismatch** (VPN, cloud network, jumbo frames)\n- With **TCP Packet Loss or Corruption** → broader stack problems\n- Intermittent app errors → **Replica Members Ping**, **TCP Connection Issues**\n\nCorrelate with **TCP InSegs / OutSegs** and **Replica Members Health**.\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)\n- Linux [SNMP / netstat counters](https://www.kernel.org/doc/html/latest/networking/snmp_counter.html)", "fieldConfig": { "defaults": { "color": { @@ -6215,8 +6312,9 @@ "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "", + "axisLabel": "events/s", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -6245,6 +6343,7 @@ "mode": "off" } }, + "decimals": 2, "mappings": [], "thresholds": { "mode": "absolute", @@ -6252,22 +6351,55 @@ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] }, "unit": "none" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*OutNoRoutes$" + }, + "properties": [ + { + "id": "displayName", + "value": "IP no route (out)" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*FragFails$" + }, + "properties": [ + { + "id": "displayName", + "value": "IP fragmentation failures" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*ReasmFails$" + }, + "properties": [ + { + "id": "displayName", + "value": "IP reassembly failures" + } + ] + } + ] }, "gridPos": { - "h": 5, + "h": 7, "w": 8, - "x": 0, - "y": 180 + "x": 8, + "y": 179 }, "id": 44, "options": { @@ -6294,7 +6426,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"systemMetrics.netstat.Ip:OutNoRoutes\" )\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> derivative(unit: v.windowPeriod)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"systemMetrics.netstat.Ip:OutNoRoutes\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Ip:FragFails\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Ip:ReasmFails\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", "refId": "A" } ], @@ -6306,7 +6438,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Disk I/O latency per device from systemMetrics.disks.*.read_time_ms and write_time_ms. These counters represent cumulative milliseconds spent on I/O operations (systemMetrics).\n\nHow to read: lines showing time spent on reads and writes per device, as a rate (ms of I/O time per second). Values approaching 1000 ms/s indicate the device is saturated.\n\nTroubleshooting: high disk latency directly causes operation latency increases in MongoDB. Correlate with 'Disk Writes and Reads' (IOPS) and 'Disk I/O' (throughput) to understand if latency is from volume or device saturation.", + "description": "### TCP packet loss or corruption\n\nPer-second rates of host **TCP/IP error and discard counters** from FTDC **systemMetrics** (`/proc/net/snmp`).\n\n**Unit:** events/sec — `derivative(unit: 1s)` on cumulative counters\n\n**Not mongod-only:** whole-host TCP/IP stack (all processes).\n\n**Expectation:** normally **flat at zero**; any sustained non-zero rate warrants investigation.\n\n---\n\n### Metrics\n\n**Fields:**\n\n`systemMetrics.netstat.Tcp:InCsumErrors`\n`systemMetrics.netstat.Tcp:InErrs`\n`systemMetrics.netstat.Tcp:RetransSegs`\n`systemMetrics.netstat.Ip:InDiscards`\n`systemMetrics.netstat.Ip:OutDiscards`\n\n**Query type:** cumulative counter → `derivative(unit: 1s, nonNegative: true)` → `aggregateWindow(mean)`\n\n**Optional (in `metrics_to_get.txt`, not plotted):** `Ip:FragFails` · `Ip:ReasmFails` (fragmentation/reassembly failures)\n\n---\n\n### How to read\n\n- **TCP retransmissions** — packet loss / timeout on the wire (most common “loss” signal)\n- **TCP checksum errors** — corrupted frames received\n- **TCP input errors** — other inbound TCP errors\n- **IP discards** — kernel dropped packets (buffers, policy, congestion)\n\n---\n\n### Troubleshooting\n\n- **RetransSegs** sustained → loss/congestion · **Replica Members Ping**, **Average Operation Latency**\n- **InCsumErrors** → bad cable/NIC/driver, offload bugs\n- **InDiscards** / **OutDiscards** → buffer pressure · NIC ring, `netdev_max_backlog`, flood traffic\n- Errors + high **TCP InSegs / OutSegs** → saturated or failing path\n- Errors on one member only → host/NIC/path to that node · **Replica Members Health**\n\nCorrelate with **TCP InSegs / OutSegs** and **TCP Connection Issues**.\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)\n- Linux [SNMP / netstat counters](https://www.kernel.org/doc/html/latest/networking/snmp_counter.html)", "fieldConfig": { "defaults": { "color": { @@ -6316,8 +6448,9 @@ "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "", + "axisLabel": "events/s", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -6346,6 +6479,7 @@ "mode": "off" } }, + "decimals": 2, "mappings": [], "thresholds": { "mode": "absolute", @@ -6353,24 +6487,81 @@ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "ms" + "unit": "none" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*InCsumErrors$" + }, + "properties": [ + { + "id": "displayName", + "value": "TCP checksum errors" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*Tcp:InErrs$" + }, + "properties": [ + { + "id": "displayName", + "value": "TCP input errors" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*RetransSegs$" + }, + "properties": [ + { + "id": "displayName", + "value": "TCP retransmissions" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*Ip:InDiscards$" + }, + "properties": [ + { + "id": "displayName", + "value": "IP inbound discards" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*Ip:OutDiscards$" + }, + "properties": [ + { + "id": "displayName", + "value": "IP outbound discards" + } + ] + } + ] }, "gridPos": { - "h": 6, + "h": 7, "w": 8, - "x": 0, - "y": 185 + "x": 16, + "y": 179 }, - "id": 50, + "id": 42, "options": { "legend": { "calcs": [ @@ -6395,11 +6586,11 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..*\\.read_time_ms$/ or r._field =~ /^systemMetrics\\.disks\\..*\\.write_time_ms$/)\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> derivative(unit: 1s)\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:InCsumErrors\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:InErrs\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:RetransSegs\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Ip:InDiscards\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Ip:OutDiscards\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", "refId": "A" } ], - "title": "Disk Latency", + "title": "TCP Packet Loss or Corruption", "type": "timeseries" } ], @@ -6454,4 +6645,4 @@ "title": "ftdc_dashboard", "uid": "ddnw277huiv40ae", "version": 1 -} +} \ No newline at end of file diff --git a/metrics_to_get.txt b/metrics_to_get.txt index 2a0ce40..72e37d9 100644 --- a/metrics_to_get.txt +++ b/metrics_to_get.txt @@ -155,3 +155,8 @@ replSetGetStatus.members.1.state replSetGetStatus.members.2.state replSetGetStatus.members.3.state replSetGetStatus.members.4.state +replSetGetStatus.members.0.name +replSetGetStatus.members.1.name +replSetGetStatus.members.2.name +replSetGetStatus.members.3.name +replSetGetStatus.members.4.name \ No newline at end of file From c41f99f7e7282363b5f818fd3feaeaadc223dc88 Mon Sep 17 00:00:00 2001 From: Jean da Silva Date: Thu, 16 Jul 2026 20:22:37 -0300 Subject: [PATCH 7/9] Fix Pages Evicted legend override matching unmodified series. --- grafana/dashboards/dashboard.json | 52 +++++++++++++++---------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/grafana/dashboards/dashboard.json b/grafana/dashboards/dashboard.json index ee4795c..082b540 100644 --- a/grafana/dashboards/dashboard.json +++ b/grafana/dashboards/dashboard.json @@ -1031,31 +1031,31 @@ "unit": "pages/s" }, "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*modified pages evicted$" - }, - "properties": [ - { - "id": "displayName", - "value": "Modified pages evicted (dirty)" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*unmodified pages evicted$" - }, - "properties": [ - { - "id": "displayName", - "value": "Unmodified pages evicted (clean)" - } - ] - } - ] + { + "matcher": { + "id": "byName", + "options": "serverStatus.wiredTiger.cache.modified pages evicted" + }, + "properties": [ + { + "id": "displayName", + "value": "Modified pages evicted (dirty)" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "serverStatus.wiredTiger.cache.unmodified pages evicted" + }, + "properties": [ + { + "id": "displayName", + "value": "Unmodified pages evicted (clean)" + } + ] + } +] }, "gridPos": { "h": 7, @@ -6645,4 +6645,4 @@ "title": "ftdc_dashboard", "uid": "ddnw277huiv40ae", "version": 1 -} \ No newline at end of file +} From 4aa930312a4c7b9a3bc8a98fa139ee0f5978c1ed Mon Sep 17 00:00:00 2001 From: jenunes Date: Fri, 17 Jul 2026 07:58:27 -0300 Subject: [PATCH 8/9] Improve Flux dashboard legends, multi-host queries, and load performance. Fix union-panel legends and broken CPU/Swap/Query Targeting queries, rename fields after aggregateWindow, and set a safer default time range plus datasource timeout. --- grafana/dashboards/dashboard.json | 1702 ++++------------------------- grafana/datasources.yaml | 1 + 2 files changed, 185 insertions(+), 1518 deletions(-) diff --git a/grafana/dashboards/dashboard.json b/grafana/dashboards/dashboard.json index 082b540..83d980c 100644 --- a/grafana/dashboards/dashboard.json +++ b/grafana/dashboards/dashboard.json @@ -39,7 +39,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Server uptime\n\n**`serverStatus.uptime`** — seconds the current **mongod** process has been running.\n\n**Unit:** seconds (displayed as duration)\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.uptime`\n\n---\n\n### How to read\n\n- Monotonic increase while the process is up\n- **Drop** or reset → **restart** (mark the timestamp)\n- May not start at zero if the dashboard time range begins mid-run\n\n---\n\n### Troubleshooting\n\n- Restart + memory drop → OOM · **mongod Memory (RSS / Virtual)**, **Swap Memory**\n- Restart + assert spike → crash · **Asserts**\n- Planned maintenance → correlate with deploy window\n- After restart, counters reset · **Operations**, **Page Faults** rates may look odd briefly\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)", + "description": "### Server uptime\n\n**`serverStatus.uptime`** \u2014 seconds the current **mongod** process has been running.\n\n**Unit:** seconds (displayed as duration)\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.uptime`\n\n---\n\n### How to read\n\n- Monotonic increase while the process is up\n- **Drop** or reset \u2192 **restart** (mark the timestamp)\n- May not start at zero if the dashboard time range begins mid-run\n\n---\n\n### Troubleshooting\n\n- Restart + memory drop \u2192 OOM \u00b7 **mongod Memory (RSS / Virtual)**, **Swap Memory**\n- Restart + assert spike \u2192 crash \u00b7 **Asserts**\n- Planned maintenance \u2192 correlate with deploy window\n- After restart, counters reset \u00b7 **Operations**, **Page Faults** rates may look odd briefly\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)", "fieldConfig": { "defaults": { "color": { @@ -93,20 +93,7 @@ }, "unit": "dtdurations" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*\\.uptime$" - }, - "properties": [ - { - "id": "displayName", - "value": "Server uptime" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 5, @@ -135,7 +122,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.uptime\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.uptime\")\r\n |> drop(columns: [\"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.uptime\" then \"Server uptime\" else r._field\r\n }))", "refId": "A" } ], @@ -147,7 +134,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### ReplSet member states\n\n**`replSetGetStatus.members.N.state`** over the selected time range (same codes as [replica states](https://www.mongodb.com/docs/manual/reference/replica-states/)).\n\n**Note:** Row labels use member **array index** unless `members.N.name` is ingested. Failover shows PRIMARY moving between rows. Primary-only FTDC filter via **hostname**.\n\n---\n\n### Metrics\n\n**Fields:** `replSetGetStatus.members.0.state` … (through your max index)\n\n---\n\n### How to read\n\n- One row per member index\n- Colored band = role · **PRIMARY** (1), **SECONDARY** (2), etc.\n- Red bands = DOWN, RECOVERING, ROLLBACK, …\n\n---\n\n### Docs\n\n- [replSetGetStatus](https://www.mongodb.com/docs/manual/reference/command/replSetGetStatus/)", + "description": "### ReplSet member states\n\n**`replSetGetStatus.members.N.state`** over the selected time range (same codes as [replica states](https://www.mongodb.com/docs/manual/reference/replica-states/)).\n\n**Note:** Row labels use member **array index** unless `members.N.name` is ingested. Failover shows PRIMARY moving between rows. Primary-only FTDC filter via **hostname**.\n\n---\n\n### Metrics\n\n**Fields:** `replSetGetStatus.members.0.state` \u2026 (through your max index)\n\n---\n\n### How to read\n\n- One row per member index\n- Colored band = role \u00b7 **PRIMARY** (1), **SECONDARY** (2), etc.\n- Red bands = DOWN, RECOVERING, ROLLBACK, \u2026\n\n---\n\n### Docs\n\n- [replSetGetStatus](https://www.mongodb.com/docs/manual/reference/command/replSetGetStatus/)", "fieldConfig": { "defaults": { "color": { @@ -244,68 +231,7 @@ }, "unit": "short" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*members\\.0\\.state$" - }, - "properties": [ - { - "id": "displayName", - "value": "Member 0" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*members\\.1\\.state$" - }, - "properties": [ - { - "id": "displayName", - "value": "Member 1" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*members\\.2\\.state$" - }, - "properties": [ - { - "id": "displayName", - "value": "Member 2" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*members\\.3\\.state$" - }, - "properties": [ - { - "id": "displayName", - "value": "Member 3" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*members\\.4\\.state$" - }, - "properties": [ - { - "id": "displayName", - "value": "Member 4" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 5, @@ -337,7 +263,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] =~ /replSetGetStatus\\.members\\.\\d+\\.state/)\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] =~ /replSetGetStatus\\.members\\.\\d+\\.state/)\r\n |> drop(columns: [\"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"replSetGetStatus.members.0.state\" then \"Member 0\" else if r._field == \"replSetGetStatus.members.1.state\" then \"Member 1\" else if r._field == \"replSetGetStatus.members.2.state\" then \"Member 2\" else if r._field == \"replSetGetStatus.members.3.state\" then \"Member 3\" else if r._field == \"replSetGetStatus.members.4.state\" then \"Member 4\" else r._field\r\n }))\r\n", "refId": "A" } ], @@ -349,7 +275,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Replica members health\n\n**`replSetGetStatus.members.N.health`** — per-member reachability (1 = healthy, 0 = down).\n\n**Unit:** 0/1 (gauge)\n\n**Note:** **N** is the member array index, not hostname order.\n\n---\n\n### Metrics\n\n**Fields:** `replSetGetStatus.members.0.health` … `replSetGetStatus.members.4.health`\n\n---\n\n### How to read\n\n- One line per member index\n- **stepAfter** · flat at **1** = healthy · drop to **0** = member lost\n- Do **not** stack — each line is independent 0/1\n\n---\n\n### Troubleshooting\n\n- **1 → 0** → network partition, crash, OOM · **Server Uptime**, **Asserts**, system **CPU/Memory/Disk** on that host\n- Healthy but high lag → **Replica Members Lag**, **Replica Members Ping**\n\n---\n\n### Docs\n\n- [replSetGetStatus](https://www.mongodb.com/docs/manual/reference/command/replSetGetStatus/)", + "description": "### Replica members health\n\n**`replSetGetStatus.members.N.health`** \u2014 per-member reachability (1 = healthy, 0 = down).\n\n**Unit:** 0/1 (gauge)\n\n**Note:** **N** is the member array index, not hostname order.\n\n---\n\n### Metrics\n\n**Fields:** `replSetGetStatus.members.0.health` \u2026 `replSetGetStatus.members.4.health`\n\n---\n\n### How to read\n\n- One line per member index\n- **stepAfter** \u00b7 flat at **1** = healthy \u00b7 drop to **0** = member lost\n- Do **not** stack \u2014 each line is independent 0/1\n\n---\n\n### Troubleshooting\n\n- **1 \u2192 0** \u2192 network partition, crash, OOM \u00b7 **Server Uptime**, **Asserts**, system **CPU/Memory/Disk** on that host\n- Healthy but high lag \u2192 **Replica Members Lag**, **Replica Members Ping**\n\n---\n\n### Docs\n\n- [replSetGetStatus](https://www.mongodb.com/docs/manual/reference/command/replSetGetStatus/)", "fieldConfig": { "defaults": { "color": { @@ -420,68 +346,7 @@ }, "unit": "short" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*members\\.0\\.health$" - }, - "properties": [ - { - "id": "displayName", - "value": "Member 0 health" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*members\\.1\\.health$" - }, - "properties": [ - { - "id": "displayName", - "value": "Member 1 health" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*members\\.2\\.health$" - }, - "properties": [ - { - "id": "displayName", - "value": "Member 2 health" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*members\\.3\\.health$" - }, - "properties": [ - { - "id": "displayName", - "value": "Member 3 health" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*members\\.4\\.health$" - }, - "properties": [ - { - "id": "displayName", - "value": "Member 4 health" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 5, @@ -513,7 +378,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] =~ /replSetGetStatus\\.members\\.\\d+\\.health/)\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] =~ /replSetGetStatus\\.members\\.\\d+\\.health/)\r\n |> drop(columns: [\"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"replSetGetStatus.members.0.health\" then \"Member 0 health\" else if r._field == \"replSetGetStatus.members.1.health\" then \"Member 1 health\" else if r._field == \"replSetGetStatus.members.2.health\" then \"Member 2 health\" else if r._field == \"replSetGetStatus.members.3.health\" then \"Member 3 health\" else if r._field == \"replSetGetStatus.members.4.health\" then \"Member 4 health\" else r._field\r\n }))\r\n", "refId": "A" } ], @@ -525,7 +390,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Asserts\n\nPer-second rate of **`serverStatus.asserts`** — mongod internal checks and user-triggered assertion counters.\n\n**Unit:** asserts/sec (`ops`)\n\n**Expectation:** normally **flat at zero**; any sustained non-zero rate warrants investigation.\n\n---\n\n### Metrics\n\n**Fields:**\n\n`serverStatus.asserts.regular`\n`serverStatus.asserts.warning`\n`serverStatus.asserts.msg`\n`serverStatus.asserts.user`\n`serverStatus.asserts.rollovers`\n\n**Query type:** cumulative counter → `derivative(unit: 1s, nonNegative: true)` → `aggregateWindow(mean)`\n\n**Rollovers:** count of times assert counters wrapped past 2³⁰ — spikes are rare and distinct from error asserts.\n\n---\n\n### How to read\n\n- **Regular** / **Msg** — internal mongod assertion failures (unusual)\n- **Warning** — non-fatal internal warnings\n- **User** — client-visible errors (duplicate key, validation, disk full, etc.)\n- **Rollovers** — counter wrap events, not application errors\n- Brief spikes on restart or deploy may appear as counter resets\n\n---\n\n### Troubleshooting\n\n- **Regular** or **Msg** sustained → possible bug or corruption · check mongod logs immediately\n- **User** spikes → application errors · correlate with app logs and **Operations**\n- Spike + **Server uptime** drop → crash/assert abort · **Replica Members Health**\n- Zero across all lines → healthy (typical)\n\nCorrelate with **Server uptime** and **Average Operation Latency**.\n\n---\n\n### Docs\n\n- [serverStatus — asserts](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.asserts)", + "description": "### Asserts\n\nPer-second rate of **`serverStatus.asserts`** \u2014 mongod internal checks and user-triggered assertion counters.\n\n**Unit:** asserts/sec (`ops`)\n\n**Expectation:** normally **flat at zero**; any sustained non-zero rate warrants investigation.\n\n---\n\n### Metrics\n\n**Fields:**\n\n`serverStatus.asserts.regular`\n`serverStatus.asserts.warning`\n`serverStatus.asserts.msg`\n`serverStatus.asserts.user`\n`serverStatus.asserts.rollovers`\n\n**Query type:** cumulative counter \u2192 `derivative(unit: 1s, nonNegative: true)` \u2192 `aggregateWindow(mean)`\n\n**Rollovers:** count of times assert counters wrapped past 2\u00b3\u2070 \u2014 spikes are rare and distinct from error asserts.\n\n---\n\n### How to read\n\n- **Regular** / **Msg** \u2014 internal mongod assertion failures (unusual)\n- **Warning** \u2014 non-fatal internal warnings\n- **User** \u2014 client-visible errors (duplicate key, validation, disk full, etc.)\n- **Rollovers** \u2014 counter wrap events, not application errors\n- Brief spikes on restart or deploy may appear as counter resets\n\n---\n\n### Troubleshooting\n\n- **Regular** or **Msg** sustained \u2192 possible bug or corruption \u00b7 check mongod logs immediately\n- **User** spikes \u2192 application errors \u00b7 correlate with app logs and **Operations**\n- Spike + **Server uptime** drop \u2192 crash/assert abort \u00b7 **Replica Members Health**\n- Zero across all lines \u2192 healthy (typical)\n\nCorrelate with **Server uptime** and **Average Operation Latency**.\n\n---\n\n### Docs\n\n- [serverStatus \u2014 asserts](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.asserts)", "fieldConfig": { "defaults": { "color": { @@ -579,66 +444,6 @@ "unit": "ops" }, "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*\\.regular$" - }, - "properties": [ - { - "id": "displayName", - "value": "Regular" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*\\.warning$" - }, - "properties": [ - { - "id": "displayName", - "value": "Warning" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*\\.msg$" - }, - "properties": [ - { - "id": "displayName", - "value": "Msg" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*\\.user$" - }, - "properties": [ - { - "id": "displayName", - "value": "User" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*\\.rollovers$" - }, - "properties": [ - { - "id": "displayName", - "value": "Rollovers" - } - ] - }, { "__systemRef": "hideSeriesFrom", "matcher": { @@ -696,7 +501,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.asserts.regular\" or\r\n r[\"_field\"] == \"serverStatus.asserts.warning\" or\r\n r[\"_field\"] == \"serverStatus.asserts.msg\" or\r\n r[\"_field\"] == \"serverStatus.asserts.user\" or\r\n r[\"_field\"] == \"serverStatus.asserts.rollovers\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.asserts.regular\" or\r\n r[\"_field\"] == \"serverStatus.asserts.warning\" or\r\n r[\"_field\"] == \"serverStatus.asserts.msg\" or\r\n r[\"_field\"] == \"serverStatus.asserts.user\" or\r\n r[\"_field\"] == \"serverStatus.asserts.rollovers\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.asserts.regular\" then \"Regular\" else if r._field == \"serverStatus.asserts.warning\" then \"Warning\" else if r._field == \"serverStatus.asserts.msg\" then \"Msg\" else if r._field == \"serverStatus.asserts.user\" then \"User\" else if r._field == \"serverStatus.asserts.rollovers\" then \"Rollovers\" else r._field\r\n }))", "refId": "A" } ], @@ -704,7 +509,7 @@ "type": "timeseries" }, { - "collapsed": false, + "collapsed": true, "gridPos": { "h": 1, "w": 24, @@ -721,7 +526,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### WiredTiger ticket availability\n\nShows how many **read** and **write** execution tickets are still available in the storage engine. Each ticket allows one concurrent operation; when **available** reaches **0**, new operations wait until a ticket is freed.\n\n**Unit:** ticket count (not bytes)\n\n---\n\n### Metrics (same concept, two paths)\n\n**MongoDB 8.x+** \n`serverStatus.queues.execution.read.available` \n`serverStatus.queues.execution.write.available`\n\n**Legacy** \n`serverStatus.wiredTiger.concurrentTransactions.read.available` \n`serverStatus.wiredTiger.concurrentTransactions.write.available`\n\nBoth paths queried for compatibility. Usually only one has data per version.\n\n---\n\n### How to read\n\n- Two lines: read and write **available** tickets \n- **stepAfter** reflects discrete ticket changes \n- Typical max ~**128** (dynamic adjustment on 7.0+)\n\n---\n\n### Troubleshooting\n\n- Sustained **low or zero** → concurrency saturation \n- Correlate with **Current Queue writers / readers** and **Average Operation Latency**\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)\n- [Performance tuning — WiredTiger tickets](https://www.mongodb.com/docs/manual/administration/performance-tuning/)", + "description": "### WiredTiger ticket availability\n\nShows how many **read** and **write** execution tickets are still available in the storage engine. Each ticket allows one concurrent operation; when **available** reaches **0**, new operations wait until a ticket is freed.\n\n**Unit:** ticket count (not bytes)\n\n---\n\n### Metrics (same concept, two paths)\n\n**MongoDB 8.x+** \n`serverStatus.queues.execution.read.available` \n`serverStatus.queues.execution.write.available`\n\n**Legacy** \n`serverStatus.wiredTiger.concurrentTransactions.read.available` \n`serverStatus.wiredTiger.concurrentTransactions.write.available`\n\nBoth paths queried for compatibility. Usually only one has data per version.\n\n---\n\n### How to read\n\n- Two lines: read and write **available** tickets \n- **stepAfter** reflects discrete ticket changes \n- Typical max ~**128** (dynamic adjustment on 7.0+)\n\n---\n\n### Troubleshooting\n\n- Sustained **low or zero** \u2192 concurrency saturation \n- Correlate with **Current Queue writers / readers** and **Average Operation Latency**\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)\n- [Performance tuning \u2014 WiredTiger tickets](https://www.mongodb.com/docs/manual/administration/performance-tuning/)", "fieldConfig": { "defaults": { "color": { @@ -779,32 +584,7 @@ }, "unit": "short" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*\\.read\\.available$" - }, - "properties": [ - { - "id": "displayName", - "value": "Tickets Available Read" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*\\.write\\.available$" - }, - "properties": [ - { - "id": "displayName", - "value": "Tickets Available Write" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 8, @@ -837,7 +617,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.concurrentTransactions.read.available\" or r[\"_field\"] == \"serverStatus.wiredTiger.concurrentTransactions.write.available\" or r[\"_field\"] == \"serverStatus.queues.execution.read.available\" or r[\"_field\"] == \"serverStatus.queues.execution.write.available\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.concurrentTransactions.read.available\" or r[\"_field\"] == \"serverStatus.wiredTiger.concurrentTransactions.write.available\" or r[\"_field\"] == \"serverStatus.queues.execution.read.available\" or r[\"_field\"] == \"serverStatus.queues.execution.write.available\")\r\n |> drop(columns: [\"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _field: if r._field == \"serverStatus.wiredTiger.concurrentTransactions.read.available\" then \"Tickets Available Read\" else if r._field == \"serverStatus.wiredTiger.concurrentTransactions.write.available\" then \"Tickets Available Write\" else if r._field == \"serverStatus.queues.execution.read.available\" then \"Tickets Available Read\" else if r._field == \"serverStatus.queues.execution.write.available\" then \"Tickets Available Write\" else r._field }))\r\n", "refId": "A" } ], @@ -849,7 +629,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### WiredTiger cache size\n\n**maximum bytes configured** vs **bytes currently in the cache** — WiredTiger internal cache limit and current usage. **Unit:** bytes (per MongoDB docs). Current usage should not exceed the configured maximum.\n\n**Metrics:** `serverStatus.wiredTiger.cache.maximum bytes configured` · `serverStatus.wiredTiger.cache.bytes currently in the cache`\n\nTo change the limit, see `wiredTigerCacheSizeGB` / `storage.wiredTiger.engineConfig.cacheSizeGB` (config in GB; these stats are still bytes).\n\n---\n\n### How to read\n\n- Two lines: configured max and current usage\n- WiredTiger often targets ~80% of max under load; sustained saturation → eviction pressure\n\n---\n\n### Troubleshooting\n\n- Usage near max → **Pages Evicted**, **Dirty Cache Bytes**\n- Consistently at max → raise `cacheSizeGB` or reduce working set\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)\n- [WiredTiger storage engine](https://www.mongodb.com/docs/manual/core/wiredtiger/)", + "description": "### WiredTiger cache size\n\n**maximum bytes configured** vs **bytes currently in the cache** \u2014 WiredTiger internal cache limit and current usage. **Unit:** bytes (per MongoDB docs). Current usage should not exceed the configured maximum.\n\n**Metrics:** `serverStatus.wiredTiger.cache.maximum bytes configured` \u00b7 `serverStatus.wiredTiger.cache.bytes currently in the cache`\n\nTo change the limit, see `wiredTigerCacheSizeGB` / `storage.wiredTiger.engineConfig.cacheSizeGB` (config in GB; these stats are still bytes).\n\n---\n\n### How to read\n\n- Two lines: configured max and current usage\n- WiredTiger often targets ~80% of max under load; sustained saturation \u2192 eviction pressure\n\n---\n\n### Troubleshooting\n\n- Usage near max \u2192 **Pages Evicted**, **Dirty Cache Bytes**\n- Consistently at max \u2192 raise `cacheSizeGB` or reduce working set\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)\n- [WiredTiger storage engine](https://www.mongodb.com/docs/manual/core/wiredtiger/)", "fieldConfig": { "defaults": { "color": { @@ -905,32 +685,7 @@ }, "unit": "bytes" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*maximum bytes configured$" - }, - "properties": [ - { - "id": "displayName", - "value": "Maximum cache configured" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*bytes currently in the cache$" - }, - "properties": [ - { - "id": "displayName", - "value": "Current cache usage" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 8, @@ -964,7 +719,7 @@ "uid": "P3C6603E967DC8568" }, "hide": false, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.cache.maximum bytes configured\" or r[\"_field\"] == \"serverStatus.wiredTiger.cache.bytes currently in the cache\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.cache.maximum bytes configured\" or r[\"_field\"] == \"serverStatus.wiredTiger.cache.bytes currently in the cache\")\r\n |> drop(columns: [\"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.wiredTiger.cache.maximum bytes configured\" then \"Maximum cache configured\" else if r._field == \"serverStatus.wiredTiger.cache.bytes currently in the cache\" then \"Current cache usage\" else r._field\r\n }))", "refId": "A" } ], @@ -976,7 +731,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### WiredTiger Pages Evicted\n\nRate of **modified** (dirty) and **unmodified** (clean) cache pages evicted per second. MongoDB docs note unmodified evictions as the main eviction statistic.\n\n**Unit:** pages/s (derivative of cumulative counters)\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.wiredTiger.cache.modified pages evicted` · `serverStatus.wiredTiger.cache.unmodified pages evicted`\n\n---\n\n### How to read\n\n- **Modified pages evicted (dirty)** — evicted after write/reconciliation\n- **Unmodified pages evicted (clean)** — discarded without write\n\n---\n\n### Troubleshooting\n\n- Dirty eviction spikes → cache pressure, write latency · **WiredTiger Cache**, **Dirty Cache Bytes**\n- Eviction can't keep up → app threads may evict (performance cliff)\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)\n- [WiredTiger storage engine](https://www.mongodb.com/docs/manual/core/wiredtiger/)", + "description": "### WiredTiger Pages Evicted\n\nRate of **modified** (dirty) and **unmodified** (clean) cache pages evicted per second. MongoDB docs note unmodified evictions as the main eviction statistic.\n\n**Unit:** pages/s (derivative of cumulative counters)\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.wiredTiger.cache.modified pages evicted` \u00b7 `serverStatus.wiredTiger.cache.unmodified pages evicted`\n\n---\n\n### How to read\n\n- **Modified pages evicted (dirty)** \u2014 evicted after write/reconciliation\n- **Unmodified pages evicted (clean)** \u2014 discarded without write\n\n---\n\n### Troubleshooting\n\n- Dirty eviction spikes \u2192 cache pressure, write latency \u00b7 **WiredTiger Cache**, **Dirty Cache Bytes**\n- Eviction can't keep up \u2192 app threads may evict (performance cliff)\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)\n- [WiredTiger storage engine](https://www.mongodb.com/docs/manual/core/wiredtiger/)", "fieldConfig": { "defaults": { "color": { @@ -1030,32 +785,7 @@ }, "unit": "pages/s" }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "serverStatus.wiredTiger.cache.modified pages evicted" - }, - "properties": [ - { - "id": "displayName", - "value": "Modified pages evicted (dirty)" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "serverStatus.wiredTiger.cache.unmodified pages evicted" - }, - "properties": [ - { - "id": "displayName", - "value": "Unmodified pages evicted (clean)" - } - ] - } -] + "overrides": [] }, "gridPos": { "h": 7, @@ -1088,7 +818,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.cache.modified pages evicted\" or r[\"_field\"] == \"serverStatus.wiredTiger.cache.unmodified pages evicted\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.cache.modified pages evicted\" or r[\"_field\"] == \"serverStatus.wiredTiger.cache.unmodified pages evicted\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _field: if r._field == \"serverStatus.wiredTiger.cache.modified pages evicted\" then \"Modified pages evicted (dirty)\" else if r._field == \"serverStatus.wiredTiger.cache.unmodified pages evicted\" then \"Unmodified pages evicted (clean)\" else r._field }))\r\n", "refId": "A" } ], @@ -1100,7 +830,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### WiredTiger data handles\n\nCount of **connection data handles currently active** — WiredTiger dhandles for open collections, indexes, and other named data sources.\n\n**Unit:** count (gauge)\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.wiredTiger.data-handle.connection data handles currently active`\n\n---\n\n### How to read\n\n- Single line: active dhandles over time\n- Sudden drop → collections closed or **mongod** restart (**Server Uptime**)\n\n---\n\n### Troubleshooting\n\n- Very high count → many collections/indexes in use, memory overhead, dhandle list lock contention\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)\n- [WiredTiger storage engine](https://www.mongodb.com/docs/manual/core/wiredtiger/)", + "description": "### WiredTiger data handles\n\nCount of **connection data handles currently active** \u2014 WiredTiger dhandles for open collections, indexes, and other named data sources.\n\n**Unit:** count (gauge)\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.wiredTiger.data-handle.connection data handles currently active`\n\n---\n\n### How to read\n\n- Single line: active dhandles over time\n- Sudden drop \u2192 collections closed or **mongod** restart (**Server Uptime**)\n\n---\n\n### Troubleshooting\n\n- Very high count \u2192 many collections/indexes in use, memory overhead, dhandle list lock contention\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)\n- [WiredTiger storage engine](https://www.mongodb.com/docs/manual/core/wiredtiger/)", "fieldConfig": { "defaults": { "color": { @@ -1154,20 +884,7 @@ }, "unit": "short" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*data handles currently active$" - }, - "properties": [ - { - "id": "displayName", - "value": "Active data handles" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 5, @@ -1200,7 +917,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.data-handle.connection data handles currently active\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.data-handle.connection data handles currently active\")\r\n |> drop(columns: [\"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _field: if r._field == \"serverStatus.wiredTiger.data-handle.connection data handles currently active\" then \"Active data handles\" else r._field }))", "refId": "A" } ], @@ -1212,7 +929,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Dirty cache bytes\n\n**tracked dirty bytes in the cache** — modified data in the WiredTiger cache not yet flushed via checkpoint or eviction. Should stay below **bytes currently in the cache**.\n\n**Unit:** bytes (gauge)\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.wiredTiger.cache.tracked dirty bytes in the cache`\n\n---\n\n### How to read\n\n- Single line: dirty bytes over time\n- ~5% of cache → eviction pressure · ~20% → aggressive eviction / stalls\n\n---\n\n### Troubleshooting\n\n- Dirty bytes high vs **WiredTiger Cache** max → eviction pressure\n- Correlate with **Pages Evicted**, **Checkpoint Write Throughput**\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)\n- [WiredTiger storage engine](https://www.mongodb.com/docs/manual/core/wiredtiger/)", + "description": "### Dirty cache bytes\n\n**tracked dirty bytes in the cache** \u2014 modified data in the WiredTiger cache not yet flushed via checkpoint or eviction. Should stay below **bytes currently in the cache**.\n\n**Unit:** bytes (gauge)\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.wiredTiger.cache.tracked dirty bytes in the cache`\n\n---\n\n### How to read\n\n- Single line: dirty bytes over time\n- ~5% of cache \u2192 eviction pressure \u00b7 ~20% \u2192 aggressive eviction / stalls\n\n---\n\n### Troubleshooting\n\n- Dirty bytes high vs **WiredTiger Cache** max \u2192 eviction pressure\n- Correlate with **Pages Evicted**, **Checkpoint Write Throughput**\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)\n- [WiredTiger storage engine](https://www.mongodb.com/docs/manual/core/wiredtiger/)", "fieldConfig": { "defaults": { "color": { @@ -1266,20 +983,7 @@ }, "unit": "bytes" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*tracked dirty bytes in the cache$" - }, - "properties": [ - { - "id": "displayName", - "value": "Dirty cache bytes" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 9, @@ -1312,7 +1016,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.cache.tracked dirty bytes in the cache\" )\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.cache.tracked dirty bytes in the cache\" )\r\n |> drop(columns: [\"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _field: if r._field == \"serverStatus.wiredTiger.cache.tracked dirty bytes in the cache\" then \"Dirty cache bytes\" else r._field }))", "refId": "A" } ], @@ -1324,7 +1028,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### WiredTiger block manager I/O\n\nRead/write throughput through WiredTiger's block manager (disk I/O for data files). **Unit:** MB/s (from bytes/s ÷ 1024²) — cumulative byte counters differentiated.\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.wiredTiger.block-manager.bytes read` · `serverStatus.wiredTiger.block-manager.bytes written`\n\n---\n\n### How to read\n\n- **Bytes read** — data read from disk (cache misses)\n- **Bytes written** — data written via block manager (eviction, checkpoints)\n\n---\n\n### Troubleshooting\n\n- High reads → cache pressure · **WiredTiger Cache**, **Pages Evicted**\n- High writes → checkpoints/eviction · **Checkpoint Write Throughput**, **Disk I/O**\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)\n- [WiredTiger storage engine](https://www.mongodb.com/docs/manual/core/wiredtiger/)", + "description": "### WiredTiger block manager I/O\n\nRead/write throughput through WiredTiger's block manager (disk I/O for data files). **Unit:** MB/s (from bytes/s \u00f7 1024\u00b2) \u2014 cumulative byte counters differentiated.\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.wiredTiger.block-manager.bytes read` \u00b7 `serverStatus.wiredTiger.block-manager.bytes written`\n\n---\n\n### How to read\n\n- **Bytes read** \u2014 data read from disk (cache misses)\n- **Bytes written** \u2014 data written via block manager (eviction, checkpoints)\n\n---\n\n### Troubleshooting\n\n- High reads \u2192 cache pressure \u00b7 **WiredTiger Cache**, **Pages Evicted**\n- High writes \u2192 checkpoints/eviction \u00b7 **Checkpoint Write Throughput**, **Disk I/O**\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)\n- [WiredTiger storage engine](https://www.mongodb.com/docs/manual/core/wiredtiger/)", "fieldConfig": { "defaults": { "color": { @@ -1378,32 +1082,7 @@ }, "unit": "MBs" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*block-manager\\.bytes read$" - }, - "properties": [ - { - "id": "displayName", - "value": "Bytes read" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*block-manager\\.bytes written$" - }, - "properties": [ - { - "id": "displayName", - "value": "Bytes written" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 7, @@ -1436,7 +1115,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.block-manager.bytes read\" or r[\"_field\"] == \"serverStatus.wiredTiger.block-manager.bytes written\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) / 1024.0 / 1024.0 }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.block-manager.bytes read\" or r[\"_field\"] == \"serverStatus.wiredTiger.block-manager.bytes written\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _field: if r._field == \"serverStatus.wiredTiger.block-manager.bytes read\" then \"Bytes read\" else if r._field == \"serverStatus.wiredTiger.block-manager.bytes written\" then \"Bytes written\" else r._field }))\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) / 1024.0 / 1024.0 }))", "refId": "A" } ], @@ -1448,7 +1127,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Checkpoint write throughput\n\nRate of **bytes written for checkpoint** — disk write throughput during WiredTiger checkpoints (~every 60s). Recovery point after crash/shutdown.\n\n**Unit:** B/s (`Bps`) — derivative of cumulative bytes\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.wiredTiger.block-manager.bytes written for checkpoint`\n\n---\n\n### How to read\n\n- Spikes during checkpoint runs · baseline low between checkpoints\n\n---\n\n### Troubleshooting\n\n- Large spikes + latency → checkpoint competing for disk I/O\n- Correlate with **Disk I/O**, **Disk Writes**, **Average Operation Latency**, **Checkpoint Duration**\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)\n- [WiredTiger storage engine](https://www.mongodb.com/docs/manual/core/wiredtiger/)", + "description": "### Checkpoint write throughput\n\nRate of **bytes written for checkpoint** \u2014 disk write throughput during WiredTiger checkpoints (~every 60s). Recovery point after crash/shutdown.\n\n**Unit:** B/s (`Bps`) \u2014 derivative of cumulative bytes\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.wiredTiger.block-manager.bytes written for checkpoint`\n\n---\n\n### How to read\n\n- Spikes during checkpoint runs \u00b7 baseline low between checkpoints\n\n---\n\n### Troubleshooting\n\n- Large spikes + latency \u2192 checkpoint competing for disk I/O\n- Correlate with **Disk I/O**, **Disk Writes**, **Average Operation Latency**, **Checkpoint Duration**\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)\n- [WiredTiger storage engine](https://www.mongodb.com/docs/manual/core/wiredtiger/)", "fieldConfig": { "defaults": { "color": { @@ -1501,20 +1180,7 @@ }, "unit": "Bps" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*bytes written for checkpoint$" - }, - "properties": [ - { - "id": "displayName", - "value": "Checkpoint write throughput" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 7, @@ -1547,7 +1213,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.block-manager.bytes written for checkpoint\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.block-manager.bytes written for checkpoint\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _field: if r._field == \"serverStatus.wiredTiger.block-manager.bytes written for checkpoint\" then \"Checkpoint write throughput\" else r._field }))", "refId": "A" } ], @@ -1559,7 +1225,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Cache read/write throughput\n\nRate of bytes **read into cache** (cold data from disk) and **written from cache** (flush to disk). Overview of WiredTiger cache I/O activity.\n\n**Unit:** B/s (`Bps`) — derivative of cumulative byte counters\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.wiredTiger.cache.bytes read into cache` · `serverStatus.wiredTiger.cache.bytes written from cache`\n\n---\n\n### How to read\n\n- **Read into cache** — cache misses / data pulled from disk\n- **Written from cache** — dirty data flushed (eviction, checkpoints)\n\n---\n\n### Troubleshooting\n\n- High reads → working set may exceed cache · **WiredTiger Cache**, **Pages Evicted**\n- High writes → write load or eviction pressure · **Dirty Cache Bytes**, **WiredTiger Block Manager**\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)\n- [WiredTiger storage engine](https://www.mongodb.com/docs/manual/core/wiredtiger/)", + "description": "### Cache read/write throughput\n\nRate of bytes **read into cache** (cold data from disk) and **written from cache** (flush to disk). Overview of WiredTiger cache I/O activity.\n\n**Unit:** B/s (`Bps`) \u2014 derivative of cumulative byte counters\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.wiredTiger.cache.bytes read into cache` \u00b7 `serverStatus.wiredTiger.cache.bytes written from cache`\n\n---\n\n### How to read\n\n- **Read into cache** \u2014 cache misses / data pulled from disk\n- **Written from cache** \u2014 dirty data flushed (eviction, checkpoints)\n\n---\n\n### Troubleshooting\n\n- High reads \u2192 working set may exceed cache \u00b7 **WiredTiger Cache**, **Pages Evicted**\n- High writes \u2192 write load or eviction pressure \u00b7 **Dirty Cache Bytes**, **WiredTiger Block Manager**\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)\n- [WiredTiger storage engine](https://www.mongodb.com/docs/manual/core/wiredtiger/)", "fieldConfig": { "defaults": { "color": { @@ -1612,32 +1278,7 @@ }, "unit": "Bps" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*bytes read into cache$" - }, - "properties": [ - { - "id": "displayName", - "value": "Read into cache" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*bytes written from cache$" - }, - "properties": [ - { - "id": "displayName", - "value": "Written from cache" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 7, @@ -1670,7 +1311,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.cache.bytes written from cache\" or r[\"_field\"] == \"serverStatus.wiredTiger.cache.bytes read into cache\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.cache.bytes written from cache\" or r[\"_field\"] == \"serverStatus.wiredTiger.cache.bytes read into cache\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _field: if r._field == \"serverStatus.wiredTiger.cache.bytes written from cache\" then \"Written from cache\" else if r._field == \"serverStatus.wiredTiger.cache.bytes read into cache\" then \"Read into cache\" else r._field }))", "refId": "A" } ], @@ -1682,7 +1323,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Checkpoint duration\n\nDuration in **milliseconds** of the **most recent** WiredTiger checkpoint. Value updates when a checkpoint completes (~every 60s), then holds until the next.\n\n**Unit:** ms (gauge)\n\n---\n\n### Metrics (same concept, two paths)\n\n**Legacy:** `serverStatus.wiredTiger.transaction.transaction checkpoint most recent time (msecs)` \n**MongoDB 8.x+:** `serverStatus.wiredTiger.checkpoint.most recent time (msecs)`\n\nBoth queried; usually one path has data per version.\n\n---\n\n### How to read\n\n- Single line · rising trend under steady write load → I/O pressure\n\n---\n\n### Troubleshooting\n\n- Increasing duration → I/O saturation · **Checkpoint Write Throughput**, **Disk I/O**, **Average Operation Latency**\n- Long checkpoints → possible write stalls\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)\n- [WiredTiger storage engine](https://www.mongodb.com/docs/manual/core/wiredtiger/)", + "description": "### Checkpoint duration\n\nDuration in **milliseconds** of the **most recent** WiredTiger checkpoint. Value updates when a checkpoint completes (~every 60s), then holds until the next.\n\n**Unit:** ms (gauge)\n\n---\n\n### Metrics (same concept, two paths)\n\n**Legacy:** `serverStatus.wiredTiger.transaction.transaction checkpoint most recent time (msecs)` \n**MongoDB 8.x+:** `serverStatus.wiredTiger.checkpoint.most recent time (msecs)`\n\nBoth queried; usually one path has data per version.\n\n---\n\n### How to read\n\n- Single line \u00b7 rising trend under steady write load \u2192 I/O pressure\n\n---\n\n### Troubleshooting\n\n- Increasing duration \u2192 I/O saturation \u00b7 **Checkpoint Write Throughput**, **Disk I/O**, **Average Operation Latency**\n- Long checkpoints \u2192 possible write stalls\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)\n- [WiredTiger storage engine](https://www.mongodb.com/docs/manual/core/wiredtiger/)", "fieldConfig": { "defaults": { "color": { @@ -1736,20 +1377,7 @@ }, "unit": "ms" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*checkpoint.most recent time \\(msecs\\)$" - }, - "properties": [ - { - "id": "displayName", - "value": "Checkpoint duration" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 6, @@ -1784,7 +1412,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.transaction.transaction checkpoint most recent time (msecs)\" or r[\"_field\"] == \"serverStatus.wiredTiger.checkpoint.most recent time (msecs)\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.transaction.transaction checkpoint most recent time (msecs)\" or r[\"_field\"] == \"serverStatus.wiredTiger.checkpoint.most recent time (msecs)\")\r\n |> drop(columns: [\"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _field: if r._field == \"serverStatus.wiredTiger.transaction.transaction checkpoint most recent time (msecs)\" then \"Checkpoint duration\" else if r._field == \"serverStatus.wiredTiger.checkpoint.most recent time (msecs)\" then \"Checkpoint duration\" else r._field }))\r\n", "refId": "A" } ], @@ -1796,7 +1424,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Journal bytes written\n\nRate of WAL/journal bytes written by WiredTiger (`wiredTiger.log`). **Unit:** B/s (`Bps`) — derivative of cumulative bytes.\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.wiredTiger.log.log bytes written`\n\n---\n\n### How to read\n\n- Single line · sustained high rate → heavy write workload\n\n---\n\n### Troubleshooting\n\n- Compare with **Checkpoint Write Throughput** (journal vs checkpoint balance)\n- Sustained high values → storage may be a bottleneck\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)\n- [WiredTiger storage engine](https://www.mongodb.com/docs/manual/core/wiredtiger/)", + "description": "### Journal bytes written\n\nRate of WAL/journal bytes written by WiredTiger (`wiredTiger.log`). **Unit:** B/s (`Bps`) \u2014 derivative of cumulative bytes.\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.wiredTiger.log.log bytes written`\n\n---\n\n### How to read\n\n- Single line \u00b7 sustained high rate \u2192 heavy write workload\n\n---\n\n### Troubleshooting\n\n- Compare with **Checkpoint Write Throughput** (journal vs checkpoint balance)\n- Sustained high values \u2192 storage may be a bottleneck\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)\n- [WiredTiger storage engine](https://www.mongodb.com/docs/manual/core/wiredtiger/)", "fieldConfig": { "defaults": { "color": { @@ -1849,20 +1477,7 @@ }, "unit": "Bps" }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "serverStatus.wiredTiger.log.log bytes written" - }, - "properties": [ - { - "id": "displayName", - "value": "Journal bytes written" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 6, @@ -1895,7 +1510,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.log.log bytes written\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.log.log bytes written\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _field: if r._field == \"serverStatus.wiredTiger.log.log bytes written\" then \"Journal bytes written\" else r._field }))\r\n", "refId": "A" } ], @@ -1903,7 +1518,7 @@ "type": "timeseries" }, { - "collapsed": false, + "collapsed": true, "gridPos": { "h": 1, "w": 24, @@ -1920,7 +1535,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### mongod memory (RSS / virtual)\n\n**mem.resident** (physical RAM) and **mem.virtual** (virtual address space) used by the mongod process.\n\n**Unit:** IEC bytes on chart (auto-scaled GiB/MiB). Source fields are **MiB** in `serverStatus.mem` (`db.serverStatus().mem`); query multiplies by 1024² for display.\n\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.mem.resident` · `serverStatus.mem.virtual`\n\n---\n\n### How to read\n\n- Two lines: resident vs virtual\n- Virtual ≫ resident is normal (mapped files)\n- Sudden resident drop → restart · **Server Uptime**\n- Tooltip/axis values are auto-scaled; divide by 1024² or compare trends, not raw shell integers\n\n---\n\n### Troubleshooting\n\n- Resident near system RAM → swap risk · **Swap Memory**, **Page Faults**\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)", + "description": "### mongod memory (RSS / virtual)\n\n**mem.resident** (physical RAM) and **mem.virtual** (virtual address space) used by the mongod process.\n\n**Unit:** IEC bytes on chart (auto-scaled GiB/MiB). Source fields are **MiB** in `serverStatus.mem` (`db.serverStatus().mem`); query multiplies by 1024\u00b2 for display.\n\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.mem.resident` \u00b7 `serverStatus.mem.virtual`\n\n---\n\n### How to read\n\n- Two lines: resident vs virtual\n- Virtual \u226b resident is normal (mapped files)\n- Sudden resident drop \u2192 restart \u00b7 **Server Uptime**\n- Tooltip/axis values are auto-scaled; divide by 1024\u00b2 or compare trends, not raw shell integers\n\n---\n\n### Troubleshooting\n\n- Resident near system RAM \u2192 swap risk \u00b7 **Swap Memory**, **Page Faults**\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)", "fieldConfig": { "defaults": { "color": { @@ -1974,32 +1589,7 @@ }, "unit": "bytes" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*mem\\.resident$" - }, - "properties": [ - { - "id": "displayName", - "value": "Resident (RSS)" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*mem\\.virtual$" - }, - "properties": [ - { - "id": "displayName", - "value": "Virtual memory" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 7, @@ -2032,7 +1622,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.mem.virtual\" or r[\"_field\"] == \"serverStatus.mem.resident\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) * 1024.0 * 1024.0 }))\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.mem.virtual\" or r[\"_field\"] == \"serverStatus.mem.resident\")\r\n |> drop(columns: [\"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _field: if r._field == \"serverStatus.mem.virtual\" then \"Virtual memory\" else if r._field == \"serverStatus.mem.resident\" then \"Resident (RSS)\" else r._field }))\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) * 1024.0 * 1024.0 }))\r\n", "refId": "A" } ], @@ -2044,7 +1634,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Network requests\n\nRate of **network.numRequests** — distinct client requests received by mongod.\n\nUse with **Network In / Out** to relate request volume to logical byte throughput.\n\n**Unit:** requests/sec (`reqps`)\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.network.numRequests`\n\n---\n\n### How to read\n\n- Single line · request arrival rate\n- Not the same as **Operations** (opcounters) — one request can trigger multiple ops\n\n---\n\n### Troubleshooting\n\n- Sudden drop → client/network issue or load shift\n- Spike without matching **Operations** → lightweight commands (heartbeats, metadata)\n- High rate + high **Network In / Out** bytesOut → large responses · check projections/limits\n\n---\n\n### Docs\n\n- [serverStatus — network](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.network)", + "description": "### Network requests\n\nRate of **network.numRequests** \u2014 distinct client requests received by mongod.\n\nUse with **Network In / Out** to relate request volume to logical byte throughput.\n\n**Unit:** requests/sec (`reqps`)\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.network.numRequests`\n\n---\n\n### How to read\n\n- Single line \u00b7 request arrival rate\n- Not the same as **Operations** (opcounters) \u2014 one request can trigger multiple ops\n\n---\n\n### Troubleshooting\n\n- Sudden drop \u2192 client/network issue or load shift\n- Spike without matching **Operations** \u2192 lightweight commands (heartbeats, metadata)\n- High rate + high **Network In / Out** bytesOut \u2192 large responses \u00b7 check projections/limits\n\n---\n\n### Docs\n\n- [serverStatus \u2014 network](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.network)", "fieldConfig": { "defaults": { "color": { @@ -2098,20 +1688,7 @@ }, "unit": "reqps" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*numRequests$" - }, - "properties": [ - { - "id": "displayName", - "value": "Network requests (/sec)" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 5, @@ -2144,7 +1721,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.network.numRequests\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.network.numRequests\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.network.numRequests\" then \"Network requests (/sec)\" else r._field\r\n }))", "refId": "A" } ], @@ -2156,7 +1733,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Connections\n\n**current**, **active**, and **available** connection counts.\n\n**Unit:** count (gauge)\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.connections.current` · `serverStatus.connections.active` · `serverStatus.connections.available`\n\n---\n\n### How to read\n\n- **Current** — all open connections (incl. idle)\n- **Active** — connections running operations\n- **Available** — unused connection slots\n\n---\n\n### Troubleshooting\n\n- **Current** near `maxIncomingConnections` (65536) → pool exhaustion\n- **Active** ≪ **Current** → idle connections · check app pool settings\n- For connection **churn**, see **Connections Created**\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)", + "description": "### Connections\n\n**current**, **active**, and **available** connection counts.\n\n**Unit:** count (gauge)\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.connections.current` \u00b7 `serverStatus.connections.active` \u00b7 `serverStatus.connections.available`\n\n---\n\n### How to read\n\n- **Current** \u2014 all open connections (incl. idle)\n- **Active** \u2014 connections running operations\n- **Available** \u2014 unused connection slots\n\n---\n\n### Troubleshooting\n\n- **Current** near `maxIncomingConnections` (65536) \u2192 pool exhaustion\n- **Active** \u226a **Current** \u2192 idle connections \u00b7 check app pool settings\n- For connection **churn**, see **Connections Created**\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)", "fieldConfig": { "defaults": { "color": { @@ -2211,42 +1788,6 @@ "unit": "short" }, "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*connections\\.current$" - }, - "properties": [ - { - "id": "displayName", - "value": "Current connections" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*connections\\.active$" - }, - "properties": [ - { - "id": "displayName", - "value": "Active connections" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*connections\\.available$" - }, - "properties": [ - { - "id": "displayName", - "value": "Available connections" - } - ] - }, { "__systemRef": "hideSeriesFrom", "matcher": { @@ -2305,7 +1846,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.connections.current\" or\r\n r[\"_field\"] == \"serverStatus.connections.active\" or\r\n r[\"_field\"] == \"serverStatus.connections.available\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.connections.current\" or\r\n r[\"_field\"] == \"serverStatus.connections.active\" or\r\n r[\"_field\"] == \"serverStatus.connections.available\")\r\n |> drop(columns: [\"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.connections.current\" then \"Current connections\" else if r._field == \"serverStatus.connections.active\" then \"Active connections\" else if r._field == \"serverStatus.connections.available\" then \"Available connections\" else r._field\r\n }))", "refId": "A" } ], @@ -2317,7 +1858,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Page faults\n\nRate of `extra_info.page_faults` (cumulative counter since process start). Page faults occur when MongoDB must access data not in physical memory. Per MongoDB docs, limited sporadic faults are normal; sustained high rates often mean the working set exceeds RAM.\n\n**Unit:** page faults/sec (Grafana `ops`)\n\n**Note:** On Windows, hard and soft page faults are both counted; fields in `extra_info` vary by platform.\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.extra_info.page_faults`\n\n---\n\n### How to read\n\n- Single line · spikes → memory pressure or cold data access\n- Flat near zero → working set mostly in memory\n\n---\n\n### Troubleshooting\n\n- Sustained high rate → working set > RAM · correlate with **WiredTiger Cache**, **mongod Memory (RSS / Virtual)**, **Swap Memory**\n- Spikes during scans/index builds → expected · check query patterns if persistent\n- High faults + active swap → add RAM or reduce working set\n\n---\n\n### Docs\n\n- [serverStatus — extra_info](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#extra_info)", + "description": "### Page faults\n\nRate of `extra_info.page_faults` (cumulative counter since process start). Page faults occur when MongoDB must access data not in physical memory. Per MongoDB docs, limited sporadic faults are normal; sustained high rates often mean the working set exceeds RAM.\n\n**Unit:** page faults/sec (Grafana `ops`)\n\n**Note:** On Windows, hard and soft page faults are both counted; fields in `extra_info` vary by platform.\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.extra_info.page_faults`\n\n---\n\n### How to read\n\n- Single line \u00b7 spikes \u2192 memory pressure or cold data access\n- Flat near zero \u2192 working set mostly in memory\n\n---\n\n### Troubleshooting\n\n- Sustained high rate \u2192 working set > RAM \u00b7 correlate with **WiredTiger Cache**, **mongod Memory (RSS / Virtual)**, **Swap Memory**\n- Spikes during scans/index builds \u2192 expected \u00b7 check query patterns if persistent\n- High faults + active swap \u2192 add RAM or reduce working set\n\n---\n\n### Docs\n\n- [serverStatus \u2014 extra_info](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#extra_info)", "fieldConfig": { "defaults": { "color": { @@ -2371,20 +1912,7 @@ }, "unit": "ops" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*page_faults$" - }, - "properties": [ - { - "id": "displayName", - "value": "Page faults (/sec)" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 6, @@ -2417,7 +1945,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.extra_info.page_faults\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.extra_info.page_faults\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.extra_info.page_faults\" then \"Page faults (/sec)\" else r._field\r\n }))", "refId": "A" } ], @@ -2429,7 +1957,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Connections created\n\nRate of new connections (`connections.totalCreated` derivative). Indicates connection **churn** — apps opening/closing connections frequently.\n\n**Unit:** connections/sec (`ops`)\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.connections.totalCreated`\n\n---\n\n### How to read\n\n- Single line · spikes → bursts of new connections\n- Sustained high rate → pool churn · check app connection pool settings\n\n---\n\n### Troubleshooting\n\n- High rate + high **Current** on **Connections** → many short-lived connections\n- Correlate with **Connections** (counts)\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)", + "description": "### Connections created\n\nRate of new connections (`connections.totalCreated` derivative). Indicates connection **churn** \u2014 apps opening/closing connections frequently.\n\n**Unit:** connections/sec (`ops`)\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.connections.totalCreated`\n\n---\n\n### How to read\n\n- Single line \u00b7 spikes \u2192 bursts of new connections\n- Sustained high rate \u2192 pool churn \u00b7 check app connection pool settings\n\n---\n\n### Troubleshooting\n\n- High rate + high **Current** on **Connections** \u2192 many short-lived connections\n- Correlate with **Connections** (counts)\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)", "fieldConfig": { "defaults": { "color": { @@ -2482,20 +2010,7 @@ }, "unit": "ops" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*connections\\.totalCreated$" - }, - "properties": [ - { - "id": "displayName", - "value": "Connections created (/sec)" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 6, @@ -2528,7 +2043,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.connections.totalCreated\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.connections.totalCreated\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.connections.totalCreated\" then \"Connections created (/sec)\" else r._field\r\n }))", "refId": "A" } ], @@ -2540,7 +2055,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Average operation latency\n\nRolling average latency per operation type, derived from cumulative `opLatencies` counters.\n\n**Formula:** `derivative(*.latency) / derivative(*.ops)` → **µs per operation** (same as Δlatency / Δops over each window).\n\n**Unit:** microseconds (`µs`)\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.opLatencies.reads.latency` · `serverStatus.opLatencies.reads.ops`\n- `serverStatus.opLatencies.writes.latency` · `serverStatus.opLatencies.writes.ops`\n- `serverStatus.opLatencies.commands.latency` · `serverStatus.opLatencies.commands.ops`\n\n---\n\n### How to read\n\n- Three lines: read, write, command average latency\n- Spikes → slower operations in that class\n- Flat at **0** → no ops in window (idle), not zero latency\n\n---\n\n### Troubleshooting\n\n- Spikes + low **WiredTiger Tickets** → concurrency contention\n- Spikes + disk I/O → storage bottleneck · **Page Faults**, **Disk I/O**\n- Latency up, ops down → **Operations Latencies Op**, **Operations**\n- Commands line often highest → includes admin/aggregate work\n\n---\n\n### Docs\n\n- [serverStatus — opLatencies](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.opLatencies)", + "description": "### Average operation latency\n\nRolling average latency per operation type, derived from cumulative `opLatencies` counters.\n\n**Formula:** `derivative(*.latency) / derivative(*.ops)` \u2192 **\u00b5s per operation** (same as \u0394latency / \u0394ops over each window).\n\n**Unit:** microseconds (`\u00b5s`)\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.opLatencies.reads.latency` \u00b7 `serverStatus.opLatencies.reads.ops`\n- `serverStatus.opLatencies.writes.latency` \u00b7 `serverStatus.opLatencies.writes.ops`\n- `serverStatus.opLatencies.commands.latency` \u00b7 `serverStatus.opLatencies.commands.ops`\n\n---\n\n### How to read\n\n- Three lines: read, write, command average latency\n- Spikes \u2192 slower operations in that class\n- Flat at **0** \u2192 no ops in window (idle), not zero latency\n\n---\n\n### Troubleshooting\n\n- Spikes + low **WiredTiger Tickets** \u2192 concurrency contention\n- Spikes + disk I/O \u2192 storage bottleneck \u00b7 **Page Faults**, **Disk I/O**\n- Latency up, ops down \u2192 **Operations Latencies Op**, **Operations**\n- Commands line often highest \u2192 includes admin/aggregate work\n\n---\n\n### Docs\n\n- [serverStatus \u2014 opLatencies](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.opLatencies)", "fieldConfig": { "defaults": { "color": { @@ -2592,46 +2107,9 @@ } ] }, - "unit": "µs" + "unit": "\u00b5s" }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "reads_avg_us" - }, - "properties": [ - { - "id": "displayName", - "value": "Read latency (avg)" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "writes_avg_us" - }, - "properties": [ - { - "id": "displayName", - "value": "Write latency (avg)" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "commands_avg_us" - }, - "properties": [ - { - "id": "displayName", - "value": "Command latency (avg)" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 10, @@ -2664,7 +2142,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.opLatencies.reads.latency\" or\r\n r[\"_field\"] == \"serverStatus.opLatencies.reads.ops\" or\r\n r[\"_field\"] == \"serverStatus.opLatencies.writes.latency\" or\r\n r[\"_field\"] == \"serverStatus.opLatencies.writes.ops\" or\r\n r[\"_field\"] == \"serverStatus.opLatencies.commands.latency\" or\r\n r[\"_field\"] == \"serverStatus.opLatencies.commands.ops\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> pivot(rowKey: [\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n reads_avg_us: if float(v: r[\"serverStatus.opLatencies.reads.ops\"]) > 0.0\r\n then float(v: r[\"serverStatus.opLatencies.reads.latency\"]) / float(v: r[\"serverStatus.opLatencies.reads.ops\"])\r\n else 0.0,\r\n writes_avg_us: if float(v: r[\"serverStatus.opLatencies.writes.ops\"]) > 0.0\r\n then float(v: r[\"serverStatus.opLatencies.writes.latency\"]) / float(v: r[\"serverStatus.opLatencies.writes.ops\"])\r\n else 0.0,\r\n commands_avg_us: if float(v: r[\"serverStatus.opLatencies.commands.ops\"]) > 0.0\r\n then float(v: r[\"serverStatus.opLatencies.commands.latency\"]) / float(v: r[\"serverStatus.opLatencies.commands.ops\"])\r\n else 0.0,\r\n }))", + "query": "wide = from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.opLatencies.reads.latency\" or\r\n r[\"_field\"] == \"serverStatus.opLatencies.reads.ops\" or\r\n r[\"_field\"] == \"serverStatus.opLatencies.writes.latency\" or\r\n r[\"_field\"] == \"serverStatus.opLatencies.writes.ops\" or\r\n r[\"_field\"] == \"serverStatus.opLatencies.commands.latency\" or\r\n r[\"_field\"] == \"serverStatus.opLatencies.commands.ops\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> pivot(rowKey: [\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n hostname: r.hostname,\r\n read: if float(v: r[\"serverStatus.opLatencies.reads.ops\"]) > 0.0\r\n then float(v: r[\"serverStatus.opLatencies.reads.latency\"]) / float(v: r[\"serverStatus.opLatencies.reads.ops\"])\r\n else 0.0,\r\n write: if float(v: r[\"serverStatus.opLatencies.writes.ops\"]) > 0.0\r\n then float(v: r[\"serverStatus.opLatencies.writes.latency\"]) / float(v: r[\"serverStatus.opLatencies.writes.ops\"])\r\n else 0.0,\r\n command: if float(v: r[\"serverStatus.opLatencies.commands.ops\"]) > 0.0\r\n then float(v: r[\"serverStatus.opLatencies.commands.latency\"]) / float(v: r[\"serverStatus.opLatencies.commands.ops\"])\r\n else 0.0,\r\n }))\r\n\r\nunion(tables: [\r\n wide |> map(fn: (r) => ({ _time: r._time, hostname: r.hostname, _field: \"Read latency (avg)\", _value: r.read })),\r\n wide |> map(fn: (r) => ({ _time: r._time, hostname: r.hostname, _field: \"Write latency (avg)\", _value: r.write })),\r\n wide |> map(fn: (r) => ({ _time: r._time, hostname: r.hostname, _field: \"Command latency (avg)\", _value: r.command })),\r\n])\r\n |> group(columns: [\"_field\", \"hostname\"])\r\n", "refId": "A" } ], @@ -2676,7 +2154,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Cursors\n\n**Open cursors** from **`metrics.cursor.open.*`** (current count, gauge). **`metrics.cursor.timedOut`** is a **cumulative counter** — shown here as **timeouts per second** (derivative).\n\n**Unit:** count (`short`) for open · timeouts/sec (`ops`) for timed out rate\n\n---\n\n### Metrics\n\n**Fields (open — gauge, `last`):**\n\n- `serverStatus.metrics.cursor.open.total`\n- `serverStatus.metrics.cursor.open.noTimeout`\n- `serverStatus.metrics.cursor.open.pinned`\n\n**Field (timeouts — counter, `derivative(1s)`):**\n\n- `serverStatus.metrics.cursor.timedOut`\n\n---\n\n### How to read\n\n- **Open total** — cursors mongod holds for clients; usually small\n- **Open (no timeout)** / **Open (pinned)** — subsets of total\n- **Timed out (/sec)** — idle cursors killed (default cursor timeout ~10 min)\n- **stepAfter** on open lines reflects discrete count changes\n\n---\n\n### Troubleshooting\n\n- **Open total** rising → possible cursor leak (app not closing/exhausting cursors)\n- **Timed out (/sec)** rising → clients abandoning cursors · correlate with **Operations** (getmore/query)\n- Tailable / pinned cursors can keep **open** elevated without a leak\n\n---\n\n### Docs\n\n- [serverStatus — metrics.cursor](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.metrics.cursor)", + "description": "### Cursors\n\n**Open cursors** from **`metrics.cursor.open.*`** (current count, gauge). **`metrics.cursor.timedOut`** is a **cumulative counter** \u2014 shown here as **timeouts per second** (derivative).\n\n**Unit:** count (`short`) for open \u00b7 timeouts/sec (`ops`) for timed out rate\n\n---\n\n### Metrics\n\n**Fields (open \u2014 gauge, `last`):**\n\n- `serverStatus.metrics.cursor.open.total`\n- `serverStatus.metrics.cursor.open.noTimeout`\n- `serverStatus.metrics.cursor.open.pinned`\n\n**Field (timeouts \u2014 counter, `derivative(1s)`):**\n\n- `serverStatus.metrics.cursor.timedOut`\n\n---\n\n### How to read\n\n- **Open total** \u2014 cursors mongod holds for clients; usually small\n- **Open (no timeout)** / **Open (pinned)** \u2014 subsets of total\n- **Timed out (/sec)** \u2014 idle cursors killed (default cursor timeout ~10 min)\n- **stepAfter** on open lines reflects discrete count changes\n\n---\n\n### Troubleshooting\n\n- **Open total** rising \u2192 possible cursor leak (app not closing/exhausting cursors)\n- **Timed out (/sec)** rising \u2192 clients abandoning cursors \u00b7 correlate with **Operations** (getmore/query)\n- Tailable / pinned cursors can keep **open** elevated without a leak\n\n---\n\n### Docs\n\n- [serverStatus \u2014 metrics.cursor](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.metrics.cursor)", "fieldConfig": { "defaults": { "color": { @@ -2734,14 +2212,10 @@ "overrides": [ { "matcher": { - "id": "byRegexp", - "options": ".*\\.open\\.total$" + "id": "byName", + "options": "Open total" }, "properties": [ - { - "id": "displayName", - "value": "Open total" - }, { "id": "unit", "value": "short" @@ -2758,14 +2232,10 @@ }, { "matcher": { - "id": "byRegexp", - "options": ".*\\.open\\.noTimeout$" + "id": "byName", + "options": "Open (no timeout)" }, "properties": [ - { - "id": "displayName", - "value": "Open (no timeout)" - }, { "id": "unit", "value": "short" @@ -2782,14 +2252,10 @@ }, { "matcher": { - "id": "byRegexp", - "options": ".*\\.open\\.pinned$" + "id": "byName", + "options": "Open (pinned)" }, "properties": [ - { - "id": "displayName", - "value": "Open (pinned)" - }, { "id": "unit", "value": "short" @@ -2806,14 +2272,10 @@ }, { "matcher": { - "id": "byRegexp", - "options": ".*\\.timedOut$" + "id": "byName", + "options": "Timed out (/sec)" }, "properties": [ - { - "id": "displayName", - "value": "Timed out (/sec)" - }, { "id": "unit", "value": "ops" @@ -2864,7 +2326,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.metrics.cursor.open.total\" or\r\n r[\"_field\"] == \"serverStatus.metrics.cursor.open.noTimeout\" or\r\n r[\"_field\"] == \"serverStatus.metrics.cursor.open.pinned\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.metrics.cursor.open.total\" or\r\n r[\"_field\"] == \"serverStatus.metrics.cursor.open.noTimeout\" or\r\n r[\"_field\"] == \"serverStatus.metrics.cursor.open.pinned\")\r\n |> drop(columns: [\"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.metrics.cursor.open.total\" then \"Open total\" else if r._field == \"serverStatus.metrics.cursor.open.noTimeout\" then \"Open (no timeout)\" else if r._field == \"serverStatus.metrics.cursor.open.pinned\" then \"Open (pinned)\" else r._field\r\n }))", "refId": "A" }, { @@ -2873,7 +2335,7 @@ "uid": "P3C6603E967DC8568" }, "hide": false, - "query": "from(bucket: v.defaultBucket)\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.metrics.cursor.timedOut\")\n |> drop(columns: [\"hostname\", \"version\"])\n |> derivative(unit: 1s, nonNegative: true)\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", + "query": "from(bucket: v.defaultBucket)\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.metrics.cursor.timedOut\")\n |> drop(columns: [\"version\"])\n |> derivative(unit: 1s, nonNegative: true)\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> map(fn: (r) => ({ r with\n _value: float(v: r._value),\n _field: if r._field == \"serverStatus.metrics.cursor.timedOut\" then \"Timed out (/sec)\" else r._field\n }))", "refId": "B" } ], @@ -2885,7 +2347,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Admin Commands (metrics.commands)\n\nPer-second rate of selected **`serverStatus.metrics.commands.*.total`** counters — admin/catalog-style commands tracked individually by MongoDB.\n\n**Not** the same as **Operations** (`opcounters.command`), which rolls up non-opcounter command traffic (**including aggregate**); this panel tracks specific `metrics.commands.*` counters individually.\n\n**Unit:** commands per second (`ops`)\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.metrics.commands.createIndexes.total`\n- `serverStatus.metrics.commands.dataSize.total`\n- `serverStatus.metrics.commands.dropIndexes.total`\n- `serverStatus.metrics.commands.listCollections.total`\n- `serverStatus.metrics.commands.listDatabases.total`\n- `serverStatus.metrics.commands.listIndexes.total`\n\n**Query:** `derivative(..., unit: 1s, nonNegative: true)` → cmds/sec\n\n---\n\n### How to read\n\n- One line per command counter above\n- Spikes on **listCollections** / **listDatabases** / **listIndexes** → metadata/catalog traffic (drivers, tools, monitoring)\n- Spikes on **createIndexes** / **dropIndexes** → schema/index maintenance\n- **dataSize** → `dataSize` command usage\n\n---\n\n### Troubleshooting\n\n- Sustained high list* rates → chatty clients or health checks · compare with **Network Requests**\n- Index build bursts → **createIndexes** + **CPU Usage** + **Disk I/O**\n- For overall op mix, use **Operations**; for document volume, **Documents**\n\n---\n\n### Docs\n\n- [serverStatus — metrics.commands](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.metrics.commands)", + "description": "### Admin Commands (metrics.commands)\n\nPer-second rate of selected **`serverStatus.metrics.commands.*.total`** counters \u2014 admin/catalog-style commands tracked individually by MongoDB.\n\n**Not** the same as **Operations** (`opcounters.command`), which rolls up non-opcounter command traffic (**including aggregate**); this panel tracks specific `metrics.commands.*` counters individually.\n\n**Unit:** commands per second (`ops`)\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.metrics.commands.createIndexes.total`\n- `serverStatus.metrics.commands.dataSize.total`\n- `serverStatus.metrics.commands.dropIndexes.total`\n- `serverStatus.metrics.commands.listCollections.total`\n- `serverStatus.metrics.commands.listDatabases.total`\n- `serverStatus.metrics.commands.listIndexes.total`\n\n**Query:** `derivative(..., unit: 1s, nonNegative: true)` \u2192 cmds/sec\n\n---\n\n### How to read\n\n- One line per command counter above\n- Spikes on **listCollections** / **listDatabases** / **listIndexes** \u2192 metadata/catalog traffic (drivers, tools, monitoring)\n- Spikes on **createIndexes** / **dropIndexes** \u2192 schema/index maintenance\n- **dataSize** \u2192 `dataSize` command usage\n\n---\n\n### Troubleshooting\n\n- Sustained high list* rates \u2192 chatty clients or health checks \u00b7 compare with **Network Requests**\n- Index build bursts \u2192 **createIndexes** + **CPU Usage** + **Disk I/O**\n- For overall op mix, use **Operations**; for document volume, **Documents**\n\n---\n\n### Docs\n\n- [serverStatus \u2014 metrics.commands](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.metrics.commands)", "fieldConfig": { "defaults": { "color": { @@ -2938,80 +2400,7 @@ }, "unit": "ops" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*\\.createIndexes\\.total$" - }, - "properties": [ - { - "id": "displayName", - "value": "createIndexes" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*\\.dropIndexes\\.total$" - }, - "properties": [ - { - "id": "displayName", - "value": "dropIndexes" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*\\.listCollections\\.total$" - }, - "properties": [ - { - "id": "displayName", - "value": "listCollections" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*\\.listDatabases\\.total$" - }, - "properties": [ - { - "id": "displayName", - "value": "listDatabases" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*\\.listIndexes\\.total$" - }, - "properties": [ - { - "id": "displayName", - "value": "listIndexes" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*\\.dataSize\\.total$" - }, - "properties": [ - { - "id": "displayName", - "value": "dataSize" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 8, @@ -3044,7 +2433,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.metrics.commands.createIndexes.total\" or\r\n r[\"_field\"] == \"serverStatus.metrics.commands.dataSize.total\" or\r\n r[\"_field\"] == \"serverStatus.metrics.commands.dropIndexes.total\" or\r\n r[\"_field\"] == \"serverStatus.metrics.commands.listCollections.total\" or\r\n r[\"_field\"] == \"serverStatus.metrics.commands.listDatabases.total\" or\r\n r[\"_field\"] == \"serverStatus.metrics.commands.listIndexes.total\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.metrics.commands.createIndexes.total\" or\r\n r[\"_field\"] == \"serverStatus.metrics.commands.dataSize.total\" or\r\n r[\"_field\"] == \"serverStatus.metrics.commands.dropIndexes.total\" or\r\n r[\"_field\"] == \"serverStatus.metrics.commands.listCollections.total\" or\r\n r[\"_field\"] == \"serverStatus.metrics.commands.listDatabases.total\" or\r\n r[\"_field\"] == \"serverStatus.metrics.commands.listIndexes.total\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.metrics.commands.createIndexes.total\" then \"createIndexes\" else if r._field == \"serverStatus.metrics.commands.dataSize.total\" then \"dataSize\" else if r._field == \"serverStatus.metrics.commands.dropIndexes.total\" then \"dropIndexes\" else if r._field == \"serverStatus.metrics.commands.listCollections.total\" then \"listCollections\" else if r._field == \"serverStatus.metrics.commands.listDatabases.total\" then \"listDatabases\" else if r._field == \"serverStatus.metrics.commands.listIndexes.total\" then \"listIndexes\" else r._field\r\n }))", "refId": "A" } ], @@ -3056,7 +2445,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Replica members lag\n\nWall-clock replication lag per member:\n\n**Formula:** `(serverStatus.localTime − members.N.lastAppliedWallTime) / 1000` → **seconds**\n\nBoth timestamps are **milliseconds** in FTDC; the query converts the delta to seconds.\n\n**Unit:** seconds (`s`)\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.localTime`\n- `replSetGetStatus.members.0.lastAppliedWallTime` … `replSetGetStatus.members.4.lastAppliedWallTime`\n\nMember **N** is the array index in `replSetGetStatus.members` (not necessarily hostname order).\n\n---\n\n### How to read\n\n- One line per member index (0–4)\n- **~0** → caught up (primary or healthy secondary)\n- Rising trend on a secondary → replication falling behind\n- **Gap** (not 0) for missing member slots — no `lastAppliedWallTime` in FTDC for that index\n\n---\n\n### Troubleshooting\n\n- Secondary lag rising → disk I/O, CPU, network on that node · **Opcounters Replica**, **Disk I/O**\n- Lag spikes during checkpoints → I/O contention\n- Primary throttling → **FlowControl isLagged** · **Replica Members Ping**\n- Member down → **Replica Members Health**\n\n---\n\n### Docs\n\n- [replSetGetStatus](https://www.mongodb.com/docs/manual/reference/command/replSetGetStatus/)\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)", + "description": "### Replica members lag\n\nWall-clock replication lag per member:\n\n**Formula:** `(serverStatus.localTime \u2212 members.N.lastAppliedWallTime) / 1000` \u2192 **seconds**\n\nBoth timestamps are **milliseconds** in FTDC; the query converts the delta to seconds.\n\n**Unit:** seconds (`s`)\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.localTime`\n- `replSetGetStatus.members.0.lastAppliedWallTime` \u2026 `replSetGetStatus.members.4.lastAppliedWallTime`\n\nMember **N** is the array index in `replSetGetStatus.members` (not necessarily hostname order).\n\n---\n\n### How to read\n\n- One line per member index (0\u20134)\n- **~0** \u2192 caught up (primary or healthy secondary)\n- Rising trend on a secondary \u2192 replication falling behind\n- **Gap** (not 0) for missing member slots \u2014 no `lastAppliedWallTime` in FTDC for that index\n\n---\n\n### Troubleshooting\n\n- Secondary lag rising \u2192 disk I/O, CPU, network on that node \u00b7 **Opcounters Replica**, **Disk I/O**\n- Lag spikes during checkpoints \u2192 I/O contention\n- Primary throttling \u2192 **FlowControl isLagged** \u00b7 **Replica Members Ping**\n- Member down \u2192 **Replica Members Health**\n\n---\n\n### Docs\n\n- [replSetGetStatus](https://www.mongodb.com/docs/manual/reference/command/replSetGetStatus/)\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)", "fieldConfig": { "defaults": { "color": { @@ -3110,68 +2499,7 @@ }, "unit": "s" }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "member_0" - }, - "properties": [ - { - "id": "displayName", - "value": "Member 0 lag" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "member_1" - }, - "properties": [ - { - "id": "displayName", - "value": "Member 1 lag" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "member_2" - }, - "properties": [ - { - "id": "displayName", - "value": "Member 2 lag" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "member_3" - }, - "properties": [ - { - "id": "displayName", - "value": "Member 3 lag" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "member_4" - }, - "properties": [ - { - "id": "displayName", - "value": "Member 4 lag" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 7, @@ -3204,7 +2532,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] =~ /replSetGetStatus\\.members\\.\\d+\\.lastAppliedWallTime|serverStatus\\.localTime/)\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> pivot(rowKey: [\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n member_0: if exists r[\"replSetGetStatus.members.0.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.0.lastAppliedWallTime\"])) / 1000.0 else float(v: \"NaN\"),\r\n member_1: if exists r[\"replSetGetStatus.members.1.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.1.lastAppliedWallTime\"])) / 1000.0 else float(v: \"NaN\"),\r\n member_2: if exists r[\"replSetGetStatus.members.2.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.2.lastAppliedWallTime\"])) / 1000.0 else float(v: \"NaN\"),\r\n member_3: if exists r[\"replSetGetStatus.members.3.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.3.lastAppliedWallTime\"])) / 1000.0 else float(v: \"NaN\"),\r\n member_4: if exists r[\"replSetGetStatus.members.4.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.4.lastAppliedWallTime\"])) / 1000.0 else float(v: \"NaN\"),\r\n }))", + "query": "wide = from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] =~ /replSetGetStatus\\.members\\.\\d+\\.lastAppliedWallTime|serverStatus\\.localTime/)\r\n |> drop(columns: [\"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> pivot(rowKey: [\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n hostname: r.hostname,\r\n member_0: if exists r[\"replSetGetStatus.members.0.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.0.lastAppliedWallTime\"])) / 1000.0 else float(v: \"NaN\"),\r\n member_1: if exists r[\"replSetGetStatus.members.1.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.1.lastAppliedWallTime\"])) / 1000.0 else float(v: \"NaN\"),\r\n member_2: if exists r[\"replSetGetStatus.members.2.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.2.lastAppliedWallTime\"])) / 1000.0 else float(v: \"NaN\"),\r\n member_3: if exists r[\"replSetGetStatus.members.3.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.3.lastAppliedWallTime\"])) / 1000.0 else float(v: \"NaN\"),\r\n member_4: if exists r[\"replSetGetStatus.members.4.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.4.lastAppliedWallTime\"])) / 1000.0 else float(v: \"NaN\"),\r\n }))\r\n\r\nunion(tables: [\r\n wide |> map(fn: (r) => ({ _time: r._time, hostname: r.hostname, _field: \"Member 0 lag\", _value: r.member_0 })),\r\n wide |> map(fn: (r) => ({ _time: r._time, hostname: r.hostname, _field: \"Member 1 lag\", _value: r.member_1 })),\r\n wide |> map(fn: (r) => ({ _time: r._time, hostname: r.hostname, _field: \"Member 2 lag\", _value: r.member_2 })),\r\n wide |> map(fn: (r) => ({ _time: r._time, hostname: r.hostname, _field: \"Member 3 lag\", _value: r.member_3 })),\r\n wide |> map(fn: (r) => ({ _time: r._time, hostname: r.hostname, _field: \"Member 4 lag\", _value: r.member_4 })),\r\n])\r\n |> group(columns: [\"_field\", \"hostname\"])\r\n", "refId": "A" } ], @@ -3216,7 +2544,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Replica members ping\n\n**`replSetGetStatus.members.N.pingMs`** — round-trip time in **milliseconds** from **this FTDC host** to each replica set member (repl heartbeat).\n\n**Unit:** milliseconds (`ms`)\n\n**Note:** **N** is the member array index in `replSetGetStatus.members`, not hostname order. Use the **hostname** template variable to pick which mongod’s view of the set you are viewing.\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `replSetGetStatus.members.0.pingMs`\n- `replSetGetStatus.members.1.pingMs`\n- `replSetGetStatus.members.2.pingMs`\n- `replSetGetStatus.members.3.pingMs`\n- `replSetGetStatus.members.4.pingMs`\n\n---\n\n### How to read\n\n- One line per member index (**Member 0 ping** … **Member 4 ping**)\n- **stepAfter** reflects discrete heartbeat samples\n- Low stable values → healthy paths between members\n- Spikes → latency or congestion between nodes\n\n---\n\n### Troubleshooting\n\n- Ping rising with **Replica Members Lag** → network may limit replication\n- Ping OK but lag high → disk/CPU on secondaries, not network\n- Correlate with **TCP** panels (RetransSegs, InSegs/OutSegs) and **Replica Members Health**\n\n---\n\n### Docs\n\n- [replSetGetStatus](https://www.mongodb.com/docs/manual/reference/command/replSetGetStatus/)", + "description": "### Replica members ping\n\n**`replSetGetStatus.members.N.pingMs`** \u2014 round-trip time in **milliseconds** from **this FTDC host** to each replica set member (repl heartbeat).\n\n**Unit:** milliseconds (`ms`)\n\n**Note:** **N** is the member array index in `replSetGetStatus.members`, not hostname order. Use the **hostname** template variable to pick which mongod\u2019s view of the set you are viewing.\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `replSetGetStatus.members.0.pingMs`\n- `replSetGetStatus.members.1.pingMs`\n- `replSetGetStatus.members.2.pingMs`\n- `replSetGetStatus.members.3.pingMs`\n- `replSetGetStatus.members.4.pingMs`\n\n---\n\n### How to read\n\n- One line per member index (**Member 0 ping** \u2026 **Member 4 ping**)\n- **stepAfter** reflects discrete heartbeat samples\n- Low stable values \u2192 healthy paths between members\n- Spikes \u2192 latency or congestion between nodes\n\n---\n\n### Troubleshooting\n\n- Ping rising with **Replica Members Lag** \u2192 network may limit replication\n- Ping OK but lag high \u2192 disk/CPU on secondaries, not network\n- Correlate with **TCP** panels (RetransSegs, InSegs/OutSegs) and **Replica Members Health**\n\n---\n\n### Docs\n\n- [replSetGetStatus](https://www.mongodb.com/docs/manual/reference/command/replSetGetStatus/)", "fieldConfig": { "defaults": { "color": { @@ -3270,68 +2598,7 @@ }, "unit": "ms" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*members\\.0\\.pingMs$" - }, - "properties": [ - { - "id": "displayName", - "value": "Member 0 ping" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*members\\.1\\.pingMs$" - }, - "properties": [ - { - "id": "displayName", - "value": "Member 1 ping" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*members\\.2\\.pingMs$" - }, - "properties": [ - { - "id": "displayName", - "value": "Member 2 ping" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*members\\.3\\.pingMs$" - }, - "properties": [ - { - "id": "displayName", - "value": "Member 3 ping" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*members\\.4\\.pingMs$" - }, - "properties": [ - { - "id": "displayName", - "value": "Member 4 ping" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 6, @@ -3364,7 +2631,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"replSetGetStatus.members.0.pingMs\" or\r\n r[\"_field\"] == \"replSetGetStatus.members.1.pingMs\" or\r\n r[\"_field\"] == \"replSetGetStatus.members.2.pingMs\" or\r\n r[\"_field\"] == \"replSetGetStatus.members.3.pingMs\" or\r\n r[\"_field\"] == \"replSetGetStatus.members.4.pingMs\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"replSetGetStatus.members.0.pingMs\" or\r\n r[\"_field\"] == \"replSetGetStatus.members.1.pingMs\" or\r\n r[\"_field\"] == \"replSetGetStatus.members.2.pingMs\" or\r\n r[\"_field\"] == \"replSetGetStatus.members.3.pingMs\" or\r\n r[\"_field\"] == \"replSetGetStatus.members.4.pingMs\")\r\n |> drop(columns: [\"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"replSetGetStatus.members.0.pingMs\" then \"Member 0 ping\" else if r._field == \"replSetGetStatus.members.1.pingMs\" then \"Member 1 ping\" else if r._field == \"replSetGetStatus.members.2.pingMs\" then \"Member 2 ping\" else if r._field == \"replSetGetStatus.members.3.pingMs\" then \"Member 3 ping\" else if r._field == \"replSetGetStatus.members.4.pingMs\" then \"Member 4 ping\" else r._field\r\n }))", "refId": "A" } ], @@ -3376,7 +2643,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Document metrics (deleted / inserted / returned / updated)\n\nPer-second rate of **`serverStatus.metrics.document`** counters — document-level access and modification (not the same as **Operations** / opcounters: one op can touch many documents).\n\n**Unit:** documents per second (`ops`)\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.metrics.document.deleted`\n- `serverStatus.metrics.document.inserted`\n- `serverStatus.metrics.document.returned`\n- `serverStatus.metrics.document.updated`\n\n**Query:** `derivative(..., unit: 1s, nonNegative: true)` → docs/sec\n\n---\n\n### How to read\n\n- Four lines: delete, insert, return, update **rates**\n- **Returned** high with low **Scanned / Returned Ratio** → efficient reads\n- **Returned** high with high ratio → scans without good targeting · **Query Executor**\n\n---\n\n### Troubleshooting\n\n- High **deleted** → TTL indexes, bulk cleanup, or churn\n- High **returned** + high **Scanned / Returned Ratio** → missing/suboptimal indexes · run `explain()`\n- Compare with **Operations** (op-level) and **Opcounters Replica** on secondaries\n\n---\n\n### Docs\n\n- [serverStatus — metrics.document](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.metrics.document)", + "description": "### Document metrics (deleted / inserted / returned / updated)\n\nPer-second rate of **`serverStatus.metrics.document`** counters \u2014 document-level access and modification (not the same as **Operations** / opcounters: one op can touch many documents).\n\n**Unit:** documents per second (`ops`)\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.metrics.document.deleted`\n- `serverStatus.metrics.document.inserted`\n- `serverStatus.metrics.document.returned`\n- `serverStatus.metrics.document.updated`\n\n**Query:** `derivative(..., unit: 1s, nonNegative: true)` \u2192 docs/sec\n\n---\n\n### How to read\n\n- Four lines: delete, insert, return, update **rates**\n- **Returned** high with low **Scanned / Returned Ratio** \u2192 efficient reads\n- **Returned** high with high ratio \u2192 scans without good targeting \u00b7 **Query Executor**\n\n---\n\n### Troubleshooting\n\n- High **deleted** \u2192 TTL indexes, bulk cleanup, or churn\n- High **returned** + high **Scanned / Returned Ratio** \u2192 missing/suboptimal indexes \u00b7 run `explain()`\n- Compare with **Operations** (op-level) and **Opcounters Replica** on secondaries\n\n---\n\n### Docs\n\n- [serverStatus \u2014 metrics.document](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.metrics.document)", "fieldConfig": { "defaults": { "color": { @@ -3429,56 +2696,7 @@ }, "unit": "ops" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*\\.document\\.deleted$" - }, - "properties": [ - { - "id": "displayName", - "value": "Documents deleted" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*\\.document\\.inserted$" - }, - "properties": [ - { - "id": "displayName", - "value": "Documents inserted" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*\\.document\\.returned$" - }, - "properties": [ - { - "id": "displayName", - "value": "Documents returned" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*\\.document\\.updated$" - }, - "properties": [ - { - "id": "displayName", - "value": "Documents updated" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 12, @@ -3511,7 +2729,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.metrics.document.deleted\" or\r\n r[\"_field\"] == \"serverStatus.metrics.document.inserted\" or\r\n r[\"_field\"] == \"serverStatus.metrics.document.returned\" or\r\n r[\"_field\"] == \"serverStatus.metrics.document.updated\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.metrics.document.deleted\" or\r\n r[\"_field\"] == \"serverStatus.metrics.document.inserted\" or\r\n r[\"_field\"] == \"serverStatus.metrics.document.returned\" or\r\n r[\"_field\"] == \"serverStatus.metrics.document.updated\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.metrics.document.deleted\" then \"Documents deleted\" else if r._field == \"serverStatus.metrics.document.inserted\" then \"Documents inserted\" else if r._field == \"serverStatus.metrics.document.returned\" then \"Documents returned\" else if r._field == \"serverStatus.metrics.document.updated\" then \"Documents updated\" else r._field\r\n }))", "refId": "A" } ], @@ -3523,7 +2741,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Flow control is lagged\n\nBoolean **`flowControl.isLagged`** — whether flow control is **currently** throttling writes on the **primary**.\n\n**Unit:** 0 = off · 1 = on\n\n**Note:** Primary only; secondaries return placeholders. For **time spent** throttled, see **Flow control lagged time**.\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.flowControl.isLagged`\n\n---\n\n### How to read\n\n- **0** → not engaging (even if secondaries lag)\n- **1** → primary write rate limited so secondaries can catch up\n\n---\n\n### Troubleshooting\n\n- **1** sustained → **Replica Members Lag**, **Opcounters Replica**, secondary disk/network\n- Lag without **1** → lag below `flowControlTargetLagSeconds` threshold\n\n---\n\n### Docs\n\n- [serverStatus — flowControl](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.flowControl)", + "description": "### Flow control is lagged\n\nBoolean **`flowControl.isLagged`** \u2014 whether flow control is **currently** throttling writes on the **primary**.\n\n**Unit:** 0 = off \u00b7 1 = on\n\n**Note:** Primary only; secondaries return placeholders. For **time spent** throttled, see **Flow control lagged time**.\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.flowControl.isLagged`\n\n---\n\n### How to read\n\n- **0** \u2192 not engaging (even if secondaries lag)\n- **1** \u2192 primary write rate limited so secondaries can catch up\n\n---\n\n### Troubleshooting\n\n- **1** sustained \u2192 **Replica Members Lag**, **Opcounters Replica**, secondary disk/network\n- Lag without **1** \u2192 lag below `flowControlTargetLagSeconds` threshold\n\n---\n\n### Docs\n\n- [serverStatus \u2014 flowControl](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.flowControl)", "fieldConfig": { "defaults": { "color": { @@ -3594,20 +2812,7 @@ }, "unit": "short" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*isLagged$" - }, - "properties": [ - { - "id": "displayName", - "value": "Flow control engaged" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 6, @@ -3640,7 +2845,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.flowControl.isLagged\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.flowControl.isLagged\")\r\n |> drop(columns: [\"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.flowControl.isLagged\" then \"Flow control engaged\" else r._field\r\n }))", "refId": "A" } ], @@ -3652,7 +2857,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Flow control lagged time\n\n**Fraction of each window** the primary spent with flow control engaged, from **`flowControl.isLaggedTimeMicros`**.\n\n**Formula:** `derivative(isLaggedTimeMicros, 1s) / 1e6` → 0–100%\n\n**Note:** Primary only. For **on/off now**, see **FlowControl isLagged**.\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.flowControl.isLaggedTimeMicros`\n\n---\n\n### How to read\n\n- **0%** → no throttling in that window\n- **100%** → flow control engaged the entire window\n- Spikes → brief throttling bursts\n\n---\n\n### Troubleshooting\n\n- Sustained **>0%** → correlate with **Replica Members Lag**, **FlowControl isLagged**\n- High % + rising lag → secondaries cannot keep up · disk/CPU/network\n\n---\n\n### Docs\n\n- [serverStatus — flowControl](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.flowControl)", + "description": "### Flow control lagged time\n\n**Fraction of each window** the primary spent with flow control engaged, from **`flowControl.isLaggedTimeMicros`**.\n\n**Formula:** `derivative(isLaggedTimeMicros, 1s) / 1e6` \u2192 0\u2013100%\n\n**Note:** Primary only. For **on/off now**, see **FlowControl isLagged**.\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.flowControl.isLaggedTimeMicros`\n\n---\n\n### How to read\n\n- **0%** \u2192 no throttling in that window\n- **100%** \u2192 flow control engaged the entire window\n- Spikes \u2192 brief throttling bursts\n\n---\n\n### Troubleshooting\n\n- Sustained **>0%** \u2192 correlate with **Replica Members Lag**, **FlowControl isLagged**\n- High % + rising lag \u2192 secondaries cannot keep up \u00b7 disk/CPU/network\n\n---\n\n### Docs\n\n- [serverStatus \u2014 flowControl](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.flowControl)", "fieldConfig": { "defaults": { "color": { @@ -3709,20 +2914,7 @@ }, "unit": "percentunit" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*isLaggedTimeMicros$" - }, - "properties": [ - { - "id": "displayName", - "value": "Flow control engaged (% of time)" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 6, @@ -3755,7 +2947,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.flowControl.isLaggedTimeMicros\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) / 1000000.0 }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.flowControl.isLaggedTimeMicros\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _field: if r._field == \"serverStatus.flowControl.isLaggedTimeMicros\" then \"Flow control engaged (% of time)\" else r._field }))\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) / 1000000.0 }))", "refId": "A" } ], @@ -3767,7 +2959,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Oplog storage size\n\nDisk space allocated to the **`local.oplog.rs`** capped collection (`collStats` / `storageStats`). **Oplog storage size** is total allocation on disk; **Oplog free storage** is unused space within that allocation.\n\n**Unit:** bytes (`bytes`)\n\n**Not `dbStats.storageSize`:** FTDC exposes oplog **collStats** here, not per-database storage totals.\n\n---\n\n### Metrics\n\n**Fields:**\n\n`local.oplog.rs.stats.storageStats.storageSize`\n`local.oplog.rs.stats.storageStats.freeStorageSize`\n\nSnapshot gauge — **`aggregateWindow(..., fn: last)`**, no derivative.\n\n---\n\n### How to read\n\n- **Oplog storage size** — grows as the capped oplog fills its on-disk allocation (does not shrink when entries roll off)\n- **Oplog free storage** — headroom within that allocation\n- **Used ≈ storage size − free storage** at each point\n- Flat **storage size** with falling **free storage** → oplog nearing capacity\n\n---\n\n### Troubleshooting\n\n- **Free storage** near zero → oplog cap pressure · replication lag risk on secondaries · **Oplog average entry size**, **Replica Members Lag**\n- Step growth in **storage size** → oplog resize or first-time allocation growth\n- High **storage size** + large **Oplog average entry size** → heavy write payloads · more replication I/O\n- **Arbiter / standalone** may show no or stale series\n\nCorrelate with **Operations (write source)**, **Oplog average entry size**, and **Documents**.\n\n---\n\n### Docs\n\n- [collStats — storageSize / freeStorageSize](https://www.mongodb.com/docs/manual/reference/command/collStats/)\n- [Change oplog size](https://www.mongodb.com/docs/manual/tutorial/change-oplog-size/)", + "description": "### Oplog storage size\n\nDisk space allocated to the **`local.oplog.rs`** capped collection (`collStats` / `storageStats`). **Oplog storage size** is total allocation on disk; **Oplog free storage** is unused space within that allocation.\n\n**Unit:** bytes (`bytes`)\n\n**Not `dbStats.storageSize`:** FTDC exposes oplog **collStats** here, not per-database storage totals.\n\n---\n\n### Metrics\n\n**Fields:**\n\n`local.oplog.rs.stats.storageStats.storageSize`\n`local.oplog.rs.stats.storageStats.freeStorageSize`\n\nSnapshot gauge \u2014 **`aggregateWindow(..., fn: last)`**, no derivative.\n\n---\n\n### How to read\n\n- **Oplog storage size** \u2014 grows as the capped oplog fills its on-disk allocation (does not shrink when entries roll off)\n- **Oplog free storage** \u2014 headroom within that allocation\n- **Used \u2248 storage size \u2212 free storage** at each point\n- Flat **storage size** with falling **free storage** \u2192 oplog nearing capacity\n\n---\n\n### Troubleshooting\n\n- **Free storage** near zero \u2192 oplog cap pressure \u00b7 replication lag risk on secondaries \u00b7 **Oplog average entry size**, **Replica Members Lag**\n- Step growth in **storage size** \u2192 oplog resize or first-time allocation growth\n- High **storage size** + large **Oplog average entry size** \u2192 heavy write payloads \u00b7 more replication I/O\n- **Arbiter / standalone** may show no or stale series\n\nCorrelate with **Operations (write source)**, **Oplog average entry size**, and **Documents**.\n\n---\n\n### Docs\n\n- [collStats \u2014 storageSize / freeStorageSize](https://www.mongodb.com/docs/manual/reference/command/collStats/)\n- [Change oplog size](https://www.mongodb.com/docs/manual/tutorial/change-oplog-size/)", "fieldConfig": { "defaults": { "color": { @@ -3821,32 +3013,7 @@ }, "unit": "bytes" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*\\.storageStats\\.storageSize$" - }, - "properties": [ - { - "id": "displayName", - "value": "Oplog storage size" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*freeStorageSize$" - }, - "properties": [ - { - "id": "displayName", - "value": "Oplog free storage" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 8, @@ -3879,7 +3046,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"local.oplog.rs.stats.storageStats.storageSize\" or\r\n r[\"_field\"] == \"local.oplog.rs.stats.storageStats.freeStorageSize\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"local.oplog.rs.stats.storageStats.storageSize\" or\r\n r[\"_field\"] == \"local.oplog.rs.stats.storageStats.freeStorageSize\")\r\n |> drop(columns: [\"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"local.oplog.rs.stats.storageStats.storageSize\" then \"Oplog storage size\" else if r._field == \"local.oplog.rs.stats.storageStats.freeStorageSize\" then \"Oplog free storage\" else r._field\r\n }))", "refId": "A" } ], @@ -3891,7 +3058,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Oplog average entry size\n\nAverage size in **bytes** of documents in the **`local.oplog.rs`** capped collection (`collStats` / `storageStats.avgObjSize`).\n\n**Unit:** bytes (`bytes`)\n\n**Not `dbStats.avgObjSize`:** FTDC exposes oplog collStats here, not per-database average user document size.\n\n---\n\n### Metrics\n\n**Field:** `local.oplog.rs.stats.storageStats.avgObjSize`\n\nSnapshot gauge — **`aggregateWindow(..., fn: last)`**, no derivative.\n\n---\n\n### How to read\n\n- Single line · **Oplog avg entry size**\n- Rises when writes carry larger payloads (big documents, wide updates, bulk ops)\n- Slow drift over days → changing write patterns or schema\n- Compare with **Oplog storage size** (`local.oplog.rs` `storageSize` / `freeStorageSize`)\n\n---\n\n### Troubleshooting\n\n- Step change + write workload shift → larger oplog entries · more replication network/disk\n- High avg size + lag → secondary apply pressure · check **Replica Members Lag**\n- **Arbiter / standalone** may show no or stale series (no meaningful oplog)\n- For **user collection** document sizing → `db.collection.stats().avgObjSize` (not in this FTDC export)\n\n---\n\n### Docs\n\n- [collStats — avgObjSize](https://www.mongodb.com/docs/manual/reference/command/collStats/)\n- [Change oplog size](https://www.mongodb.com/docs/manual/tutorial/change-oplog-size/)", + "description": "### Oplog average entry size\n\nAverage size in **bytes** of documents in the **`local.oplog.rs`** capped collection (`collStats` / `storageStats.avgObjSize`).\n\n**Unit:** bytes (`bytes`)\n\n**Not `dbStats.avgObjSize`:** FTDC exposes oplog collStats here, not per-database average user document size.\n\n---\n\n### Metrics\n\n**Field:** `local.oplog.rs.stats.storageStats.avgObjSize`\n\nSnapshot gauge \u2014 **`aggregateWindow(..., fn: last)`**, no derivative.\n\n---\n\n### How to read\n\n- Single line \u00b7 **Oplog avg entry size**\n- Rises when writes carry larger payloads (big documents, wide updates, bulk ops)\n- Slow drift over days \u2192 changing write patterns or schema\n- Compare with **Oplog storage size** (`local.oplog.rs` `storageSize` / `freeStorageSize`)\n\n---\n\n### Troubleshooting\n\n- Step change + write workload shift \u2192 larger oplog entries \u00b7 more replication network/disk\n- High avg size + lag \u2192 secondary apply pressure \u00b7 check **Replica Members Lag**\n- **Arbiter / standalone** may show no or stale series (no meaningful oplog)\n- For **user collection** document sizing \u2192 `db.collection.stats().avgObjSize` (not in this FTDC export)\n\n---\n\n### Docs\n\n- [collStats \u2014 avgObjSize](https://www.mongodb.com/docs/manual/reference/command/collStats/)\n- [Change oplog size](https://www.mongodb.com/docs/manual/tutorial/change-oplog-size/)", "fieldConfig": { "defaults": { "color": { @@ -3945,20 +3112,7 @@ }, "unit": "bytes" }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "local.oplog.rs.stats.storageStats.avgObjSize" - }, - "properties": [ - { - "id": "displayName", - "value": "Oplog avg entry size" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 8, @@ -3991,7 +3145,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"local.oplog.rs.stats.storageStats.avgObjSize\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"local.oplog.rs.stats.storageStats.avgObjSize\")\r\n |> drop(columns: [\"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"local.oplog.rs.stats.storageStats.avgObjSize\" then \"Oplog avg entry size\" else if r._field == \"local.oplog.rs.stats.storageStats.avgObjSize\" then \"Oplog avg entry size\" else r._field\r\n }))", "refId": "A" } ], @@ -4003,7 +3157,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Current queue (writers / readers)\n\nOperations **waiting** for the global read or write lock (`globalLock.currentQueue`). Snapshot gauge — not a rate.\n\n**Unit:** count (`short`)\n\n**Note:** On MongoDB 4.2+, these queues are often **0**; use **WiredTiger Tickets** and **Average Operation Latency** for concurrency pressure on modern releases.\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.globalLock.currentQueue.readers` · `serverStatus.globalLock.currentQueue.writers`\n\n---\n\n### How to read\n\n- Two lines: queued readers vs writers\n- Brief spikes can be normal · sustained non-zero → contention\n\n---\n\n### Troubleshooting\n\n- Persistent write queue → **WiredTiger Tickets**, **Average Operation Latency**, **Operations**\n- Persistent read queue → long-running reads blocking others\n- Always zero but slow DB → focus on tickets/latency, not this panel alone\n\n---\n\n### Docs\n\n- [serverStatus — globalLock](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.globalLock)", + "description": "### Current queue (writers / readers)\n\nOperations **waiting** for the global read or write lock (`globalLock.currentQueue`). Snapshot gauge \u2014 not a rate.\n\n**Unit:** count (`short`)\n\n**Note:** On MongoDB 4.2+, these queues are often **0**; use **WiredTiger Tickets** and **Average Operation Latency** for concurrency pressure on modern releases.\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.globalLock.currentQueue.readers` \u00b7 `serverStatus.globalLock.currentQueue.writers`\n\n---\n\n### How to read\n\n- Two lines: queued readers vs writers\n- Brief spikes can be normal \u00b7 sustained non-zero \u2192 contention\n\n---\n\n### Troubleshooting\n\n- Persistent write queue \u2192 **WiredTiger Tickets**, **Average Operation Latency**, **Operations**\n- Persistent read queue \u2192 long-running reads blocking others\n- Always zero but slow DB \u2192 focus on tickets/latency, not this panel alone\n\n---\n\n### Docs\n\n- [serverStatus \u2014 globalLock](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.globalLock)", "fieldConfig": { "defaults": { "color": { @@ -4057,32 +3211,7 @@ }, "unit": "short" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*currentQueue\\.readers$" - }, - "properties": [ - { - "id": "displayName", - "value": "Queued readers" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*currentQueue\\.writers$" - }, - "properties": [ - { - "id": "displayName", - "value": "Queued writers" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 7, @@ -4115,7 +3244,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.globalLock.currentQueue.writers\" or\r\n r[\"_field\"] == \"serverStatus.globalLock.currentQueue.readers\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.globalLock.currentQueue.writers\" or\r\n r[\"_field\"] == \"serverStatus.globalLock.currentQueue.readers\")\r\n |> drop(columns: [\"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.globalLock.currentQueue.writers\" then \"Queued writers\" else if r._field == \"serverStatus.globalLock.currentQueue.readers\" then \"Queued readers\" else r._field\r\n }))", "refId": "A" } ], @@ -4127,7 +3256,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Query executor\n\nPer-second rates from **queryExecutor** counters — how much work the query engine does examining indexes and documents.\n\n**Unit:** operations/sec (`ops`) — keys scanned, docs scanned, and collection-scan events are all counter derivatives.\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.metrics.queryExecutor.scanned` — index keys examined (`totalKeysExamined` in explain())\n- `serverStatus.metrics.queryExecutor.scannedObjects` — documents examined (`totalDocsExamined`)\n- `serverStatus.metrics.queryExecutor.collectionScans.total` — queries that performed a **collection scan** (COLLSCAN)\n\nAll are cumulative counters; query uses **`derivative(unit: 1s, nonNegative: true)`**.\n\n---\n\n### How to read\n\n- **Index keys scanned / sec** — index traversal volume\n- **Documents scanned / sec** — document examination volume (often dominates I/O)\n- **Collection scans / sec** (right axis) — queries **not** using a useful index; should stay near **0** in production\n\n**Keys vs docs:** high docs with low keys can mean wide index scans or many docs per key; high docs with high **Collection scans** confirms COLLSCAN.\n\nCompare with **Query Targeting: Scanned Objects / Returned** (ratio) and **Documents** (`document.returned`).\n\n---\n\n### Troubleshooting\n\n- High **Documents scanned** + high targeting ratio → missing/poor indexes · run `explain()` on slow queries\n- Sustained **Collection scans / sec** → enable profiler / Performance Advisor · fix query patterns or add indexes\n- High scan rates + **Page Faults** / **Disk I/O** → storage pressure from inefficient reads\n- **Operations** `opcounters.query` rising with scan rates → read-heavy workload under index stress\n\n---\n\n### Docs\n\n- [serverStatus — metrics.queryExecutor](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.metrics.queryExecutor)", + "description": "### Query executor\n\nPer-second rates from **queryExecutor** counters \u2014 how much work the query engine does examining indexes and documents.\n\n**Unit:** operations/sec (`ops`) \u2014 keys scanned, docs scanned, and collection-scan events are all counter derivatives.\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.metrics.queryExecutor.scanned` \u2014 index keys examined (`totalKeysExamined` in explain())\n- `serverStatus.metrics.queryExecutor.scannedObjects` \u2014 documents examined (`totalDocsExamined`)\n- `serverStatus.metrics.queryExecutor.collectionScans.total` \u2014 queries that performed a **collection scan** (COLLSCAN)\n\nAll are cumulative counters; query uses **`derivative(unit: 1s, nonNegative: true)`**.\n\n---\n\n### How to read\n\n- **Index keys scanned / sec** \u2014 index traversal volume\n- **Documents scanned / sec** \u2014 document examination volume (often dominates I/O)\n- **Collection scans / sec** (right axis) \u2014 queries **not** using a useful index; should stay near **0** in production\n\n**Keys vs docs:** high docs with low keys can mean wide index scans or many docs per key; high docs with high **Collection scans** confirms COLLSCAN.\n\nCompare with **Query Targeting: Scanned Objects / Returned** (ratio) and **Documents** (`document.returned`).\n\n---\n\n### Troubleshooting\n\n- High **Documents scanned** + high targeting ratio \u2192 missing/poor indexes \u00b7 run `explain()` on slow queries\n- Sustained **Collection scans / sec** \u2192 enable profiler / Performance Advisor \u00b7 fix query patterns or add indexes\n- High scan rates + **Page Faults** / **Disk I/O** \u2192 storage pressure from inefficient reads\n- **Operations** `opcounters.query` rising with scan rates \u2192 read-heavy workload under index stress\n\n---\n\n### Docs\n\n- [serverStatus \u2014 metrics.queryExecutor](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.metrics.queryExecutor)", "fieldConfig": { "defaults": { "color": { @@ -4184,37 +3313,9 @@ { "matcher": { "id": "byName", - "options": "serverStatus.metrics.queryExecutor.scanned" - }, - "properties": [ - { - "id": "displayName", - "value": "Index keys scanned / sec" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "serverStatus.metrics.queryExecutor.scannedObjects" - }, - "properties": [ - { - "id": "displayName", - "value": "Documents scanned / sec" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "serverStatus.metrics.queryExecutor.collectionScans.total" + "options": "Collection scans / sec" }, "properties": [ - { - "id": "displayName", - "value": "Collection scans / sec" - }, { "id": "custom.axisPlacement", "value": "right" @@ -4254,7 +3355,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.metrics.queryExecutor.scannedObjects\" or\r\n r[\"_field\"] == \"serverStatus.metrics.queryExecutor.scanned\" or\r\n r[\"_field\"] == \"serverStatus.metrics.queryExecutor.collectionScans.total\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.metrics.queryExecutor.scannedObjects\" or\r\n r[\"_field\"] == \"serverStatus.metrics.queryExecutor.scanned\" or\r\n r[\"_field\"] == \"serverStatus.metrics.queryExecutor.collectionScans.total\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _field: if r._field == \"serverStatus.metrics.queryExecutor.scanned\" then \"Index keys scanned / sec\" else if r._field == \"serverStatus.metrics.queryExecutor.scannedObjects\" then \"Documents scanned / sec\" else if r._field == \"serverStatus.metrics.queryExecutor.collectionScans.total\" then \"Collection scans / sec\" else r._field }))", "refId": "A" } ], @@ -4266,7 +3367,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Operations (opcounters)\n\nPer-second rate of **`serverStatus.opcounters`** — **operation-level** counts (one bulk/multi-doc op = one op). Not the same as **Documents** (`metrics.document`), which counts documents touched.\n\n**Unit:** operations per second (`ops`)\n\n**Note:** `opcounters.command` counts commands **not** already tracked as insert, update, delete, query, or getmore — **including aggregate** and other admin/catalog commands.\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.opcounters.command`\n- `serverStatus.opcounters.delete`\n- `serverStatus.opcounters.getmore`\n- `serverStatus.opcounters.insert`\n- `serverStatus.opcounters.query`\n- `serverStatus.opcounters.update`\n\n**Query:** `derivative(..., unit: 1s, nonNegative: true)` → ops/sec\n\n---\n\n### How to read\n\n- Six lines: command, delete, getmore, insert, query, update **rates**\n- Sudden drop → blocking or idle period\n- Spike → load burst; check **Average Operation Latency** and **WiredTiger Tickets**\n\n---\n\n### Troubleshooting\n\n- Sustained high write ops + latency → **Current Queue writers / readers**, disk I/O\n- High **query** + **getmore** → cursor-heavy workload\n- On secondaries, use **Opcounters Replica** instead of this panel\n\n---\n\n### Docs\n\n- [serverStatus — opcounters](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.opcounters)", + "description": "### Operations (opcounters)\n\nPer-second rate of **`serverStatus.opcounters`** \u2014 **operation-level** counts (one bulk/multi-doc op = one op). Not the same as **Documents** (`metrics.document`), which counts documents touched.\n\n**Unit:** operations per second (`ops`)\n\n**Note:** `opcounters.command` counts commands **not** already tracked as insert, update, delete, query, or getmore \u2014 **including aggregate** and other admin/catalog commands.\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.opcounters.command`\n- `serverStatus.opcounters.delete`\n- `serverStatus.opcounters.getmore`\n- `serverStatus.opcounters.insert`\n- `serverStatus.opcounters.query`\n- `serverStatus.opcounters.update`\n\n**Query:** `derivative(..., unit: 1s, nonNegative: true)` \u2192 ops/sec\n\n---\n\n### How to read\n\n- Six lines: command, delete, getmore, insert, query, update **rates**\n- Sudden drop \u2192 blocking or idle period\n- Spike \u2192 load burst; check **Average Operation Latency** and **WiredTiger Tickets**\n\n---\n\n### Troubleshooting\n\n- Sustained high write ops + latency \u2192 **Current Queue writers / readers**, disk I/O\n- High **query** + **getmore** \u2192 cursor-heavy workload\n- On secondaries, use **Opcounters Replica** instead of this panel\n\n---\n\n### Docs\n\n- [serverStatus \u2014 opcounters](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.opcounters)", "fieldConfig": { "defaults": { "color": { @@ -4319,80 +3420,7 @@ }, "unit": "ops" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*\\.opcounters\\.command$" - }, - "properties": [ - { - "id": "displayName", - "value": "Command" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*\\.opcounters\\.delete$" - }, - "properties": [ - { - "id": "displayName", - "value": "Delete" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*\\.opcounters\\.getmore$" - }, - "properties": [ - { - "id": "displayName", - "value": "Getmore" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*\\.opcounters\\.insert$" - }, - "properties": [ - { - "id": "displayName", - "value": "Insert" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*\\.opcounters\\.query$" - }, - "properties": [ - { - "id": "displayName", - "value": "Query" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*\\.opcounters\\.update$" - }, - "properties": [ - { - "id": "displayName", - "value": "Update" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 11, @@ -4425,7 +3453,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.opcounters.command\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.delete\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.getmore\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.insert\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.query\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.update\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.opcounters.command\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.delete\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.getmore\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.insert\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.query\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.update\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.opcounters.command\" then \"Command\" else if r._field == \"serverStatus.opcounters.delete\" then \"Delete\" else if r._field == \"serverStatus.opcounters.getmore\" then \"Getmore\" else if r._field == \"serverStatus.opcounters.insert\" then \"Insert\" else if r._field == \"serverStatus.opcounters.query\" then \"Query\" else if r._field == \"serverStatus.opcounters.update\" then \"Update\" else r._field\r\n }))", "refId": "A" } ], @@ -4437,7 +3465,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Replication opcounters (If secondary)\n\nPer-second rates from **`serverStatus.opcountersRepl`** — operations applied when this member is acting as a **secondary** (oplog apply).\n\n**Unit:** operations/sec (`ops`)\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.opcountersRepl.insert`\n- `serverStatus.opcountersRepl.query`\n- `serverStatus.opcountersRepl.update`\n- `serverStatus.opcountersRepl.delete`\n- `serverStatus.opcountersRepl.getmore`\n- `serverStatus.opcountersRepl.command`\n\nCumulative counters; query uses **`derivative(unit: 1s, nonNegative: true)`**.\n\n---\n\n### How to read\n\n- Six lines · replication apply rate by type\n- **View a secondary hostname** — answers “how fast is this node applying replication?”\n- **Primary / write source** — expect near-zero or unhelpful lines; switch host or open **Operations (write source)**\n- Fractional ops/sec → normal with sparse FTDC sampling\n- **0** → idle or no replication apply in window\n\nCompare with **Operations (write source)** on the primary, **Replica Members Lag**, **Flow control is lagged**.\n\n---\n\n### Troubleshooting\n\n- Repl rates drop + **lag rising** → secondary cannot keep up · disk I/O, CPU, **Disk I/O**\n- Primary **Operations (write source)** high, secondary repl low → bottleneck on secondary\n- Repl **update** ≫ primary **update** → expected replication amplification\n- **Flow control is lagged** = 1 + lag → primary throttling writes\n\n---\n\n### Docs\n\n- [serverStatus — opcountersRepl](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.opcountersRepl)\n- [serverStatus — opcounters](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.opcounters)", + "description": "### Replication opcounters (If secondary)\n\nPer-second rates from **`serverStatus.opcountersRepl`** \u2014 operations applied when this member is acting as a **secondary** (oplog apply).\n\n**Unit:** operations/sec (`ops`)\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.opcountersRepl.insert`\n- `serverStatus.opcountersRepl.query`\n- `serverStatus.opcountersRepl.update`\n- `serverStatus.opcountersRepl.delete`\n- `serverStatus.opcountersRepl.getmore`\n- `serverStatus.opcountersRepl.command`\n\nCumulative counters; query uses **`derivative(unit: 1s, nonNegative: true)`**.\n\n---\n\n### How to read\n\n- Six lines \u00b7 replication apply rate by type\n- **View a secondary hostname** \u2014 answers \u201chow fast is this node applying replication?\u201d\n- **Primary / write source** \u2014 expect near-zero or unhelpful lines; switch host or open **Operations (write source)**\n- Fractional ops/sec \u2192 normal with sparse FTDC sampling\n- **0** \u2192 idle or no replication apply in window\n\nCompare with **Operations (write source)** on the primary, **Replica Members Lag**, **Flow control is lagged**.\n\n---\n\n### Troubleshooting\n\n- Repl rates drop + **lag rising** \u2192 secondary cannot keep up \u00b7 disk I/O, CPU, **Disk I/O**\n- Primary **Operations (write source)** high, secondary repl low \u2192 bottleneck on secondary\n- Repl **update** \u226b primary **update** \u2192 expected replication amplification\n- **Flow control is lagged** = 1 + lag \u2192 primary throttling writes\n\n---\n\n### Docs\n\n- [serverStatus \u2014 opcountersRepl](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.opcountersRepl)\n- [serverStatus \u2014 opcounters](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.opcounters)", "fieldConfig": { "defaults": { "color": { @@ -4469,101 +3497,28 @@ }, "showPoints": "auto", "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "ops" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "serverStatus.opcountersRepl.insert" - }, - "properties": [ - { - "id": "displayName", - "value": "Insert / sec" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "serverStatus.opcountersRepl.query" - }, - "properties": [ - { - "id": "displayName", - "value": "Query / sec" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "serverStatus.opcountersRepl.update" - }, - "properties": [ - { - "id": "displayName", - "value": "Update / sec" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "serverStatus.opcountersRepl.delete" - }, - "properties": [ - { - "id": "displayName", - "value": "Delete / sec" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "serverStatus.opcountersRepl.getmore" + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "properties": [ - { - "id": "displayName", - "value": "Getmore / sec" - } - ] + "thresholdsStyle": { + "mode": "off" + } }, - { - "matcher": { - "id": "byName", - "options": "serverStatus.opcountersRepl.command" - }, - "properties": [ + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "id": "displayName", - "value": "Command / sec" + "color": "green", + "value": 0 } ] - } - ] + }, + "unit": "ops" + }, + "overrides": [] }, "gridPos": { "h": 11, @@ -4596,7 +3551,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.insert\" or\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.query\" or\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.update\" or\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.delete\" or\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.getmore\" or\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.command\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.insert\" or\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.query\" or\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.update\" or\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.delete\" or\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.getmore\" or\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.command\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _field: if r._field == \"serverStatus.opcountersRepl.insert\" then \"Insert / sec\" else if r._field == \"serverStatus.opcountersRepl.query\" then \"Query / sec\" else if r._field == \"serverStatus.opcountersRepl.update\" then \"Update / sec\" else if r._field == \"serverStatus.opcountersRepl.delete\" then \"Delete / sec\" else if r._field == \"serverStatus.opcountersRepl.getmore\" then \"Getmore / sec\" else if r._field == \"serverStatus.opcountersRepl.command\" then \"Command / sec\" else r._field }))", "refId": "A" } ], @@ -4608,7 +3563,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Operation latency ops\n\nPer-second rates from **opLatencies.*.ops** counters — how many read, write, and command operations ran in each latency class.\n\n**Unit:** operations/sec (`ops`)\n\nThis is the **denominator** for **Average Operation Latency** (`derivative(latency) / derivative(ops)` → µs/op).\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.opLatencies.reads.ops`\n- `serverStatus.opLatencies.writes.ops`\n- `serverStatus.opLatencies.commands.ops`\n\nCumulative counters; query uses **`derivative(unit: 1s, nonNegative: true)`**.\n\n**Not the same as opcounters:** `opLatencies` groups reads / writes / commands for latency stats; **Operations** uses `opcounters.*` (query, insert, update, delete, getmore, command).\n\n---\n\n### How to read\n\n- **Read / write / command ops / sec** — throughput per latency class\n- **Commands** often includes admin, aggregate, and other command work\n- Fractional values → normal with sparse FTDC sampling\n- **0** → idle window for that class\n\nCompare with **Average Operation Latency** (numerator/denominator) and **Operations** (opcounters workload mix).\n\n---\n\n### Troubleshooting\n\n- **Ops up, avg latency up** → load + slower per-op work\n- **Ops down, avg latency up** → fewer ops but each slower (contention, I/O)\n- **Ops up, avg latency flat** → healthy scale-out of throughput\n- Read/write imbalance → workload skew · correlate with **Query Executor**, **Documents**, **Disk I/O**\n\n---\n\n### Docs\n\n- [serverStatus — opLatencies](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.opLatencies)", + "description": "### Operation latency ops\n\nPer-second rates from **opLatencies.*.ops** counters \u2014 how many read, write, and command operations ran in each latency class.\n\n**Unit:** operations/sec (`ops`)\n\nThis is the **denominator** for **Average Operation Latency** (`derivative(latency) / derivative(ops)` \u2192 \u00b5s/op).\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.opLatencies.reads.ops`\n- `serverStatus.opLatencies.writes.ops`\n- `serverStatus.opLatencies.commands.ops`\n\nCumulative counters; query uses **`derivative(unit: 1s, nonNegative: true)`**.\n\n**Not the same as opcounters:** `opLatencies` groups reads / writes / commands for latency stats; **Operations** uses `opcounters.*` (query, insert, update, delete, getmore, command).\n\n---\n\n### How to read\n\n- **Read / write / command ops / sec** \u2014 throughput per latency class\n- **Commands** often includes admin, aggregate, and other command work\n- Fractional values \u2192 normal with sparse FTDC sampling\n- **0** \u2192 idle window for that class\n\nCompare with **Average Operation Latency** (numerator/denominator) and **Operations** (opcounters workload mix).\n\n---\n\n### Troubleshooting\n\n- **Ops up, avg latency up** \u2192 load + slower per-op work\n- **Ops down, avg latency up** \u2192 fewer ops but each slower (contention, I/O)\n- **Ops up, avg latency flat** \u2192 healthy scale-out of throughput\n- Read/write imbalance \u2192 workload skew \u00b7 correlate with **Query Executor**, **Documents**, **Disk I/O**\n\n---\n\n### Docs\n\n- [serverStatus \u2014 opLatencies](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.opLatencies)", "fieldConfig": { "defaults": { "color": { @@ -4661,44 +3616,7 @@ }, "unit": "ops" }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "serverStatus.opLatencies.reads.ops" - }, - "properties": [ - { - "id": "displayName", - "value": "Read ops / sec" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "serverStatus.opLatencies.writes.ops" - }, - "properties": [ - { - "id": "displayName", - "value": "Write ops / sec" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "serverStatus.opLatencies.commands.ops" - }, - "properties": [ - { - "id": "displayName", - "value": "Command ops / sec" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 9, @@ -4731,7 +3649,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.opLatencies.reads.ops\" or\r\n r[\"_field\"] == \"serverStatus.opLatencies.writes.ops\" or\r\n r[\"_field\"] == \"serverStatus.opLatencies.commands.ops\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.opLatencies.reads.ops\" or\r\n r[\"_field\"] == \"serverStatus.opLatencies.writes.ops\" or\r\n r[\"_field\"] == \"serverStatus.opLatencies.commands.ops\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _field: if r._field == \"serverStatus.opLatencies.reads.ops\" then \"Read ops / sec\" else if r._field == \"serverStatus.opLatencies.writes.ops\" then \"Write ops / sec\" else if r._field == \"serverStatus.opLatencies.commands.ops\" then \"Command ops / sec\" else r._field }))", "refId": "A" } ], @@ -4743,7 +3661,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Query targeting: scanned objects / returned\n\nRatio of **document scan rate** to **document return rate**:\n\n**Formula:** `derivative(scannedObjects, 1s) / derivative(document.returned, 1s)`\n\nSame idea as **totalDocsExamined / docs returned** in explain() — aggregated over time.\n\n**Unit:** dimensionless ratio (`none`) — **1.0** ≈ one doc examined per doc returned\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.metrics.queryExecutor.scannedObjects`\n- `serverStatus.metrics.document.returned`\n\nBoth are cumulative counters; query uses **`derivative(unit: 1s, nonNegative: true)`** before the ratio.\n\n---\n\n### How to read\n\n- Single line · **Scanned / returned ratio**\n- **~1** → strong targeting (most examined docs were returned)\n- **> 1** (especially **10+**) → scans dominate · missing/poor indexes or large in-memory filters\n- **Gap** (not 0) → no returned docs in window or zero denominator — **not** “perfect”\n\n---\n\n### Troubleshooting\n\n- High ratio + high **Documents returned** → **Query Executor** · run `explain()` on hot queries\n- Correlate with **Page Faults**, **Disk I/O**, **Average Operation Latency**\n- **Collection scans** on **Query Executor** (right axis) confirms full collection scans\n\n---\n\n### Docs\n\n- [serverStatus — metrics.queryExecutor](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.metrics.queryExecutor)\n- [serverStatus — metrics.document](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.metrics.document)", + "description": "### Query targeting: scanned objects / returned\n\nRatio of **document scan rate** to **document return rate**:\n\n**Formula:** `derivative(scannedObjects, 1s) / derivative(document.returned, 1s)`\n\nSame idea as **totalDocsExamined / docs returned** in explain() \u2014 aggregated over time.\n\n**Unit:** dimensionless ratio (`none`) \u2014 **1.0** \u2248 one doc examined per doc returned\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.metrics.queryExecutor.scannedObjects`\n- `serverStatus.metrics.document.returned`\n\nBoth are cumulative counters; query uses **`derivative(unit: 1s, nonNegative: true)`** before the ratio.\n\n---\n\n### How to read\n\n- Single line \u00b7 **Scanned / returned ratio**\n- **~1** \u2192 strong targeting (most examined docs were returned)\n- **> 1** (especially **10+**) \u2192 scans dominate \u00b7 missing/poor indexes or large in-memory filters\n- **Gap** (not 0) \u2192 no returned docs in window or zero denominator \u2014 **not** \u201cperfect\u201d\n\n---\n\n### Troubleshooting\n\n- High ratio + high **Documents returned** \u2192 **Query Executor** \u00b7 run `explain()` on hot queries\n- Correlate with **Page Faults**, **Disk I/O**, **Average Operation Latency**\n- **Collection scans** on **Query Executor** (right axis) confirms full collection scans\n\n---\n\n### Docs\n\n- [serverStatus \u2014 metrics.queryExecutor](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.metrics.queryExecutor)\n- [serverStatus \u2014 metrics.document](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.metrics.document)", "fieldConfig": { "defaults": { "color": { @@ -4797,20 +3715,7 @@ }, "unit": "none" }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "scanned_returned_ratio" - }, - "properties": [ - { - "id": "displayName", - "value": "Scanned / returned ratio" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 6, @@ -4843,7 +3748,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.metrics.document.returned\" or\r\n r[\"_field\"] == \"serverStatus.metrics.queryExecutor.scannedObjects\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> pivot(rowKey: [\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n scanned_returned_ratio:\r\n if exists r[\"serverStatus.metrics.document.returned\"]\r\n and float(v: r[\"serverStatus.metrics.document.returned\"]) > 0.0\r\n and exists r[\"serverStatus.metrics.queryExecutor.scannedObjects\"]\r\n then float(v: r[\"serverStatus.metrics.queryExecutor.scannedObjects\"])\r\n / float(v: r[\"serverStatus.metrics.document.returned\"])\r\n else float(v: \"NaN\")\r\n }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.metrics.document.returned\" or\r\n r[\"_field\"] == \"serverStatus.metrics.queryExecutor.scannedObjects\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> pivot(rowKey: [\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n hostname: r.hostname,\r\n _field: \"Scanned / returned ratio\",\r\n _value:\r\n if exists r[\"serverStatus.metrics.document.returned\"]\r\n and float(v: r[\"serverStatus.metrics.document.returned\"]) > 0.0\r\n and exists r[\"serverStatus.metrics.queryExecutor.scannedObjects\"]\r\n then float(v: r[\"serverStatus.metrics.queryExecutor.scannedObjects\"])\r\n / float(v: r[\"serverStatus.metrics.document.returned\"])\r\n else float(v: \"NaN\")\r\n }))\r\n |> group(columns: [\"_field\", \"hostname\"])\r\n", "refId": "A" } ], @@ -4851,7 +3756,7 @@ "type": "timeseries" }, { - "collapsed": false, + "collapsed": true, "gridPos": { "h": 1, "w": 24, @@ -4868,7 +3773,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### CPU usage\n\nHost CPU time by mode, from FTDC **systemMetrics** (OS counters in milliseconds). Each line is that mode’s share of total CPU time in the window after per-mode `derivative(1s)`.\n\n**Formula:** `aggregateWindow(last)` → `pivot` → `derivative(*_ms, 1s)` → `(mode_ms / total_ms) × 100`\n\n**Unit:** percent (`percent`, 0–100)\n\n---\n\n### Metrics\n\n**Fields:**\n\n`systemMetrics.cpu.user_ms`\n`systemMetrics.cpu.system_ms`\n`systemMetrics.cpu.iowait_ms`\n`systemMetrics.cpu.idle_ms`\n`systemMetrics.cpu.nice_ms`\n`systemMetrics.cpu.softirq_ms`\n`systemMetrics.cpu.steal_ms`\n\n**Plotted:** user, system, iowait percentages only. Idle, nice, softirq, and steal are included in **total_ms** (denominator) but not shown as separate lines.\n\n**Query type:** computed ratio (not a single counter derivative). Output uses `_field` / `_value` via `union` for Grafana.\n\n---\n\n### How to read\n\n- **User %** — application time (mongod, queries, aggregations)\n- **System (kernel) %** — kernel time, syscalls, context switching\n- **I/O wait %** — CPU idle while waiting on disk\n- Three lines sum to **≤ 100%** (unplotted modes account for the rest)\n\n---\n\n### Troubleshooting\n\n- High **User %** with **Operations** or **Query Executor** → CPU-bound workload\n- High **I/O wait %** → disk bottleneck · **Disk I/O**, **WiredTiger Block Manager**, **Page Faults**\n- High **System %** with many **Connections** → thread/syscall overhead\n- VM steal not plotted — inspect raw `systemMetrics.cpu.steal_ms` in Explore if host may be oversubscribed\n\nCorrelate with **mongod Memory (RSS / Virtual)** and **System Memory**.\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)", + "description": "### CPU usage\n\nHost CPU time by mode, from FTDC **systemMetrics** (OS counters in milliseconds). Each line is that mode\u2019s share of total CPU time in the window after per-mode `derivative(1s)`.\n\n**Formula:** `aggregateWindow(last)` \u2192 `pivot` \u2192 `derivative(*_ms, 1s)` \u2192 `(mode_ms / total_ms) \u00d7 100`\n\n**Unit:** percent (`percent`, 0\u2013100)\n\n---\n\n### Metrics\n\n**Fields:**\n\n`systemMetrics.cpu.user_ms`\n`systemMetrics.cpu.system_ms`\n`systemMetrics.cpu.iowait_ms`\n`systemMetrics.cpu.idle_ms`\n`systemMetrics.cpu.nice_ms`\n`systemMetrics.cpu.softirq_ms`\n`systemMetrics.cpu.steal_ms`\n\n**Plotted:** user, system, iowait percentages only. Idle, nice, softirq, and steal are included in **total_ms** (denominator) but not shown as separate lines.\n\n**Query type:** computed ratio (not a single counter derivative). Output uses `_field` / `_value` via `union` for Grafana.\n\n---\n\n### How to read\n\n- **User %** \u2014 application time (mongod, queries, aggregations)\n- **System (kernel) %** \u2014 kernel time, syscalls, context switching\n- **I/O wait %** \u2014 CPU idle while waiting on disk\n- Three lines sum to **\u2264 100%** (unplotted modes account for the rest)\n\n---\n\n### Troubleshooting\n\n- High **User %** with **Operations** or **Query Executor** \u2192 CPU-bound workload\n- High **I/O wait %** \u2192 disk bottleneck \u00b7 **Disk I/O**, **WiredTiger Block Manager**, **Page Faults**\n- High **System %** with many **Connections** \u2192 thread/syscall overhead\n- VM steal not plotted \u2014 inspect raw `systemMetrics.cpu.steal_ms` in Explore if host may be oversubscribed\n\nCorrelate with **mongod Memory (RSS / Virtual)** and **System Memory**.\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)", "fieldConfig": { "defaults": { "color": { @@ -4954,7 +3859,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "base =\r\n from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r._field == \"systemMetrics.cpu.idle_ms\" or\r\n r._field == \"systemMetrics.cpu.user_ms\" or\r\n r._field == \"systemMetrics.cpu.iowait_ms\" or\r\n r._field == \"systemMetrics.cpu.nice_ms\" or\r\n r._field == \"systemMetrics.cpu.softirq_ms\" or\r\n r._field == \"systemMetrics.cpu.steal_ms\" or\r\n r._field == \"systemMetrics.cpu.system_ms\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> pivot(rowKey: [\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\r\n |> derivative(\r\n unit: 1s,\r\n nonNegative: true,\r\n columns: [\r\n \"systemMetrics.cpu.idle_ms\",\r\n \"systemMetrics.cpu.user_ms\",\r\n \"systemMetrics.cpu.iowait_ms\",\r\n \"systemMetrics.cpu.nice_ms\",\r\n \"systemMetrics.cpu.softirq_ms\",\r\n \"systemMetrics.cpu.steal_ms\",\r\n \"systemMetrics.cpu.system_ms\",\r\n ],\r\n )\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n user_ms: float(v: r[\"systemMetrics.cpu.user_ms\"]),\r\n system_ms: float(v: r[\"systemMetrics.cpu.system_ms\"]),\r\n iowait_ms: float(v: r[\"systemMetrics.cpu.iowait_ms\"]),\r\n total_ms:\r\n float(v: r[\"systemMetrics.cpu.idle_ms\"]) +\r\n float(v: r[\"systemMetrics.cpu.user_ms\"]) +\r\n float(v: r[\"systemMetrics.cpu.iowait_ms\"]) +\r\n float(v: r[\"systemMetrics.cpu.nice_ms\"]) +\r\n float(v: r[\"systemMetrics.cpu.softirq_ms\"]) +\r\n float(v: r[\"systemMetrics.cpu.steal_ms\"]) +\r\n float(v: r[\"systemMetrics.cpu.system_ms\"]),\r\n }))\r\n\r\nunion(tables: [\r\n base |> map(fn: (r) => ({\r\n _time: r._time,\r\n _field: \"User %\",\r\n _value: if r.total_ms > 0.0 then 100.0 * r.user_ms / r.total_ms else 0.0,\r\n })),\r\n base |> map(fn: (r) => ({\r\n _time: r._time,\r\n _field: \"System (kernel) %\",\r\n _value: if r.total_ms > 0.0 then 100.0 * r.system_ms / r.total_ms else 0.0,\r\n })),\r\n base |> map(fn: (r) => ({\r\n _time: r._time,\r\n _field: \"I/O wait %\",\r\n _value: if r.total_ms > 0.0 then 100.0 * r.iowait_ms / r.total_ms else 0.0,\r\n })),\r\n])\r\n |> group(columns: [\"_field\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", + "query": "base =\r\n from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r._field == \"systemMetrics.cpu.idle_ms\" or\r\n r._field == \"systemMetrics.cpu.user_ms\" or\r\n r._field == \"systemMetrics.cpu.iowait_ms\" or\r\n r._field == \"systemMetrics.cpu.nice_ms\" or\r\n r._field == \"systemMetrics.cpu.softirq_ms\" or\r\n r._field == \"systemMetrics.cpu.steal_ms\" or\r\n r._field == \"systemMetrics.cpu.system_ms\")\r\n |> drop(columns: [\"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> pivot(rowKey: [\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\r\n |> derivative(\r\n unit: 1s,\r\n nonNegative: true,\r\n columns: [\r\n \"systemMetrics.cpu.idle_ms\",\r\n \"systemMetrics.cpu.user_ms\",\r\n \"systemMetrics.cpu.iowait_ms\",\r\n \"systemMetrics.cpu.nice_ms\",\r\n \"systemMetrics.cpu.softirq_ms\",\r\n \"systemMetrics.cpu.steal_ms\",\r\n \"systemMetrics.cpu.system_ms\",\r\n ],\r\n )\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n hostname: r.hostname,\r\n user_ms: float(v: r[\"systemMetrics.cpu.user_ms\"]),\r\n system_ms: float(v: r[\"systemMetrics.cpu.system_ms\"]),\r\n iowait_ms: float(v: r[\"systemMetrics.cpu.iowait_ms\"]),\r\n total_ms:\r\n float(v: r[\"systemMetrics.cpu.idle_ms\"]) +\r\n float(v: r[\"systemMetrics.cpu.user_ms\"]) +\r\n float(v: r[\"systemMetrics.cpu.iowait_ms\"]) +\r\n float(v: r[\"systemMetrics.cpu.nice_ms\"]) +\r\n float(v: r[\"systemMetrics.cpu.softirq_ms\"]) +\r\n float(v: r[\"systemMetrics.cpu.steal_ms\"]) +\r\n float(v: r[\"systemMetrics.cpu.system_ms\"]),\r\n }))\r\n\r\nunion(tables: [\r\n base |> map(fn: (r) => ({\r\n _time: r._time, hostname: r.hostname,\r\n _field: \"User %\",\r\n _value: if r.total_ms > 0.0 then 100.0 * r.user_ms / r.total_ms else 0.0,\r\n })),\r\n base |> map(fn: (r) => ({\r\n _time: r._time, hostname: r.hostname,\r\n _field: \"System (kernel) %\",\r\n _value: if r.total_ms > 0.0 then 100.0 * r.system_ms / r.total_ms else 0.0,\r\n })),\r\n base |> map(fn: (r) => ({\r\n _time: r._time, hostname: r.hostname,\r\n _field: \"I/O wait %\",\r\n _value: if r.total_ms > 0.0 then 100.0 * r.iowait_ms / r.total_ms else 0.0,\r\n })),\r\n])\r\n |> group(columns: [\"_field\", \"hostname\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", "refId": "A" } ], @@ -4966,7 +3871,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### System memory\n\nHost RAM from FTDC **systemMetrics** (`/proc/meminfo`, values stored as **kB**). Shows how much memory the OS reports as **available** vs **free**.\n\n**Unit:** bytes (Grafana auto-scale; FTDC fields are `*_kb`, query multiplies by 1024)\n\n---\n\n### Metrics\n\n**Fields:**\n\n`systemMetrics.memory.MemAvailable_kb`\n`systemMetrics.memory.MemFree_kb`\n\n**MemAvailable** — estimate of memory startable for new workloads without swapping (includes reclaimable cache).\n\n**MemFree** — completely unallocated pages (often much lower than available).\n\n---\n\n### How to read\n\n- **Memory available** — primary signal for OS headroom\n- **Memory free** — strictly idle RAM; low free + healthy available is normal on Linux\n- Both lines fall when mongod, page cache, or other services consume RAM\n\n---\n\n### Troubleshooting\n\n- **Available** trending toward zero → memory pressure · **Swap Memory**, **Page Faults**, **WiredTiger Cache**\n- Available low but **mongod Memory (RSS / Virtual)** moderate → other processes or OS page cache\n- Available and free both low + high **CPU Usage** I/O wait → possible thrashing\n- After mongod restart, available may jump as RSS is released\n\nCorrelate with **mongod Memory (RSS / Virtual)** and **Swap Memory**.\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)\n- Linux [meminfo](https://www.kernel.org/doc/html/latest/admin-guide/mm/memory-model.html) — `MemAvailable` vs `MemFree`", + "description": "### System memory\n\nHost RAM from FTDC **systemMetrics** (`/proc/meminfo`, values stored as **kB**). Shows how much memory the OS reports as **available** vs **free**.\n\n**Unit:** bytes (Grafana auto-scale; FTDC fields are `*_kb`, query multiplies by 1024)\n\n---\n\n### Metrics\n\n**Fields:**\n\n`systemMetrics.memory.MemAvailable_kb`\n`systemMetrics.memory.MemFree_kb`\n\n**MemAvailable** \u2014 estimate of memory startable for new workloads without swapping (includes reclaimable cache).\n\n**MemFree** \u2014 completely unallocated pages (often much lower than available).\n\n---\n\n### How to read\n\n- **Memory available** \u2014 primary signal for OS headroom\n- **Memory free** \u2014 strictly idle RAM; low free + healthy available is normal on Linux\n- Both lines fall when mongod, page cache, or other services consume RAM\n\n---\n\n### Troubleshooting\n\n- **Available** trending toward zero \u2192 memory pressure \u00b7 **Swap Memory**, **Page Faults**, **WiredTiger Cache**\n- Available low but **mongod Memory (RSS / Virtual)** moderate \u2192 other processes or OS page cache\n- Available and free both low + high **CPU Usage** I/O wait \u2192 possible thrashing\n- After mongod restart, available may jump as RSS is released\n\nCorrelate with **mongod Memory (RSS / Virtual)** and **Swap Memory**.\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)\n- Linux [meminfo](https://www.kernel.org/doc/html/latest/admin-guide/mm/memory-model.html) \u2014 `MemAvailable` vs `MemFree`", "fieldConfig": { "defaults": { "color": { @@ -5020,32 +3925,7 @@ }, "unit": "bytes" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*MemAvailable_kb$" - }, - "properties": [ - { - "id": "displayName", - "value": "Memory available" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*MemFree_kb$" - }, - "properties": [ - { - "id": "displayName", - "value": "Memory free" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 8, @@ -5078,7 +3958,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"systemMetrics.memory.MemAvailable_kb\" or\r\n r[\"_field\"] == \"systemMetrics.memory.MemFree_kb\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) * 1024.0 }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"systemMetrics.memory.MemAvailable_kb\" or\r\n r[\"_field\"] == \"systemMetrics.memory.MemFree_kb\")\r\n |> drop(columns: [\"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value) * 1024.0,\r\n _field: if r._field == \"systemMetrics.memory.MemAvailable_kb\" then \"Memory available\" else if r._field == \"systemMetrics.memory.MemFree_kb\" then \"Memory free\" else r._field\r\n }))", "refId": "A" } ], @@ -5090,7 +3970,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Swap memory\n\nHost swap from FTDC **systemMetrics** (`/proc/meminfo`, values stored as **kB**). Shows **used**, **free**, and **total** swap capacity. **Swap used** is computed as `SwapTotal − SwapFree`.\n\n**Unit:** bytes (Grafana auto-scale; FTDC fields are `*_kb`, query multiplies by 1024)\n\n---\n\n### Metrics\n\n**Fields:**\n\n`systemMetrics.memory.SwapTotal_kb`\n`systemMetrics.memory.SwapFree_kb`\n\n**Computed:** Swap used = `SwapTotal_kb − SwapFree_kb` (not a separate FTDC field)\n\n**Plotted:** Swap used (computed) · Swap free · Swap total\n\n---\n\n### How to read\n\n- **Swap used** — should stay at **0** on a MongoDB host\n- **Swap free** — unused swap capacity\n- **Swap total** — configured swap size (may be **0** if swap is disabled)\n- **Swap used + Swap free ≈ Swap total** at each point\n\n---\n\n### Troubleshooting\n\n- **Any sustained Swap used > 0** → critical · RAM pressure · latency spikes\n- Correlate with **System Memory**, **Page Faults**, **mongod Memory (RSS / Virtual)**, **WiredTiger Cache**\n- High **CPU Usage** I/O wait + swap → severe thrashing\n- Remediation: add RAM, reduce `--wiredTigerCacheSizeGB`, fix memory leaks, stop co-located heavy processes\n- **Swap total = 0** → swap disabled (common on tuned MongoDB VMs)\n\nCorrelate with **System Memory** and **Average Operation Latency**.\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)\n- [MongoDB production notes — memory](https://www.mongodb.com/docs/manual/administration/production-notes/#std-label-production-notes-memory)", + "description": "### Swap memory\n\nHost swap from FTDC **systemMetrics** (`/proc/meminfo`, values stored as **kB**). Shows **used**, **free**, and **total** swap capacity. **Swap used** is computed as `SwapTotal \u2212 SwapFree`.\n\n**Unit:** bytes (Grafana auto-scale; FTDC fields are `*_kb`, query multiplies by 1024)\n\n---\n\n### Metrics\n\n**Fields:**\n\n`systemMetrics.memory.SwapTotal_kb`\n`systemMetrics.memory.SwapFree_kb`\n\n**Computed:** Swap used = `SwapTotal_kb \u2212 SwapFree_kb` (not a separate FTDC field)\n\n**Plotted:** Swap used (computed) \u00b7 Swap free \u00b7 Swap total\n\n---\n\n### How to read\n\n- **Swap used** \u2014 should stay at **0** on a MongoDB host\n- **Swap free** \u2014 unused swap capacity\n- **Swap total** \u2014 configured swap size (may be **0** if swap is disabled)\n- **Swap used + Swap free \u2248 Swap total** at each point\n\n---\n\n### Troubleshooting\n\n- **Any sustained Swap used > 0** \u2192 critical \u00b7 RAM pressure \u00b7 latency spikes\n- Correlate with **System Memory**, **Page Faults**, **mongod Memory (RSS / Virtual)**, **WiredTiger Cache**\n- High **CPU Usage** I/O wait + swap \u2192 severe thrashing\n- Remediation: add RAM, reduce `--wiredTigerCacheSizeGB`, fix memory leaks, stop co-located heavy processes\n- **Swap total = 0** \u2192 swap disabled (common on tuned MongoDB VMs)\n\nCorrelate with **System Memory** and **Average Operation Latency**.\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)\n- [MongoDB production notes \u2014 memory](https://www.mongodb.com/docs/manual/administration/production-notes/#std-label-production-notes-memory)", "fieldConfig": { "defaults": { "color": { @@ -5177,7 +4057,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "base =\r\n from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"systemMetrics.memory.SwapFree_kb\" or\r\n r[\"_field\"] == \"systemMetrics.memory.SwapTotal_kb\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> pivot(rowKey: [\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n swap_total_kb: float(v: r[\"systemMetrics.memory.SwapTotal_kb\"]),\r\n swap_free_kb: float(v: r[\"systemMetrics.memory.SwapFree_kb\"]),\r\n swap_used_kb:\r\n float(v: r[\"systemMetrics.memory.SwapTotal_kb\"]) -\r\n float(v: r[\"systemMetrics.memory.SwapFree_kb\"]),\r\n }))\r\n\r\nunion(tables: [\r\n base |> map(fn: (r) => ({\r\n _time: r._time,\r\n _field: \"Swap used\",\r\n _value: r.swap_used_kb * 1024.0,\r\n })),\r\n base |> map(fn: (r) => ({\r\n _time: r._time,\r\n _field: \"Swap free\",\r\n _value: r.swap_free_kb * 1024.0,\r\n })),\r\n base |> map(fn: (r) => ({\r\n _time: r._time,\r\n _field: \"Swap total\",\r\n _value: r.swap_total_kb * 1024.0,\r\n })),\r\n])\r\n |> group(columns: [\"_field\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)", + "query": "base =\r\n from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"systemMetrics.memory.SwapFree_kb\" or\r\n r[\"_field\"] == \"systemMetrics.memory.SwapTotal_kb\")\r\n |> drop(columns: [\"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> pivot(rowKey: [\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n hostname: r.hostname,\r\n swap_total_kb: float(v: r[\"systemMetrics.memory.SwapTotal_kb\"]),\r\n swap_free_kb: float(v: r[\"systemMetrics.memory.SwapFree_kb\"]),\r\n swap_used_kb:\r\n float(v: r[\"systemMetrics.memory.SwapTotal_kb\"]) -\r\n float(v: r[\"systemMetrics.memory.SwapFree_kb\"]),\r\n }))\r\n\r\nunion(tables: [\r\n base |> map(fn: (r) => ({\r\n _time: r._time, hostname: r.hostname,\r\n _field: \"Swap used\",\r\n _value: r.swap_used_kb * 1024.0,\r\n })),\r\n base |> map(fn: (r) => ({\r\n _time: r._time, hostname: r.hostname,\r\n _field: \"Swap free\",\r\n _value: r.swap_free_kb * 1024.0,\r\n })),\r\n base |> map(fn: (r) => ({\r\n _time: r._time, hostname: r.hostname,\r\n _field: \"Swap total\",\r\n _value: r.swap_total_kb * 1024.0,\r\n })),\r\n])\r\n |> group(columns: [\"_field\", \"hostname\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)", "refId": "A" } ], @@ -5189,7 +4069,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Disk writes and reads\n\nPer-device **IOPS** (read/write operations per second) from FTDC **systemMetrics** (`/proc/diskstats` cumulative `reads` / `writes` counters).\n\n**Unit:** IOPS (`iops`) — `derivative(unit: 1s)` on OS counters\n\n**Not mongod-only:** includes all processes on the host (OS view).\n\n---\n\n### Metrics\n\n**Pattern (device names vary by host):**\n\n`systemMetrics.disks..reads`\n`systemMetrics.disks..writes`\n\nExample: `systemMetrics.disks.xvda.reads` · `systemMetrics.disks.sdb.writes`\n\nAll disk stats matching the exporter’s disk allowlist are ingested automatically (no per-device entries in `metrics_to_get.txt`).\n\n**Query:** `derivative(..., unit: 1s, nonNegative: true)` → IOPS\n\n---\n\n### How to read\n\n- One line per **device · reads** and **device · writes**\n- Spikes during WiredTiger **checkpoints** (~60s) are normal on the data volume\n- Many small IOPS + low byte throughput → small random I/O · compare **Disk I/O**\n\n---\n\n### Troubleshooting\n\n- Sustained high IOPS + high **CPU Usage** I/O wait → disk saturation\n- High reads → cache misses · **WiredTiger Block Manager**, **Page Faults**\n- High writes → checkpoints/eviction · **Checkpoint Write Throughput**, **WiredTiger Pages Evicted**\n- Correlate with **Disk I/O** (bytes/sec) and **Disk I/O Busy Time / Disk Queue Time** panels\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)\n- Linux [diskstats](https://www.kernel.org/doc/html/latest/admin-guide/iostats.html)", + "description": "### Disk writes and reads\n\nPer-device **IOPS** (read/write operations per second) from FTDC **systemMetrics** (`/proc/diskstats` cumulative `reads` / `writes` counters).\n\n**Unit:** IOPS (`iops`) \u2014 `derivative(unit: 1s)` on OS counters\n\n**Not mongod-only:** includes all processes on the host (OS view).\n\n---\n\n### Metrics\n\n**Pattern (device names vary by host):**\n\n`systemMetrics.disks..reads`\n`systemMetrics.disks..writes`\n\nExample: `systemMetrics.disks.xvda.reads` \u00b7 `systemMetrics.disks.sdb.writes`\n\nAll disk stats matching the exporter\u2019s disk allowlist are ingested automatically (no per-device entries in `metrics_to_get.txt`).\n\n**Query:** `derivative(..., unit: 1s, nonNegative: true)` \u2192 IOPS\n\n---\n\n### How to read\n\n- One line per **device \u00b7 reads** and **device \u00b7 writes**\n- Spikes during WiredTiger **checkpoints** (~60s) are normal on the data volume\n- Many small IOPS + low byte throughput \u2192 small random I/O \u00b7 compare **Disk I/O**\n\n---\n\n### Troubleshooting\n\n- Sustained high IOPS + high **CPU Usage** I/O wait \u2192 disk saturation\n- High reads \u2192 cache misses \u00b7 **WiredTiger Block Manager**, **Page Faults**\n- High writes \u2192 checkpoints/eviction \u00b7 **Checkpoint Write Throughput**, **WiredTiger Pages Evicted**\n- Correlate with **Disk I/O** (bytes/sec) and **Disk I/O Busy Time / Disk Queue Time** panels\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)\n- Linux [diskstats](https://www.kernel.org/doc/html/latest/admin-guide/iostats.html)", "fieldConfig": { "defaults": { "color": { @@ -5275,7 +4155,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "import \"strings\"\r\n\r\nfrom(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r._field =~ /^systemMetrics\\.disks\\..+\\.reads$/ or\r\n r._field =~ /^systemMetrics\\.disks\\..+\\.writes$/)\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({\r\n r with\r\n _value: float(v: r._value),\r\n _field:\r\n if strings.hasSuffix(v: r._field, suffix: \".reads\") then\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".reads\",\r\n ) + \" reads\"\r\n else\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".writes\",\r\n ) + \" writes\",\r\n }))", + "query": "import \"strings\"\r\n\r\nfrom(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r._field =~ /^systemMetrics\\.disks\\..+\\.reads$/ or\r\n r._field =~ /^systemMetrics\\.disks\\..+\\.writes$/)\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({\r\n r with\r\n _value: float(v: r._value),\r\n _field:\r\n if strings.hasSuffix(v: r._field, suffix: \".reads\") then\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".reads\",\r\n ) + \" reads\"\r\n else\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".writes\",\r\n ) + \" writes\",\r\n }))", "refId": "A" } ], @@ -5287,7 +4167,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Disk I/O\n\nPer-device **read/write throughput** from FTDC **systemMetrics** (`/proc/diskstats` cumulative sector counters).\n\n**Formula:** `derivative(read_sectors|write_sectors, 1s) × 512` → **bytes/sec**\n\n**Unit:** B/s (`Bps`) — Linux diskstats sectors are **512 bytes** each\n\n**Not mongod-only:** OS view of all processes on the host.\n\n---\n\n### Metrics\n\n**Pattern (device names vary by host):**\n\n`systemMetrics.disks..read_sectors`\n`systemMetrics.disks..write_sectors`\n\nExample: `systemMetrics.disks.xvda.read_sectors` · `systemMetrics.disks.sdb.write_sectors`\n\nDisk metrics are ingested via the exporter disk allowlist (no per-device lines in `metrics_to_get.txt`).\n\n**Query:** `derivative(..., unit: 1s, nonNegative: true)` → sectors/s → × 512 → B/s\n\n---\n\n### How to read\n\n- One line per **device · read** and **device · write** throughput\n- Write spikes every ~**60s** → WiredTiger **checkpoints** (normal on data volume)\n- High B/s + low IOPS → large sequential I/O · compare **Disk writes and reads**\n- High B/s + high IOPS → random or mixed workload\n\n---\n\n### Troubleshooting\n\n- Sustained high throughput + **CPU Usage** I/O wait → disk bandwidth saturation\n- Read-heavy → cache pressure · **WiredTiger Block Manager**, **Page Faults**\n- Write-heavy → **Checkpoint Write Throughput**, **Pages Evicted**\n- Queueing / busy time → **Disk Queue Time**, **Disk I/O Busy Time** (not this panel)\n\nCorrelate with **Disk writes and reads** (IOPS) and **WiredTiger Block Manager**.\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)\n- Linux [diskstats](https://www.kernel.org/doc/html/latest/admin-guide/iostats.html)", + "description": "### Disk I/O\n\nPer-device **read/write throughput** from FTDC **systemMetrics** (`/proc/diskstats` cumulative sector counters).\n\n**Formula:** `derivative(read_sectors|write_sectors, 1s) \u00d7 512` \u2192 **bytes/sec**\n\n**Unit:** B/s (`Bps`) \u2014 Linux diskstats sectors are **512 bytes** each\n\n**Not mongod-only:** OS view of all processes on the host.\n\n---\n\n### Metrics\n\n**Pattern (device names vary by host):**\n\n`systemMetrics.disks..read_sectors`\n`systemMetrics.disks..write_sectors`\n\nExample: `systemMetrics.disks.xvda.read_sectors` \u00b7 `systemMetrics.disks.sdb.write_sectors`\n\nDisk metrics are ingested via the exporter disk allowlist (no per-device lines in `metrics_to_get.txt`).\n\n**Query:** `derivative(..., unit: 1s, nonNegative: true)` \u2192 sectors/s \u2192 \u00d7 512 \u2192 B/s\n\n---\n\n### How to read\n\n- One line per **device \u00b7 read** and **device \u00b7 write** throughput\n- Write spikes every ~**60s** \u2192 WiredTiger **checkpoints** (normal on data volume)\n- High B/s + low IOPS \u2192 large sequential I/O \u00b7 compare **Disk writes and reads**\n- High B/s + high IOPS \u2192 random or mixed workload\n\n---\n\n### Troubleshooting\n\n- Sustained high throughput + **CPU Usage** I/O wait \u2192 disk bandwidth saturation\n- Read-heavy \u2192 cache pressure \u00b7 **WiredTiger Block Manager**, **Page Faults**\n- Write-heavy \u2192 **Checkpoint Write Throughput**, **Pages Evicted**\n- Queueing / busy time \u2192 **Disk Queue Time**, **Disk I/O Busy Time** (not this panel)\n\nCorrelate with **Disk writes and reads** (IOPS) and **WiredTiger Block Manager**.\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)\n- Linux [diskstats](https://www.kernel.org/doc/html/latest/admin-guide/iostats.html)", "fieldConfig": { "defaults": { "color": { @@ -5374,7 +4254,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "import \"strings\"\r\n\r\nfrom(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r._field =~ /^systemMetrics\\.disks\\..+\\.read_sectors$/ or\r\n r._field =~ /^systemMetrics\\.disks\\..+\\.write_sectors$/)\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({\r\n r with\r\n _value: float(v: r._value) * 512.0,\r\n _field:\r\n if strings.hasSuffix(v: r._field, suffix: \".read_sectors\") then\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".read_sectors\",\r\n ) + \" read\"\r\n else\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".write_sectors\",\r\n ) + \" write\",\r\n }))", + "query": "import \"strings\"\r\n\r\nfrom(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r._field =~ /^systemMetrics\\.disks\\..+\\.read_sectors$/ or\r\n r._field =~ /^systemMetrics\\.disks\\..+\\.write_sectors$/)\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({\r\n r with\r\n _value: float(v: r._value) * 512.0,\r\n _field:\r\n if strings.hasSuffix(v: r._field, suffix: \".read_sectors\") then\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".read_sectors\",\r\n ) + \" read\"\r\n else\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".write_sectors\",\r\n ) + \" write\",\r\n }))", "refId": "A" } ], @@ -5386,7 +4266,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Disk reads\n\nPer-device **read throughput** from FTDC **systemMetrics** (`/proc/diskstats` cumulative `read_sectors` counter).\n\n**Formula:** `derivative(read_sectors, 1s) × 512` → **bytes/sec**\n\n**Unit:** B/s (`Bps`) — Linux diskstats sectors are **512 bytes** each\n\n**Not mongod-only:** OS view of all processes on the host (not `serverStatus`).\n\n---\n\n### Metrics\n\n**Pattern (device names vary by host — regex filter):**\n\n`systemMetrics.disks..read_sectors`\n\nExample: `systemMetrics.disks.xvda.read_sectors`\n\nIngested via exporter disk allowlist (`isDiskMetric`); no per-device entries in `metrics_to_get.txt`.\n\n**Query type:** cumulative counter → `derivative(unit: 1s, nonNegative: true)` → `aggregateWindow(mean)` → × 512\n\n**Not this panel:** `read_time_ms` → **Disk I/O Busy Time** · `.reads` IOPS → **Disk writes and reads**\n\n---\n\n### How to read\n\n- One line per device · legend **`{device} read`**\n- Sustained high reads → cold data / cache misses · working set vs RAM\n- High B/s + low read IOPS → large sequential reads · compare **Disk writes and reads**\n\n---\n\n### Troubleshooting\n\n- High reads + **Page Faults** → memory pressure\n- High reads + **WiredTiger Cache** full → increase cache or RAM\n- High reads + **CPU Usage** I/O wait → disk bandwidth · **Disk I/O**, **Disk I/O Busy Time**\n- Mongod-specific reads → **WiredTiger Block Manager** (bytes read)\n\nCorrelate with **Disk I/O** (combined read/write B/s).\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)\n- Linux [diskstats](https://www.kernel.org/doc/html/latest/admin-guide/iostats.html)", + "description": "### Disk reads\n\nPer-device **read throughput** from FTDC **systemMetrics** (`/proc/diskstats` cumulative `read_sectors` counter).\n\n**Formula:** `derivative(read_sectors, 1s) \u00d7 512` \u2192 **bytes/sec**\n\n**Unit:** B/s (`Bps`) \u2014 Linux diskstats sectors are **512 bytes** each\n\n**Not mongod-only:** OS view of all processes on the host (not `serverStatus`).\n\n---\n\n### Metrics\n\n**Pattern (device names vary by host \u2014 regex filter):**\n\n`systemMetrics.disks..read_sectors`\n\nExample: `systemMetrics.disks.xvda.read_sectors`\n\nIngested via exporter disk allowlist (`isDiskMetric`); no per-device entries in `metrics_to_get.txt`.\n\n**Query type:** cumulative counter \u2192 `derivative(unit: 1s, nonNegative: true)` \u2192 `aggregateWindow(mean)` \u2192 \u00d7 512\n\n**Not this panel:** `read_time_ms` \u2192 **Disk I/O Busy Time** \u00b7 `.reads` IOPS \u2192 **Disk writes and reads**\n\n---\n\n### How to read\n\n- One line per device \u00b7 legend **`{device} read`**\n- Sustained high reads \u2192 cold data / cache misses \u00b7 working set vs RAM\n- High B/s + low read IOPS \u2192 large sequential reads \u00b7 compare **Disk writes and reads**\n\n---\n\n### Troubleshooting\n\n- High reads + **Page Faults** \u2192 memory pressure\n- High reads + **WiredTiger Cache** full \u2192 increase cache or RAM\n- High reads + **CPU Usage** I/O wait \u2192 disk bandwidth \u00b7 **Disk I/O**, **Disk I/O Busy Time**\n- Mongod-specific reads \u2192 **WiredTiger Block Manager** (bytes read)\n\nCorrelate with **Disk I/O** (combined read/write B/s).\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)\n- Linux [diskstats](https://www.kernel.org/doc/html/latest/admin-guide/iostats.html)", "fieldConfig": { "defaults": { "color": { @@ -5473,7 +4353,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "import \"strings\"\r\n\r\nfrom(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..+\\.read_sectors$/)\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({\r\n r with\r\n _value: float(v: r._value) * 512.0,\r\n _field:\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".read_sectors\",\r\n ) + \" read\",\r\n }))", + "query": "import \"strings\"\r\n\r\nfrom(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..+\\.read_sectors$/)\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({\r\n r with\r\n _value: float(v: r._value) * 512.0,\r\n _field:\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".read_sectors\",\r\n ) + \" read\",\r\n }))", "refId": "A" } ], @@ -5485,7 +4365,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Disk writes\n\nPer-device **write throughput** from FTDC **systemMetrics** (`/proc/diskstats` cumulative `write_sectors` counter).\n\n**Formula:** `derivative(write_sectors, 1s) × 512` → **bytes/sec**\n\n**Unit:** B/s (`Bps`) — Linux diskstats sectors are **512 bytes** each\n\n**Not mongod-only:** OS view of all processes on the host (not `serverStatus`).\n\n---\n\n### Metrics\n\n**Pattern (device names vary by host — regex filter):**\n\n`systemMetrics.disks..write_sectors`\n\nExample: `systemMetrics.disks.xvda.write_sectors`\n\nIngested via exporter disk allowlist (`isDiskMetric`).\n\n**Query type:** cumulative counter → `derivative(unit: 1s, nonNegative: true)` → `aggregateWindow(mean)` → × 512\n\n**Not this panel:** `write_time_ms` → **Disk I/O Busy Time** · `.writes` IOPS → **Disk writes and reads**\n\n---\n\n### How to read\n\n- One line per device · legend **`{device} write`**\n- Spikes every ~**60s** → WiredTiger **checkpoints** on data volume (normal)\n- Sustained high writes between spikes → heavy write load or eviction\n\n---\n\n### Troubleshooting\n\n- Write spikes + latency → checkpoint I/O · **Checkpoint Write Throughput**, **Checkpoint Duration**\n- Sustained high writes → **Dirty Cache Bytes**, **Pages Evicted**, **Operations**\n- High write B/s + high **CPU Usage** I/O wait → disk bandwidth · **Disk I/O**, **Disk I/O Busy Time**\n\nCorrelate with **Disk I/O** (combined read/write B/s) and **WiredTiger Block Manager**.\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)\n- Linux [diskstats](https://www.kernel.org/doc/html/latest/admin-guide/iostats.html)", + "description": "### Disk writes\n\nPer-device **write throughput** from FTDC **systemMetrics** (`/proc/diskstats` cumulative `write_sectors` counter).\n\n**Formula:** `derivative(write_sectors, 1s) \u00d7 512` \u2192 **bytes/sec**\n\n**Unit:** B/s (`Bps`) \u2014 Linux diskstats sectors are **512 bytes** each\n\n**Not mongod-only:** OS view of all processes on the host (not `serverStatus`).\n\n---\n\n### Metrics\n\n**Pattern (device names vary by host \u2014 regex filter):**\n\n`systemMetrics.disks..write_sectors`\n\nExample: `systemMetrics.disks.xvda.write_sectors`\n\nIngested via exporter disk allowlist (`isDiskMetric`).\n\n**Query type:** cumulative counter \u2192 `derivative(unit: 1s, nonNegative: true)` \u2192 `aggregateWindow(mean)` \u2192 \u00d7 512\n\n**Not this panel:** `write_time_ms` \u2192 **Disk I/O Busy Time** \u00b7 `.writes` IOPS \u2192 **Disk writes and reads**\n\n---\n\n### How to read\n\n- One line per device \u00b7 legend **`{device} write`**\n- Spikes every ~**60s** \u2192 WiredTiger **checkpoints** on data volume (normal)\n- Sustained high writes between spikes \u2192 heavy write load or eviction\n\n---\n\n### Troubleshooting\n\n- Write spikes + latency \u2192 checkpoint I/O \u00b7 **Checkpoint Write Throughput**, **Checkpoint Duration**\n- Sustained high writes \u2192 **Dirty Cache Bytes**, **Pages Evicted**, **Operations**\n- High write B/s + high **CPU Usage** I/O wait \u2192 disk bandwidth \u00b7 **Disk I/O**, **Disk I/O Busy Time**\n\nCorrelate with **Disk I/O** (combined read/write B/s) and **WiredTiger Block Manager**.\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)\n- Linux [diskstats](https://www.kernel.org/doc/html/latest/admin-guide/iostats.html)", "fieldConfig": { "defaults": { "color": { @@ -5572,7 +4452,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "import \"strings\"\r\n\r\nfrom(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..+\\.write_sectors$/)\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({\r\n r with\r\n _value: float(v: r._value) * 512.0,\r\n _field:\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".write_sectors\",\r\n ) + \" write\",\r\n }))", + "query": "import \"strings\"\r\n\r\nfrom(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..+\\.write_sectors$/)\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({\r\n r with\r\n _value: float(v: r._value) * 512.0,\r\n _field:\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".write_sectors\",\r\n ) + \" write\",\r\n }))", "refId": "A" } ], @@ -5584,7 +4464,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Disk queue time\n\nPer-device **I/O queue wait time rate** from FTDC **systemMetrics** (`/proc/diskstats` cumulative `io_queued_ms` counter).\n\n**Formula:** `derivative(io_queued_ms, 1s)` → **ms/s** (milliseconds of queue wait accrued per wall-clock second)\n\n**Unit:** ms/s — **not** in-flight I/O count · **not** `read_time_ms` / `write_time_ms` (**Disk I/O Busy Time**)\n\n**Not mongod-only:** OS view of all processes on the host.\n\n**Saturation hint:** sustained values toward **~1000 ms/s** → persistent queueing on that device.\n\n---\n\n### Metrics\n\n**Pattern (device names vary by host — regex filter):**\n\n`systemMetrics.disks..io_queued_ms`\n\nExample: `systemMetrics.disks.xvda.io_queued_ms`\n\nIngested via exporter disk allowlist (`isDiskMetric`).\n\n**Query type:** cumulative counter → `derivative(unit: 1s, nonNegative: true)` → `aggregateWindow(mean)`\n\n**In-flight I/O count:** `systemMetrics.disks..io_in_progress` (gauge) — not plotted here.\n\n---\n\n### How to read\n\n- One line per device · **`{device} queue time`** (ms/s)\n- Rising queue time + flat throughput → device falling behind\n- Spikes with checkpoint windows → normal on data volume if brief\n\n---\n\n### Troubleshooting\n\n- Sustained high ms/s → disk saturation · **Disk I/O Busy Time**, **Disk I/O**, **CPU Usage** I/O wait\n- Queue time up + **Average Operation Latency** up → storage bottleneck\n- Queue time without high B/s → random/small I/O or slow device · **Disk writes and reads**\n\nCorrelate with **Disk I/O Busy Time** and **Disk I/O**.\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)\n- Linux [diskstats](https://www.kernel.org/doc/html/latest/admin-guide/iostats.html)", + "description": "### Disk queue time\n\nPer-device **I/O queue wait time rate** from FTDC **systemMetrics** (`/proc/diskstats` cumulative `io_queued_ms` counter).\n\n**Formula:** `derivative(io_queued_ms, 1s)` \u2192 **ms/s** (milliseconds of queue wait accrued per wall-clock second)\n\n**Unit:** ms/s \u2014 **not** in-flight I/O count \u00b7 **not** `read_time_ms` / `write_time_ms` (**Disk I/O Busy Time**)\n\n**Not mongod-only:** OS view of all processes on the host.\n\n**Saturation hint:** sustained values toward **~1000 ms/s** \u2192 persistent queueing on that device.\n\n---\n\n### Metrics\n\n**Pattern (device names vary by host \u2014 regex filter):**\n\n`systemMetrics.disks..io_queued_ms`\n\nExample: `systemMetrics.disks.xvda.io_queued_ms`\n\nIngested via exporter disk allowlist (`isDiskMetric`).\n\n**Query type:** cumulative counter \u2192 `derivative(unit: 1s, nonNegative: true)` \u2192 `aggregateWindow(mean)`\n\n**In-flight I/O count:** `systemMetrics.disks..io_in_progress` (gauge) \u2014 not plotted here.\n\n---\n\n### How to read\n\n- One line per device \u00b7 **`{device} queue time`** (ms/s)\n- Rising queue time + flat throughput \u2192 device falling behind\n- Spikes with checkpoint windows \u2192 normal on data volume if brief\n\n---\n\n### Troubleshooting\n\n- Sustained high ms/s \u2192 disk saturation \u00b7 **Disk I/O Busy Time**, **Disk I/O**, **CPU Usage** I/O wait\n- Queue time up + **Average Operation Latency** up \u2192 storage bottleneck\n- Queue time without high B/s \u2192 random/small I/O or slow device \u00b7 **Disk writes and reads**\n\nCorrelate with **Disk I/O Busy Time** and **Disk I/O**.\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)\n- Linux [diskstats](https://www.kernel.org/doc/html/latest/admin-guide/iostats.html)", "fieldConfig": { "defaults": { "color": { @@ -5670,7 +4550,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "import \"strings\"\r\n\r\nfrom(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..+\\.io_queued_ms$/)\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({\r\n r with\r\n _value: float(v: r._value),\r\n _field:\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".io_queued_ms\",\r\n ) + \" queue time\",\r\n }))", + "query": "import \"strings\"\r\n\r\nfrom(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..+\\.io_queued_ms$/)\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({\r\n r with\r\n _value: float(v: r._value),\r\n _field:\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".io_queued_ms\",\r\n ) + \" queue time\",\r\n }))", "refId": "A" } ], @@ -5682,7 +4562,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Disk I/O busy time\n\nPer-device **I/O service time rate** from FTDC **systemMetrics** (`/proc/diskstats` cumulative `read_time_ms` / `write_time_ms` counters).\n\n**Formula:** `derivative(read_time_ms|write_time_ms, 1s)` → **ms/s** (milliseconds of I/O time accrued per wall-clock second)\n\n**Unit:** ms/s — **not** average latency per operation · **not** `serverStatus.opLatencies`\n\n**Not mongod-only:** OS view of all processes on the host.\n\n**Saturation hint:** values approaching **~1000 ms/s** on a line → device spending ~100% of wall time in that I/O direction.\n\n---\n\n### Metrics\n\n**Pattern (device names vary by host — regex filter):**\n\n`systemMetrics.disks..read_time_ms`\n`systemMetrics.disks..write_time_ms`\n\nExample: `systemMetrics.disks.xvda.read_time_ms` · `systemMetrics.disks.sdb.write_time_ms`\n\nIngested via exporter disk allowlist (`isDiskMetric`).\n\n**Query type:** cumulative counter → `derivative(unit: 1s, nonNegative: true)` → `aggregateWindow(mean)`\n\n**Not this panel:** queue wait → **Disk Queue Time** (`io_queued_ms`) · throughput → **Disk I/O** · IOPS → **Disk writes and reads**\n\n---\n\n### How to read\n\n- One line per device · **read time** / **write time** (ms/s)\n- Write spikes ~every **60s** → checkpoint I/O on data volume (normal)\n- High ms/s + high **Disk I/O** B/s → bandwidth-heavy workload\n- High ms/s + high IOPS → random/small I/O or saturation\n\n---\n\n### Troubleshooting\n\n- Sustained high ms/s → disk busy · **Disk Queue Time**, **CPU Usage** I/O wait\n- High ms/s + rising **Average Operation Latency** → storage bottleneck\n- High read time + **Page Faults** → cache / memory pressure\n- High write time + **Checkpoint Write Throughput** → checkpoint / eviction pressure\n\nCorrelate with **Disk I/O**, **Disk writes and reads**, and **Disk Queue Time**.\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)\n- Linux [diskstats](https://www.kernel.org/doc/html/latest/admin-guide/iostats.html)", + "description": "### Disk I/O busy time\n\nPer-device **I/O service time rate** from FTDC **systemMetrics** (`/proc/diskstats` cumulative `read_time_ms` / `write_time_ms` counters).\n\n**Formula:** `derivative(read_time_ms|write_time_ms, 1s)` \u2192 **ms/s** (milliseconds of I/O time accrued per wall-clock second)\n\n**Unit:** ms/s \u2014 **not** average latency per operation \u00b7 **not** `serverStatus.opLatencies`\n\n**Not mongod-only:** OS view of all processes on the host.\n\n**Saturation hint:** values approaching **~1000 ms/s** on a line \u2192 device spending ~100% of wall time in that I/O direction.\n\n---\n\n### Metrics\n\n**Pattern (device names vary by host \u2014 regex filter):**\n\n`systemMetrics.disks..read_time_ms`\n`systemMetrics.disks..write_time_ms`\n\nExample: `systemMetrics.disks.xvda.read_time_ms` \u00b7 `systemMetrics.disks.sdb.write_time_ms`\n\nIngested via exporter disk allowlist (`isDiskMetric`).\n\n**Query type:** cumulative counter \u2192 `derivative(unit: 1s, nonNegative: true)` \u2192 `aggregateWindow(mean)`\n\n**Not this panel:** queue wait \u2192 **Disk Queue Time** (`io_queued_ms`) \u00b7 throughput \u2192 **Disk I/O** \u00b7 IOPS \u2192 **Disk writes and reads**\n\n---\n\n### How to read\n\n- One line per device \u00b7 **read time** / **write time** (ms/s)\n- Write spikes ~every **60s** \u2192 checkpoint I/O on data volume (normal)\n- High ms/s + high **Disk I/O** B/s \u2192 bandwidth-heavy workload\n- High ms/s + high IOPS \u2192 random/small I/O or saturation\n\n---\n\n### Troubleshooting\n\n- Sustained high ms/s \u2192 disk busy \u00b7 **Disk Queue Time**, **CPU Usage** I/O wait\n- High ms/s + rising **Average Operation Latency** \u2192 storage bottleneck\n- High read time + **Page Faults** \u2192 cache / memory pressure\n- High write time + **Checkpoint Write Throughput** \u2192 checkpoint / eviction pressure\n\nCorrelate with **Disk I/O**, **Disk writes and reads**, and **Disk Queue Time**.\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)\n- Linux [diskstats](https://www.kernel.org/doc/html/latest/admin-guide/iostats.html)", "fieldConfig": { "defaults": { "color": { @@ -5769,7 +4649,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "import \"strings\"\r\n\r\nfrom(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r._field =~ /^systemMetrics\\.disks\\..+\\.read_time_ms$/ or\r\n r._field =~ /^systemMetrics\\.disks\\..+\\.write_time_ms$/)\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({\r\n r with\r\n _value: float(v: r._value),\r\n _field:\r\n if strings.hasSuffix(v: r._field, suffix: \".read_time_ms\") then\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".read_time_ms\",\r\n ) + \" read time\"\r\n else\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".write_time_ms\",\r\n ) + \" write time\",\r\n }))", + "query": "import \"strings\"\r\n\r\nfrom(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r._field =~ /^systemMetrics\\.disks\\..+\\.read_time_ms$/ or\r\n r._field =~ /^systemMetrics\\.disks\\..+\\.write_time_ms$/)\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({\r\n r with\r\n _value: float(v: r._value),\r\n _field:\r\n if strings.hasSuffix(v: r._field, suffix: \".read_time_ms\") then\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".read_time_ms\",\r\n ) + \" read time\"\r\n else\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".write_time_ms\",\r\n ) + \" write time\",\r\n }))", "refId": "A" } ], @@ -5781,7 +4661,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### TCP segments in / out\n\nPer-second rates of **Tcp:InSegs** and **Tcp:OutSegs** from FTDC **systemMetrics** (`/proc/net/snmp`).\n\n**Unit:** segments/s (`ops`)\n\n**Not mongod-only:** whole-host TCP stack.\n\n---\n\n### Metrics\n\n**Fields:**\n\n`systemMetrics.netstat.Tcp:InSegs`\n`systemMetrics.netstat.Tcp:OutSegs`\n\n**Query:** `derivative(unit: 1s, nonNegative: true)` → `aggregateWindow(mean)`\n\n---\n\n### How to read\n\n- **TCP segments in** / **TCP segments out** — raw TCP-level volume\n- High In/Out ratio on a secondary → receives more replication than it sends\n\n---\n\n### Troubleshooting\n\n- Sudden drops → network outage\n- Compare with **Network In / Out** (application bytes) and **Replica Members Ping**\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)", + "description": "### TCP segments in / out\n\nPer-second rates of **Tcp:InSegs** and **Tcp:OutSegs** from FTDC **systemMetrics** (`/proc/net/snmp`).\n\n**Unit:** segments/s (`ops`)\n\n**Not mongod-only:** whole-host TCP stack.\n\n---\n\n### Metrics\n\n**Fields:**\n\n`systemMetrics.netstat.Tcp:InSegs`\n`systemMetrics.netstat.Tcp:OutSegs`\n\n**Query:** `derivative(unit: 1s, nonNegative: true)` \u2192 `aggregateWindow(mean)`\n\n---\n\n### How to read\n\n- **TCP segments in** / **TCP segments out** \u2014 raw TCP-level volume\n- High In/Out ratio on a secondary \u2192 receives more replication than it sends\n\n---\n\n### Troubleshooting\n\n- Sudden drops \u2192 network outage\n- Compare with **Network In / Out** (application bytes) and **Replica Members Ping**\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)", "fieldConfig": { "defaults": { "color": { @@ -5835,32 +4715,7 @@ }, "unit": "ops" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*InSegs$" - }, - "properties": [ - { - "id": "displayName", - "value": "TCP segments in" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*OutSegs$" - }, - "properties": [ - { - "id": "displayName", - "value": "TCP segments out" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 7, @@ -5893,7 +4748,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:InSegs\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:OutSegs\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:InSegs\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:OutSegs\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"systemMetrics.netstat.Tcp:InSegs\" then \"TCP segments in\" else if r._field == \"systemMetrics.netstat.Tcp:OutSegs\" then \"TCP segments out\" else r._field\r\n }))", "refId": "A" } ], @@ -5905,7 +4760,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Network In / Out\n\nPer-second **logical** network throughput for **mongod** — cumulative **`serverStatus.network.bytesIn`** / **`bytesOut`**.\n\n**Unit:** B/s (`Bps`) — `derivative(unit: 1s)` on byte counters\n\n**Logical bytes:** BSON/document size as counted by mongod (not necessarily on-wire size). Compare with **Network Physical In / Out** for compression/wire bytes.\n\n---\n\n### Metrics\n\n**Fields:**\n\n`serverStatus.network.bytesIn`\n`serverStatus.network.bytesOut`\n\n**Query type:** cumulative counter → `derivative(unit: 1s, nonNegative: true)` → `aggregateWindow(mean)`\n\n---\n\n### How to read\n\n- **Logical bytes in** — data received by mongod\n- **Logical bytes out** — data sent to clients / other nodes\n- Secondary: often **in ≥ out** during replication catch-up\n- Primary: **out** rises with large query results or replication fan-out\n\n---\n\n### Troubleshooting\n\n- High **bytes out** + high **Documents** returned → large result sets · check projections/`limit`\n- High bytes + low **Operations** → big payloads per op\n- Logical ≫ physical on **Network Physical In / Out** → compression effective (normal)\n- Logical ≈ physical → little compression · replication or already-compressed payloads\n- Bytes flat but lag → **Replica Members Lag**, **TCP InSegs / OutSegs**, **Replica Members Ping**\n\nCorrelate with **Network requests** and **Operations**.\n\n---\n\n### Docs\n\n- [serverStatus — network](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.network)", + "description": "### Network In / Out\n\nPer-second **logical** network throughput for **mongod** \u2014 cumulative **`serverStatus.network.bytesIn`** / **`bytesOut`**.\n\n**Unit:** B/s (`Bps`) \u2014 `derivative(unit: 1s)` on byte counters\n\n**Logical bytes:** BSON/document size as counted by mongod (not necessarily on-wire size). Compare with **Network Physical In / Out** for compression/wire bytes.\n\n---\n\n### Metrics\n\n**Fields:**\n\n`serverStatus.network.bytesIn`\n`serverStatus.network.bytesOut`\n\n**Query type:** cumulative counter \u2192 `derivative(unit: 1s, nonNegative: true)` \u2192 `aggregateWindow(mean)`\n\n---\n\n### How to read\n\n- **Logical bytes in** \u2014 data received by mongod\n- **Logical bytes out** \u2014 data sent to clients / other nodes\n- Secondary: often **in \u2265 out** during replication catch-up\n- Primary: **out** rises with large query results or replication fan-out\n\n---\n\n### Troubleshooting\n\n- High **bytes out** + high **Documents** returned \u2192 large result sets \u00b7 check projections/`limit`\n- High bytes + low **Operations** \u2192 big payloads per op\n- Logical \u226b physical on **Network Physical In / Out** \u2192 compression effective (normal)\n- Logical \u2248 physical \u2192 little compression \u00b7 replication or already-compressed payloads\n- Bytes flat but lag \u2192 **Replica Members Lag**, **TCP InSegs / OutSegs**, **Replica Members Ping**\n\nCorrelate with **Network requests** and **Operations**.\n\n---\n\n### Docs\n\n- [serverStatus \u2014 network](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.network)", "fieldConfig": { "defaults": { "color": { @@ -5959,32 +4814,7 @@ }, "unit": "Bps" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*\\.bytesIn$" - }, - "properties": [ - { - "id": "displayName", - "value": "Logical bytes in" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*\\.bytesOut$" - }, - "properties": [ - { - "id": "displayName", - "value": "Logical bytes out" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 7, @@ -6017,7 +4847,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.network.bytesIn\" or\r\n r[\"_field\"] == \"serverStatus.network.bytesOut\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.network.bytesIn\" or\r\n r[\"_field\"] == \"serverStatus.network.bytesOut\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.network.bytesIn\" then \"Logical bytes in\" else if r._field == \"serverStatus.network.bytesOut\" then \"Logical bytes out\" else r._field\r\n }))", "refId": "A" } ], @@ -6029,7 +4859,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### Network In / Out Physical\n\nPer-second **physical (on-the-wire)** network throughput for **mongod** — cumulative **`serverStatus.network.physicalBytesIn`** / **`physicalBytesOut`**.\n\n**Unit:** B/s (`Bps`) — `derivative(unit: 1s)` on byte counters\n\n**Physical vs logical:** **Physical** = bytes sent/received on network connections (wire size, after compression where applicable). **Logical** = document/BSON bytes as counted by mongod · compare **Network In / Out**.\n\n**Not disk I/O:** network bytes, not storage · see **Disk I/O** for block device throughput.\n\n---\n\n### Metrics\n\n**Fields:**\n\n`serverStatus.network.physicalBytesIn`\n`serverStatus.network.physicalBytesOut`\n\n**Query type:** cumulative counter → `derivative(unit: 1s, nonNegative: true)` → `aggregateWindow(mean)`\n\n---\n\n### How to read\n\n- **Physical bytes in** — wire bytes received by mongod\n- **Physical bytes out** — wire bytes sent by mongod\n- **Logical > physical** on paired panels → compression saving bandwidth (common on replication)\n- **Logical ≈ physical** → little compression benefit on the wire\n\n---\n\n### Troubleshooting\n\n- High physical **out** on primary → replication + client traffic · **Replica Members Lag**, **TCP InSegs / OutSegs**\n- High physical **in** on secondary → replication catch-up\n- Physical throughput near link capacity → timeouts, lag · **Replica Members Ping**\n- High physical + low **Operations** → large compressed payloads or oplog batches\n\nCorrelate with **Network In / Out** (logical) and **Network requests**.\n\n---\n\n### Docs\n\n- [serverStatus — network](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.network)", + "description": "### Network In / Out Physical\n\nPer-second **physical (on-the-wire)** network throughput for **mongod** \u2014 cumulative **`serverStatus.network.physicalBytesIn`** / **`physicalBytesOut`**.\n\n**Unit:** B/s (`Bps`) \u2014 `derivative(unit: 1s)` on byte counters\n\n**Physical vs logical:** **Physical** = bytes sent/received on network connections (wire size, after compression where applicable). **Logical** = document/BSON bytes as counted by mongod \u00b7 compare **Network In / Out**.\n\n**Not disk I/O:** network bytes, not storage \u00b7 see **Disk I/O** for block device throughput.\n\n---\n\n### Metrics\n\n**Fields:**\n\n`serverStatus.network.physicalBytesIn`\n`serverStatus.network.physicalBytesOut`\n\n**Query type:** cumulative counter \u2192 `derivative(unit: 1s, nonNegative: true)` \u2192 `aggregateWindow(mean)`\n\n---\n\n### How to read\n\n- **Physical bytes in** \u2014 wire bytes received by mongod\n- **Physical bytes out** \u2014 wire bytes sent by mongod\n- **Logical > physical** on paired panels \u2192 compression saving bandwidth (common on replication)\n- **Logical \u2248 physical** \u2192 little compression benefit on the wire\n\n---\n\n### Troubleshooting\n\n- High physical **out** on primary \u2192 replication + client traffic \u00b7 **Replica Members Lag**, **TCP InSegs / OutSegs**\n- High physical **in** on secondary \u2192 replication catch-up\n- Physical throughput near link capacity \u2192 timeouts, lag \u00b7 **Replica Members Ping**\n- High physical + low **Operations** \u2192 large compressed payloads or oplog batches\n\nCorrelate with **Network In / Out** (logical) and **Network requests**.\n\n---\n\n### Docs\n\n- [serverStatus \u2014 network](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.network)", "fieldConfig": { "defaults": { "color": { @@ -6083,32 +4913,7 @@ }, "unit": "Bps" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*physicalBytesIn$" - }, - "properties": [ - { - "id": "displayName", - "value": "Physical bytes in" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*physicalBytesOut$" - }, - "properties": [ - { - "id": "displayName", - "value": "Physical bytes out" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 7, @@ -6141,7 +4946,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.network.physicalBytesIn\" or\r\n r[\"_field\"] == \"serverStatus.network.physicalBytesOut\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.network.physicalBytesIn\" or\r\n r[\"_field\"] == \"serverStatus.network.physicalBytesOut\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.network.physicalBytesIn\" then \"Physical bytes in\" else if r._field == \"serverStatus.network.physicalBytesOut\" then \"Physical bytes out\" else r._field\r\n }))", "refId": "A" } ], @@ -6153,7 +4958,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### TCP connection lifecycle\n\nPer-second rates of host **TCP connection lifecycle counters** from FTDC **systemMetrics** (`/proc/net/snmp`).\n\n**Unit:** events/sec — `derivative(unit: 1s, nonNegative: true)` on cumulative counters\n\n**Not mongod-only:** whole-host TCP stack.\n\n---\n\n### Metrics\n\n**Fields:**\n\n`systemMetrics.netstat.Tcp:ActiveOpens`\n`systemMetrics.netstat.Tcp:AttemptFails`\n`systemMetrics.netstat.Tcp:EstabResets`\n`systemMetrics.netstat.Tcp:OutRsts`\n\n**Query type:** cumulative counter → `derivative(unit: 1s, nonNegative: true)` → `aggregateWindow(mean)`\n\n**Not this panel:** `Tcp:CurrEstab` (open connection **count**, gauge — use `last`, not derivative)\n\n---\n\n### How to read\n\n- **TCP active opens** — successful new connections initiated (normally **> 0** on a busy host)\n- **TCP connect failures** — failed connection attempts (expect **~0**)\n- **TCP established resets** — established connections reset (expect **~0**)\n- **TCP RST sent** — segments with RST flag (expect **~0**)\n- Brief spikes during deploys/restarts are normal\n\n---\n\n### Troubleshooting\n\n- Sustained **AttemptFails** → refused connections · firewall, `ulimit`, mongod down, wrong port\n- **EstabResets** / **RST sent** → mid-connection drops · **Replica Members Health**, node restarts\n- High **ActiveOpens** with errors → churny clients or replication mesh · compare **TCP InSegs / OutSegs**\n- With **TCP Packet Loss or Corruption** → loss before reset\n- With lag → **Replica Members Lag**, **Replica Members Ping**\n\nCorrelate with **TCP InSegs / OutSegs** and **Replica Members Health**.\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)\n- Linux [SNMP-TCP-MIB / netstat](https://www.kernel.org/doc/html/latest/networking/snmp_counter.html)", + "description": "### TCP connection lifecycle\n\nPer-second rates of host **TCP connection lifecycle counters** from FTDC **systemMetrics** (`/proc/net/snmp`).\n\n**Unit:** events/sec \u2014 `derivative(unit: 1s, nonNegative: true)` on cumulative counters\n\n**Not mongod-only:** whole-host TCP stack.\n\n---\n\n### Metrics\n\n**Fields:**\n\n`systemMetrics.netstat.Tcp:ActiveOpens`\n`systemMetrics.netstat.Tcp:AttemptFails`\n`systemMetrics.netstat.Tcp:EstabResets`\n`systemMetrics.netstat.Tcp:OutRsts`\n\n**Query type:** cumulative counter \u2192 `derivative(unit: 1s, nonNegative: true)` \u2192 `aggregateWindow(mean)`\n\n**Not this panel:** `Tcp:CurrEstab` (open connection **count**, gauge \u2014 use `last`, not derivative)\n\n---\n\n### How to read\n\n- **TCP active opens** \u2014 successful new connections initiated (normally **> 0** on a busy host)\n- **TCP connect failures** \u2014 failed connection attempts (expect **~0**)\n- **TCP established resets** \u2014 established connections reset (expect **~0**)\n- **TCP RST sent** \u2014 segments with RST flag (expect **~0**)\n- Brief spikes during deploys/restarts are normal\n\n---\n\n### Troubleshooting\n\n- Sustained **AttemptFails** \u2192 refused connections \u00b7 firewall, `ulimit`, mongod down, wrong port\n- **EstabResets** / **RST sent** \u2192 mid-connection drops \u00b7 **Replica Members Health**, node restarts\n- High **ActiveOpens** with errors \u2192 churny clients or replication mesh \u00b7 compare **TCP InSegs / OutSegs**\n- With **TCP Packet Loss or Corruption** \u2192 loss before reset\n- With lag \u2192 **Replica Members Lag**, **Replica Members Ping**\n\nCorrelate with **TCP InSegs / OutSegs** and **Replica Members Health**.\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)\n- Linux [SNMP-TCP-MIB / netstat](https://www.kernel.org/doc/html/latest/networking/snmp_counter.html)", "fieldConfig": { "defaults": { "color": { @@ -6208,56 +5013,7 @@ }, "unit": "none" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*AttemptFails$" - }, - "properties": [ - { - "id": "displayName", - "value": "TCP connect failures" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*EstabResets$" - }, - "properties": [ - { - "id": "displayName", - "value": "TCP established resets" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*ActiveOpens$" - }, - "properties": [ - { - "id": "displayName", - "value": "TCP active opens" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*OutRsts$" - }, - "properties": [ - { - "id": "displayName", - "value": "TCP RST sent" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 7, @@ -6290,7 +5046,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:ActiveOpens\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:AttemptFails\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:EstabResets\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:OutRsts\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:ActiveOpens\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:AttemptFails\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:EstabResets\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:OutRsts\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"systemMetrics.netstat.Tcp:ActiveOpens\" then \"TCP active opens\" else if r._field == \"systemMetrics.netstat.Tcp:AttemptFails\" then \"TCP connect failures\" else if r._field == \"systemMetrics.netstat.Tcp:EstabResets\" then \"TCP established resets\" else if r._field == \"systemMetrics.netstat.Tcp:OutRsts\" then \"TCP RST sent\" else r._field\r\n }))", "refId": "A" } ], @@ -6302,7 +5058,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### TCP routing issues\n\nPer-second rates of host **IP routing and fragmentation error counters** from FTDC **systemMetrics** (`/proc/net/snmp`).\n\n**Unit:** events/sec — `derivative(unit: 1s)` on cumulative counters\n\n**Layer note:** these are **IP** counters (title says “TCP routing” historically). **Not mongod-only.**\n\n**Expectation:** normally **flat at zero**; any sustained rate → routing, MTU, or path problems.\n\n---\n\n### Metrics\n\n**Fields:**\n\n`systemMetrics.netstat.Ip:OutNoRoutes`\n`systemMetrics.netstat.Ip:FragFails`\n`systemMetrics.netstat.Ip:ReasmFails`\n\n**Query type:** cumulative counter → `derivative(unit: 1s, nonNegative: true)` → `aggregateWindow(mean)`\n\n**Not ingested (optional FTDC `IpExt` paths):** `IpExt:InNoRoutes` · `IpExt:InTruncatedPkts` — add to `metrics_to_get.txt` if present in your `metrics.txt`\n\n**Not this panel:** `Ip:InDiscards` / `Ip:OutDiscards` → **TCP Packet Loss or Corruption**\n\n---\n\n### How to read\n\n- **IP no route (out)** — packets dropped with no route to destination\n- **IP fragmentation failures** — could not fragment (often **MTU / PMTU** issues)\n- **IP reassembly failures** — fragmented packets could not be reassembled\n\n---\n\n### Troubleshooting\n\n- **OutNoRoutes** → wrong/missing routes, asymmetric routing, firewall blackholes\n- **FragFails** / **ReasmFails** → **MTU mismatch** (VPN, cloud network, jumbo frames)\n- With **TCP Packet Loss or Corruption** → broader stack problems\n- Intermittent app errors → **Replica Members Ping**, **TCP Connection Issues**\n\nCorrelate with **TCP InSegs / OutSegs** and **Replica Members Health**.\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)\n- Linux [SNMP / netstat counters](https://www.kernel.org/doc/html/latest/networking/snmp_counter.html)", + "description": "### TCP routing issues\n\nPer-second rates of host **IP routing and fragmentation error counters** from FTDC **systemMetrics** (`/proc/net/snmp`).\n\n**Unit:** events/sec \u2014 `derivative(unit: 1s)` on cumulative counters\n\n**Layer note:** these are **IP** counters (title says \u201cTCP routing\u201d historically). **Not mongod-only.**\n\n**Expectation:** normally **flat at zero**; any sustained rate \u2192 routing, MTU, or path problems.\n\n---\n\n### Metrics\n\n**Fields:**\n\n`systemMetrics.netstat.Ip:OutNoRoutes`\n`systemMetrics.netstat.Ip:FragFails`\n`systemMetrics.netstat.Ip:ReasmFails`\n\n**Query type:** cumulative counter \u2192 `derivative(unit: 1s, nonNegative: true)` \u2192 `aggregateWindow(mean)`\n\n**Not ingested (optional FTDC `IpExt` paths):** `IpExt:InNoRoutes` \u00b7 `IpExt:InTruncatedPkts` \u2014 add to `metrics_to_get.txt` if present in your `metrics.txt`\n\n**Not this panel:** `Ip:InDiscards` / `Ip:OutDiscards` \u2192 **TCP Packet Loss or Corruption**\n\n---\n\n### How to read\n\n- **IP no route (out)** \u2014 packets dropped with no route to destination\n- **IP fragmentation failures** \u2014 could not fragment (often **MTU / PMTU** issues)\n- **IP reassembly failures** \u2014 fragmented packets could not be reassembled\n\n---\n\n### Troubleshooting\n\n- **OutNoRoutes** \u2192 wrong/missing routes, asymmetric routing, firewall blackholes\n- **FragFails** / **ReasmFails** \u2192 **MTU mismatch** (VPN, cloud network, jumbo frames)\n- With **TCP Packet Loss or Corruption** \u2192 broader stack problems\n- Intermittent app errors \u2192 **Replica Members Ping**, **TCP Connection Issues**\n\nCorrelate with **TCP InSegs / OutSegs** and **Replica Members Health**.\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)\n- Linux [SNMP / netstat counters](https://www.kernel.org/doc/html/latest/networking/snmp_counter.html)", "fieldConfig": { "defaults": { "color": { @@ -6356,44 +5112,7 @@ }, "unit": "none" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*OutNoRoutes$" - }, - "properties": [ - { - "id": "displayName", - "value": "IP no route (out)" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*FragFails$" - }, - "properties": [ - { - "id": "displayName", - "value": "IP fragmentation failures" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*ReasmFails$" - }, - "properties": [ - { - "id": "displayName", - "value": "IP reassembly failures" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 7, @@ -6426,7 +5145,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"systemMetrics.netstat.Ip:OutNoRoutes\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Ip:FragFails\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Ip:ReasmFails\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"systemMetrics.netstat.Ip:OutNoRoutes\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Ip:FragFails\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Ip:ReasmFails\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"systemMetrics.netstat.Ip:OutNoRoutes\" then \"IP no route (out)\" else if r._field == \"systemMetrics.netstat.Ip:FragFails\" then \"IP fragmentation failures\" else if r._field == \"systemMetrics.netstat.Ip:ReasmFails\" then \"IP reassembly failures\" else r._field\r\n }))", "refId": "A" } ], @@ -6438,7 +5157,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "### TCP packet loss or corruption\n\nPer-second rates of host **TCP/IP error and discard counters** from FTDC **systemMetrics** (`/proc/net/snmp`).\n\n**Unit:** events/sec — `derivative(unit: 1s)` on cumulative counters\n\n**Not mongod-only:** whole-host TCP/IP stack (all processes).\n\n**Expectation:** normally **flat at zero**; any sustained non-zero rate warrants investigation.\n\n---\n\n### Metrics\n\n**Fields:**\n\n`systemMetrics.netstat.Tcp:InCsumErrors`\n`systemMetrics.netstat.Tcp:InErrs`\n`systemMetrics.netstat.Tcp:RetransSegs`\n`systemMetrics.netstat.Ip:InDiscards`\n`systemMetrics.netstat.Ip:OutDiscards`\n\n**Query type:** cumulative counter → `derivative(unit: 1s, nonNegative: true)` → `aggregateWindow(mean)`\n\n**Optional (in `metrics_to_get.txt`, not plotted):** `Ip:FragFails` · `Ip:ReasmFails` (fragmentation/reassembly failures)\n\n---\n\n### How to read\n\n- **TCP retransmissions** — packet loss / timeout on the wire (most common “loss” signal)\n- **TCP checksum errors** — corrupted frames received\n- **TCP input errors** — other inbound TCP errors\n- **IP discards** — kernel dropped packets (buffers, policy, congestion)\n\n---\n\n### Troubleshooting\n\n- **RetransSegs** sustained → loss/congestion · **Replica Members Ping**, **Average Operation Latency**\n- **InCsumErrors** → bad cable/NIC/driver, offload bugs\n- **InDiscards** / **OutDiscards** → buffer pressure · NIC ring, `netdev_max_backlog`, flood traffic\n- Errors + high **TCP InSegs / OutSegs** → saturated or failing path\n- Errors on one member only → host/NIC/path to that node · **Replica Members Health**\n\nCorrelate with **TCP InSegs / OutSegs** and **TCP Connection Issues**.\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)\n- Linux [SNMP / netstat counters](https://www.kernel.org/doc/html/latest/networking/snmp_counter.html)", + "description": "### TCP packet loss or corruption\n\nPer-second rates of host **TCP/IP error and discard counters** from FTDC **systemMetrics** (`/proc/net/snmp`).\n\n**Unit:** events/sec \u2014 `derivative(unit: 1s)` on cumulative counters\n\n**Not mongod-only:** whole-host TCP/IP stack (all processes).\n\n**Expectation:** normally **flat at zero**; any sustained non-zero rate warrants investigation.\n\n---\n\n### Metrics\n\n**Fields:**\n\n`systemMetrics.netstat.Tcp:InCsumErrors`\n`systemMetrics.netstat.Tcp:InErrs`\n`systemMetrics.netstat.Tcp:RetransSegs`\n`systemMetrics.netstat.Ip:InDiscards`\n`systemMetrics.netstat.Ip:OutDiscards`\n\n**Query type:** cumulative counter \u2192 `derivative(unit: 1s, nonNegative: true)` \u2192 `aggregateWindow(mean)`\n\n**Optional (in `metrics_to_get.txt`, not plotted):** `Ip:FragFails` \u00b7 `Ip:ReasmFails` (fragmentation/reassembly failures)\n\n---\n\n### How to read\n\n- **TCP retransmissions** \u2014 packet loss / timeout on the wire (most common \u201closs\u201d signal)\n- **TCP checksum errors** \u2014 corrupted frames received\n- **TCP input errors** \u2014 other inbound TCP errors\n- **IP discards** \u2014 kernel dropped packets (buffers, policy, congestion)\n\n---\n\n### Troubleshooting\n\n- **RetransSegs** sustained \u2192 loss/congestion \u00b7 **Replica Members Ping**, **Average Operation Latency**\n- **InCsumErrors** \u2192 bad cable/NIC/driver, offload bugs\n- **InDiscards** / **OutDiscards** \u2192 buffer pressure \u00b7 NIC ring, `netdev_max_backlog`, flood traffic\n- Errors + high **TCP InSegs / OutSegs** \u2192 saturated or failing path\n- Errors on one member only \u2192 host/NIC/path to that node \u00b7 **Replica Members Health**\n\nCorrelate with **TCP InSegs / OutSegs** and **TCP Connection Issues**.\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)\n- Linux [SNMP / netstat counters](https://www.kernel.org/doc/html/latest/networking/snmp_counter.html)", "fieldConfig": { "defaults": { "color": { @@ -6492,68 +5211,7 @@ }, "unit": "none" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*InCsumErrors$" - }, - "properties": [ - { - "id": "displayName", - "value": "TCP checksum errors" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*Tcp:InErrs$" - }, - "properties": [ - { - "id": "displayName", - "value": "TCP input errors" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*RetransSegs$" - }, - "properties": [ - { - "id": "displayName", - "value": "TCP retransmissions" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*Ip:InDiscards$" - }, - "properties": [ - { - "id": "displayName", - "value": "IP inbound discards" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*Ip:OutDiscards$" - }, - "properties": [ - { - "id": "displayName", - "value": "IP outbound discards" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 7, @@ -6586,7 +5244,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:InCsumErrors\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:InErrs\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:RetransSegs\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Ip:InDiscards\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Ip:OutDiscards\")\r\n |> drop(columns: [\"hostname\", \"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:InCsumErrors\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:InErrs\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:RetransSegs\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Ip:InDiscards\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Ip:OutDiscards\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"systemMetrics.netstat.Tcp:InCsumErrors\" then \"TCP checksum errors\" else if r._field == \"systemMetrics.netstat.Tcp:InErrs\" then \"TCP input errors\" else if r._field == \"systemMetrics.netstat.Tcp:RetransSegs\" then \"TCP retransmissions\" else if r._field == \"systemMetrics.netstat.Ip:InDiscards\" then \"IP inbound discards\" else if r._field == \"systemMetrics.netstat.Ip:OutDiscards\" then \"IP outbound discards\" else r._field\r\n }))", "refId": "A" } ], @@ -6603,7 +5261,7 @@ { "allValue": ".*", "allowCustomValue": false, - "definition": "import \"influxdata/influxdb/schema\"\n\nschema.tagValues(\n bucket: v.bucket,\n tag: \"hostname\"\n)", + "definition": "import \"influxdata/influxdb/schema\"\n\nschema.tagValues(\n bucket: v.defaultBucket,\n tag: \"hostname\",\n start: v.timeRangeStart,\n stop: v.timeRangeStop\n)", "description": "", "includeAll": true, "label": "hostname", @@ -6611,16 +5269,20 @@ "name": "hostname", "options": [], "query": { - "query": "import \"influxdata/influxdb/schema\"\n\nschema.tagValues(\n bucket: v.bucket,\n tag: \"hostname\"\n)" + "query": "import \"influxdata/influxdb/schema\"\n\nschema.tagValues(\n bucket: v.defaultBucket,\n tag: \"hostname\",\n start: v.timeRangeStart,\n stop: v.timeRangeStop\n)" }, "refresh": 1, "regex": "", - "type": "query" + "type": "query", + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + } }, { "allValue": ".*", "allowCustomValue": false, - "definition": "import \"influxdata/influxdb/schema\"\n\nschema.tagValues(\n bucket: v.bucket,\n tag: \"version\"\n)", + "definition": "import \"influxdata/influxdb/schema\"\n\nschema.tagValues(\n bucket: v.defaultBucket,\n tag: \"version\",\n start: v.timeRangeStart,\n stop: v.timeRangeStop\n)", "description": "version", "includeAll": true, "label": "version", @@ -6628,17 +5290,21 @@ "name": "version", "options": [], "query": { - "query": "import \"influxdata/influxdb/schema\"\n\nschema.tagValues(\n bucket: v.bucket,\n tag: \"version\"\n)" + "query": "import \"influxdata/influxdb/schema\"\n\nschema.tagValues(\n bucket: v.defaultBucket,\n tag: \"version\",\n start: v.timeRangeStart,\n stop: v.timeRangeStop\n)" }, "refresh": 1, "regex": "", - "type": "query" + "type": "query", + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + } } ] }, "time": { - "from": "2024-05-25T19:23:09.607Z", - "to": "2031-05-26T00:01:28.240Z" + "from": "now-6h", + "to": "now" }, "timepicker": {}, "timezone": "UTC", diff --git a/grafana/datasources.yaml b/grafana/datasources.yaml index 00092ff..002a2a2 100644 --- a/grafana/datasources.yaml +++ b/grafana/datasources.yaml @@ -10,5 +10,6 @@ datasources: organization: ${INFLUX_ORG} defaultBucket: ${INFLUX_BUCKET} tlsSkipVerify: true + timeout: 60 secureJsonData: token: ${INFLUX_API_TOKEN} From 1f8a9ac83bea4c8f8b7f5cb2aa648b92cdc1f195 Mon Sep 17 00:00:00 2001 From: jenunes Date: Fri, 17 Jul 2026 08:46:51 -0300 Subject: [PATCH 9/9] Sort before derivative and unwrap MongoDB 8 sharded FTDC role keys --- ftdc_exporter/ftdc/normalize.go | 61 +++++++++-- ftdc_exporter/ftdc/normalize_test.go | 156 +++++++++++++++++++++++++++ grafana/dashboards/dashboard.json | 64 +++++------ 3 files changed, 243 insertions(+), 38 deletions(-) diff --git a/ftdc_exporter/ftdc/normalize.go b/ftdc_exporter/ftdc/normalize.go index 06d6991..8a0002f 100644 --- a/ftdc_exporter/ftdc/normalize.go +++ b/ftdc_exporter/ftdc/normalize.go @@ -69,6 +69,45 @@ func joinMetricKey(prefix, key string) string { return prefix + "." + key } +// isTopLevelRoleWrapper reports MongoDB 8+ FTDC role document keys that must not +// appear in the flattened metric path. Only matches at the metrics document root +// so nested fields named "shard" on older versions are left alone. +func isTopLevelRoleWrapper(prefix, key string) bool { + if prefix != "" { + return false + } + switch key { + case "shard", "router", "common": + return true + default: + return false + } +} + +// stripRoleKeyPrefix removes a leading role prefix from flattened metric keys. +// github.com/mongodb/ftdc often emits already-dotted keys such as +// "shard.replSetGetStatus.members.0.state" at the document root. +// Nested keys (prefix != "") are left unchanged except for the historical +// "common." trim applied by the caller. +func stripRoleKeyPrefix(prefix, key string) string { + if prefix != "" { + return key + } + for _, p := range []string{"shard.", "router.", "common."} { + if strings.HasPrefix(key, p) { + return strings.TrimPrefix(key, p) + } + } + return key +} + +func setIfAbsent(dst map[string]interface{}, key string, value interface{}) { + if _, exists := dst[key]; exists { + return + } + dst[key] = value +} + // normalizeDocument preserves nested maps (used for FTDC metadata). func normalizeDocument(document *birch.Document, includedPatterns map[string]struct{}) map[string]interface{} { normalized := make(map[string]interface{}) @@ -80,7 +119,7 @@ func normalizeDocument(document *birch.Document, includedPatterns map[string]str key = strings.TrimPrefix(key, "common.") val := elem.Value() if isIncluded(key, includedPatterns) { - normalized[key] = normalizeValue(val, includedPatterns) + setIfAbsent(normalized, key, normalizeValue(val, includedPatterns)) } } return normalized @@ -99,19 +138,29 @@ func flattenDocument(document *birch.Document, prefix string, includedPatterns m elem := iter.Element() key := elem.Key() key = strings.TrimPrefix(key, "common.") - fullKey := joinMetricKey(prefix, key) + key = stripRoleKeyPrefix(prefix, key) val := elem.Value() + // MongoDB 8+ sharded FTDC wraps mongod/mongos collectors under shard/router/common. + if isTopLevelRoleWrapper(prefix, key) && val.Type() == bsontype.EmbeddedDocument { + for k, v := range flattenDocument(val.MutableDocument(), prefix, includedPatterns) { + setIfAbsent(normalized, k, v) + } + continue + } + + fullKey := joinMetricKey(prefix, key) + switch val.Type() { case bsontype.EmbeddedDocument: for k, v := range flattenDocument(val.MutableDocument(), fullKey, includedPatterns) { - normalized[k] = v + setIfAbsent(normalized, k, v) } case bsontype.Array: flattenArray(val.MutableArray(), fullKey, includedPatterns, normalized) default: if isIncluded(fullKey, includedPatterns) { - normalized[fullKey] = normalizeScalar(val) + setIfAbsent(normalized, fullKey, normalizeScalar(val)) } } } @@ -127,13 +176,13 @@ func flattenArray(arr *birch.Array, prefix string, includedPatterns map[string]s switch val.Type() { case bsontype.EmbeddedDocument: for k, v := range flattenDocument(val.MutableDocument(), indexKey, includedPatterns) { - out[k] = v + setIfAbsent(out, k, v) } case bsontype.Array: flattenArray(val.MutableArray(), indexKey, includedPatterns, out) default: if isIncluded(indexKey, includedPatterns) { - out[indexKey] = normalizeScalar(val) + setIfAbsent(out, indexKey, normalizeScalar(val)) } } i++ diff --git a/ftdc_exporter/ftdc/normalize_test.go b/ftdc_exporter/ftdc/normalize_test.go index 8a631f5..2b57954 100644 --- a/ftdc_exporter/ftdc/normalize_test.go +++ b/ftdc_exporter/ftdc/normalize_test.go @@ -99,3 +99,159 @@ func TestNormalizeDocument_preservesNestedMetadata(t *testing.T) { t.Fatalf("version = %q, want 8.0.17-6", version) } } + +func TestNormalizeMetricsDocument_unwrapsTopLevelShardRole(t *testing.T) { + doc := birch.NewDocument( + birch.EC.SubDocument("shard", birch.NewDocument( + birch.EC.SubDocument("replSetGetStatus", birch.NewDocument( + birch.EC.ArrayFromElements("members", + birch.VC.DocumentFromElements( + birch.EC.Int32("state", 1), + birch.EC.Int32("health", 1), + birch.EC.Int32("pingMs", 2), + ), + ), + )), + birch.EC.SubDocument("local", birch.NewDocument( + birch.EC.SubDocument("oplog", birch.NewDocument( + birch.EC.SubDocument("rs", birch.NewDocument( + birch.EC.SubDocument("stats", birch.NewDocument( + birch.EC.SubDocument("storageStats", birch.NewDocument( + birch.EC.Int64("storageSize", 1024), + )), + )), + )), + )), + )), + )), + ) + + include := map[string]struct{}{ + "replSetGetStatus.members.0.state": {}, + "replSetGetStatus.members.0.health": {}, + "replSetGetStatus.members.0.pingMs": {}, + "local.oplog.rs.stats.storageStats.storageSize": {}, + "shard.replSetGetStatus.members.0.state": {}, // must not be required + } + + got := normalizeMetricsDocument(doc, include) + + if got["replSetGetStatus.members.0.state"] != int32(1) { + t.Fatalf("state = %#v", got["replSetGetStatus.members.0.state"]) + } + if got["replSetGetStatus.members.0.health"] != int32(1) { + t.Fatalf("health = %#v", got["replSetGetStatus.members.0.health"]) + } + if got["replSetGetStatus.members.0.pingMs"] != int32(2) { + t.Fatalf("pingMs = %#v", got["replSetGetStatus.members.0.pingMs"]) + } + if got["local.oplog.rs.stats.storageStats.storageSize"] != int64(1024) { + t.Fatalf("storageSize = %#v", got["local.oplog.rs.stats.storageStats.storageSize"]) + } + if _, ok := got["shard.replSetGetStatus.members.0.state"]; ok { + t.Fatalf("unexpected shard-prefixed key in output: %#v", got) + } +} + +func TestNormalizeMetricsDocument_roleCollisionKeepsTopLevelFirst(t *testing.T) { + doc := birch.NewDocument( + birch.EC.SubDocument("serverStatus", birch.NewDocument( + birch.EC.Int32("uptime", 100), + )), + birch.EC.SubDocument("shard", birch.NewDocument( + birch.EC.SubDocument("serverStatus", birch.NewDocument( + birch.EC.Int32("uptime", 999), + )), + )), + ) + + include := map[string]struct{}{ + "serverStatus.uptime": {}, + } + + got := normalizeMetricsDocument(doc, include) + if got["serverStatus.uptime"] != int32(100) { + t.Fatalf("uptime = %#v, want top-level 100 (first-write-wins)", got["serverStatus.uptime"]) + } +} + +func TestNormalizeMetricsDocument_nestedShardFieldNotUnwrapped(t *testing.T) { + // Salvaguarda: "shard" nested under a non-empty prefix must remain in the path. + doc := birch.NewDocument( + birch.EC.SubDocument("serverStatus", birch.NewDocument( + birch.EC.SubDocument("shard", birch.NewDocument( + birch.EC.Int32("something", 7), + )), + )), + ) + + include := map[string]struct{}{ + "serverStatus.shard.something": {}, + } + + got := normalizeMetricsDocument(doc, include) + if got["serverStatus.shard.something"] != int32(7) { + t.Fatalf("got %#v", got) + } + if _, ok := got["something"]; ok { + t.Fatalf("nested shard was incorrectly unwrapped: %#v", got) + } +} + +func TestNormalizeMetricsDocument_stripsFlatShardDottedKeys(t *testing.T) { + // github.com/mongodb/ftdc often emits already-flattened dotted keys at the root. + doc := birch.NewDocument( + birch.EC.Int32("shard.replSetGetStatus.members.0.state", 1), + birch.EC.Int64("shard.local.oplog.rs.stats.storageStats.storageSize", 2048), + birch.EC.Int32("serverStatus.uptime", 10), + ) + + include := map[string]struct{}{ + "replSetGetStatus.members.0.state": {}, + "local.oplog.rs.stats.storageStats.storageSize": {}, + "serverStatus.uptime": {}, + } + + got := normalizeMetricsDocument(doc, include) + if got["replSetGetStatus.members.0.state"] != int32(1) { + t.Fatalf("state = %#v", got["replSetGetStatus.members.0.state"]) + } + if got["local.oplog.rs.stats.storageStats.storageSize"] != int64(2048) { + t.Fatalf("storageSize = %#v", got["local.oplog.rs.stats.storageStats.storageSize"]) + } + if _, ok := got["shard.replSetGetStatus.members.0.state"]; ok { + t.Fatalf("shard-prefixed key remained: %#v", got) + } +} + +func TestNormalizeMetricsDocument_flatPre80LayoutUnchanged(t *testing.T) { + // MongoDB 5/6/7 style: collectors at the top level, no role wrappers. + doc := birch.NewDocument( + birch.EC.SubDocument("replSetGetStatus", birch.NewDocument( + birch.EC.ArrayFromElements("members", + birch.VC.DocumentFromElements( + birch.EC.Int32("state", 2), + ), + ), + )), + birch.EC.SubDocument("serverStatus", birch.NewDocument( + birch.EC.Int32("uptime", 42), + )), + ) + + include := map[string]struct{}{ + "replSetGetStatus.members.0.state": {}, + "serverStatus.uptime": {}, + } + + got := normalizeMetricsDocument(doc, include) + if got["replSetGetStatus.members.0.state"] != int32(2) { + t.Fatalf("state = %#v", got["replSetGetStatus.members.0.state"]) + } + if got["serverStatus.uptime"] != int32(42) { + t.Fatalf("uptime = %#v", got["serverStatus.uptime"]) + } + if len(got) != 2 { + t.Fatalf("unexpected keys: %#v", got) + } +} diff --git a/grafana/dashboards/dashboard.json b/grafana/dashboards/dashboard.json index 83d980c..7c88552 100644 --- a/grafana/dashboards/dashboard.json +++ b/grafana/dashboards/dashboard.json @@ -501,7 +501,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.asserts.regular\" or\r\n r[\"_field\"] == \"serverStatus.asserts.warning\" or\r\n r[\"_field\"] == \"serverStatus.asserts.msg\" or\r\n r[\"_field\"] == \"serverStatus.asserts.user\" or\r\n r[\"_field\"] == \"serverStatus.asserts.rollovers\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.asserts.regular\" then \"Regular\" else if r._field == \"serverStatus.asserts.warning\" then \"Warning\" else if r._field == \"serverStatus.asserts.msg\" then \"Msg\" else if r._field == \"serverStatus.asserts.user\" then \"User\" else if r._field == \"serverStatus.asserts.rollovers\" then \"Rollovers\" else r._field\r\n }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.asserts.regular\" or\r\n r[\"_field\"] == \"serverStatus.asserts.warning\" or\r\n r[\"_field\"] == \"serverStatus.asserts.msg\" or\r\n r[\"_field\"] == \"serverStatus.asserts.user\" or\r\n r[\"_field\"] == \"serverStatus.asserts.rollovers\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.asserts.regular\" then \"Regular\" else if r._field == \"serverStatus.asserts.warning\" then \"Warning\" else if r._field == \"serverStatus.asserts.msg\" then \"Msg\" else if r._field == \"serverStatus.asserts.user\" then \"User\" else if r._field == \"serverStatus.asserts.rollovers\" then \"Rollovers\" else r._field\r\n }))", "refId": "A" } ], @@ -818,7 +818,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.cache.modified pages evicted\" or r[\"_field\"] == \"serverStatus.wiredTiger.cache.unmodified pages evicted\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _field: if r._field == \"serverStatus.wiredTiger.cache.modified pages evicted\" then \"Modified pages evicted (dirty)\" else if r._field == \"serverStatus.wiredTiger.cache.unmodified pages evicted\" then \"Unmodified pages evicted (clean)\" else r._field }))\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.cache.modified pages evicted\" or r[\"_field\"] == \"serverStatus.wiredTiger.cache.unmodified pages evicted\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _field: if r._field == \"serverStatus.wiredTiger.cache.modified pages evicted\" then \"Modified pages evicted (dirty)\" else if r._field == \"serverStatus.wiredTiger.cache.unmodified pages evicted\" then \"Unmodified pages evicted (clean)\" else r._field }))\r\n", "refId": "A" } ], @@ -1115,7 +1115,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.block-manager.bytes read\" or r[\"_field\"] == \"serverStatus.wiredTiger.block-manager.bytes written\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _field: if r._field == \"serverStatus.wiredTiger.block-manager.bytes read\" then \"Bytes read\" else if r._field == \"serverStatus.wiredTiger.block-manager.bytes written\" then \"Bytes written\" else r._field }))\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) / 1024.0 / 1024.0 }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.block-manager.bytes read\" or r[\"_field\"] == \"serverStatus.wiredTiger.block-manager.bytes written\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _field: if r._field == \"serverStatus.wiredTiger.block-manager.bytes read\" then \"Bytes read\" else if r._field == \"serverStatus.wiredTiger.block-manager.bytes written\" then \"Bytes written\" else r._field }))\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) / 1024.0 / 1024.0 }))", "refId": "A" } ], @@ -1213,7 +1213,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.block-manager.bytes written for checkpoint\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _field: if r._field == \"serverStatus.wiredTiger.block-manager.bytes written for checkpoint\" then \"Checkpoint write throughput\" else r._field }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.block-manager.bytes written for checkpoint\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _field: if r._field == \"serverStatus.wiredTiger.block-manager.bytes written for checkpoint\" then \"Checkpoint write throughput\" else r._field }))", "refId": "A" } ], @@ -1311,7 +1311,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.cache.bytes written from cache\" or r[\"_field\"] == \"serverStatus.wiredTiger.cache.bytes read into cache\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _field: if r._field == \"serverStatus.wiredTiger.cache.bytes written from cache\" then \"Written from cache\" else if r._field == \"serverStatus.wiredTiger.cache.bytes read into cache\" then \"Read into cache\" else r._field }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.cache.bytes written from cache\" or r[\"_field\"] == \"serverStatus.wiredTiger.cache.bytes read into cache\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _field: if r._field == \"serverStatus.wiredTiger.cache.bytes written from cache\" then \"Written from cache\" else if r._field == \"serverStatus.wiredTiger.cache.bytes read into cache\" then \"Read into cache\" else r._field }))", "refId": "A" } ], @@ -1510,7 +1510,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.log.log bytes written\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _field: if r._field == \"serverStatus.wiredTiger.log.log bytes written\" then \"Journal bytes written\" else r._field }))\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.log.log bytes written\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _field: if r._field == \"serverStatus.wiredTiger.log.log bytes written\" then \"Journal bytes written\" else r._field }))\r\n", "refId": "A" } ], @@ -1721,7 +1721,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.network.numRequests\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.network.numRequests\" then \"Network requests (/sec)\" else r._field\r\n }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.network.numRequests\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.network.numRequests\" then \"Network requests (/sec)\" else r._field\r\n }))", "refId": "A" } ], @@ -1945,7 +1945,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.extra_info.page_faults\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.extra_info.page_faults\" then \"Page faults (/sec)\" else r._field\r\n }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.extra_info.page_faults\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.extra_info.page_faults\" then \"Page faults (/sec)\" else r._field\r\n }))", "refId": "A" } ], @@ -2043,7 +2043,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.connections.totalCreated\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.connections.totalCreated\" then \"Connections created (/sec)\" else r._field\r\n }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.connections.totalCreated\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.connections.totalCreated\" then \"Connections created (/sec)\" else r._field\r\n }))", "refId": "A" } ], @@ -2142,7 +2142,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "wide = from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.opLatencies.reads.latency\" or\r\n r[\"_field\"] == \"serverStatus.opLatencies.reads.ops\" or\r\n r[\"_field\"] == \"serverStatus.opLatencies.writes.latency\" or\r\n r[\"_field\"] == \"serverStatus.opLatencies.writes.ops\" or\r\n r[\"_field\"] == \"serverStatus.opLatencies.commands.latency\" or\r\n r[\"_field\"] == \"serverStatus.opLatencies.commands.ops\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> pivot(rowKey: [\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n hostname: r.hostname,\r\n read: if float(v: r[\"serverStatus.opLatencies.reads.ops\"]) > 0.0\r\n then float(v: r[\"serverStatus.opLatencies.reads.latency\"]) / float(v: r[\"serverStatus.opLatencies.reads.ops\"])\r\n else 0.0,\r\n write: if float(v: r[\"serverStatus.opLatencies.writes.ops\"]) > 0.0\r\n then float(v: r[\"serverStatus.opLatencies.writes.latency\"]) / float(v: r[\"serverStatus.opLatencies.writes.ops\"])\r\n else 0.0,\r\n command: if float(v: r[\"serverStatus.opLatencies.commands.ops\"]) > 0.0\r\n then float(v: r[\"serverStatus.opLatencies.commands.latency\"]) / float(v: r[\"serverStatus.opLatencies.commands.ops\"])\r\n else 0.0,\r\n }))\r\n\r\nunion(tables: [\r\n wide |> map(fn: (r) => ({ _time: r._time, hostname: r.hostname, _field: \"Read latency (avg)\", _value: r.read })),\r\n wide |> map(fn: (r) => ({ _time: r._time, hostname: r.hostname, _field: \"Write latency (avg)\", _value: r.write })),\r\n wide |> map(fn: (r) => ({ _time: r._time, hostname: r.hostname, _field: \"Command latency (avg)\", _value: r.command })),\r\n])\r\n |> group(columns: [\"_field\", \"hostname\"])\r\n", + "query": "wide = from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.opLatencies.reads.latency\" or\r\n r[\"_field\"] == \"serverStatus.opLatencies.reads.ops\" or\r\n r[\"_field\"] == \"serverStatus.opLatencies.writes.latency\" or\r\n r[\"_field\"] == \"serverStatus.opLatencies.writes.ops\" or\r\n r[\"_field\"] == \"serverStatus.opLatencies.commands.latency\" or\r\n r[\"_field\"] == \"serverStatus.opLatencies.commands.ops\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> pivot(rowKey: [\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n hostname: r.hostname,\r\n read: if float(v: r[\"serverStatus.opLatencies.reads.ops\"]) > 0.0\r\n then float(v: r[\"serverStatus.opLatencies.reads.latency\"]) / float(v: r[\"serverStatus.opLatencies.reads.ops\"])\r\n else 0.0,\r\n write: if float(v: r[\"serverStatus.opLatencies.writes.ops\"]) > 0.0\r\n then float(v: r[\"serverStatus.opLatencies.writes.latency\"]) / float(v: r[\"serverStatus.opLatencies.writes.ops\"])\r\n else 0.0,\r\n command: if float(v: r[\"serverStatus.opLatencies.commands.ops\"]) > 0.0\r\n then float(v: r[\"serverStatus.opLatencies.commands.latency\"]) / float(v: r[\"serverStatus.opLatencies.commands.ops\"])\r\n else 0.0,\r\n }))\r\n\r\nunion(tables: [\r\n wide |> map(fn: (r) => ({ _time: r._time, hostname: r.hostname, _field: \"Read latency (avg)\", _value: r.read })),\r\n wide |> map(fn: (r) => ({ _time: r._time, hostname: r.hostname, _field: \"Write latency (avg)\", _value: r.write })),\r\n wide |> map(fn: (r) => ({ _time: r._time, hostname: r.hostname, _field: \"Command latency (avg)\", _value: r.command })),\r\n])\r\n |> group(columns: [\"_field\", \"hostname\"])\r\n", "refId": "A" } ], @@ -2335,7 +2335,7 @@ "uid": "P3C6603E967DC8568" }, "hide": false, - "query": "from(bucket: v.defaultBucket)\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.metrics.cursor.timedOut\")\n |> drop(columns: [\"version\"])\n |> derivative(unit: 1s, nonNegative: true)\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> map(fn: (r) => ({ r with\n _value: float(v: r._value),\n _field: if r._field == \"serverStatus.metrics.cursor.timedOut\" then \"Timed out (/sec)\" else r._field\n }))", + "query": "from(bucket: v.defaultBucket)\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.metrics.cursor.timedOut\")\n |> drop(columns: [\"version\"])\n |> sort(columns: [\"_time\"])\n |> derivative(unit: 1s, nonNegative: true)\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> map(fn: (r) => ({ r with\n _value: float(v: r._value),\n _field: if r._field == \"serverStatus.metrics.cursor.timedOut\" then \"Timed out (/sec)\" else r._field\n }))", "refId": "B" } ], @@ -2433,7 +2433,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.metrics.commands.createIndexes.total\" or\r\n r[\"_field\"] == \"serverStatus.metrics.commands.dataSize.total\" or\r\n r[\"_field\"] == \"serverStatus.metrics.commands.dropIndexes.total\" or\r\n r[\"_field\"] == \"serverStatus.metrics.commands.listCollections.total\" or\r\n r[\"_field\"] == \"serverStatus.metrics.commands.listDatabases.total\" or\r\n r[\"_field\"] == \"serverStatus.metrics.commands.listIndexes.total\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.metrics.commands.createIndexes.total\" then \"createIndexes\" else if r._field == \"serverStatus.metrics.commands.dataSize.total\" then \"dataSize\" else if r._field == \"serverStatus.metrics.commands.dropIndexes.total\" then \"dropIndexes\" else if r._field == \"serverStatus.metrics.commands.listCollections.total\" then \"listCollections\" else if r._field == \"serverStatus.metrics.commands.listDatabases.total\" then \"listDatabases\" else if r._field == \"serverStatus.metrics.commands.listIndexes.total\" then \"listIndexes\" else r._field\r\n }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.metrics.commands.createIndexes.total\" or\r\n r[\"_field\"] == \"serverStatus.metrics.commands.dataSize.total\" or\r\n r[\"_field\"] == \"serverStatus.metrics.commands.dropIndexes.total\" or\r\n r[\"_field\"] == \"serverStatus.metrics.commands.listCollections.total\" or\r\n r[\"_field\"] == \"serverStatus.metrics.commands.listDatabases.total\" or\r\n r[\"_field\"] == \"serverStatus.metrics.commands.listIndexes.total\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.metrics.commands.createIndexes.total\" then \"createIndexes\" else if r._field == \"serverStatus.metrics.commands.dataSize.total\" then \"dataSize\" else if r._field == \"serverStatus.metrics.commands.dropIndexes.total\" then \"dropIndexes\" else if r._field == \"serverStatus.metrics.commands.listCollections.total\" then \"listCollections\" else if r._field == \"serverStatus.metrics.commands.listDatabases.total\" then \"listDatabases\" else if r._field == \"serverStatus.metrics.commands.listIndexes.total\" then \"listIndexes\" else r._field\r\n }))", "refId": "A" } ], @@ -2729,7 +2729,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.metrics.document.deleted\" or\r\n r[\"_field\"] == \"serverStatus.metrics.document.inserted\" or\r\n r[\"_field\"] == \"serverStatus.metrics.document.returned\" or\r\n r[\"_field\"] == \"serverStatus.metrics.document.updated\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.metrics.document.deleted\" then \"Documents deleted\" else if r._field == \"serverStatus.metrics.document.inserted\" then \"Documents inserted\" else if r._field == \"serverStatus.metrics.document.returned\" then \"Documents returned\" else if r._field == \"serverStatus.metrics.document.updated\" then \"Documents updated\" else r._field\r\n }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.metrics.document.deleted\" or\r\n r[\"_field\"] == \"serverStatus.metrics.document.inserted\" or\r\n r[\"_field\"] == \"serverStatus.metrics.document.returned\" or\r\n r[\"_field\"] == \"serverStatus.metrics.document.updated\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.metrics.document.deleted\" then \"Documents deleted\" else if r._field == \"serverStatus.metrics.document.inserted\" then \"Documents inserted\" else if r._field == \"serverStatus.metrics.document.returned\" then \"Documents returned\" else if r._field == \"serverStatus.metrics.document.updated\" then \"Documents updated\" else r._field\r\n }))", "refId": "A" } ], @@ -2947,7 +2947,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.flowControl.isLaggedTimeMicros\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _field: if r._field == \"serverStatus.flowControl.isLaggedTimeMicros\" then \"Flow control engaged (% of time)\" else r._field }))\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) / 1000000.0 }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.flowControl.isLaggedTimeMicros\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _field: if r._field == \"serverStatus.flowControl.isLaggedTimeMicros\" then \"Flow control engaged (% of time)\" else r._field }))\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) / 1000000.0 }))", "refId": "A" } ], @@ -3355,7 +3355,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.metrics.queryExecutor.scannedObjects\" or\r\n r[\"_field\"] == \"serverStatus.metrics.queryExecutor.scanned\" or\r\n r[\"_field\"] == \"serverStatus.metrics.queryExecutor.collectionScans.total\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _field: if r._field == \"serverStatus.metrics.queryExecutor.scanned\" then \"Index keys scanned / sec\" else if r._field == \"serverStatus.metrics.queryExecutor.scannedObjects\" then \"Documents scanned / sec\" else if r._field == \"serverStatus.metrics.queryExecutor.collectionScans.total\" then \"Collection scans / sec\" else r._field }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.metrics.queryExecutor.scannedObjects\" or\r\n r[\"_field\"] == \"serverStatus.metrics.queryExecutor.scanned\" or\r\n r[\"_field\"] == \"serverStatus.metrics.queryExecutor.collectionScans.total\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _field: if r._field == \"serverStatus.metrics.queryExecutor.scanned\" then \"Index keys scanned / sec\" else if r._field == \"serverStatus.metrics.queryExecutor.scannedObjects\" then \"Documents scanned / sec\" else if r._field == \"serverStatus.metrics.queryExecutor.collectionScans.total\" then \"Collection scans / sec\" else r._field }))", "refId": "A" } ], @@ -3453,7 +3453,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.opcounters.command\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.delete\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.getmore\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.insert\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.query\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.update\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.opcounters.command\" then \"Command\" else if r._field == \"serverStatus.opcounters.delete\" then \"Delete\" else if r._field == \"serverStatus.opcounters.getmore\" then \"Getmore\" else if r._field == \"serverStatus.opcounters.insert\" then \"Insert\" else if r._field == \"serverStatus.opcounters.query\" then \"Query\" else if r._field == \"serverStatus.opcounters.update\" then \"Update\" else r._field\r\n }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.opcounters.command\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.delete\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.getmore\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.insert\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.query\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.update\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.opcounters.command\" then \"Command\" else if r._field == \"serverStatus.opcounters.delete\" then \"Delete\" else if r._field == \"serverStatus.opcounters.getmore\" then \"Getmore\" else if r._field == \"serverStatus.opcounters.insert\" then \"Insert\" else if r._field == \"serverStatus.opcounters.query\" then \"Query\" else if r._field == \"serverStatus.opcounters.update\" then \"Update\" else r._field\r\n }))", "refId": "A" } ], @@ -3551,7 +3551,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.insert\" or\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.query\" or\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.update\" or\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.delete\" or\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.getmore\" or\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.command\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _field: if r._field == \"serverStatus.opcountersRepl.insert\" then \"Insert / sec\" else if r._field == \"serverStatus.opcountersRepl.query\" then \"Query / sec\" else if r._field == \"serverStatus.opcountersRepl.update\" then \"Update / sec\" else if r._field == \"serverStatus.opcountersRepl.delete\" then \"Delete / sec\" else if r._field == \"serverStatus.opcountersRepl.getmore\" then \"Getmore / sec\" else if r._field == \"serverStatus.opcountersRepl.command\" then \"Command / sec\" else r._field }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.insert\" or\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.query\" or\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.update\" or\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.delete\" or\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.getmore\" or\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.command\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _field: if r._field == \"serverStatus.opcountersRepl.insert\" then \"Insert / sec\" else if r._field == \"serverStatus.opcountersRepl.query\" then \"Query / sec\" else if r._field == \"serverStatus.opcountersRepl.update\" then \"Update / sec\" else if r._field == \"serverStatus.opcountersRepl.delete\" then \"Delete / sec\" else if r._field == \"serverStatus.opcountersRepl.getmore\" then \"Getmore / sec\" else if r._field == \"serverStatus.opcountersRepl.command\" then \"Command / sec\" else r._field }))", "refId": "A" } ], @@ -3649,7 +3649,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.opLatencies.reads.ops\" or\r\n r[\"_field\"] == \"serverStatus.opLatencies.writes.ops\" or\r\n r[\"_field\"] == \"serverStatus.opLatencies.commands.ops\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _field: if r._field == \"serverStatus.opLatencies.reads.ops\" then \"Read ops / sec\" else if r._field == \"serverStatus.opLatencies.writes.ops\" then \"Write ops / sec\" else if r._field == \"serverStatus.opLatencies.commands.ops\" then \"Command ops / sec\" else r._field }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.opLatencies.reads.ops\" or\r\n r[\"_field\"] == \"serverStatus.opLatencies.writes.ops\" or\r\n r[\"_field\"] == \"serverStatus.opLatencies.commands.ops\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _field: if r._field == \"serverStatus.opLatencies.reads.ops\" then \"Read ops / sec\" else if r._field == \"serverStatus.opLatencies.writes.ops\" then \"Write ops / sec\" else if r._field == \"serverStatus.opLatencies.commands.ops\" then \"Command ops / sec\" else r._field }))", "refId": "A" } ], @@ -3748,7 +3748,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.metrics.document.returned\" or\r\n r[\"_field\"] == \"serverStatus.metrics.queryExecutor.scannedObjects\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> pivot(rowKey: [\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n hostname: r.hostname,\r\n _field: \"Scanned / returned ratio\",\r\n _value:\r\n if exists r[\"serverStatus.metrics.document.returned\"]\r\n and float(v: r[\"serverStatus.metrics.document.returned\"]) > 0.0\r\n and exists r[\"serverStatus.metrics.queryExecutor.scannedObjects\"]\r\n then float(v: r[\"serverStatus.metrics.queryExecutor.scannedObjects\"])\r\n / float(v: r[\"serverStatus.metrics.document.returned\"])\r\n else float(v: \"NaN\")\r\n }))\r\n |> group(columns: [\"_field\", \"hostname\"])\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.metrics.document.returned\" or\r\n r[\"_field\"] == \"serverStatus.metrics.queryExecutor.scannedObjects\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> pivot(rowKey: [\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n hostname: r.hostname,\r\n _field: \"Scanned / returned ratio\",\r\n _value:\r\n if exists r[\"serverStatus.metrics.document.returned\"]\r\n and float(v: r[\"serverStatus.metrics.document.returned\"]) > 0.0\r\n and exists r[\"serverStatus.metrics.queryExecutor.scannedObjects\"]\r\n then float(v: r[\"serverStatus.metrics.queryExecutor.scannedObjects\"])\r\n / float(v: r[\"serverStatus.metrics.document.returned\"])\r\n else float(v: \"NaN\")\r\n }))\r\n |> group(columns: [\"_field\", \"hostname\"])\r\n", "refId": "A" } ], @@ -3859,7 +3859,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "base =\r\n from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r._field == \"systemMetrics.cpu.idle_ms\" or\r\n r._field == \"systemMetrics.cpu.user_ms\" or\r\n r._field == \"systemMetrics.cpu.iowait_ms\" or\r\n r._field == \"systemMetrics.cpu.nice_ms\" or\r\n r._field == \"systemMetrics.cpu.softirq_ms\" or\r\n r._field == \"systemMetrics.cpu.steal_ms\" or\r\n r._field == \"systemMetrics.cpu.system_ms\")\r\n |> drop(columns: [\"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> pivot(rowKey: [\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\r\n |> derivative(\r\n unit: 1s,\r\n nonNegative: true,\r\n columns: [\r\n \"systemMetrics.cpu.idle_ms\",\r\n \"systemMetrics.cpu.user_ms\",\r\n \"systemMetrics.cpu.iowait_ms\",\r\n \"systemMetrics.cpu.nice_ms\",\r\n \"systemMetrics.cpu.softirq_ms\",\r\n \"systemMetrics.cpu.steal_ms\",\r\n \"systemMetrics.cpu.system_ms\",\r\n ],\r\n )\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n hostname: r.hostname,\r\n user_ms: float(v: r[\"systemMetrics.cpu.user_ms\"]),\r\n system_ms: float(v: r[\"systemMetrics.cpu.system_ms\"]),\r\n iowait_ms: float(v: r[\"systemMetrics.cpu.iowait_ms\"]),\r\n total_ms:\r\n float(v: r[\"systemMetrics.cpu.idle_ms\"]) +\r\n float(v: r[\"systemMetrics.cpu.user_ms\"]) +\r\n float(v: r[\"systemMetrics.cpu.iowait_ms\"]) +\r\n float(v: r[\"systemMetrics.cpu.nice_ms\"]) +\r\n float(v: r[\"systemMetrics.cpu.softirq_ms\"]) +\r\n float(v: r[\"systemMetrics.cpu.steal_ms\"]) +\r\n float(v: r[\"systemMetrics.cpu.system_ms\"]),\r\n }))\r\n\r\nunion(tables: [\r\n base |> map(fn: (r) => ({\r\n _time: r._time, hostname: r.hostname,\r\n _field: \"User %\",\r\n _value: if r.total_ms > 0.0 then 100.0 * r.user_ms / r.total_ms else 0.0,\r\n })),\r\n base |> map(fn: (r) => ({\r\n _time: r._time, hostname: r.hostname,\r\n _field: \"System (kernel) %\",\r\n _value: if r.total_ms > 0.0 then 100.0 * r.system_ms / r.total_ms else 0.0,\r\n })),\r\n base |> map(fn: (r) => ({\r\n _time: r._time, hostname: r.hostname,\r\n _field: \"I/O wait %\",\r\n _value: if r.total_ms > 0.0 then 100.0 * r.iowait_ms / r.total_ms else 0.0,\r\n })),\r\n])\r\n |> group(columns: [\"_field\", \"hostname\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", + "query": "base =\r\n from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r._field == \"systemMetrics.cpu.idle_ms\" or\r\n r._field == \"systemMetrics.cpu.user_ms\" or\r\n r._field == \"systemMetrics.cpu.iowait_ms\" or\r\n r._field == \"systemMetrics.cpu.nice_ms\" or\r\n r._field == \"systemMetrics.cpu.softirq_ms\" or\r\n r._field == \"systemMetrics.cpu.steal_ms\" or\r\n r._field == \"systemMetrics.cpu.system_ms\")\r\n |> drop(columns: [\"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> pivot(rowKey: [\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(\r\n unit: 1s,\r\n nonNegative: true,\r\n columns: [\r\n \"systemMetrics.cpu.idle_ms\",\r\n \"systemMetrics.cpu.user_ms\",\r\n \"systemMetrics.cpu.iowait_ms\",\r\n \"systemMetrics.cpu.nice_ms\",\r\n \"systemMetrics.cpu.softirq_ms\",\r\n \"systemMetrics.cpu.steal_ms\",\r\n \"systemMetrics.cpu.system_ms\",\r\n ],\r\n )\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n hostname: r.hostname,\r\n user_ms: float(v: r[\"systemMetrics.cpu.user_ms\"]),\r\n system_ms: float(v: r[\"systemMetrics.cpu.system_ms\"]),\r\n iowait_ms: float(v: r[\"systemMetrics.cpu.iowait_ms\"]),\r\n total_ms:\r\n float(v: r[\"systemMetrics.cpu.idle_ms\"]) +\r\n float(v: r[\"systemMetrics.cpu.user_ms\"]) +\r\n float(v: r[\"systemMetrics.cpu.iowait_ms\"]) +\r\n float(v: r[\"systemMetrics.cpu.nice_ms\"]) +\r\n float(v: r[\"systemMetrics.cpu.softirq_ms\"]) +\r\n float(v: r[\"systemMetrics.cpu.steal_ms\"]) +\r\n float(v: r[\"systemMetrics.cpu.system_ms\"]),\r\n }))\r\n\r\nunion(tables: [\r\n base |> map(fn: (r) => ({\r\n _time: r._time, hostname: r.hostname,\r\n _field: \"User %\",\r\n _value: if r.total_ms > 0.0 then 100.0 * r.user_ms / r.total_ms else 0.0,\r\n })),\r\n base |> map(fn: (r) => ({\r\n _time: r._time, hostname: r.hostname,\r\n _field: \"System (kernel) %\",\r\n _value: if r.total_ms > 0.0 then 100.0 * r.system_ms / r.total_ms else 0.0,\r\n })),\r\n base |> map(fn: (r) => ({\r\n _time: r._time, hostname: r.hostname,\r\n _field: \"I/O wait %\",\r\n _value: if r.total_ms > 0.0 then 100.0 * r.iowait_ms / r.total_ms else 0.0,\r\n })),\r\n])\r\n |> group(columns: [\"_field\", \"hostname\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", "refId": "A" } ], @@ -4155,7 +4155,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "import \"strings\"\r\n\r\nfrom(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r._field =~ /^systemMetrics\\.disks\\..+\\.reads$/ or\r\n r._field =~ /^systemMetrics\\.disks\\..+\\.writes$/)\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({\r\n r with\r\n _value: float(v: r._value),\r\n _field:\r\n if strings.hasSuffix(v: r._field, suffix: \".reads\") then\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".reads\",\r\n ) + \" reads\"\r\n else\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".writes\",\r\n ) + \" writes\",\r\n }))", + "query": "import \"strings\"\r\n\r\nfrom(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r._field =~ /^systemMetrics\\.disks\\..+\\.reads$/ or\r\n r._field =~ /^systemMetrics\\.disks\\..+\\.writes$/)\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({\r\n r with\r\n _value: float(v: r._value),\r\n _field:\r\n if strings.hasSuffix(v: r._field, suffix: \".reads\") then\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".reads\",\r\n ) + \" reads\"\r\n else\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".writes\",\r\n ) + \" writes\",\r\n }))", "refId": "A" } ], @@ -4254,7 +4254,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "import \"strings\"\r\n\r\nfrom(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r._field =~ /^systemMetrics\\.disks\\..+\\.read_sectors$/ or\r\n r._field =~ /^systemMetrics\\.disks\\..+\\.write_sectors$/)\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({\r\n r with\r\n _value: float(v: r._value) * 512.0,\r\n _field:\r\n if strings.hasSuffix(v: r._field, suffix: \".read_sectors\") then\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".read_sectors\",\r\n ) + \" read\"\r\n else\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".write_sectors\",\r\n ) + \" write\",\r\n }))", + "query": "import \"strings\"\r\n\r\nfrom(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r._field =~ /^systemMetrics\\.disks\\..+\\.read_sectors$/ or\r\n r._field =~ /^systemMetrics\\.disks\\..+\\.write_sectors$/)\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({\r\n r with\r\n _value: float(v: r._value) * 512.0,\r\n _field:\r\n if strings.hasSuffix(v: r._field, suffix: \".read_sectors\") then\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".read_sectors\",\r\n ) + \" read\"\r\n else\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".write_sectors\",\r\n ) + \" write\",\r\n }))", "refId": "A" } ], @@ -4353,7 +4353,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "import \"strings\"\r\n\r\nfrom(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..+\\.read_sectors$/)\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({\r\n r with\r\n _value: float(v: r._value) * 512.0,\r\n _field:\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".read_sectors\",\r\n ) + \" read\",\r\n }))", + "query": "import \"strings\"\r\n\r\nfrom(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..+\\.read_sectors$/)\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({\r\n r with\r\n _value: float(v: r._value) * 512.0,\r\n _field:\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".read_sectors\",\r\n ) + \" read\",\r\n }))", "refId": "A" } ], @@ -4452,7 +4452,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "import \"strings\"\r\n\r\nfrom(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..+\\.write_sectors$/)\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({\r\n r with\r\n _value: float(v: r._value) * 512.0,\r\n _field:\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".write_sectors\",\r\n ) + \" write\",\r\n }))", + "query": "import \"strings\"\r\n\r\nfrom(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..+\\.write_sectors$/)\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({\r\n r with\r\n _value: float(v: r._value) * 512.0,\r\n _field:\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".write_sectors\",\r\n ) + \" write\",\r\n }))", "refId": "A" } ], @@ -4550,7 +4550,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "import \"strings\"\r\n\r\nfrom(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..+\\.io_queued_ms$/)\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({\r\n r with\r\n _value: float(v: r._value),\r\n _field:\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".io_queued_ms\",\r\n ) + \" queue time\",\r\n }))", + "query": "import \"strings\"\r\n\r\nfrom(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..+\\.io_queued_ms$/)\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({\r\n r with\r\n _value: float(v: r._value),\r\n _field:\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".io_queued_ms\",\r\n ) + \" queue time\",\r\n }))", "refId": "A" } ], @@ -4649,7 +4649,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "import \"strings\"\r\n\r\nfrom(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r._field =~ /^systemMetrics\\.disks\\..+\\.read_time_ms$/ or\r\n r._field =~ /^systemMetrics\\.disks\\..+\\.write_time_ms$/)\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({\r\n r with\r\n _value: float(v: r._value),\r\n _field:\r\n if strings.hasSuffix(v: r._field, suffix: \".read_time_ms\") then\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".read_time_ms\",\r\n ) + \" read time\"\r\n else\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".write_time_ms\",\r\n ) + \" write time\",\r\n }))", + "query": "import \"strings\"\r\n\r\nfrom(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r._field =~ /^systemMetrics\\.disks\\..+\\.read_time_ms$/ or\r\n r._field =~ /^systemMetrics\\.disks\\..+\\.write_time_ms$/)\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({\r\n r with\r\n _value: float(v: r._value),\r\n _field:\r\n if strings.hasSuffix(v: r._field, suffix: \".read_time_ms\") then\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".read_time_ms\",\r\n ) + \" read time\"\r\n else\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".write_time_ms\",\r\n ) + \" write time\",\r\n }))", "refId": "A" } ], @@ -4748,7 +4748,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:InSegs\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:OutSegs\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"systemMetrics.netstat.Tcp:InSegs\" then \"TCP segments in\" else if r._field == \"systemMetrics.netstat.Tcp:OutSegs\" then \"TCP segments out\" else r._field\r\n }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:InSegs\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:OutSegs\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"systemMetrics.netstat.Tcp:InSegs\" then \"TCP segments in\" else if r._field == \"systemMetrics.netstat.Tcp:OutSegs\" then \"TCP segments out\" else r._field\r\n }))", "refId": "A" } ], @@ -4847,7 +4847,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.network.bytesIn\" or\r\n r[\"_field\"] == \"serverStatus.network.bytesOut\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.network.bytesIn\" then \"Logical bytes in\" else if r._field == \"serverStatus.network.bytesOut\" then \"Logical bytes out\" else r._field\r\n }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.network.bytesIn\" or\r\n r[\"_field\"] == \"serverStatus.network.bytesOut\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.network.bytesIn\" then \"Logical bytes in\" else if r._field == \"serverStatus.network.bytesOut\" then \"Logical bytes out\" else r._field\r\n }))", "refId": "A" } ], @@ -4946,7 +4946,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.network.physicalBytesIn\" or\r\n r[\"_field\"] == \"serverStatus.network.physicalBytesOut\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.network.physicalBytesIn\" then \"Physical bytes in\" else if r._field == \"serverStatus.network.physicalBytesOut\" then \"Physical bytes out\" else r._field\r\n }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.network.physicalBytesIn\" or\r\n r[\"_field\"] == \"serverStatus.network.physicalBytesOut\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.network.physicalBytesIn\" then \"Physical bytes in\" else if r._field == \"serverStatus.network.physicalBytesOut\" then \"Physical bytes out\" else r._field\r\n }))", "refId": "A" } ], @@ -5046,7 +5046,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:ActiveOpens\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:AttemptFails\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:EstabResets\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:OutRsts\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"systemMetrics.netstat.Tcp:ActiveOpens\" then \"TCP active opens\" else if r._field == \"systemMetrics.netstat.Tcp:AttemptFails\" then \"TCP connect failures\" else if r._field == \"systemMetrics.netstat.Tcp:EstabResets\" then \"TCP established resets\" else if r._field == \"systemMetrics.netstat.Tcp:OutRsts\" then \"TCP RST sent\" else r._field\r\n }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:ActiveOpens\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:AttemptFails\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:EstabResets\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:OutRsts\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"systemMetrics.netstat.Tcp:ActiveOpens\" then \"TCP active opens\" else if r._field == \"systemMetrics.netstat.Tcp:AttemptFails\" then \"TCP connect failures\" else if r._field == \"systemMetrics.netstat.Tcp:EstabResets\" then \"TCP established resets\" else if r._field == \"systemMetrics.netstat.Tcp:OutRsts\" then \"TCP RST sent\" else r._field\r\n }))", "refId": "A" } ], @@ -5145,7 +5145,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"systemMetrics.netstat.Ip:OutNoRoutes\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Ip:FragFails\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Ip:ReasmFails\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"systemMetrics.netstat.Ip:OutNoRoutes\" then \"IP no route (out)\" else if r._field == \"systemMetrics.netstat.Ip:FragFails\" then \"IP fragmentation failures\" else if r._field == \"systemMetrics.netstat.Ip:ReasmFails\" then \"IP reassembly failures\" else r._field\r\n }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"systemMetrics.netstat.Ip:OutNoRoutes\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Ip:FragFails\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Ip:ReasmFails\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"systemMetrics.netstat.Ip:OutNoRoutes\" then \"IP no route (out)\" else if r._field == \"systemMetrics.netstat.Ip:FragFails\" then \"IP fragmentation failures\" else if r._field == \"systemMetrics.netstat.Ip:ReasmFails\" then \"IP reassembly failures\" else r._field\r\n }))", "refId": "A" } ], @@ -5244,7 +5244,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:InCsumErrors\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:InErrs\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:RetransSegs\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Ip:InDiscards\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Ip:OutDiscards\")\r\n |> drop(columns: [\"version\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"systemMetrics.netstat.Tcp:InCsumErrors\" then \"TCP checksum errors\" else if r._field == \"systemMetrics.netstat.Tcp:InErrs\" then \"TCP input errors\" else if r._field == \"systemMetrics.netstat.Tcp:RetransSegs\" then \"TCP retransmissions\" else if r._field == \"systemMetrics.netstat.Ip:InDiscards\" then \"IP inbound discards\" else if r._field == \"systemMetrics.netstat.Ip:OutDiscards\" then \"IP outbound discards\" else r._field\r\n }))", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:InCsumErrors\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:InErrs\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:RetransSegs\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Ip:InDiscards\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Ip:OutDiscards\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"systemMetrics.netstat.Tcp:InCsumErrors\" then \"TCP checksum errors\" else if r._field == \"systemMetrics.netstat.Tcp:InErrs\" then \"TCP input errors\" else if r._field == \"systemMetrics.netstat.Tcp:RetransSegs\" then \"TCP retransmissions\" else if r._field == \"systemMetrics.netstat.Ip:InDiscards\" then \"IP inbound discards\" else if r._field == \"systemMetrics.netstat.Ip:OutDiscards\" then \"IP outbound discards\" else r._field\r\n }))", "refId": "A" } ],