diff --git a/.gitignore b/.gitignore index 220f221..43b1a79 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,10 @@ diagnostic.data/ .idea .DS_Store influxdb_data/ -./ftdc_exporter/vendor \ No newline at end of file +./ftdc_exporter/vendor +.cursor/ +mongodb_ftdc_decoder/ +grafana/backups/ +grafana/DASHBOARD.md +grafana/dashboards/bkp.json +grafana/dashboards/my.jsonscripts/ diff --git a/ftdc_exporter/ftdc/iterator.go b/ftdc_exporter/ftdc/iterator.go index 9a475eb..222b535 100644 --- a/ftdc_exporter/ftdc/iterator.go +++ b/ftdc_exporter/ftdc/iterator.go @@ -1,7 +1,7 @@ package ftdc func (it *FTDCDataIterator) NormalisedDocument(includedPatterns map[string]struct{}) map[string]interface{} { - return normalizeDocument(it.doc, includedPatterns) + return normalizeMetricsDocument(it.doc, includedPatterns) } func (it *FTDCDataIterator) Close() { diff --git a/ftdc_exporter/ftdc/normalize.go b/ftdc_exporter/ftdc/normalize.go index c05339e..8a0002f 100644 --- a/ftdc_exporter/ftdc/normalize.go +++ b/ftdc_exporter/ftdc/normalize.go @@ -1,6 +1,8 @@ package ftdc import ( + "strconv" + "github.com/evergreen-ci/birch" "github.com/evergreen-ci/birch/bsontype" "strings" @@ -60,6 +62,53 @@ func isIncluded(key string, includedPatterns map[string]struct{}) bool { return false } +func joinMetricKey(prefix, key string) string { + if prefix == "" { + return key + } + return prefix + "." + key +} + +// isTopLevelRoleWrapper reports MongoDB 8+ FTDC role document keys that must not +// appear in the flattened metric path. Only matches at the metrics document root +// so nested fields named "shard" on older versions are left alone. +func isTopLevelRoleWrapper(prefix, key string) bool { + if prefix != "" { + return false + } + switch key { + case "shard", "router", "common": + return true + default: + return false + } +} + +// stripRoleKeyPrefix removes a leading role prefix from flattened metric keys. +// github.com/mongodb/ftdc often emits already-dotted keys such as +// "shard.replSetGetStatus.members.0.state" at the document root. +// Nested keys (prefix != "") are left unchanged except for the historical +// "common." trim applied by the caller. +func stripRoleKeyPrefix(prefix, key string) string { + if prefix != "" { + return key + } + for _, p := range []string{"shard.", "router.", "common."} { + if strings.HasPrefix(key, p) { + return strings.TrimPrefix(key, p) + } + } + return key +} + +func setIfAbsent(dst map[string]interface{}, key string, value interface{}) { + if _, exists := dst[key]; exists { + return + } + dst[key] = value +} + +// normalizeDocument preserves nested maps (used for FTDC metadata). func normalizeDocument(document *birch.Document, includedPatterns map[string]struct{}) map[string]interface{} { normalized := make(map[string]interface{}) iter := document.Iterator() @@ -67,17 +116,102 @@ func normalizeDocument(document *birch.Document, includedPatterns map[string]str for iter.Next() { elem := iter.Element() key := elem.Key() - // on some versions, the metrics are starting with a common. prefix - // I decided to get rid of it so that we do not have to change anything in the grafana dashboards. key = strings.TrimPrefix(key, "common.") val := elem.Value() if isIncluded(key, includedPatterns) { - normalized[key] = normalizeValue(val, includedPatterns) + setIfAbsent(normalized, key, normalizeValue(val, includedPatterns)) + } + } + return normalized +} + +// normalizeMetricsDocument flattens nested FTDC samples into dotted field names for InfluxDB. +func normalizeMetricsDocument(document *birch.Document, includedPatterns map[string]struct{}) map[string]interface{} { + return flattenDocument(document, "", includedPatterns) +} + +func flattenDocument(document *birch.Document, prefix string, includedPatterns map[string]struct{}) map[string]interface{} { + normalized := make(map[string]interface{}) + iter := document.Iterator() + + for iter.Next() { + elem := iter.Element() + key := elem.Key() + key = strings.TrimPrefix(key, "common.") + key = stripRoleKeyPrefix(prefix, key) + val := elem.Value() + + // MongoDB 8+ sharded FTDC wraps mongod/mongos collectors under shard/router/common. + if isTopLevelRoleWrapper(prefix, key) && val.Type() == bsontype.EmbeddedDocument { + for k, v := range flattenDocument(val.MutableDocument(), prefix, includedPatterns) { + setIfAbsent(normalized, k, v) + } + continue + } + + fullKey := joinMetricKey(prefix, key) + + switch val.Type() { + case bsontype.EmbeddedDocument: + for k, v := range flattenDocument(val.MutableDocument(), fullKey, includedPatterns) { + setIfAbsent(normalized, k, v) + } + case bsontype.Array: + flattenArray(val.MutableArray(), fullKey, includedPatterns, normalized) + default: + if isIncluded(fullKey, includedPatterns) { + setIfAbsent(normalized, fullKey, normalizeScalar(val)) + } } } return normalized } +func flattenArray(arr *birch.Array, prefix string, includedPatterns map[string]struct{}, out map[string]interface{}) { + it := arr.Iterator() + i := 0 + for it.Next() { + indexKey := joinMetricKey(prefix, strconv.Itoa(i)) + val := it.Value() + switch val.Type() { + case bsontype.EmbeddedDocument: + for k, v := range flattenDocument(val.MutableDocument(), indexKey, includedPatterns) { + setIfAbsent(out, k, v) + } + case bsontype.Array: + flattenArray(val.MutableArray(), indexKey, includedPatterns, out) + default: + if isIncluded(indexKey, includedPatterns) { + setIfAbsent(out, indexKey, normalizeScalar(val)) + } + } + i++ + } +} + +func normalizeScalar(val *birch.Value) interface{} { + switch val.Type() { + case bsontype.Double: + return val.Double() + case bsontype.String: + return val.StringValue() + case bsontype.Boolean: + return val.Boolean() + case bsontype.Int32: + return val.Int32() + case bsontype.Int64: + return val.Int64() + case bsontype.Null: + return -1 + case bsontype.ObjectID: + return val.ObjectID().Hex() + case bsontype.DateTime: + return time.UnixMilli(val.DateTime()).UnixMilli() + default: + return val.Interface() + } +} + func normalizeValue(val *birch.Value, includedPatterns map[string]struct{}) interface{} { switch val.Type() { case bsontype.Double: @@ -99,16 +233,13 @@ func normalizeValue(val *birch.Value, includedPatterns map[string]struct{}) inte case bsontype.Array: out := []interface{}{} it := val.MutableArray().Iterator() - i := 0 for it.Next() { out = append(out, normalizeValue(it.Value(), includedPatterns)) - i++ } return out case bsontype.DateTime: return time.UnixMilli(val.DateTime()).UnixMilli() default: - // Handle unsupported types as raw or string, or skip - return val.Interface() // fallback + return val.Interface() } } diff --git a/ftdc_exporter/ftdc/normalize_test.go b/ftdc_exporter/ftdc/normalize_test.go new file mode 100644 index 0000000..2b57954 --- /dev/null +++ b/ftdc_exporter/ftdc/normalize_test.go @@ -0,0 +1,257 @@ +package ftdc + +import ( + "testing" + + "github.com/evergreen-ci/birch" +) + +func TestNormalizeMetricsDocument_flatAndNestedKeys(t *testing.T) { + doc := birch.NewDocument( + birch.EC.Int64("start", 1700000000000), + birch.EC.Int64("serverStatus.wiredTiger.cache.bytes currently in the cache", 4096), + birch.EC.SubDocument("serverStatus", birch.NewDocument( + birch.EC.SubDocument("wiredTiger", birch.NewDocument( + birch.EC.SubDocument("concurrentTransactions", birch.NewDocument( + birch.EC.SubDocument("monitor", birch.NewDocument( + birch.EC.Int32("available", 100), + birch.EC.Int32("totalTickets", 128), + birch.EC.Int32("queueLength", 0), + )), + )), + )), + )), + ) + + include := map[string]struct{}{ + "start": {}, + "serverStatus.wiredTiger.cache.bytes currently in the cache": {}, + "serverStatus.wiredTiger.concurrentTransactions.monitor.available": {}, + "serverStatus.wiredTiger.concurrentTransactions.monitor.totalTickets": {}, + "serverStatus.wiredTiger.concurrentTransactions.monitor.queueLength": {}, + } + + got := normalizeMetricsDocument(doc, include) + + want := map[string]interface{}{ + "start": int64(1700000000000), + "serverStatus.wiredTiger.cache.bytes currently in the cache": int64(4096), + "serverStatus.wiredTiger.concurrentTransactions.monitor.available": int32(100), + "serverStatus.wiredTiger.concurrentTransactions.monitor.totalTickets": int32(128), + "serverStatus.wiredTiger.concurrentTransactions.monitor.queueLength": int32(0), + } + + for key, wantVal := range want { + gotVal, ok := got[key] + if !ok { + t.Fatalf("missing key %q; got %#v", key, got) + } + if gotVal != wantVal { + t.Fatalf("key %q: got %v (%T), want %v (%T)", key, gotVal, gotVal, wantVal, wantVal) + } + } + + if len(got) != len(want) { + t.Fatalf("unexpected extra keys: %#v", got) + } +} + +func TestNormalizeMetricsDocument_flatQueueExecutionKeys(t *testing.T) { + doc := birch.NewDocument( + birch.EC.Int32("serverStatus.queues.execution.read.available", 100), + birch.EC.Int32("serverStatus.queues.execution.write.available", 95), + birch.EC.Int32("serverStatus.queues.execution.read.totalTickets", 128), + ) + + include := map[string]struct{}{ + "serverStatus.queues.execution.read.available": {}, + "serverStatus.queues.execution.write.available": {}, + "serverStatus.queues.execution.read.totalTickets": {}, + } + + got := normalizeMetricsDocument(doc, include) + if got["serverStatus.queues.execution.read.available"] != int32(100) { + t.Fatalf("read.available = %#v", got) + } +} + +func TestNormalizeDocument_preservesNestedMetadata(t *testing.T) { + doc := birch.NewDocument( + birch.EC.SubDocument("doc", birch.NewDocument( + birch.EC.SubDocument("hostInfo", birch.NewDocument( + birch.EC.SubDocument("system", birch.NewDocument( + birch.EC.String("hostname", "mongod1.example:27017"), + )), + )), + birch.EC.SubDocument("buildInfo", birch.NewDocument( + birch.EC.String("version", "8.0.17-6"), + )), + )), + ) + + got := normalizeDocument(doc, map[string]struct{}{}) + hostname := getNestedString(got, "doc.hostInfo.system.hostname") + if hostname != "mongod1.example:27017" { + t.Fatalf("hostname = %q, want mongod1.example:27017", hostname) + } + version := getNestedString(got, "doc.buildInfo.version") + if version != "8.0.17-6" { + t.Fatalf("version = %q, want 8.0.17-6", version) + } +} + +func TestNormalizeMetricsDocument_unwrapsTopLevelShardRole(t *testing.T) { + doc := birch.NewDocument( + birch.EC.SubDocument("shard", birch.NewDocument( + birch.EC.SubDocument("replSetGetStatus", birch.NewDocument( + birch.EC.ArrayFromElements("members", + birch.VC.DocumentFromElements( + birch.EC.Int32("state", 1), + birch.EC.Int32("health", 1), + birch.EC.Int32("pingMs", 2), + ), + ), + )), + birch.EC.SubDocument("local", birch.NewDocument( + birch.EC.SubDocument("oplog", birch.NewDocument( + birch.EC.SubDocument("rs", birch.NewDocument( + birch.EC.SubDocument("stats", birch.NewDocument( + birch.EC.SubDocument("storageStats", birch.NewDocument( + birch.EC.Int64("storageSize", 1024), + )), + )), + )), + )), + )), + )), + ) + + include := map[string]struct{}{ + "replSetGetStatus.members.0.state": {}, + "replSetGetStatus.members.0.health": {}, + "replSetGetStatus.members.0.pingMs": {}, + "local.oplog.rs.stats.storageStats.storageSize": {}, + "shard.replSetGetStatus.members.0.state": {}, // must not be required + } + + got := normalizeMetricsDocument(doc, include) + + if got["replSetGetStatus.members.0.state"] != int32(1) { + t.Fatalf("state = %#v", got["replSetGetStatus.members.0.state"]) + } + if got["replSetGetStatus.members.0.health"] != int32(1) { + t.Fatalf("health = %#v", got["replSetGetStatus.members.0.health"]) + } + if got["replSetGetStatus.members.0.pingMs"] != int32(2) { + t.Fatalf("pingMs = %#v", got["replSetGetStatus.members.0.pingMs"]) + } + if got["local.oplog.rs.stats.storageStats.storageSize"] != int64(1024) { + t.Fatalf("storageSize = %#v", got["local.oplog.rs.stats.storageStats.storageSize"]) + } + if _, ok := got["shard.replSetGetStatus.members.0.state"]; ok { + t.Fatalf("unexpected shard-prefixed key in output: %#v", got) + } +} + +func TestNormalizeMetricsDocument_roleCollisionKeepsTopLevelFirst(t *testing.T) { + doc := birch.NewDocument( + birch.EC.SubDocument("serverStatus", birch.NewDocument( + birch.EC.Int32("uptime", 100), + )), + birch.EC.SubDocument("shard", birch.NewDocument( + birch.EC.SubDocument("serverStatus", birch.NewDocument( + birch.EC.Int32("uptime", 999), + )), + )), + ) + + include := map[string]struct{}{ + "serverStatus.uptime": {}, + } + + got := normalizeMetricsDocument(doc, include) + if got["serverStatus.uptime"] != int32(100) { + t.Fatalf("uptime = %#v, want top-level 100 (first-write-wins)", got["serverStatus.uptime"]) + } +} + +func TestNormalizeMetricsDocument_nestedShardFieldNotUnwrapped(t *testing.T) { + // Salvaguarda: "shard" nested under a non-empty prefix must remain in the path. + doc := birch.NewDocument( + birch.EC.SubDocument("serverStatus", birch.NewDocument( + birch.EC.SubDocument("shard", birch.NewDocument( + birch.EC.Int32("something", 7), + )), + )), + ) + + include := map[string]struct{}{ + "serverStatus.shard.something": {}, + } + + got := normalizeMetricsDocument(doc, include) + if got["serverStatus.shard.something"] != int32(7) { + t.Fatalf("got %#v", got) + } + if _, ok := got["something"]; ok { + t.Fatalf("nested shard was incorrectly unwrapped: %#v", got) + } +} + +func TestNormalizeMetricsDocument_stripsFlatShardDottedKeys(t *testing.T) { + // github.com/mongodb/ftdc often emits already-flattened dotted keys at the root. + doc := birch.NewDocument( + birch.EC.Int32("shard.replSetGetStatus.members.0.state", 1), + birch.EC.Int64("shard.local.oplog.rs.stats.storageStats.storageSize", 2048), + birch.EC.Int32("serverStatus.uptime", 10), + ) + + include := map[string]struct{}{ + "replSetGetStatus.members.0.state": {}, + "local.oplog.rs.stats.storageStats.storageSize": {}, + "serverStatus.uptime": {}, + } + + got := normalizeMetricsDocument(doc, include) + if got["replSetGetStatus.members.0.state"] != int32(1) { + t.Fatalf("state = %#v", got["replSetGetStatus.members.0.state"]) + } + if got["local.oplog.rs.stats.storageStats.storageSize"] != int64(2048) { + t.Fatalf("storageSize = %#v", got["local.oplog.rs.stats.storageStats.storageSize"]) + } + if _, ok := got["shard.replSetGetStatus.members.0.state"]; ok { + t.Fatalf("shard-prefixed key remained: %#v", got) + } +} + +func TestNormalizeMetricsDocument_flatPre80LayoutUnchanged(t *testing.T) { + // MongoDB 5/6/7 style: collectors at the top level, no role wrappers. + doc := birch.NewDocument( + birch.EC.SubDocument("replSetGetStatus", birch.NewDocument( + birch.EC.ArrayFromElements("members", + birch.VC.DocumentFromElements( + birch.EC.Int32("state", 2), + ), + ), + )), + birch.EC.SubDocument("serverStatus", birch.NewDocument( + birch.EC.Int32("uptime", 42), + )), + ) + + include := map[string]struct{}{ + "replSetGetStatus.members.0.state": {}, + "serverStatus.uptime": {}, + } + + got := normalizeMetricsDocument(doc, include) + if got["replSetGetStatus.members.0.state"] != int32(2) { + t.Fatalf("state = %#v", got["replSetGetStatus.members.0.state"]) + } + if got["serverStatus.uptime"] != int32(42) { + t.Fatalf("uptime = %#v", got["serverStatus.uptime"]) + } + if len(got) != 2 { + t.Fatalf("unexpected keys: %#v", got) + } +} diff --git a/ftdc_exporter/ftdc/testdata/metrics_to_get.txt b/ftdc_exporter/ftdc/testdata/metrics_to_get.txt index 6265f78..2a0ce40 100644 --- a/ftdc_exporter/ftdc/testdata/metrics_to_get.txt +++ b/ftdc_exporter/ftdc/testdata/metrics_to_get.txt @@ -35,6 +35,9 @@ serverStatus.flowControl.targetRateLimit serverStatus.wiredTiger.block-manager.bytes read serverStatus.wiredTiger.block-manager.bytes written serverStatus.wiredTiger.block-manager.bytes written for checkpoint +serverStatus.wiredTiger.log.log bytes written +serverStatus.wiredTiger.transaction.transaction checkpoint most recent time (msecs) +serverStatus.wiredTiger.checkpoint.most recent time (msecs) serverStatus.wiredTiger.cache.bytes currently in the cache serverStatus.wiredTiger.cache.maximum bytes configured serverStatus.wiredTiger.cache.modified pages evicted @@ -45,6 +48,15 @@ serverStatus.wiredTiger.cache.unmodified pages evicted serverStatus.wiredTiger.data-handle.connection data handles currently active serverStatus.wiredTiger.concurrentTransactions.read.available serverStatus.wiredTiger.concurrentTransactions.write.available +serverStatus.wiredTiger.concurrentTransactions.monitor.available +serverStatus.wiredTiger.concurrentTransactions.monitor.totalTickets +serverStatus.wiredTiger.concurrentTransactions.monitor.queueLength +serverStatus.queues.execution.read.available +serverStatus.queues.execution.read.totalTickets +serverStatus.queues.execution.read.normalPriority.queueLength +serverStatus.queues.execution.write.available +serverStatus.queues.execution.write.totalTickets +serverStatus.queues.execution.write.normalPriority.queueLength serverStatus.flowControl.enabled serverStatus.flowControl.isLagged serverStatus.flowControl.isLaggedCount @@ -138,3 +150,8 @@ replSetGetStatus.members.1.lastAppliedWallTime replSetGetStatus.members.2.lastAppliedWallTime replSetGetStatus.members.3.lastAppliedWallTime replSetGetStatus.members.4.lastAppliedWallTime +replSetGetStatus.members.0.state +replSetGetStatus.members.1.state +replSetGetStatus.members.2.state +replSetGetStatus.members.3.state +replSetGetStatus.members.4.state diff --git a/grafana/dashboards/dashboard.json b/grafana/dashboards/dashboard.json index 57b7944..7c88552 100644 --- a/grafana/dashboards/dashboard.json +++ b/grafana/dashboards/dashboard.json @@ -29,9 +29,9 @@ "x": 0, "y": 0 }, - "id": 24, + "id": 55, "panels": [], - "title": "WiredTiger", + "title": "Mongo Health", "type": "row" }, { @@ -39,7 +39,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Tickets Available Read displays the number of read tickets available to the WiredTiger storage engine. Read tickets represent the number of concurrent read operations allowed into the storage engine. When this value reaches zero, new read requests might queue until a read ticket becomes available.\n\nTickets Available Write displays the number of write tickets available to the WiredTiger storage engine. Write tickets represent the number of concurrent write operations allowed into the storage engine. When this value reaches zero, new write requests might queue until a write ticket becomes available.", + "description": "### Server uptime\n\n**`serverStatus.uptime`** \u2014 seconds the current **mongod** process has been running.\n\n**Unit:** seconds (displayed as duration)\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.uptime`\n\n---\n\n### How to read\n\n- Monotonic increase while the process is up\n- **Drop** or reset \u2192 **restart** (mark the timestamp)\n- May not start at zero if the dashboard time range begins mid-run\n\n---\n\n### Troubleshooting\n\n- Restart + memory drop \u2192 OOM \u00b7 **mongod Memory (RSS / Virtual)**, **Swap Memory**\n- Restart + assert spike \u2192 crash \u00b7 **Asserts**\n- Planned maintenance \u2192 correlate with deploy window\n- After restart, counters reset \u00b7 **Operations**, **Page Faults** rates may look odd briefly\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)", "fieldConfig": { "defaults": { "color": { @@ -50,12 +50,12 @@ "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", - "axisPlacement": "left", + "axisPlacement": "auto", "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 14, + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, @@ -63,12 +63,9 @@ "viz": false }, "insertNulls": false, - "lineInterpolation": "stepAfter", - "lineStyle": { - "fill": "solid" - }, + "lineInterpolation": "linear", "lineWidth": 1, - "pointSize": 4, + "pointSize": 5, "scaleDistribution": { "type": "linear" }, @@ -83,8 +80,7 @@ "mode": "off" } }, - "decimals": 0, - "fieldMinMax": false, + "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", @@ -92,49 +88,45 @@ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "none" + "unit": "dtdurations" }, "overrides": [] }, "gridPos": { - "h": 6, - "w": 14, + "h": 5, + "w": 12, "x": 0, "y": 1 }, - "id": 1, + "id": 20, "options": { "legend": { "calcs": [], - "displayMode": "list", + "displayMode": "table", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "hideZeros": false, - "mode": "multi", + "mode": "single", "sort": "none" } }, - "pluginVersion": "12.2.0", + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", - "uid": "fdnvtvpv64tfkc" + "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.concurrentTransactions.write.available\" or r[\"_field\"] == \"serverStatus.wiredTiger.concurrentTransactions.read.available\")\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.uptime\")\r\n |> drop(columns: [\"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.uptime\" then \"Server uptime\" else r._field\r\n }))", "refId": "A" } ], - "title": "WiredTiger Tickets", + "title": "Server Uptime", "type": "timeseries" }, { @@ -142,107 +134,148 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "used displays the total bytes cached in memory for serving reads and writes.\n\nThese metrics include both indexes and data from the working set.\n\nSustained high cache usage indicates the RAM is too small for your workloads. Optimize your queries to avoid frequent disk reads. If write operations make cache usage high, throttle them.", + "description": "### ReplSet member states\n\n**`replSetGetStatus.members.N.state`** over the selected time range (same codes as [replica states](https://www.mongodb.com/docs/manual/reference/replica-states/)).\n\n**Note:** Row labels use member **array index** unless `members.N.name` is ingested. Failover shows PRIMARY moving between rows. Primary-only FTDC filter via **hostname**.\n\n---\n\n### Metrics\n\n**Fields:** `replSetGetStatus.members.0.state` \u2026 (through your max index)\n\n---\n\n### How to read\n\n- One row per member index\n- Colored band = role \u00b7 **PRIMARY** (1), **SECONDARY** (2), etc.\n- Red bands = DOWN, RECOVERING, ROLLBACK, \u2026\n\n---\n\n### Docs\n\n- [replSetGetStatus](https://www.mongodb.com/docs/manual/reference/command/replSetGetStatus/)", "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" + "fixedColor": "blue", + "mode": "fixed" }, "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", + "fillOpacity": 100, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "lineWidth": 0, + "spanNulls": false }, - "fieldMinMax": true, - "mappings": [], + "mappings": [ + { + "options": { + "0": { + "color": "semi-dark-red", + "index": 11, + "text": "STARTUP" + }, + "1": { + "color": "rgb(107, 152, 102)", + "index": 4, + "text": "PRIMARY" + }, + "2": { + "color": "rgb(193, 159, 20)", + "index": 3, + "text": "SECONDARY" + }, + "3": { + "color": "semi-dark-red", + "index": 10, + "text": "RECOVERING" + }, + "4": { + "color": "semi-dark-red", + "index": 12, + "text": "FATAL" + }, + "5": { + "color": "semi-dark-red", + "index": 9, + "text": "STARTUP2" + }, + "6": { + "color": "semi-dark-red", + "index": 8, + "text": "UNKNOWN" + }, + "7": { + "color": "#8B8000", + "index": 1, + "text": "ARBITER" + }, + "8": { + "color": "semi-dark-red", + "index": 7, + "text": "DOWN" + }, + "9": { + "color": "semi-dark-red", + "index": 6, + "text": "ROLLBACK" + }, + "10": { + "color": "semi-dark-red", + "index": 5, + "text": "REMOVED" + }, + "null": { + "color": "#FF7383", + "index": 2, + "text": "No data" + } + }, + "type": "value" + } + ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "decgbytes" + "unit": "short" }, "overrides": [] }, "gridPos": { - "h": 6, - "w": 10, - "x": 14, + "h": 5, + "w": 12, + "x": 12, "y": 1 }, - "id": 6, + "id": 54, "options": { + "alignValue": "right", "legend": { - "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, + "mergeValues": true, + "rowHeight": 0.9, + "showValue": "auto", "tooltip": { "hideZeros": false, - "mode": "multi", + "mode": "single", "sort": "none" } }, - "pluginVersion": "12.2.0", + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "hide": false, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.cache.maximum bytes configured\" or r[\"_field\"] == \"serverStatus.wiredTiger.cache.bytes currently in the cache\" )\r\n |> map(fn: (r) => ({ r with _value: r._value / 1024 / 1024 / 1024}))\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] =~ /replSetGetStatus\\.members\\.\\d+\\.state/)\r\n |> drop(columns: [\"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"replSetGetStatus.members.0.state\" then \"Member 0\" else if r._field == \"replSetGetStatus.members.1.state\" then \"Member 1\" else if r._field == \"replSetGetStatus.members.2.state\" then \"Member 2\" else if r._field == \"replSetGetStatus.members.3.state\" then \"Member 3\" else if r._field == \"replSetGetStatus.members.4.state\" then \"Member 4\" else r._field\r\n }))\r\n", "refId": "A" } ], - "title": "WiredTiger cache", - "type": "timeseries" + "title": "Member States", + "type": "state-timeline" }, { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "read Into displays the rate in bytes per second of data read from disk into memory to service queries.\n\nwritten From displays the rate in bytes per second of data written from memory into disk to service writes.", + "description": "### Replica members health\n\n**`replSetGetStatus.members.N.health`** \u2014 per-member reachability (1 = healthy, 0 = down).\n\n**Unit:** 0/1 (gauge)\n\n**Note:** **N** is the member array index, not hostname order.\n\n---\n\n### Metrics\n\n**Fields:** `replSetGetStatus.members.0.health` \u2026 `replSetGetStatus.members.4.health`\n\n---\n\n### How to read\n\n- One line per member index\n- **stepAfter** \u00b7 flat at **1** = healthy \u00b7 drop to **0** = member lost\n- Do **not** stack \u2014 each line is independent 0/1\n\n---\n\n### Troubleshooting\n\n- **1 \u2192 0** \u2192 network partition, crash, OOM \u00b7 **Server Uptime**, **Asserts**, system **CPU/Memory/Disk** on that host\n- Healthy but high lag \u2192 **Replica Members Lag**, **Replica Members Ping**\n\n---\n\n### Docs\n\n- [replSetGetStatus](https://www.mongodb.com/docs/manual/reference/command/replSetGetStatus/)", "fieldConfig": { "defaults": { "color": { @@ -254,6 +287,8 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMax": 1, + "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -265,7 +300,7 @@ "viz": false }, "insertNulls": false, - "lineInterpolation": "linear", + "lineInterpolation": "stepAfter", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -282,35 +317,51 @@ "mode": "off" } }, - "mappings": [], + "decimals": 0, + "mappings": [ + { + "options": { + "0": { + "index": 0, + "text": "Down" + }, + "1": { + "index": 1, + "text": "Up" + } + }, + "type": "value" + } + ], + "max": 1, + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "decbytes" + "unit": "short" }, "overrides": [] }, "gridPos": { - "h": 6, - "w": 6, + "h": 5, + "w": 12, "x": 0, - "y": 7 + "y": 6 }, - "id": 8, + "id": 16, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, @@ -320,18 +371,18 @@ "sort": "none" } }, - "pluginVersion": "12.2.0", + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.cache.bytes written from cache\" or r[\"_field\"] == \"serverStatus.wiredTiger.cache.bytes read into cache\")\r\n |> derivative(unit: 1s)\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] =~ /replSetGetStatus\\.members\\.\\d+\\.health/)\r\n |> drop(columns: [\"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"replSetGetStatus.members.0.health\" then \"Member 0 health\" else if r._field == \"replSetGetStatus.members.1.health\" then \"Member 1 health\" else if r._field == \"replSetGetStatus.members.2.health\" then \"Member 2 health\" else if r._field == \"replSetGetStatus.members.3.health\" then \"Member 3 health\" else if r._field == \"replSetGetStatus.members.4.health\" then \"Member 4 health\" else r._field\r\n }))\r\n", "refId": "A" } ], - "title": "Mb read into cache / written from cache", + "title": "Replica Members Health", "type": "timeseries" }, { @@ -339,7 +390,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Clean vs dirty data\n\nClean refers to data in the cache that is identical to the version stored on disk.\nDirty refers to data in the cache that has been modified and needs to be reconciled with the data stored on the disk.\n\nClean vs dirty eviction\n\nIn the case of clean eviction, there is no dirty content on the page and it is directly removed from the memory. After that, the page remains solely on the disk unchanged.\nIn the case of dirty eviction, the dirty page goes through reconciliation. The obsolete content on the page is thrown out, the latest value goes to the data store and older values go to the history store.", + "description": "### Asserts\n\nPer-second rate of **`serverStatus.asserts`** \u2014 mongod internal checks and user-triggered assertion counters.\n\n**Unit:** asserts/sec (`ops`)\n\n**Expectation:** normally **flat at zero**; any sustained non-zero rate warrants investigation.\n\n---\n\n### Metrics\n\n**Fields:**\n\n`serverStatus.asserts.regular`\n`serverStatus.asserts.warning`\n`serverStatus.asserts.msg`\n`serverStatus.asserts.user`\n`serverStatus.asserts.rollovers`\n\n**Query type:** cumulative counter \u2192 `derivative(unit: 1s, nonNegative: true)` \u2192 `aggregateWindow(mean)`\n\n**Rollovers:** count of times assert counters wrapped past 2\u00b3\u2070 \u2014 spikes are rare and distinct from error asserts.\n\n---\n\n### How to read\n\n- **Regular** / **Msg** \u2014 internal mongod assertion failures (unusual)\n- **Warning** \u2014 non-fatal internal warnings\n- **User** \u2014 client-visible errors (duplicate key, validation, disk full, etc.)\n- **Rollovers** \u2014 counter wrap events, not application errors\n- Brief spikes on restart or deploy may appear as counter resets\n\n---\n\n### Troubleshooting\n\n- **Regular** or **Msg** sustained \u2192 possible bug or corruption \u00b7 check mongod logs immediately\n- **User** spikes \u2192 application errors \u00b7 correlate with app logs and **Operations**\n- Spike + **Server uptime** drop \u2192 crash/assert abort \u00b7 **Replica Members Health**\n- Zero across all lines \u2192 healthy (typical)\n\nCorrelate with **Server uptime** and **Average Operation Latency**.\n\n---\n\n### Docs\n\n- [serverStatus \u2014 asserts](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.asserts)", "fieldConfig": { "defaults": { "color": { @@ -351,6 +402,7 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -379,7 +431,6 @@ "mode": "off" } }, - "decimals": 0, "mappings": [], "thresholds": { "mode": "absolute", @@ -387,29 +438,54 @@ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "short" + "unit": "ops" }, - "overrides": [] + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Msg" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": true, + "viz": true + } + } + ] + } + ] }, "gridPos": { - "h": 6, - "w": 8, - "x": 6, - "y": 7 + "h": 5, + "w": 12, + "x": 12, + "y": 6 }, - "id": 12, + "id": 30, "options": { "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "right", "showLegend": true }, "tooltip": { @@ -418,26 +494,39 @@ "sort": "none" } }, - "pluginVersion": "12.2.0", + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.cache.modified pages evicted\" or r[\"_field\"] == \"serverStatus.wiredTiger.cache.unmodified pages evicted\")\r\n |> map(fn: (r) => ({ r with _value: r._value }))\r\n |> derivative(unit: 1s)\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.asserts.regular\" or\r\n r[\"_field\"] == \"serverStatus.asserts.warning\" or\r\n r[\"_field\"] == \"serverStatus.asserts.msg\" or\r\n r[\"_field\"] == \"serverStatus.asserts.user\" or\r\n r[\"_field\"] == \"serverStatus.asserts.rollovers\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.asserts.regular\" then \"Regular\" else if r._field == \"serverStatus.asserts.warning\" then \"Warning\" else if r._field == \"serverStatus.asserts.msg\" then \"Msg\" else if r._field == \"serverStatus.asserts.user\" then \"User\" else if r._field == \"serverStatus.asserts.rollovers\" then \"Rollovers\" else r._field\r\n }))", "refId": "A" } ], - "title": "Pages Evicted", + "title": "Asserts", "type": "timeseries" }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 24, + "panels": [], + "title": "WiredTiger", + "type": "row" + }, { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "indicates modified data that needs to be written to the data files via a checkpoint.", + "description": "### WiredTiger ticket availability\n\nShows how many **read** and **write** execution tickets are still available in the storage engine. Each ticket allows one concurrent operation; when **available** reaches **0**, new operations wait until a ticket is freed.\n\n**Unit:** ticket count (not bytes)\n\n---\n\n### Metrics (same concept, two paths)\n\n**MongoDB 8.x+** \n`serverStatus.queues.execution.read.available` \n`serverStatus.queues.execution.write.available`\n\n**Legacy** \n`serverStatus.wiredTiger.concurrentTransactions.read.available` \n`serverStatus.wiredTiger.concurrentTransactions.write.available`\n\nBoth paths queried for compatibility. Usually only one has data per version.\n\n---\n\n### How to read\n\n- Two lines: read and write **available** tickets \n- **stepAfter** reflects discrete ticket changes \n- Typical max ~**128** (dynamic adjustment on 7.0+)\n\n---\n\n### Troubleshooting\n\n- Sustained **low or zero** \u2192 concurrency saturation \n- Correlate with **Current Queue writers / readers** and **Average Operation Latency**\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)\n- [Performance tuning \u2014 WiredTiger tickets](https://www.mongodb.com/docs/manual/administration/performance-tuning/)", "fieldConfig": { "defaults": { "color": { @@ -448,11 +537,12 @@ "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", - "axisPlacement": "auto", + "axisPlacement": "left", + "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 14, "gradientMode": "none", "hideFrom": { "legend": false, @@ -460,9 +550,12 @@ "viz": false }, "insertNulls": false, - "lineInterpolation": "linear", + "lineInterpolation": "stepAfter", + "lineStyle": { + "fill": "solid" + }, "lineWidth": 1, - "pointSize": 5, + "pointSize": 4, "scaleDistribution": { "type": "linear" }, @@ -477,82 +570,58 @@ "mode": "off" } }, + "decimals": 0, + "fieldMinMax": false, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "decbytes" + "unit": "short" }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "serverStatus.wiredTiger.cache.tracked dirty bytes in the cache" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": true, - "viz": true - } - } - ] - } - ] + "overrides": [] }, "gridPos": { - "h": 7, - "w": 10, - "x": 14, - "y": 7 + "h": 8, + "w": 13, + "x": 0, + "y": 12 }, - "id": 7, + "id": 1, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, - "mode": "single", + "mode": "multi", "sort": "none" } }, - "pluginVersion": "12.2.0", + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.cache.tracked dirty bytes in the cache\" )\r\n |> map(fn: (r) => ({ r with _value: r._value}))\r\n\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> yield(name: \"last\")", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.concurrentTransactions.read.available\" or r[\"_field\"] == \"serverStatus.wiredTiger.concurrentTransactions.write.available\" or r[\"_field\"] == \"serverStatus.queues.execution.read.available\" or r[\"_field\"] == \"serverStatus.queues.execution.write.available\")\r\n |> drop(columns: [\"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _field: if r._field == \"serverStatus.wiredTiger.concurrentTransactions.read.available\" then \"Tickets Available Read\" else if r._field == \"serverStatus.wiredTiger.concurrentTransactions.write.available\" then \"Tickets Available Write\" else if r._field == \"serverStatus.queues.execution.read.available\" then \"Tickets Available Read\" else if r._field == \"serverStatus.queues.execution.write.available\" then \"Tickets Available Write\" else r._field }))\r\n", "refId": "A" } ], - "title": "Dirty mb in the cache", + "title": "WiredTiger Tickets", "type": "timeseries" }, { @@ -560,7 +629,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "A checkpoint is a known point in time from which WiredTiger can recover in the event of a crash or unexpected shutdown.", + "description": "### WiredTiger cache size\n\n**maximum bytes configured** vs **bytes currently in the cache** \u2014 WiredTiger internal cache limit and current usage. **Unit:** bytes (per MongoDB docs). Current usage should not exceed the configured maximum.\n\n**Metrics:** `serverStatus.wiredTiger.cache.maximum bytes configured` \u00b7 `serverStatus.wiredTiger.cache.bytes currently in the cache`\n\nTo change the limit, see `wiredTigerCacheSizeGB` / `storage.wiredTiger.engineConfig.cacheSizeGB` (config in GB; these stats are still bytes).\n\n---\n\n### How to read\n\n- Two lines: configured max and current usage\n- WiredTiger often targets ~80% of max under load; sustained saturation \u2192 eviction pressure\n\n---\n\n### Troubleshooting\n\n- Usage near max \u2192 **Pages Evicted**, **Dirty Cache Bytes**\n- Consistently at max \u2192 raise `cacheSizeGB` or reduce working set\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)\n- [WiredTiger storage engine](https://www.mongodb.com/docs/manual/core/wiredtiger/)", "fieldConfig": { "defaults": { "color": { @@ -572,7 +641,7 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", - "axisSoftMax": 10, + "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -601,56 +670,60 @@ "mode": "off" } }, + "decimals": 2, + "fieldMinMax": true, "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "decbytes" + "unit": "bytes" }, "overrides": [] }, "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 13 + "h": 8, + "w": 11, + "x": 13, + "y": 12 }, - "id": 19, + "id": 6, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, - "mode": "single", + "mode": "multi", "sort": "none" } }, - "pluginVersion": "12.2.0", + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.block-manager.bytes written for checkpoint\")\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> derivative(unit: v.windowPeriod)\r\n", + "hide": false, + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.cache.maximum bytes configured\" or r[\"_field\"] == \"serverStatus.wiredTiger.cache.bytes currently in the cache\")\r\n |> drop(columns: [\"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.wiredTiger.cache.maximum bytes configured\" then \"Maximum cache configured\" else if r._field == \"serverStatus.wiredTiger.cache.bytes currently in the cache\" then \"Current cache usage\" else r._field\r\n }))", "refId": "A" } ], - "title": "Mb written for checkpoint", + "title": "WiredTiger Cache", "type": "timeseries" }, { @@ -658,7 +731,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "The WiredTiger block manager subsystem manages the reading and writing of data from the disk. It is designed to facilitate high performance, economic use of disk space and customizability.", + "description": "### WiredTiger Pages Evicted\n\nRate of **modified** (dirty) and **unmodified** (clean) cache pages evicted per second. MongoDB docs note unmodified evictions as the main eviction statistic.\n\n**Unit:** pages/s (derivative of cumulative counters)\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.wiredTiger.cache.modified pages evicted` \u00b7 `serverStatus.wiredTiger.cache.unmodified pages evicted`\n\n---\n\n### How to read\n\n- **Modified pages evicted (dirty)** \u2014 evicted after write/reconciliation\n- **Unmodified pages evicted (clean)** \u2014 discarded without write\n\n---\n\n### Troubleshooting\n\n- Dirty eviction spikes \u2192 cache pressure, write latency \u00b7 **WiredTiger Cache**, **Dirty Cache Bytes**\n- Eviction can't keep up \u2192 app threads may evict (performance cliff)\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)\n- [WiredTiger storage engine](https://www.mongodb.com/docs/manual/core/wiredtiger/)", "fieldConfig": { "defaults": { "color": { @@ -670,6 +743,7 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -698,6 +772,7 @@ "mode": "off" } }, + "decimals": 0, "mappings": [], "thresholds": { "mode": "absolute", @@ -705,28 +780,28 @@ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "MBs" + "unit": "pages/s" }, "overrides": [] }, "gridPos": { - "h": 6, - "w": 8, - "x": 6, - "y": 13 + "h": 7, + "w": 13, + "x": 0, + "y": 20 }, - "id": 23, + "id": 12, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, @@ -736,18 +811,18 @@ "sort": "none" } }, - "pluginVersion": "12.2.0", + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.block-manager.bytes read\" or r[\"_field\"] == \"serverStatus.wiredTiger.block-manager.bytes written\" )\r\n |> map(fn: (r) => ({ r with _value: r._value}))\r\n |> map(fn: (r) => ({ r with _value: r._value / 1024 /1024 }))\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> derivative(unit: 1s)\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.cache.modified pages evicted\" or r[\"_field\"] == \"serverStatus.wiredTiger.cache.unmodified pages evicted\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _field: if r._field == \"serverStatus.wiredTiger.cache.modified pages evicted\" then \"Modified pages evicted (dirty)\" else if r._field == \"serverStatus.wiredTiger.cache.unmodified pages evicted\" then \"Unmodified pages evicted (clean)\" else r._field }))\r\n", "refId": "A" } ], - "title": "wiredTiger Block Manager", + "title": "WiredTiger Pages Evicted", "type": "timeseries" }, { @@ -755,7 +830,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "A WiredTiger Data Handle (dhandle) is a generic representation of any named data source. This representation contains information such as its name, how many references there are to it, individual data source statistics and what type of underlying data object it is.\n\nWiredTiger maintains all dhandles in a global dhandle list accessed from the connection. Multiple sessions access this list, which is protected by a R/W lock. Each session also maintains a session dhandle cache which is a cache of dhandles a session has operated upon. The entries in the cache are references into the global dhandle list.", + "description": "### WiredTiger data handles\n\nCount of **connection data handles currently active** \u2014 WiredTiger dhandles for open collections, indexes, and other named data sources.\n\n**Unit:** count (gauge)\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.wiredTiger.data-handle.connection data handles currently active`\n\n---\n\n### How to read\n\n- Single line: active dhandles over time\n- Sudden drop \u2192 collections closed or **mongod** restart (**Server Uptime**)\n\n---\n\n### Troubleshooting\n\n- Very high count \u2192 many collections/indexes in use, memory overhead, dhandle list lock contention\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)\n- [WiredTiger storage engine](https://www.mongodb.com/docs/manual/core/wiredtiger/)", "fieldConfig": { "defaults": { "color": { @@ -767,6 +842,7 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -795,6 +871,7 @@ "mode": "off" } }, + "decimals": 0, "mappings": [], "thresholds": { "mode": "absolute", @@ -802,68 +879,57 @@ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] - } + }, + "unit": "short" }, "overrides": [] }, "gridPos": { "h": 5, - "w": 10, - "x": 14, - "y": 14 + "w": 11, + "x": 13, + "y": 20 }, "id": 26, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, - "mode": "single", + "mode": "multi", "sort": "none" } }, - "pluginVersion": "12.2.0", + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.data-handle.connection data handles currently active\")\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> yield(name: \"last\")", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.data-handle.connection data handles currently active\")\r\n |> drop(columns: [\"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _field: if r._field == \"serverStatus.wiredTiger.data-handle.connection data handles currently active\" then \"Active data handles\" else r._field }))", "refId": "A" } ], "title": "WiredTiger Data Handle", "type": "timeseries" }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 19 - }, - "id": 22, - "panels": [], - "title": "Server Status", - "type": "row" - }, { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "### Dirty cache bytes\n\n**tracked dirty bytes in the cache** \u2014 modified data in the WiredTiger cache not yet flushed via checkpoint or eviction. Should stay below **bytes currently in the cache**.\n\n**Unit:** bytes (gauge)\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.wiredTiger.cache.tracked dirty bytes in the cache`\n\n---\n\n### How to read\n\n- Single line: dirty bytes over time\n- ~5% of cache \u2192 eviction pressure \u00b7 ~20% \u2192 aggressive eviction / stalls\n\n---\n\n### Troubleshooting\n\n- Dirty bytes high vs **WiredTiger Cache** max \u2192 eviction pressure\n- Correlate with **Pages Evicted**, **Checkpoint Write Throughput**\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)\n- [WiredTiger storage engine](https://www.mongodb.com/docs/manual/core/wiredtiger/)", "fieldConfig": { "defaults": { "color": { @@ -875,7 +941,6 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", - "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -889,7 +954,7 @@ "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, - "pointSize": 1, + "pointSize": 5, "scaleDistribution": { "type": "linear" }, @@ -904,35 +969,37 @@ "mode": "off" } }, + "decimals": 2, "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "decgbytes" + "unit": "bytes" }, "overrides": [] }, "gridPos": { - "h": 5, - "w": 8, - "x": 0, - "y": 20 + "h": 9, + "w": 11, + "x": 13, + "y": 25 }, - "id": 3, + "id": 7, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, @@ -942,18 +1009,18 @@ "sort": "none" } }, - "pluginVersion": "12.2.0", + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", - "uid": "influxdb" + "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.mem.virtual\" or r[\"_field\"] == \"serverStatus.mem.resident\")\r\n |> aggregateWindow(every: 1m, fn: median, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _value: r._value / 1000.0 }))\r\n |> yield(name: \"median\")\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.cache.tracked dirty bytes in the cache\" )\r\n |> drop(columns: [\"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _field: if r._field == \"serverStatus.wiredTiger.cache.tracked dirty bytes in the cache\" then \"Dirty cache bytes\" else r._field }))", "refId": "A" } ], - "title": "Memory", + "title": "Dirty Cache Bytes", "type": "timeseries" }, { @@ -961,6 +1028,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "### WiredTiger block manager I/O\n\nRead/write throughput through WiredTiger's block manager (disk I/O for data files). **Unit:** MB/s (from bytes/s \u00f7 1024\u00b2) \u2014 cumulative byte counters differentiated.\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.wiredTiger.block-manager.bytes read` \u00b7 `serverStatus.wiredTiger.block-manager.bytes written`\n\n---\n\n### How to read\n\n- **Bytes read** \u2014 data read from disk (cache misses)\n- **Bytes written** \u2014 data written via block manager (eviction, checkpoints)\n\n---\n\n### Troubleshooting\n\n- High reads \u2192 cache pressure \u00b7 **WiredTiger Cache**, **Pages Evicted**\n- High writes \u2192 checkpoints/eviction \u00b7 **Checkpoint Write Throughput**, **Disk I/O**\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)\n- [WiredTiger storage engine](https://www.mongodb.com/docs/manual/core/wiredtiger/)", "fieldConfig": { "defaults": { "color": { @@ -1000,55 +1068,58 @@ "mode": "off" } }, + "decimals": 2, "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] - } + }, + "unit": "MBs" }, "overrides": [] }, "gridPos": { - "h": 5, - "w": 7, - "x": 8, - "y": 20 + "h": 7, + "w": 13, + "x": 0, + "y": 27 }, - "id": 27, + "id": 23, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, - "mode": "single", + "mode": "multi", "sort": "none" } }, - "pluginVersion": "12.2.0", + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.extra_info.page_faults\")\r\n |> derivative(unit: v.windowPeriod)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.block-manager.bytes read\" or r[\"_field\"] == \"serverStatus.wiredTiger.block-manager.bytes written\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _field: if r._field == \"serverStatus.wiredTiger.block-manager.bytes read\" then \"Bytes read\" else if r._field == \"serverStatus.wiredTiger.block-manager.bytes written\" then \"Bytes written\" else r._field }))\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) / 1024.0 / 1024.0 }))", "refId": "A" } ], - "title": "Page faults", + "title": "WiredTiger Block Manager", "type": "timeseries" }, { @@ -1056,6 +1127,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "### Checkpoint write throughput\n\nRate of **bytes written for checkpoint** \u2014 disk write throughput during WiredTiger checkpoints (~every 60s). Recovery point after crash/shutdown.\n\n**Unit:** B/s (`Bps`) \u2014 derivative of cumulative bytes\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.wiredTiger.block-manager.bytes written for checkpoint`\n\n---\n\n### How to read\n\n- Spikes during checkpoint runs \u00b7 baseline low between checkpoints\n\n---\n\n### Troubleshooting\n\n- Large spikes + latency \u2192 checkpoint competing for disk I/O\n- Correlate with **Disk I/O**, **Disk Writes**, **Average Operation Latency**, **Checkpoint Duration**\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)\n- [WiredTiger storage engine](https://www.mongodb.com/docs/manual/core/wiredtiger/)", "fieldConfig": { "defaults": { "color": { @@ -1095,6 +1167,7 @@ "mode": "off" } }, + "decimals": 2, "mappings": [], "thresholds": { "mode": "absolute", @@ -1102,27 +1175,28 @@ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] - } + }, + "unit": "Bps" }, "overrides": [] }, "gridPos": { "h": 7, - "w": 9, - "x": 15, - "y": 20 + "w": 13, + "x": 0, + "y": 34 }, - "id": 2, + "id": 19, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, @@ -1132,27 +1206,18 @@ "sort": "none" } }, - "pluginVersion": "12.2.0", + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", - "uid": "influxdb" + "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.connections.active\" or r[\"_field\"] == \"serverStatus.connections.current\" )\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> yield(name: \"last\")\r\n\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.block-manager.bytes written for checkpoint\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _field: if r._field == \"serverStatus.wiredTiger.block-manager.bytes written for checkpoint\" then \"Checkpoint write throughput\" else r._field }))", "refId": "A" - }, - { - "datasource": { - "type": "influxdb", - "uid": "influxdb" - }, - "hide": false, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.connections.totalCreated\")\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> derivative(unit: v.windowPeriod)", - "refId": "B" } ], - "title": "Connections", + "title": "Checkpoint Write Throughput", "type": "timeseries" }, { @@ -1160,6 +1225,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "### Cache read/write throughput\n\nRate of bytes **read into cache** (cold data from disk) and **written from cache** (flush to disk). Overview of WiredTiger cache I/O activity.\n\n**Unit:** B/s (`Bps`) \u2014 derivative of cumulative byte counters\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.wiredTiger.cache.bytes read into cache` \u00b7 `serverStatus.wiredTiger.cache.bytes written from cache`\n\n---\n\n### How to read\n\n- **Read into cache** \u2014 cache misses / data pulled from disk\n- **Written from cache** \u2014 dirty data flushed (eviction, checkpoints)\n\n---\n\n### Troubleshooting\n\n- High reads \u2192 working set may exceed cache \u00b7 **WiredTiger Cache**, **Pages Evicted**\n- High writes \u2192 write load or eviction pressure \u00b7 **Dirty Cache Bytes**, **WiredTiger Block Manager**\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)\n- [WiredTiger storage engine](https://www.mongodb.com/docs/manual/core/wiredtiger/)", "fieldConfig": { "defaults": { "color": { @@ -1172,6 +1238,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1188,6 +1255,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -1197,54 +1265,57 @@ "mode": "off" } }, - "decimals": 1, + "decimals": 2, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": 0 } ] }, - "unit": "ns" + "unit": "Bps" }, "overrides": [] }, "gridPos": { "h": 7, - "w": 8, - "x": 0, - "y": 25 + "w": 11, + "x": 13, + "y": 34 }, - "id": 5, + "id": 8, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.opLatencies.reads.latency\" or r[\"_field\"] == \"serverStatus.opLatencies.writes.latency\" or r[\"_field\"] == \"serverStatus.opLatencies.commands.latency\" )\r\n |> derivative(unit: v.windowPeriod)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.cache.bytes written from cache\" or r[\"_field\"] == \"serverStatus.wiredTiger.cache.bytes read into cache\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _field: if r._field == \"serverStatus.wiredTiger.cache.bytes written from cache\" then \"Written from cache\" else if r._field == \"serverStatus.wiredTiger.cache.bytes read into cache\" then \"Read into cache\" else r._field }))", "refId": "A" } ], - "title": "Latency", + "title": "Cache Read/Write Throughput", "type": "timeseries" }, { @@ -1252,7 +1323,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "", + "description": "### Checkpoint duration\n\nDuration in **milliseconds** of the **most recent** WiredTiger checkpoint. Value updates when a checkpoint completes (~every 60s), then holds until the next.\n\n**Unit:** ms (gauge)\n\n---\n\n### Metrics (same concept, two paths)\n\n**Legacy:** `serverStatus.wiredTiger.transaction.transaction checkpoint most recent time (msecs)` \n**MongoDB 8.x+:** `serverStatus.wiredTiger.checkpoint.most recent time (msecs)`\n\nBoth queried; usually one path has data per version.\n\n---\n\n### How to read\n\n- Single line \u00b7 rising trend under steady write load \u2192 I/O pressure\n\n---\n\n### Troubleshooting\n\n- Increasing duration \u2192 I/O saturation \u00b7 **Checkpoint Write Throughput**, **Disk I/O**, **Average Operation Latency**\n- Long checkpoints \u2192 possible write stalls\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)\n- [WiredTiger storage engine](https://www.mongodb.com/docs/manual/core/wiredtiger/)", "fieldConfig": { "defaults": { "color": { @@ -1265,6 +1336,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1281,6 +1353,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -1290,52 +1363,60 @@ "mode": "off" } }, + "decimals": 1, "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": 0 } ] - } + }, + "unit": "ms" }, "overrides": [] }, "gridPos": { - "h": 7, - "w": 7, - "x": 8, - "y": 25 + "h": 6, + "w": 13, + "x": 0, + "y": 41 }, - "id": 9, + "id": 48, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", "placement": "bottom", - "showLegend": true + "showLegend": true, + "sortBy": "Name", + "sortDesc": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => \r\n r[\"_field\"] == \"serverStatus.opcounters.update\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.query\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.insert\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.getmore\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.delete\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.command\")\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.transaction.transaction checkpoint most recent time (msecs)\" or r[\"_field\"] == \"serverStatus.wiredTiger.checkpoint.most recent time (msecs)\")\r\n |> drop(columns: [\"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _field: if r._field == \"serverStatus.wiredTiger.transaction.transaction checkpoint most recent time (msecs)\" then \"Checkpoint duration\" else if r._field == \"serverStatus.wiredTiger.checkpoint.most recent time (msecs)\" then \"Checkpoint duration\" else r._field }))\r\n", "refId": "A" } ], - "title": "Operations", + "title": "Checkpoint Duration", "type": "timeseries" }, { @@ -1343,6 +1424,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "### Journal bytes written\n\nRate of WAL/journal bytes written by WiredTiger (`wiredTiger.log`). **Unit:** B/s (`Bps`) \u2014 derivative of cumulative bytes.\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.wiredTiger.log.log bytes written`\n\n---\n\n### How to read\n\n- Single line \u00b7 sustained high rate \u2192 heavy write workload\n\n---\n\n### Troubleshooting\n\n- Compare with **Checkpoint Write Throughput** (journal vs checkpoint balance)\n- Sustained high values \u2192 storage may be a bottleneck\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)\n- [WiredTiger storage engine](https://www.mongodb.com/docs/manual/core/wiredtiger/)", "fieldConfig": { "defaults": { "color": { @@ -1355,6 +1437,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1371,6 +1454,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -1380,59 +1464,78 @@ "mode": "off" } }, + "decimals": 2, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": 0 } ] - } + }, + "unit": "Bps" }, "overrides": [] }, "gridPos": { - "h": 5, - "w": 9, - "x": 15, - "y": 27 + "h": 6, + "w": 11, + "x": 13, + "y": 41 }, - "id": 28, + "id": 49, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.network.numRequests\")\r\n |> derivative(unit: v.windowPeriod)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.wiredTiger.log.log bytes written\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _field: if r._field == \"serverStatus.wiredTiger.log.log bytes written\" then \"Journal bytes written\" else r._field }))\r\n", "refId": "A" } ], - "title": "Network requests", + "title": "Journal Bytes Written", "type": "timeseries" }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 47 + }, + "id": 22, + "panels": [], + "title": "Server Status", + "type": "row" + }, { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "### mongod memory (RSS / virtual)\n\n**mem.resident** (physical RAM) and **mem.virtual** (virtual address space) used by the mongod process.\n\n**Unit:** IEC bytes on chart (auto-scaled GiB/MiB). Source fields are **MiB** in `serverStatus.mem` (`db.serverStatus().mem`); query multiplies by 1024\u00b2 for display.\n\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.mem.resident` \u00b7 `serverStatus.mem.virtual`\n\n---\n\n### How to read\n\n- Two lines: resident vs virtual\n- Virtual \u226b resident is normal (mapped files)\n- Sudden resident drop \u2192 restart \u00b7 **Server Uptime**\n- Tooltip/axis values are auto-scaled; divide by 1024\u00b2 or compare trends, not raw shell integers\n\n---\n\n### Troubleshooting\n\n- Resident near system RAM \u2192 swap risk \u00b7 **Swap Memory**, **Page Faults**\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)", "fieldConfig": { "defaults": { "color": { @@ -1444,7 +1547,9 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1456,11 +1561,12 @@ "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, - "pointSize": 5, + "pointSize": 1, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -1470,52 +1576,57 @@ "mode": "off" } }, + "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": 0 } ] - } + }, + "unit": "bytes" }, "overrides": [] }, "gridPos": { - "h": 6, - "w": 8, + "h": 7, + "w": 10, "x": 0, - "y": 32 + "y": 48 }, - "id": 29, + "id": 3, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.globalLock.currentQueue.writers\" or r[\"_field\"] == \"serverStatus.globalLock.currentQueue.readers\")\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> yield(name: \"last\")", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.mem.virtual\" or r[\"_field\"] == \"serverStatus.mem.resident\")\r\n |> drop(columns: [\"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _field: if r._field == \"serverStatus.mem.virtual\" then \"Virtual memory\" else if r._field == \"serverStatus.mem.resident\" then \"Resident (RSS)\" else r._field }))\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) * 1024.0 * 1024.0 }))\r\n", "refId": "A" } ], - "title": "Current Queue writers / readers", + "title": "mongod Memory (RSS / Virtual)", "type": "timeseries" }, { @@ -1523,6 +1634,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "### Network requests\n\nRate of **network.numRequests** \u2014 distinct client requests received by mongod.\n\nUse with **Network In / Out** to relate request volume to logical byte throughput.\n\n**Unit:** requests/sec (`reqps`)\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.network.numRequests`\n\n---\n\n### How to read\n\n- Single line \u00b7 request arrival rate\n- Not the same as **Operations** (opcounters) \u2014 one request can trigger multiple ops\n\n---\n\n### Troubleshooting\n\n- Sudden drop \u2192 client/network issue or load shift\n- Spike without matching **Operations** \u2192 lightweight commands (heartbeats, metadata)\n- High rate + high **Network In / Out** bytesOut \u2192 large responses \u00b7 check projections/limits\n\n---\n\n### Docs\n\n- [serverStatus \u2014 network](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.network)", "fieldConfig": { "defaults": { "color": { @@ -1534,9 +1646,11 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 25, + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, @@ -1544,71 +1658,74 @@ "viz": false }, "insertNulls": false, - "lineInterpolation": "stepAfter", + "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", - "mode": "normal" + "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, + "decimals": 1, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": 0 } ] }, - "unit": "none" + "unit": "reqps" }, "overrides": [] }, "gridPos": { - "h": 6, - "w": 7, - "x": 8, - "y": 32 + "h": 5, + "w": 14, + "x": 10, + "y": 48 }, - "id": 16, + "id": 28, "options": { "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "right", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, - "pluginVersion": "10.4.3", + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"replSetGetStatus.members.0.health\" or r[\"_field\"] == \"replSetGetStatus.members.1.health\" or r[\"_field\"] == \"replSetGetStatus.members.2.health\")\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> yield(name: \"last\")", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.network.numRequests\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.network.numRequests\" then \"Network requests (/sec)\" else r._field\r\n }))", "refId": "A" } ], - "title": "Replica members health", + "title": "Network Requests", "type": "timeseries" }, { @@ -1616,6 +1733,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "### Connections\n\n**current**, **active**, and **available** connection counts.\n\n**Unit:** count (gauge)\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.connections.current` \u00b7 `serverStatus.connections.active` \u00b7 `serverStatus.connections.available`\n\n---\n\n### How to read\n\n- **Current** \u2014 all open connections (incl. idle)\n- **Active** \u2014 connections running operations\n- **Available** \u2014 unused connection slots\n\n---\n\n### Troubleshooting\n\n- **Current** near `maxIncomingConnections` (65536) \u2192 pool exhaustion\n- **Active** \u226a **Current** \u2192 idle connections \u00b7 check app pool settings\n- For connection **churn**, see **Connections Created**\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)", "fieldConfig": { "defaults": { "color": { @@ -1629,6 +1747,7 @@ "axisPlacement": "auto", "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1645,6 +1764,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -1654,53 +1774,83 @@ "mode": "off" } }, + "decimals": 0, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": 0 } ] }, - "unit": "ms" + "unit": "short" }, - "overrides": [] + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Active connections", + "Current connections" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": true, + "viz": true + } + } + ] + } + ] }, "gridPos": { "h": 6, - "w": 9, - "x": 15, - "y": 32 + "w": 14, + "x": 10, + "y": 53 }, - "id": 15, + "id": 2, "options": { "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "right", "showLegend": true }, "tooltip": { - "mode": "none", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] =~ /replSetGetStatus\\.members\\.\\d+\\.lastAppliedWallTime|serverStatus\\.localTime/)\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> pivot(rowKey:[\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n member_0: if exists r[\"replSetGetStatus.members.0.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.0.lastAppliedWallTime\"])) / 1000.0 else 0.0,\r\n member_1: if exists r[\"replSetGetStatus.members.1.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.1.lastAppliedWallTime\"])) / 1000.0 else 0.0,\r\n member_2: if exists r[\"replSetGetStatus.members.2.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.2.lastAppliedWallTime\"])) / 1000.0 else 0.0,\r\n member_3: if exists r[\"replSetGetStatus.members.3.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.3.lastAppliedWallTime\"])) / 1000.0 else 0.0, // Add more members as needed\r\n }))\r\n |> yield(name: \"differences\")\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.connections.current\" or\r\n r[\"_field\"] == \"serverStatus.connections.active\" or\r\n r[\"_field\"] == \"serverStatus.connections.available\")\r\n |> drop(columns: [\"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.connections.current\" then \"Current connections\" else if r._field == \"serverStatus.connections.active\" then \"Active connections\" else if r._field == \"serverStatus.connections.available\" then \"Available connections\" else r._field\r\n }))", "refId": "A" } ], - "title": "Replica members lag", + "title": "Connections", "type": "timeseries" }, { @@ -1708,7 +1858,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "Operation latencies for the instance as a whole.", + "description": "### Page faults\n\nRate of `extra_info.page_faults` (cumulative counter since process start). Page faults occur when MongoDB must access data not in physical memory. Per MongoDB docs, limited sporadic faults are normal; sustained high rates often mean the working set exceeds RAM.\n\n**Unit:** page faults/sec (Grafana `ops`)\n\n**Note:** On Windows, hard and soft page faults are both counted; fields in `extra_info` vary by platform.\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.extra_info.page_faults`\n\n---\n\n### How to read\n\n- Single line \u00b7 spikes \u2192 memory pressure or cold data access\n- Flat near zero \u2192 working set mostly in memory\n\n---\n\n### Troubleshooting\n\n- Sustained high rate \u2192 working set > RAM \u00b7 correlate with **WiredTiger Cache**, **mongod Memory (RSS / Virtual)**, **Swap Memory**\n- Spikes during scans/index builds \u2192 expected \u00b7 check query patterns if persistent\n- High faults + active swap \u2192 add RAM or reduce working set\n\n---\n\n### Docs\n\n- [serverStatus \u2014 extra_info](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#extra_info)", "fieldConfig": { "defaults": { "color": { @@ -1720,7 +1870,9 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1737,6 +1889,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -1746,53 +1899,57 @@ "mode": "off" } }, + "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": 0 } ] }, - "unit": "µs" + "unit": "ops" }, "overrides": [] }, "gridPos": { - "h": 5, - "w": 8, + "h": 6, + "w": 10, "x": 0, - "y": 38 + "y": 55 }, - "id": 10, + "id": 27, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "multi", + "hideZeros": false, + "mode": "single", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.opLatencies.reads.ops\" or r[\"_field\"] == \"serverStatus.opLatencies.writes.ops\" or r[\"_field\"] == \"serverStatus.opLatencies.commands.ops\" )\r\n |> derivative(unit: 1s)\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.extra_info.page_faults\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.extra_info.page_faults\" then \"Page faults (/sec)\" else r._field\r\n }))", "refId": "A" } ], - "title": "Operations latencies op", + "title": "Page Faults", "type": "timeseries" }, { @@ -1800,7 +1957,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "", + "description": "### Connections created\n\nRate of new connections (`connections.totalCreated` derivative). Indicates connection **churn** \u2014 apps opening/closing connections frequently.\n\n**Unit:** connections/sec (`ops`)\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.connections.totalCreated`\n\n---\n\n### How to read\n\n- Single line \u00b7 spikes \u2192 bursts of new connections\n- Sustained high rate \u2192 pool churn \u00b7 check app connection pool settings\n\n---\n\n### Troubleshooting\n\n- High rate + high **Current** on **Connections** \u2192 many short-lived connections\n- Correlate with **Connections** (counts)\n\n---\n\n### Docs\n\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)", "fieldConfig": { "defaults": { "color": { @@ -1812,8 +1969,8 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", - "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1830,6 +1987,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -1839,54 +1997,57 @@ "mode": "off" } }, + "decimals": 2, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": 0 } ] }, - "unit": "none" + "unit": "ops" }, "overrides": [] }, "gridPos": { - "h": 5, - "w": 7, - "x": 8, - "y": 38 + "h": 6, + "w": 14, + "x": 10, + "y": 59 }, - "id": 20, + "id": 52, "options": { "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "right", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "hide": false, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.uptime\" )\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.connections.totalCreated\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.connections.totalCreated\" then \"Connections created (/sec)\" else r._field\r\n }))", "refId": "A" } ], - "title": "Server Uptime", + "title": "Connections Created", "type": "timeseries" }, { @@ -1894,6 +2055,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "### Average operation latency\n\nRolling average latency per operation type, derived from cumulative `opLatencies` counters.\n\n**Formula:** `derivative(*.latency) / derivative(*.ops)` \u2192 **\u00b5s per operation** (same as \u0394latency / \u0394ops over each window).\n\n**Unit:** microseconds (`\u00b5s`)\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.opLatencies.reads.latency` \u00b7 `serverStatus.opLatencies.reads.ops`\n- `serverStatus.opLatencies.writes.latency` \u00b7 `serverStatus.opLatencies.writes.ops`\n- `serverStatus.opLatencies.commands.latency` \u00b7 `serverStatus.opLatencies.commands.ops`\n\n---\n\n### How to read\n\n- Three lines: read, write, command average latency\n- Spikes \u2192 slower operations in that class\n- Flat at **0** \u2192 no ops in window (idle), not zero latency\n\n---\n\n### Troubleshooting\n\n- Spikes + low **WiredTiger Tickets** \u2192 concurrency contention\n- Spikes + disk I/O \u2192 storage bottleneck \u00b7 **Page Faults**, **Disk I/O**\n- Latency up, ops down \u2192 **Operations Latencies Op**, **Operations**\n- Commands line often highest \u2192 includes admin/aggregate work\n\n---\n\n### Docs\n\n- [serverStatus \u2014 opLatencies](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.opLatencies)", "fieldConfig": { "defaults": { "color": { @@ -1905,7 +2067,9 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1922,6 +2086,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -1931,53 +2096,57 @@ "mode": "off" } }, + "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": 0 } ] }, - "unit": "ms" + "unit": "\u00b5s" }, "overrides": [] }, "gridPos": { - "h": 5, - "w": 9, - "x": 15, - "y": 38 + "h": 10, + "w": 10, + "x": 0, + "y": 61 }, - "id": 25, + "id": 5, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] =~ /replSetGetStatus\\.members\\.\\d+\\.pingMs/)\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n", + "query": "wide = from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.opLatencies.reads.latency\" or\r\n r[\"_field\"] == \"serverStatus.opLatencies.reads.ops\" or\r\n r[\"_field\"] == \"serverStatus.opLatencies.writes.latency\" or\r\n r[\"_field\"] == \"serverStatus.opLatencies.writes.ops\" or\r\n r[\"_field\"] == \"serverStatus.opLatencies.commands.latency\" or\r\n r[\"_field\"] == \"serverStatus.opLatencies.commands.ops\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> pivot(rowKey: [\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n hostname: r.hostname,\r\n read: if float(v: r[\"serverStatus.opLatencies.reads.ops\"]) > 0.0\r\n then float(v: r[\"serverStatus.opLatencies.reads.latency\"]) / float(v: r[\"serverStatus.opLatencies.reads.ops\"])\r\n else 0.0,\r\n write: if float(v: r[\"serverStatus.opLatencies.writes.ops\"]) > 0.0\r\n then float(v: r[\"serverStatus.opLatencies.writes.latency\"]) / float(v: r[\"serverStatus.opLatencies.writes.ops\"])\r\n else 0.0,\r\n command: if float(v: r[\"serverStatus.opLatencies.commands.ops\"]) > 0.0\r\n then float(v: r[\"serverStatus.opLatencies.commands.latency\"]) / float(v: r[\"serverStatus.opLatencies.commands.ops\"])\r\n else 0.0,\r\n }))\r\n\r\nunion(tables: [\r\n wide |> map(fn: (r) => ({ _time: r._time, hostname: r.hostname, _field: \"Read latency (avg)\", _value: r.read })),\r\n wide |> map(fn: (r) => ({ _time: r._time, hostname: r.hostname, _field: \"Write latency (avg)\", _value: r.write })),\r\n wide |> map(fn: (r) => ({ _time: r._time, hostname: r.hostname, _field: \"Command latency (avg)\", _value: r.command })),\r\n])\r\n |> group(columns: [\"_field\", \"hostname\"])\r\n", "refId": "A" } ], - "title": "Replica members ping", + "title": "Average Operation Latency", "type": "timeseries" }, { @@ -1985,6 +2154,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "### Cursors\n\n**Open cursors** from **`metrics.cursor.open.*`** (current count, gauge). **`metrics.cursor.timedOut`** is a **cumulative counter** \u2014 shown here as **timeouts per second** (derivative).\n\n**Unit:** count (`short`) for open \u00b7 timeouts/sec (`ops`) for timed out rate\n\n---\n\n### Metrics\n\n**Fields (open \u2014 gauge, `last`):**\n\n- `serverStatus.metrics.cursor.open.total`\n- `serverStatus.metrics.cursor.open.noTimeout`\n- `serverStatus.metrics.cursor.open.pinned`\n\n**Field (timeouts \u2014 counter, `derivative(1s)`):**\n\n- `serverStatus.metrics.cursor.timedOut`\n\n---\n\n### How to read\n\n- **Open total** \u2014 cursors mongod holds for clients; usually small\n- **Open (no timeout)** / **Open (pinned)** \u2014 subsets of total\n- **Timed out (/sec)** \u2014 idle cursors killed (default cursor timeout ~10 min)\n- **stepAfter** on open lines reflects discrete count changes\n\n---\n\n### Troubleshooting\n\n- **Open total** rising \u2192 possible cursor leak (app not closing/exhausting cursors)\n- **Timed out (/sec)** rising \u2192 clients abandoning cursors \u00b7 correlate with **Operations** (getmore/query)\n- Tailable / pinned cursors can keep **open** elevated without a leak\n\n---\n\n### Docs\n\n- [serverStatus \u2014 metrics.cursor](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.metrics.cursor)", "fieldConfig": { "defaults": { "color": { @@ -1996,7 +2166,9 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2006,13 +2178,14 @@ "viz": false }, "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, + "lineInterpolation": "stepAfter", + "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -2022,43 +2195,102 @@ "mode": "off" } }, + "decimals": 0, "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": 0 } ] }, - "unit": "µs" + "unit": "short" }, "overrides": [ { - "__systemRef": "hideSeriesFrom", "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "serverStatus.flowControl.isLaggedTimeMicros" - ], - "prefix": "All except:", - "readOnly": true + "id": "byName", + "options": "Open total" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 0 + }, + { + "id": "custom.lineInterpolation", + "value": "stepAfter" } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Open (no timeout)" }, "properties": [ { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": true, - "viz": true - } + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 0 + }, + { + "id": "custom.lineInterpolation", + "value": "stepAfter" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Open (pinned)" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 0 + }, + { + "id": "custom.lineInterpolation", + "value": "stepAfter" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Timed out (/sec)" + }, + "properties": [ + { + "id": "unit", + "value": "ops" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "custom.lineInterpolation", + "value": "linear" } ] } @@ -2066,34 +2298,660 @@ }, "gridPos": { "h": 6, - "w": 8, + "w": 14, + "x": 10, + "y": 65 + }, + "id": 33, + "options": { + "legend": { + "calcs": [ + "min", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.metrics.cursor.open.total\" or\r\n r[\"_field\"] == \"serverStatus.metrics.cursor.open.noTimeout\" or\r\n r[\"_field\"] == \"serverStatus.metrics.cursor.open.pinned\")\r\n |> drop(columns: [\"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.metrics.cursor.open.total\" then \"Open total\" else if r._field == \"serverStatus.metrics.cursor.open.noTimeout\" then \"Open (no timeout)\" else if r._field == \"serverStatus.metrics.cursor.open.pinned\" then \"Open (pinned)\" else r._field\r\n }))", + "refId": "A" + }, + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "hide": false, + "query": "from(bucket: v.defaultBucket)\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.metrics.cursor.timedOut\")\n |> drop(columns: [\"version\"])\n |> sort(columns: [\"_time\"])\n |> derivative(unit: 1s, nonNegative: true)\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> map(fn: (r) => ({ r with\n _value: float(v: r._value),\n _field: if r._field == \"serverStatus.metrics.cursor.timedOut\" then \"Timed out (/sec)\" else r._field\n }))", + "refId": "B" + } + ], + "title": "Cursors", + "type": "timeseries" + }, + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "description": "### Admin Commands (metrics.commands)\n\nPer-second rate of selected **`serverStatus.metrics.commands.*.total`** counters \u2014 admin/catalog-style commands tracked individually by MongoDB.\n\n**Not** the same as **Operations** (`opcounters.command`), which rolls up non-opcounter command traffic (**including aggregate**); this panel tracks specific `metrics.commands.*` counters individually.\n\n**Unit:** commands per second (`ops`)\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.metrics.commands.createIndexes.total`\n- `serverStatus.metrics.commands.dataSize.total`\n- `serverStatus.metrics.commands.dropIndexes.total`\n- `serverStatus.metrics.commands.listCollections.total`\n- `serverStatus.metrics.commands.listDatabases.total`\n- `serverStatus.metrics.commands.listIndexes.total`\n\n**Query:** `derivative(..., unit: 1s, nonNegative: true)` \u2192 cmds/sec\n\n---\n\n### How to read\n\n- One line per command counter above\n- Spikes on **listCollections** / **listDatabases** / **listIndexes** \u2192 metadata/catalog traffic (drivers, tools, monitoring)\n- Spikes on **createIndexes** / **dropIndexes** \u2192 schema/index maintenance\n- **dataSize** \u2192 `dataSize` command usage\n\n---\n\n### Troubleshooting\n\n- Sustained high list* rates \u2192 chatty clients or health checks \u00b7 compare with **Network Requests**\n- Index build bursts \u2192 **createIndexes** + **CPU Usage** + **Disk I/O**\n- For overall op mix, use **Operations**; for document volume, **Documents**\n\n---\n\n### Docs\n\n- [serverStatus \u2014 metrics.commands](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.metrics.commands)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 10, "x": 0, - "y": 43 + "y": 71 + }, + "id": 32, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.metrics.commands.createIndexes.total\" or\r\n r[\"_field\"] == \"serverStatus.metrics.commands.dataSize.total\" or\r\n r[\"_field\"] == \"serverStatus.metrics.commands.dropIndexes.total\" or\r\n r[\"_field\"] == \"serverStatus.metrics.commands.listCollections.total\" or\r\n r[\"_field\"] == \"serverStatus.metrics.commands.listDatabases.total\" or\r\n r[\"_field\"] == \"serverStatus.metrics.commands.listIndexes.total\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.metrics.commands.createIndexes.total\" then \"createIndexes\" else if r._field == \"serverStatus.metrics.commands.dataSize.total\" then \"dataSize\" else if r._field == \"serverStatus.metrics.commands.dropIndexes.total\" then \"dropIndexes\" else if r._field == \"serverStatus.metrics.commands.listCollections.total\" then \"listCollections\" else if r._field == \"serverStatus.metrics.commands.listDatabases.total\" then \"listDatabases\" else if r._field == \"serverStatus.metrics.commands.listIndexes.total\" then \"listIndexes\" else r._field\r\n }))", + "refId": "A" + } + ], + "title": "Admin Commands", + "type": "timeseries" + }, + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "description": "### Replica members lag\n\nWall-clock replication lag per member:\n\n**Formula:** `(serverStatus.localTime \u2212 members.N.lastAppliedWallTime) / 1000` \u2192 **seconds**\n\nBoth timestamps are **milliseconds** in FTDC; the query converts the delta to seconds.\n\n**Unit:** seconds (`s`)\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.localTime`\n- `replSetGetStatus.members.0.lastAppliedWallTime` \u2026 `replSetGetStatus.members.4.lastAppliedWallTime`\n\nMember **N** is the array index in `replSetGetStatus.members` (not necessarily hostname order).\n\n---\n\n### How to read\n\n- One line per member index (0\u20134)\n- **~0** \u2192 caught up (primary or healthy secondary)\n- Rising trend on a secondary \u2192 replication falling behind\n- **Gap** (not 0) for missing member slots \u2014 no `lastAppliedWallTime` in FTDC for that index\n\n---\n\n### Troubleshooting\n\n- Secondary lag rising \u2192 disk I/O, CPU, network on that node \u00b7 **Opcounters Replica**, **Disk I/O**\n- Lag spikes during checkpoints \u2192 I/O contention\n- Primary throttling \u2192 **FlowControl isLagged** \u00b7 **Replica Members Ping**\n- Member down \u2192 **Replica Members Health**\n\n---\n\n### Docs\n\n- [replSetGetStatus](https://www.mongodb.com/docs/manual/reference/command/replSetGetStatus/)\n- [serverStatus](https://www.mongodb.com/docs/manual/reference/command/serverStatus/)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 14, + "x": 10, + "y": 71 + }, + "id": 15, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "query": "wide = from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] =~ /replSetGetStatus\\.members\\.\\d+\\.lastAppliedWallTime|serverStatus\\.localTime/)\r\n |> drop(columns: [\"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> pivot(rowKey: [\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n hostname: r.hostname,\r\n member_0: if exists r[\"replSetGetStatus.members.0.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.0.lastAppliedWallTime\"])) / 1000.0 else float(v: \"NaN\"),\r\n member_1: if exists r[\"replSetGetStatus.members.1.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.1.lastAppliedWallTime\"])) / 1000.0 else float(v: \"NaN\"),\r\n member_2: if exists r[\"replSetGetStatus.members.2.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.2.lastAppliedWallTime\"])) / 1000.0 else float(v: \"NaN\"),\r\n member_3: if exists r[\"replSetGetStatus.members.3.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.3.lastAppliedWallTime\"])) / 1000.0 else float(v: \"NaN\"),\r\n member_4: if exists r[\"replSetGetStatus.members.4.lastAppliedWallTime\"] then (float(v: r[\"serverStatus.localTime\"]) - float(v: r[\"replSetGetStatus.members.4.lastAppliedWallTime\"])) / 1000.0 else float(v: \"NaN\"),\r\n }))\r\n\r\nunion(tables: [\r\n wide |> map(fn: (r) => ({ _time: r._time, hostname: r.hostname, _field: \"Member 0 lag\", _value: r.member_0 })),\r\n wide |> map(fn: (r) => ({ _time: r._time, hostname: r.hostname, _field: \"Member 1 lag\", _value: r.member_1 })),\r\n wide |> map(fn: (r) => ({ _time: r._time, hostname: r.hostname, _field: \"Member 2 lag\", _value: r.member_2 })),\r\n wide |> map(fn: (r) => ({ _time: r._time, hostname: r.hostname, _field: \"Member 3 lag\", _value: r.member_3 })),\r\n wide |> map(fn: (r) => ({ _time: r._time, hostname: r.hostname, _field: \"Member 4 lag\", _value: r.member_4 })),\r\n])\r\n |> group(columns: [\"_field\", \"hostname\"])\r\n", + "refId": "A" + } + ], + "title": "Replica Members Lag", + "type": "timeseries" + }, + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "description": "### Replica members ping\n\n**`replSetGetStatus.members.N.pingMs`** \u2014 round-trip time in **milliseconds** from **this FTDC host** to each replica set member (repl heartbeat).\n\n**Unit:** milliseconds (`ms`)\n\n**Note:** **N** is the member array index in `replSetGetStatus.members`, not hostname order. Use the **hostname** template variable to pick which mongod\u2019s view of the set you are viewing.\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `replSetGetStatus.members.0.pingMs`\n- `replSetGetStatus.members.1.pingMs`\n- `replSetGetStatus.members.2.pingMs`\n- `replSetGetStatus.members.3.pingMs`\n- `replSetGetStatus.members.4.pingMs`\n\n---\n\n### How to read\n\n- One line per member index (**Member 0 ping** \u2026 **Member 4 ping**)\n- **stepAfter** reflects discrete heartbeat samples\n- Low stable values \u2192 healthy paths between members\n- Spikes \u2192 latency or congestion between nodes\n\n---\n\n### Troubleshooting\n\n- Ping rising with **Replica Members Lag** \u2192 network may limit replication\n- Ping OK but lag high \u2192 disk/CPU on secondaries, not network\n- Correlate with **TCP** panels (RetransSegs, InSegs/OutSegs) and **Replica Members Health**\n\n---\n\n### Docs\n\n- [replSetGetStatus](https://www.mongodb.com/docs/manual/reference/command/replSetGetStatus/)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 14, + "x": 10, + "y": 78 + }, + "id": 25, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"replSetGetStatus.members.0.pingMs\" or\r\n r[\"_field\"] == \"replSetGetStatus.members.1.pingMs\" or\r\n r[\"_field\"] == \"replSetGetStatus.members.2.pingMs\" or\r\n r[\"_field\"] == \"replSetGetStatus.members.3.pingMs\" or\r\n r[\"_field\"] == \"replSetGetStatus.members.4.pingMs\")\r\n |> drop(columns: [\"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"replSetGetStatus.members.0.pingMs\" then \"Member 0 ping\" else if r._field == \"replSetGetStatus.members.1.pingMs\" then \"Member 1 ping\" else if r._field == \"replSetGetStatus.members.2.pingMs\" then \"Member 2 ping\" else if r._field == \"replSetGetStatus.members.3.pingMs\" then \"Member 3 ping\" else if r._field == \"replSetGetStatus.members.4.pingMs\" then \"Member 4 ping\" else r._field\r\n }))", + "refId": "A" + } + ], + "title": "Replica Members Ping", + "type": "timeseries" + }, + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "description": "### Document metrics (deleted / inserted / returned / updated)\n\nPer-second rate of **`serverStatus.metrics.document`** counters \u2014 document-level access and modification (not the same as **Operations** / opcounters: one op can touch many documents).\n\n**Unit:** documents per second (`ops`)\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.metrics.document.deleted`\n- `serverStatus.metrics.document.inserted`\n- `serverStatus.metrics.document.returned`\n- `serverStatus.metrics.document.updated`\n\n**Query:** `derivative(..., unit: 1s, nonNegative: true)` \u2192 docs/sec\n\n---\n\n### How to read\n\n- Four lines: delete, insert, return, update **rates**\n- **Returned** high with low **Scanned / Returned Ratio** \u2192 efficient reads\n- **Returned** high with high ratio \u2192 scans without good targeting \u00b7 **Query Executor**\n\n---\n\n### Troubleshooting\n\n- High **deleted** \u2192 TTL indexes, bulk cleanup, or churn\n- High **returned** + high **Scanned / Returned Ratio** \u2192 missing/suboptimal indexes \u00b7 run `explain()`\n- Compare with **Operations** (op-level) and **Opcounters Replica** on secondaries\n\n---\n\n### Docs\n\n- [serverStatus \u2014 metrics.document](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.metrics.document)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 10, + "x": 0, + "y": 79 + }, + "id": 36, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.metrics.document.deleted\" or\r\n r[\"_field\"] == \"serverStatus.metrics.document.inserted\" or\r\n r[\"_field\"] == \"serverStatus.metrics.document.returned\" or\r\n r[\"_field\"] == \"serverStatus.metrics.document.updated\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.metrics.document.deleted\" then \"Documents deleted\" else if r._field == \"serverStatus.metrics.document.inserted\" then \"Documents inserted\" else if r._field == \"serverStatus.metrics.document.returned\" then \"Documents returned\" else if r._field == \"serverStatus.metrics.document.updated\" then \"Documents updated\" else r._field\r\n }))", + "refId": "A" + } + ], + "title": "Documents (deleted / inserted / returned / updated)", + "type": "timeseries" + }, + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "description": "### Flow control is lagged\n\nBoolean **`flowControl.isLagged`** \u2014 whether flow control is **currently** throttling writes on the **primary**.\n\n**Unit:** 0 = off \u00b7 1 = on\n\n**Note:** Primary only; secondaries return placeholders. For **time spent** throttled, see **Flow control lagged time**.\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.flowControl.isLagged`\n\n---\n\n### How to read\n\n- **0** \u2192 not engaging (even if secondaries lag)\n- **1** \u2192 primary write rate limited so secondaries can catch up\n\n---\n\n### Troubleshooting\n\n- **1** sustained \u2192 **Replica Members Lag**, **Opcounters Replica**, secondary disk/network\n- Lag without **1** \u2192 lag below `flowControlTargetLagSeconds` threshold\n\n---\n\n### Docs\n\n- [serverStatus \u2014 flowControl](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.flowControl)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "stepAfter", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [ + { + "options": { + "0": { + "index": 0, + "text": "Off" + }, + "1": { + "index": 1, + "text": "On" + } + }, + "type": "value" + } + ], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 7, + "x": 10, + "y": 84 + }, + "id": 14, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.2", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.flowControl.isLagged\")\r\n |> drop(columns: [\"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.flowControl.isLagged\" then \"Flow control engaged\" else r._field\r\n }))", + "refId": "A" + } + ], + "title": "FlowControl isLagged", + "type": "timeseries" + }, + { + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + }, + "description": "### Flow control lagged time\n\n**Fraction of each window** the primary spent with flow control engaged, from **`flowControl.isLaggedTimeMicros`**.\n\n**Formula:** `derivative(isLaggedTimeMicros, 1s) / 1e6` \u2192 0\u2013100%\n\n**Note:** Primary only. For **on/off now**, see **FlowControl isLagged**.\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.flowControl.isLaggedTimeMicros`\n\n---\n\n### How to read\n\n- **0%** \u2192 no throttling in that window\n- **100%** \u2192 flow control engaged the entire window\n- Spikes \u2192 brief throttling bursts\n\n---\n\n### Troubleshooting\n\n- Sustained **>0%** \u2192 correlate with **Replica Members Lag**, **FlowControl isLagged**\n- High % + rising lag \u2192 secondaries cannot keep up \u00b7 disk/CPU/network\n\n---\n\n### Docs\n\n- [serverStatus \u2014 flowControl](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.flowControl)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 7, + "x": 17, + "y": 84 }, - "id": 14, + "id": 53, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.flowControl.isLaggedTimeMicros\")\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> derivative(unit: v.windowPeriod)\r\n |> yield(name: \"last\")", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.flowControl.isLaggedTimeMicros\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _field: if r._field == \"serverStatus.flowControl.isLaggedTimeMicros\" then \"Flow control engaged (% of time)\" else r._field }))\r\n |> map(fn: (r) => ({ r with _value: float(v: r._value) / 1000000.0 }))", "refId": "A" } ], - "title": "FlowControl isLagged", + "title": "Flow control lagged time", "type": "timeseries" }, { @@ -2101,6 +2959,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "### Oplog storage size\n\nDisk space allocated to the **`local.oplog.rs`** capped collection (`collStats` / `storageStats`). **Oplog storage size** is total allocation on disk; **Oplog free storage** is unused space within that allocation.\n\n**Unit:** bytes (`bytes`)\n\n**Not `dbStats.storageSize`:** FTDC exposes oplog **collStats** here, not per-database storage totals.\n\n---\n\n### Metrics\n\n**Fields:**\n\n`local.oplog.rs.stats.storageStats.storageSize`\n`local.oplog.rs.stats.storageStats.freeStorageSize`\n\nSnapshot gauge \u2014 **`aggregateWindow(..., fn: last)`**, no derivative.\n\n---\n\n### How to read\n\n- **Oplog storage size** \u2014 grows as the capped oplog fills its on-disk allocation (does not shrink when entries roll off)\n- **Oplog free storage** \u2014 headroom within that allocation\n- **Used \u2248 storage size \u2212 free storage** at each point\n- Flat **storage size** with falling **free storage** \u2192 oplog nearing capacity\n\n---\n\n### Troubleshooting\n\n- **Free storage** near zero \u2192 oplog cap pressure \u00b7 replication lag risk on secondaries \u00b7 **Oplog average entry size**, **Replica Members Lag**\n- Step growth in **storage size** \u2192 oplog resize or first-time allocation growth\n- High **storage size** + large **Oplog average entry size** \u2192 heavy write payloads \u00b7 more replication I/O\n- **Arbiter / standalone** may show no or stale series\n\nCorrelate with **Operations (write source)**, **Oplog average entry size**, and **Documents**.\n\n---\n\n### Docs\n\n- [collStats \u2014 storageSize / freeStorageSize](https://www.mongodb.com/docs/manual/reference/command/collStats/)\n- [Change oplog size](https://www.mongodb.com/docs/manual/tutorial/change-oplog-size/)", "fieldConfig": { "defaults": { "color": { @@ -2112,7 +2971,9 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2129,6 +2990,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -2138,52 +3000,57 @@ "mode": "off" } }, + "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": 0 } ] - } + }, + "unit": "bytes" }, "overrides": [] }, "gridPos": { - "h": 7, + "h": 8, "w": 7, - "x": 8, - "y": 43 + "x": 10, + "y": 90 }, - "id": 30, + "id": 35, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.asserts.warning\" or r[\"_field\"] == \"serverStatus.asserts.user\" or r[\"_field\"] == \"serverStatus.asserts.rollovers\" or r[\"_field\"] == \"serverStatus.asserts.regular\" or r[\"_field\"] == \"serverStatus.asserts.msg\")\r\n |> derivative(unit: v.windowPeriod)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"local.oplog.rs.stats.storageStats.storageSize\" or\r\n r[\"_field\"] == \"local.oplog.rs.stats.storageStats.freeStorageSize\")\r\n |> drop(columns: [\"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"local.oplog.rs.stats.storageStats.storageSize\" then \"Oplog storage size\" else if r._field == \"local.oplog.rs.stats.storageStats.freeStorageSize\" then \"Oplog free storage\" else r._field\r\n }))", "refId": "A" } ], - "title": "Asserts", + "title": "Oplog storage size", "type": "timeseries" }, { @@ -2191,6 +3058,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "### Oplog average entry size\n\nAverage size in **bytes** of documents in the **`local.oplog.rs`** capped collection (`collStats` / `storageStats.avgObjSize`).\n\n**Unit:** bytes (`bytes`)\n\n**Not `dbStats.avgObjSize`:** FTDC exposes oplog collStats here, not per-database average user document size.\n\n---\n\n### Metrics\n\n**Field:** `local.oplog.rs.stats.storageStats.avgObjSize`\n\nSnapshot gauge \u2014 **`aggregateWindow(..., fn: last)`**, no derivative.\n\n---\n\n### How to read\n\n- Single line \u00b7 **Oplog avg entry size**\n- Rises when writes carry larger payloads (big documents, wide updates, bulk ops)\n- Slow drift over days \u2192 changing write patterns or schema\n- Compare with **Oplog storage size** (`local.oplog.rs` `storageSize` / `freeStorageSize`)\n\n---\n\n### Troubleshooting\n\n- Step change + write workload shift \u2192 larger oplog entries \u00b7 more replication network/disk\n- High avg size + lag \u2192 secondary apply pressure \u00b7 check **Replica Members Lag**\n- **Arbiter / standalone** may show no or stale series (no meaningful oplog)\n- For **user collection** document sizing \u2192 `db.collection.stats().avgObjSize` (not in this FTDC export)\n\n---\n\n### Docs\n\n- [collStats \u2014 avgObjSize](https://www.mongodb.com/docs/manual/reference/command/collStats/)\n- [Change oplog size](https://www.mongodb.com/docs/manual/tutorial/change-oplog-size/)", "fieldConfig": { "defaults": { "color": { @@ -2202,7 +3070,9 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2219,6 +3089,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -2229,52 +3100,56 @@ } }, "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": 0 } ] }, - "unit": "short" + "unit": "bytes" }, "overrides": [] }, "gridPos": { - "h": 6, - "w": 9, - "x": 15, - "y": 43 + "h": 8, + "w": 7, + "x": 17, + "y": 90 }, - "id": 31, + "id": 34, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.metrics.queryExecutor.scannedObjects\" or r[\"_field\"] == \"serverStatus.metrics.queryExecutor.scanned\" or r[\"_field\"] == \"serverStatus.metrics.queryExecutor.collectionScans.total\")\r\n |> derivative(unit: v.windowPeriod)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"local.oplog.rs.stats.storageStats.avgObjSize\")\r\n |> drop(columns: [\"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"local.oplog.rs.stats.storageStats.avgObjSize\" then \"Oplog avg entry size\" else if r._field == \"local.oplog.rs.stats.storageStats.avgObjSize\" then \"Oplog avg entry size\" else r._field\r\n }))", "refId": "A" } ], - "title": "Query Executor", + "title": "Oplog average entry size", "type": "timeseries" }, { @@ -2282,6 +3157,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "### Current queue (writers / readers)\n\nOperations **waiting** for the global read or write lock (`globalLock.currentQueue`). Snapshot gauge \u2014 not a rate.\n\n**Unit:** count (`short`)\n\n**Note:** On MongoDB 4.2+, these queues are often **0**; use **WiredTiger Tickets** and **Average Operation Latency** for concurrency pressure on modern releases.\n\n---\n\n### Metrics\n\n**Fields:** `serverStatus.globalLock.currentQueue.readers` \u00b7 `serverStatus.globalLock.currentQueue.writers`\n\n---\n\n### How to read\n\n- Two lines: queued readers vs writers\n- Brief spikes can be normal \u00b7 sustained non-zero \u2192 contention\n\n---\n\n### Troubleshooting\n\n- Persistent write queue \u2192 **WiredTiger Tickets**, **Average Operation Latency**, **Operations**\n- Persistent read queue \u2192 long-running reads blocking others\n- Always zero but slow DB \u2192 focus on tickets/latency, not this panel alone\n\n---\n\n### Docs\n\n- [serverStatus \u2014 globalLock](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.globalLock)", "fieldConfig": { "defaults": { "color": { @@ -2293,7 +3169,9 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2303,13 +3181,14 @@ "viz": false }, "insertNulls": false, - "lineInterpolation": "linear", + "lineInterpolation": "stepAfter", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -2319,52 +3198,57 @@ "mode": "off" } }, + "decimals": 0, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": 0 } ] - } + }, + "unit": "short" }, "overrides": [] }, "gridPos": { - "h": 6, - "w": 8, + "h": 7, + "w": 10, "x": 0, - "y": 49 + "y": 91 }, - "id": 32, + "id": 29, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.metrics.commands.createIndexes.total\" or r[\"_field\"] == \"serverStatus.metrics.commands.dataSize.total\" or r[\"_field\"] == \"serverStatus.metrics.commands.dropIndexes.total\" or r[\"_field\"] == \"serverStatus.metrics.commands.listCollections.total\" or r[\"_field\"] == \"serverStatus.metrics.commands.listDatabases.total\")\r\n|> derivative(unit: v.windowPeriod)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.globalLock.currentQueue.writers\" or\r\n r[\"_field\"] == \"serverStatus.globalLock.currentQueue.readers\")\r\n |> drop(columns: [\"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.globalLock.currentQueue.writers\" then \"Queued writers\" else if r._field == \"serverStatus.globalLock.currentQueue.readers\" then \"Queued readers\" else r._field\r\n }))", "refId": "A" } ], - "title": "Commands", + "title": "Current Queue writers / readers", "type": "timeseries" }, { @@ -2372,6 +3256,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "### Query executor\n\nPer-second rates from **queryExecutor** counters \u2014 how much work the query engine does examining indexes and documents.\n\n**Unit:** operations/sec (`ops`) \u2014 keys scanned, docs scanned, and collection-scan events are all counter derivatives.\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.metrics.queryExecutor.scanned` \u2014 index keys examined (`totalKeysExamined` in explain())\n- `serverStatus.metrics.queryExecutor.scannedObjects` \u2014 documents examined (`totalDocsExamined`)\n- `serverStatus.metrics.queryExecutor.collectionScans.total` \u2014 queries that performed a **collection scan** (COLLSCAN)\n\nAll are cumulative counters; query uses **`derivative(unit: 1s, nonNegative: true)`**.\n\n---\n\n### How to read\n\n- **Index keys scanned / sec** \u2014 index traversal volume\n- **Documents scanned / sec** \u2014 document examination volume (often dominates I/O)\n- **Collection scans / sec** (right axis) \u2014 queries **not** using a useful index; should stay near **0** in production\n\n**Keys vs docs:** high docs with low keys can mean wide index scans or many docs per key; high docs with high **Collection scans** confirms COLLSCAN.\n\nCompare with **Query Targeting: Scanned Objects / Returned** (ratio) and **Documents** (`document.returned`).\n\n---\n\n### Troubleshooting\n\n- High **Documents scanned** + high targeting ratio \u2192 missing/poor indexes \u00b7 run `explain()` on slow queries\n- Sustained **Collection scans / sec** \u2192 enable profiler / Performance Advisor \u00b7 fix query patterns or add indexes\n- High scan rates + **Page Faults** / **Disk I/O** \u2192 storage pressure from inefficient reads\n- **Operations** `opcounters.query` rising with scan rates \u2192 read-heavy workload under index stress\n\n---\n\n### Docs\n\n- [serverStatus \u2014 metrics.queryExecutor](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.metrics.queryExecutor)", "fieldConfig": { "defaults": { "color": { @@ -2383,7 +3268,9 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2393,13 +3280,14 @@ "viz": false }, "insertNulls": false, - "lineInterpolation": "linear", + "lineInterpolation": "stepAfter", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -2410,52 +3298,68 @@ } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" - }, + "color": "green", + "value": 0 + } + ] + }, + "unit": "ops" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Collection scans / sec" + }, + "properties": [ { - "color": "red", - "value": 80 + "id": "custom.axisPlacement", + "value": "right" } ] } - }, - "overrides": [] + ] }, "gridPos": { - "h": 6, - "w": 9, - "x": 15, - "y": 49 + "h": 7, + "w": 24, + "x": 0, + "y": 98 }, - "id": 34, + "id": 31, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"local.oplog.rs.stats.storageStats.avgObjSize\")\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.metrics.queryExecutor.scannedObjects\" or\r\n r[\"_field\"] == \"serverStatus.metrics.queryExecutor.scanned\" or\r\n r[\"_field\"] == \"serverStatus.metrics.queryExecutor.collectionScans.total\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _field: if r._field == \"serverStatus.metrics.queryExecutor.scanned\" then \"Index keys scanned / sec\" else if r._field == \"serverStatus.metrics.queryExecutor.scannedObjects\" then \"Documents scanned / sec\" else if r._field == \"serverStatus.metrics.queryExecutor.collectionScans.total\" then \"Collection scans / sec\" else r._field }))", "refId": "A" } ], - "title": "avg Obj Size", + "title": "Query Executor", "type": "timeseries" }, { @@ -2463,7 +3367,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "", + "description": "### Operations (opcounters)\n\nPer-second rate of **`serverStatus.opcounters`** \u2014 **operation-level** counts (one bulk/multi-doc op = one op). Not the same as **Documents** (`metrics.document`), which counts documents touched.\n\n**Unit:** operations per second (`ops`)\n\n**Note:** `opcounters.command` counts commands **not** already tracked as insert, update, delete, query, or getmore \u2014 **including aggregate** and other admin/catalog commands.\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.opcounters.command`\n- `serverStatus.opcounters.delete`\n- `serverStatus.opcounters.getmore`\n- `serverStatus.opcounters.insert`\n- `serverStatus.opcounters.query`\n- `serverStatus.opcounters.update`\n\n**Query:** `derivative(..., unit: 1s, nonNegative: true)` \u2192 ops/sec\n\n---\n\n### How to read\n\n- Six lines: command, delete, getmore, insert, query, update **rates**\n- Sudden drop \u2192 blocking or idle period\n- Spike \u2192 load burst; check **Average Operation Latency** and **WiredTiger Tickets**\n\n---\n\n### Troubleshooting\n\n- Sustained high write ops + latency \u2192 **Current Queue writers / readers**, disk I/O\n- High **query** + **getmore** \u2192 cursor-heavy workload\n- On secondaries, use **Opcounters Replica** instead of this panel\n\n---\n\n### Docs\n\n- [serverStatus \u2014 opcounters](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.opcounters)", "fieldConfig": { "defaults": { "color": { @@ -2475,7 +3379,9 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2485,13 +3391,14 @@ "viz": false }, "insertNulls": false, - "lineInterpolation": "linear", + "lineInterpolation": "stepAfter", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -2502,52 +3409,55 @@ } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": 0 } ] - } + }, + "unit": "ops" }, "overrides": [] }, "gridPos": { - "h": 6, - "w": 7, - "x": 8, - "y": 50 + "h": 11, + "w": 12, + "x": 0, + "y": 105 }, - "id": 33, + "id": 9, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.metrics.cursor.open.noTimeout\" or r[\"_field\"] == \"serverStatus.metrics.cursor.open.pinned\" or r[\"_field\"] == \"serverStatus.metrics.cursor.open.total\" or r[\"_field\"] == \"serverStatus.metrics.cursor.timedOut\")\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.opcounters.command\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.delete\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.getmore\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.insert\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.query\" or\r\n r[\"_field\"] == \"serverStatus.opcounters.update\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.opcounters.command\" then \"Command\" else if r._field == \"serverStatus.opcounters.delete\" then \"Delete\" else if r._field == \"serverStatus.opcounters.getmore\" then \"Getmore\" else if r._field == \"serverStatus.opcounters.insert\" then \"Insert\" else if r._field == \"serverStatus.opcounters.query\" then \"Query\" else if r._field == \"serverStatus.opcounters.update\" then \"Update\" else r._field\r\n }))", "refId": "A" } ], - "title": "Cursors", + "title": "Operations", "type": "timeseries" }, { @@ -2555,7 +3465,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "", + "description": "### Replication opcounters (If secondary)\n\nPer-second rates from **`serverStatus.opcountersRepl`** \u2014 operations applied when this member is acting as a **secondary** (oplog apply).\n\n**Unit:** operations/sec (`ops`)\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.opcountersRepl.insert`\n- `serverStatus.opcountersRepl.query`\n- `serverStatus.opcountersRepl.update`\n- `serverStatus.opcountersRepl.delete`\n- `serverStatus.opcountersRepl.getmore`\n- `serverStatus.opcountersRepl.command`\n\nCumulative counters; query uses **`derivative(unit: 1s, nonNegative: true)`**.\n\n---\n\n### How to read\n\n- Six lines \u00b7 replication apply rate by type\n- **View a secondary hostname** \u2014 answers \u201chow fast is this node applying replication?\u201d\n- **Primary / write source** \u2014 expect near-zero or unhelpful lines; switch host or open **Operations (write source)**\n- Fractional ops/sec \u2192 normal with sparse FTDC sampling\n- **0** \u2192 idle or no replication apply in window\n\nCompare with **Operations (write source)** on the primary, **Replica Members Lag**, **Flow control is lagged**.\n\n---\n\n### Troubleshooting\n\n- Repl rates drop + **lag rising** \u2192 secondary cannot keep up \u00b7 disk I/O, CPU, **Disk I/O**\n- Primary **Operations (write source)** high, secondary repl low \u2192 bottleneck on secondary\n- Repl **update** \u226b primary **update** \u2192 expected replication amplification\n- **Flow control is lagged** = 1 + lag \u2192 primary throttling writes\n\n---\n\n### Docs\n\n- [serverStatus \u2014 opcountersRepl](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.opcountersRepl)\n- [serverStatus \u2014 opcounters](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.opcounters)", "fieldConfig": { "defaults": { "color": { @@ -2567,7 +3477,9 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2584,6 +3496,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -2598,47 +3511,51 @@ "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": 0 } ] - } + }, + "unit": "ops" }, "overrides": [] }, "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 55 + "h": 11, + "w": 12, + "x": 12, + "y": 105 }, - "id": 36, + "id": 38, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.metrics.document.updated\" or r[\"_field\"] == \"serverStatus.metrics.document.returned\" or r[\"_field\"] == \"serverStatus.metrics.document.inserted\" or r[\"_field\"] == \"serverStatus.metrics.document.deleted\")\r\n |> derivative(unit: v.windowPeriod)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.insert\" or\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.query\" or\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.update\" or\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.delete\" or\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.getmore\" or\r\n r[\"_field\"] == \"serverStatus.opcountersRepl.command\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _field: if r._field == \"serverStatus.opcountersRepl.insert\" then \"Insert / sec\" else if r._field == \"serverStatus.opcountersRepl.query\" then \"Query / sec\" else if r._field == \"serverStatus.opcountersRepl.update\" then \"Update / sec\" else if r._field == \"serverStatus.opcountersRepl.delete\" then \"Delete / sec\" else if r._field == \"serverStatus.opcountersRepl.getmore\" then \"Getmore / sec\" else if r._field == \"serverStatus.opcountersRepl.command\" then \"Command / sec\" else r._field }))", "refId": "A" } ], - "title": "Documents (deleted / inserted / returned / updated)", + "title": "Opcounters Replica (If secondary)", "type": "timeseries" }, { @@ -2646,6 +3563,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "### Operation latency ops\n\nPer-second rates from **opLatencies.*.ops** counters \u2014 how many read, write, and command operations ran in each latency class.\n\n**Unit:** operations/sec (`ops`)\n\nThis is the **denominator** for **Average Operation Latency** (`derivative(latency) / derivative(ops)` \u2192 \u00b5s/op).\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.opLatencies.reads.ops`\n- `serverStatus.opLatencies.writes.ops`\n- `serverStatus.opLatencies.commands.ops`\n\nCumulative counters; query uses **`derivative(unit: 1s, nonNegative: true)`**.\n\n**Not the same as opcounters:** `opLatencies` groups reads / writes / commands for latency stats; **Operations** uses `opcounters.*` (query, insert, update, delete, getmore, command).\n\n---\n\n### How to read\n\n- **Read / write / command ops / sec** \u2014 throughput per latency class\n- **Commands** often includes admin, aggregate, and other command work\n- Fractional values \u2192 normal with sparse FTDC sampling\n- **0** \u2192 idle window for that class\n\nCompare with **Average Operation Latency** (numerator/denominator) and **Operations** (opcounters workload mix).\n\n---\n\n### Troubleshooting\n\n- **Ops up, avg latency up** \u2192 load + slower per-op work\n- **Ops down, avg latency up** \u2192 fewer ops but each slower (contention, I/O)\n- **Ops up, avg latency flat** \u2192 healthy scale-out of throughput\n- Read/write imbalance \u2192 workload skew \u00b7 correlate with **Query Executor**, **Documents**, **Disk I/O**\n\n---\n\n### Docs\n\n- [serverStatus \u2014 opLatencies](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.opLatencies)", "fieldConfig": { "defaults": { "color": { @@ -2657,7 +3575,9 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2667,13 +3587,14 @@ "viz": false }, "insertNulls": false, - "lineInterpolation": "linear", + "lineInterpolation": "stepAfter", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -2688,47 +3609,51 @@ "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": 0 } ] - } + }, + "unit": "ops" }, "overrides": [] }, "gridPos": { - "h": 7, - "w": 9, - "x": 15, - "y": 55 + "h": 9, + "w": 24, + "x": 0, + "y": 116 }, - "id": 38, + "id": 10, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.opcountersRepl.command\" or r[\"_field\"] == \"serverStatus.opcountersRepl.delete\" or r[\"_field\"] == \"serverStatus.opcountersRepl.getmore\" or r[\"_field\"] == \"serverStatus.opcountersRepl.insert\" or r[\"_field\"] == \"serverStatus.opcountersRepl.query\" or r[\"_field\"] == \"serverStatus.opcountersRepl.update\")\r\n |> derivative(unit: v.windowPeriod)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.opLatencies.reads.ops\" or\r\n r[\"_field\"] == \"serverStatus.opLatencies.writes.ops\" or\r\n r[\"_field\"] == \"serverStatus.opLatencies.commands.ops\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with _field: if r._field == \"serverStatus.opLatencies.reads.ops\" then \"Read ops / sec\" else if r._field == \"serverStatus.opLatencies.writes.ops\" then \"Write ops / sec\" else if r._field == \"serverStatus.opLatencies.commands.ops\" then \"Command ops / sec\" else r._field }))", "refId": "A" } ], - "title": "Opcounters Replica", + "title": "Operation Latency Ops", "type": "timeseries" }, { @@ -2736,6 +3661,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "### Query targeting: scanned objects / returned\n\nRatio of **document scan rate** to **document return rate**:\n\n**Formula:** `derivative(scannedObjects, 1s) / derivative(document.returned, 1s)`\n\nSame idea as **totalDocsExamined / docs returned** in explain() \u2014 aggregated over time.\n\n**Unit:** dimensionless ratio (`none`) \u2014 **1.0** \u2248 one doc examined per doc returned\n\n---\n\n### Metrics\n\n**Fields:**\n\n- `serverStatus.metrics.queryExecutor.scannedObjects`\n- `serverStatus.metrics.document.returned`\n\nBoth are cumulative counters; query uses **`derivative(unit: 1s, nonNegative: true)`** before the ratio.\n\n---\n\n### How to read\n\n- Single line \u00b7 **Scanned / returned ratio**\n- **~1** \u2192 strong targeting (most examined docs were returned)\n- **> 1** (especially **10+**) \u2192 scans dominate \u00b7 missing/poor indexes or large in-memory filters\n- **Gap** (not 0) \u2192 no returned docs in window or zero denominator \u2014 **not** \u201cperfect\u201d\n\n---\n\n### Troubleshooting\n\n- High ratio + high **Documents returned** \u2192 **Query Executor** \u00b7 run `explain()` on hot queries\n- Correlate with **Page Faults**, **Disk I/O**, **Average Operation Latency**\n- **Collection scans** on **Query Executor** (right axis) confirms full collection scans\n\n---\n\n### Docs\n\n- [serverStatus \u2014 metrics.queryExecutor](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.metrics.queryExecutor)\n- [serverStatus \u2014 metrics.document](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.metrics.document)", "fieldConfig": { "defaults": { "color": { @@ -2747,7 +3673,9 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2758,12 +3686,13 @@ }, "insertNulls": false, "lineInterpolation": "linear", - "lineWidth": 4, + "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -2775,61 +3704,64 @@ }, "decimals": 2, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": 0 } ] - } + }, + "unit": "none" }, "overrides": [] }, "gridPos": { "h": 6, - "w": 7, - "x": 8, - "y": 56 + "w": 24, + "x": 0, + "y": 125 }, "id": 37, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.metrics.document.returned\" or \r\n r[\"_field\"] == \"serverStatus.metrics.queryExecutor.scannedObjects\")\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> pivot(\r\n rowKey: [\"_time\"],\r\n columnKey: [\"_field\"],\r\n valueColumn: \"_value\"\r\n )\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n scanned_returned_ratio: r[\"serverStatus.metrics.queryExecutor.scannedObjects\"] / r[\"serverStatus.metrics.document.returned\"]\r\n }))\r\n |> yield(name: \"scanned_returned_ratio\")\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.metrics.document.returned\" or\r\n r[\"_field\"] == \"serverStatus.metrics.queryExecutor.scannedObjects\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> pivot(rowKey: [\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n hostname: r.hostname,\r\n _field: \"Scanned / returned ratio\",\r\n _value:\r\n if exists r[\"serverStatus.metrics.document.returned\"]\r\n and float(v: r[\"serverStatus.metrics.document.returned\"]) > 0.0\r\n and exists r[\"serverStatus.metrics.queryExecutor.scannedObjects\"]\r\n then float(v: r[\"serverStatus.metrics.queryExecutor.scannedObjects\"])\r\n / float(v: r[\"serverStatus.metrics.document.returned\"])\r\n else float(v: \"NaN\")\r\n }))\r\n |> group(columns: [\"_field\", \"hostname\"])\r\n", "refId": "A" } ], - "title": "Query Targeting: Scanned Objects / Returned ", + "title": "Query Targeting: Scanned Objects / Returned", "type": "timeseries" }, { - "collapsed": false, + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 62 + "y": 131 }, "id": 21, "panels": [], @@ -2841,6 +3773,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "### CPU usage\n\nHost CPU time by mode, from FTDC **systemMetrics** (OS counters in milliseconds). Each line is that mode\u2019s share of total CPU time in the window after per-mode `derivative(1s)`.\n\n**Formula:** `aggregateWindow(last)` \u2192 `pivot` \u2192 `derivative(*_ms, 1s)` \u2192 `(mode_ms / total_ms) \u00d7 100`\n\n**Unit:** percent (`percent`, 0\u2013100)\n\n---\n\n### Metrics\n\n**Fields:**\n\n`systemMetrics.cpu.user_ms`\n`systemMetrics.cpu.system_ms`\n`systemMetrics.cpu.iowait_ms`\n`systemMetrics.cpu.idle_ms`\n`systemMetrics.cpu.nice_ms`\n`systemMetrics.cpu.softirq_ms`\n`systemMetrics.cpu.steal_ms`\n\n**Plotted:** user, system, iowait percentages only. Idle, nice, softirq, and steal are included in **total_ms** (denominator) but not shown as separate lines.\n\n**Query type:** computed ratio (not a single counter derivative). Output uses `_field` / `_value` via `union` for Grafana.\n\n---\n\n### How to read\n\n- **User %** \u2014 application time (mongod, queries, aggregations)\n- **System (kernel) %** \u2014 kernel time, syscalls, context switching\n- **I/O wait %** \u2014 CPU idle while waiting on disk\n- Three lines sum to **\u2264 100%** (unplotted modes account for the rest)\n\n---\n\n### Troubleshooting\n\n- High **User %** with **Operations** or **Query Executor** \u2192 CPU-bound workload\n- High **I/O wait %** \u2192 disk bottleneck \u00b7 **Disk I/O**, **WiredTiger Block Manager**, **Page Faults**\n- High **System %** with many **Connections** \u2192 thread/syscall overhead\n- VM steal not plotted \u2014 inspect raw `systemMetrics.cpu.steal_ms` in Explore if host may be oversubscribed\n\nCorrelate with **mongod Memory (RSS / Virtual)** and **System Memory**.\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)", "fieldConfig": { "defaults": { "color": { @@ -2852,8 +3785,9 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", - "axisSoftMax": 10, + "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2870,6 +3804,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -2884,48 +3819,51 @@ "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": 0 } ] }, - "unit": "decbytes" + "unit": "percent" }, "overrides": [] }, "gridPos": { - "h": 5, - "w": 8, + "h": 9, + "w": 24, "x": 0, - "y": 63 + "y": 132 }, - "id": 11, + "id": 13, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.network.physicalBytesIn\" or r[\"_field\"] == \"serverStatus.network.physicalBytesOut\")\r\n |> derivative(unit: 1s)\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n", + "query": "base =\r\n from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r._field == \"systemMetrics.cpu.idle_ms\" or\r\n r._field == \"systemMetrics.cpu.user_ms\" or\r\n r._field == \"systemMetrics.cpu.iowait_ms\" or\r\n r._field == \"systemMetrics.cpu.nice_ms\" or\r\n r._field == \"systemMetrics.cpu.softirq_ms\" or\r\n r._field == \"systemMetrics.cpu.steal_ms\" or\r\n r._field == \"systemMetrics.cpu.system_ms\")\r\n |> drop(columns: [\"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> pivot(rowKey: [\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(\r\n unit: 1s,\r\n nonNegative: true,\r\n columns: [\r\n \"systemMetrics.cpu.idle_ms\",\r\n \"systemMetrics.cpu.user_ms\",\r\n \"systemMetrics.cpu.iowait_ms\",\r\n \"systemMetrics.cpu.nice_ms\",\r\n \"systemMetrics.cpu.softirq_ms\",\r\n \"systemMetrics.cpu.steal_ms\",\r\n \"systemMetrics.cpu.system_ms\",\r\n ],\r\n )\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n hostname: r.hostname,\r\n user_ms: float(v: r[\"systemMetrics.cpu.user_ms\"]),\r\n system_ms: float(v: r[\"systemMetrics.cpu.system_ms\"]),\r\n iowait_ms: float(v: r[\"systemMetrics.cpu.iowait_ms\"]),\r\n total_ms:\r\n float(v: r[\"systemMetrics.cpu.idle_ms\"]) +\r\n float(v: r[\"systemMetrics.cpu.user_ms\"]) +\r\n float(v: r[\"systemMetrics.cpu.iowait_ms\"]) +\r\n float(v: r[\"systemMetrics.cpu.nice_ms\"]) +\r\n float(v: r[\"systemMetrics.cpu.softirq_ms\"]) +\r\n float(v: r[\"systemMetrics.cpu.steal_ms\"]) +\r\n float(v: r[\"systemMetrics.cpu.system_ms\"]),\r\n }))\r\n\r\nunion(tables: [\r\n base |> map(fn: (r) => ({\r\n _time: r._time, hostname: r.hostname,\r\n _field: \"User %\",\r\n _value: if r.total_ms > 0.0 then 100.0 * r.user_ms / r.total_ms else 0.0,\r\n })),\r\n base |> map(fn: (r) => ({\r\n _time: r._time, hostname: r.hostname,\r\n _field: \"System (kernel) %\",\r\n _value: if r.total_ms > 0.0 then 100.0 * r.system_ms / r.total_ms else 0.0,\r\n })),\r\n base |> map(fn: (r) => ({\r\n _time: r._time, hostname: r.hostname,\r\n _field: \"I/O wait %\",\r\n _value: if r.total_ms > 0.0 then 100.0 * r.iowait_ms / r.total_ms else 0.0,\r\n })),\r\n])\r\n |> group(columns: [\"_field\", \"hostname\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)", "refId": "A" } ], - "title": "Network In / Out physical", + "title": "CPU Usage", "type": "timeseries" }, { @@ -2933,6 +3871,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "### System memory\n\nHost RAM from FTDC **systemMetrics** (`/proc/meminfo`, values stored as **kB**). Shows how much memory the OS reports as **available** vs **free**.\n\n**Unit:** bytes (Grafana auto-scale; FTDC fields are `*_kb`, query multiplies by 1024)\n\n---\n\n### Metrics\n\n**Fields:**\n\n`systemMetrics.memory.MemAvailable_kb`\n`systemMetrics.memory.MemFree_kb`\n\n**MemAvailable** \u2014 estimate of memory startable for new workloads without swapping (includes reclaimable cache).\n\n**MemFree** \u2014 completely unallocated pages (often much lower than available).\n\n---\n\n### How to read\n\n- **Memory available** \u2014 primary signal for OS headroom\n- **Memory free** \u2014 strictly idle RAM; low free + healthy available is normal on Linux\n- Both lines fall when mongod, page cache, or other services consume RAM\n\n---\n\n### Troubleshooting\n\n- **Available** trending toward zero \u2192 memory pressure \u00b7 **Swap Memory**, **Page Faults**, **WiredTiger Cache**\n- Available low but **mongod Memory (RSS / Virtual)** moderate \u2192 other processes or OS page cache\n- Available and free both low + high **CPU Usage** I/O wait \u2192 possible thrashing\n- After mongod restart, available may jump as RSS is released\n\nCorrelate with **mongod Memory (RSS / Virtual)** and **Swap Memory**.\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)\n- Linux [meminfo](https://www.kernel.org/doc/html/latest/admin-guide/mm/memory-model.html) \u2014 `MemAvailable` vs `MemFree`", "fieldConfig": { "defaults": { "color": { @@ -2944,8 +3883,9 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", - "axisSoftMax": 10, + "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2962,6 +3902,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -2971,52 +3912,57 @@ "mode": "off" } }, + "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": 0 } ] - } + }, + "unit": "bytes" }, "overrides": [] }, "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 63 + "h": 8, + "w": 12, + "x": 0, + "y": 141 }, - "id": 4, + "id": 40, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"serverStatus.network.bytesOut\" or r[\"_field\"] == \"serverStatus.network.bytesIn\")\r\n |> map(fn: (r) => ({ r with _value: r._value / 1024 /1024 }))\r\n |> derivative(unit: 1s)\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"systemMetrics.memory.MemAvailable_kb\" or\r\n r[\"_field\"] == \"systemMetrics.memory.MemFree_kb\")\r\n |> drop(columns: [\"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value) * 1024.0,\r\n _field: if r._field == \"systemMetrics.memory.MemAvailable_kb\" then \"Memory available\" else if r._field == \"systemMetrics.memory.MemFree_kb\" then \"Memory free\" else r._field\r\n }))", "refId": "A" } ], - "title": "Network In / Out", + "title": "System Memory", "type": "timeseries" }, { @@ -3024,7 +3970,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "", + "description": "### Swap memory\n\nHost swap from FTDC **systemMetrics** (`/proc/meminfo`, values stored as **kB**). Shows **used**, **free**, and **total** swap capacity. **Swap used** is computed as `SwapTotal \u2212 SwapFree`.\n\n**Unit:** bytes (Grafana auto-scale; FTDC fields are `*_kb`, query multiplies by 1024)\n\n---\n\n### Metrics\n\n**Fields:**\n\n`systemMetrics.memory.SwapTotal_kb`\n`systemMetrics.memory.SwapFree_kb`\n\n**Computed:** Swap used = `SwapTotal_kb \u2212 SwapFree_kb` (not a separate FTDC field)\n\n**Plotted:** Swap used (computed) \u00b7 Swap free \u00b7 Swap total\n\n---\n\n### How to read\n\n- **Swap used** \u2014 should stay at **0** on a MongoDB host\n- **Swap free** \u2014 unused swap capacity\n- **Swap total** \u2014 configured swap size (may be **0** if swap is disabled)\n- **Swap used + Swap free \u2248 Swap total** at each point\n\n---\n\n### Troubleshooting\n\n- **Any sustained Swap used > 0** \u2192 critical \u00b7 RAM pressure \u00b7 latency spikes\n- Correlate with **System Memory**, **Page Faults**, **mongod Memory (RSS / Virtual)**, **WiredTiger Cache**\n- High **CPU Usage** I/O wait + swap \u2192 severe thrashing\n- Remediation: add RAM, reduce `--wiredTigerCacheSizeGB`, fix memory leaks, stop co-located heavy processes\n- **Swap total = 0** \u2192 swap disabled (common on tuned MongoDB VMs)\n\nCorrelate with **System Memory** and **Average Operation Latency**.\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)\n- [MongoDB production notes \u2014 memory](https://www.mongodb.com/docs/manual/administration/production-notes/#std-label-production-notes-memory)", "fieldConfig": { "defaults": { "color": { @@ -3036,7 +3982,9 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3053,6 +4001,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -3062,16 +4011,14 @@ "mode": "off" } }, + "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": 0 } ] }, @@ -3080,35 +4027,41 @@ "overrides": [] }, "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 63 + "h": 8, + "w": 12, + "x": 12, + "y": 141 }, - "id": 35, + "id": 39, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"local.oplog.rs.stats.storageStats.freeStorageSize\" or r[\"_field\"] == \"local.oplog.rs.stats.storageStats.storageSize\")\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> yield(name: \"mean\")", + "query": "base =\r\n from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"systemMetrics.memory.SwapFree_kb\" or\r\n r[\"_field\"] == \"systemMetrics.memory.SwapTotal_kb\")\r\n |> drop(columns: [\"version\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> pivot(rowKey: [\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n hostname: r.hostname,\r\n swap_total_kb: float(v: r[\"systemMetrics.memory.SwapTotal_kb\"]),\r\n swap_free_kb: float(v: r[\"systemMetrics.memory.SwapFree_kb\"]),\r\n swap_used_kb:\r\n float(v: r[\"systemMetrics.memory.SwapTotal_kb\"]) -\r\n float(v: r[\"systemMetrics.memory.SwapFree_kb\"]),\r\n }))\r\n\r\nunion(tables: [\r\n base |> map(fn: (r) => ({\r\n _time: r._time, hostname: r.hostname,\r\n _field: \"Swap used\",\r\n _value: r.swap_used_kb * 1024.0,\r\n })),\r\n base |> map(fn: (r) => ({\r\n _time: r._time, hostname: r.hostname,\r\n _field: \"Swap free\",\r\n _value: r.swap_free_kb * 1024.0,\r\n })),\r\n base |> map(fn: (r) => ({\r\n _time: r._time, hostname: r.hostname,\r\n _field: \"Swap total\",\r\n _value: r.swap_total_kb * 1024.0,\r\n })),\r\n])\r\n |> group(columns: [\"_field\", \"hostname\"])\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)", "refId": "A" } ], - "title": "Storage Size", + "title": "Swap Memory", "type": "timeseries" }, { @@ -3116,6 +4069,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "### Disk writes and reads\n\nPer-device **IOPS** (read/write operations per second) from FTDC **systemMetrics** (`/proc/diskstats` cumulative `reads` / `writes` counters).\n\n**Unit:** IOPS (`iops`) \u2014 `derivative(unit: 1s)` on OS counters\n\n**Not mongod-only:** includes all processes on the host (OS view).\n\n---\n\n### Metrics\n\n**Pattern (device names vary by host):**\n\n`systemMetrics.disks..reads`\n`systemMetrics.disks..writes`\n\nExample: `systemMetrics.disks.xvda.reads` \u00b7 `systemMetrics.disks.sdb.writes`\n\nAll disk stats matching the exporter\u2019s disk allowlist are ingested automatically (no per-device entries in `metrics_to_get.txt`).\n\n**Query:** `derivative(..., unit: 1s, nonNegative: true)` \u2192 IOPS\n\n---\n\n### How to read\n\n- One line per **device \u00b7 reads** and **device \u00b7 writes**\n- Spikes during WiredTiger **checkpoints** (~60s) are normal on the data volume\n- Many small IOPS + low byte throughput \u2192 small random I/O \u00b7 compare **Disk I/O**\n\n---\n\n### Troubleshooting\n\n- Sustained high IOPS + high **CPU Usage** I/O wait \u2192 disk saturation\n- High reads \u2192 cache misses \u00b7 **WiredTiger Block Manager**, **Page Faults**\n- High writes \u2192 checkpoints/eviction \u00b7 **Checkpoint Write Throughput**, **WiredTiger Pages Evicted**\n- Correlate with **Disk I/O** (bytes/sec) and **Disk I/O Busy Time / Disk Queue Time** panels\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)\n- Linux [diskstats](https://www.kernel.org/doc/html/latest/admin-guide/iostats.html)", "fieldConfig": { "defaults": { "color": { @@ -3127,7 +4081,9 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3144,6 +4100,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -3158,48 +4115,51 @@ "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": 0 } ] }, - "unit": "percent" + "unit": "iops" }, "overrides": [] }, "gridPos": { - "h": 6, - "w": 8, + "h": 8, + "w": 12, "x": 0, - "y": 68 + "y": 149 }, - "id": 13, + "id": 17, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field == \"systemMetrics.cpu.idle_ms\" or r._field == \"systemMetrics.cpu.user_ms\" or r._field == \"systemMetrics.cpu.iowait_ms\" or r._field == \"systemMetrics.cpu.nice_ms\" or r._field == \"systemMetrics.cpu.softirq_ms\" or r._field == \"systemMetrics.cpu.steal_ms\" or r._field == \"systemMetrics.cpu.system_ms\")\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> pivot(rowKey:[\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")\r\n |> derivative(unit: 1s, nonNegative: true, columns: [\"systemMetrics.cpu.idle_ms\", \"systemMetrics.cpu.user_ms\", \"systemMetrics.cpu.iowait_ms\", \"systemMetrics.cpu.nice_ms\", \"systemMetrics.cpu.softirq_ms\", \"systemMetrics.cpu.steal_ms\", \"systemMetrics.cpu.system_ms\"])\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n idle_ms: r[\"systemMetrics.cpu.idle_ms\"],\r\n user_ms: r[\"systemMetrics.cpu.user_ms\"],\r\n iowait_ms: r[\"systemMetrics.cpu.iowait_ms\"],\r\n nice_ms: r[\"systemMetrics.cpu.nice_ms\"],\r\n softirq_ms: r[\"systemMetrics.cpu.softirq_ms\"],\r\n steal_ms: r[\"systemMetrics.cpu.steal_ms\"],\r\n system_ms: r[\"systemMetrics.cpu.system_ms\"],\r\n total_ms: r[\"systemMetrics.cpu.idle_ms\"] + r[\"systemMetrics.cpu.user_ms\"] + r[\"systemMetrics.cpu.iowait_ms\"] + r[\"systemMetrics.cpu.nice_ms\"] + r[\"systemMetrics.cpu.softirq_ms\"] + r[\"systemMetrics.cpu.steal_ms\"] + r[\"systemMetrics.cpu.system_ms\"]\r\n }))\r\n |> map(fn: (r) => ({\r\n _time: r._time,\r\n user: 100.0 * float(v: r.user_ms) / float(v: r.total_ms),\r\n system: 100.0 * float(v: r.system_ms) / float(v: r.total_ms),\r\n iowait: 100.0 * float(v: r.iowait_ms) / float(v: r.total_ms),\r\n }))\r\n", + "query": "import \"strings\"\r\n\r\nfrom(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r._field =~ /^systemMetrics\\.disks\\..+\\.reads$/ or\r\n r._field =~ /^systemMetrics\\.disks\\..+\\.writes$/)\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({\r\n r with\r\n _value: float(v: r._value),\r\n _field:\r\n if strings.hasSuffix(v: r._field, suffix: \".reads\") then\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".reads\",\r\n ) + \" reads\"\r\n else\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".writes\",\r\n ) + \" writes\",\r\n }))", "refId": "A" } ], - "title": "CPU Usage", + "title": "Disk Writes and Reads", "type": "timeseries" }, { @@ -3207,6 +4167,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "### Disk I/O\n\nPer-device **read/write throughput** from FTDC **systemMetrics** (`/proc/diskstats` cumulative sector counters).\n\n**Formula:** `derivative(read_sectors|write_sectors, 1s) \u00d7 512` \u2192 **bytes/sec**\n\n**Unit:** B/s (`Bps`) \u2014 Linux diskstats sectors are **512 bytes** each\n\n**Not mongod-only:** OS view of all processes on the host.\n\n---\n\n### Metrics\n\n**Pattern (device names vary by host):**\n\n`systemMetrics.disks..read_sectors`\n`systemMetrics.disks..write_sectors`\n\nExample: `systemMetrics.disks.xvda.read_sectors` \u00b7 `systemMetrics.disks.sdb.write_sectors`\n\nDisk metrics are ingested via the exporter disk allowlist (no per-device lines in `metrics_to_get.txt`).\n\n**Query:** `derivative(..., unit: 1s, nonNegative: true)` \u2192 sectors/s \u2192 \u00d7 512 \u2192 B/s\n\n---\n\n### How to read\n\n- One line per **device \u00b7 read** and **device \u00b7 write** throughput\n- Write spikes every ~**60s** \u2192 WiredTiger **checkpoints** (normal on data volume)\n- High B/s + low IOPS \u2192 large sequential I/O \u00b7 compare **Disk writes and reads**\n- High B/s + high IOPS \u2192 random or mixed workload\n\n---\n\n### Troubleshooting\n\n- Sustained high throughput + **CPU Usage** I/O wait \u2192 disk bandwidth saturation\n- Read-heavy \u2192 cache pressure \u00b7 **WiredTiger Block Manager**, **Page Faults**\n- Write-heavy \u2192 **Checkpoint Write Throughput**, **Pages Evicted**\n- Queueing / busy time \u2192 **Disk Queue Time**, **Disk I/O Busy Time** (not this panel)\n\nCorrelate with **Disk writes and reads** (IOPS) and **WiredTiger Block Manager**.\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)\n- Linux [diskstats](https://www.kernel.org/doc/html/latest/admin-guide/iostats.html)", "fieldConfig": { "defaults": { "color": { @@ -3218,7 +4179,9 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3235,6 +4198,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -3244,52 +4208,57 @@ "mode": "off" } }, + "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": 0 } ] - } + }, + "unit": "Bps" }, "overrides": [] }, "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 69 + "h": 8, + "w": 12, + "x": 12, + "y": 149 }, - "id": 17, + "id": 47, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..*\\.writes$/ or r._field =~ /^systemMetrics\\.disks\\..*\\.reads$/)\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> derivative(unit: 1s)\r\n", + "query": "import \"strings\"\r\n\r\nfrom(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r._field =~ /^systemMetrics\\.disks\\..+\\.read_sectors$/ or\r\n r._field =~ /^systemMetrics\\.disks\\..+\\.write_sectors$/)\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({\r\n r with\r\n _value: float(v: r._value) * 512.0,\r\n _field:\r\n if strings.hasSuffix(v: r._field, suffix: \".read_sectors\") then\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".read_sectors\",\r\n ) + \" read\"\r\n else\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".write_sectors\",\r\n ) + \" write\",\r\n }))", "refId": "A" } ], - "title": "Disk writes and reads", + "title": "Disk I/O", "type": "timeseries" }, { @@ -3297,6 +4266,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "### Disk reads\n\nPer-device **read throughput** from FTDC **systemMetrics** (`/proc/diskstats` cumulative `read_sectors` counter).\n\n**Formula:** `derivative(read_sectors, 1s) \u00d7 512` \u2192 **bytes/sec**\n\n**Unit:** B/s (`Bps`) \u2014 Linux diskstats sectors are **512 bytes** each\n\n**Not mongod-only:** OS view of all processes on the host (not `serverStatus`).\n\n---\n\n### Metrics\n\n**Pattern (device names vary by host \u2014 regex filter):**\n\n`systemMetrics.disks..read_sectors`\n\nExample: `systemMetrics.disks.xvda.read_sectors`\n\nIngested via exporter disk allowlist (`isDiskMetric`); no per-device entries in `metrics_to_get.txt`.\n\n**Query type:** cumulative counter \u2192 `derivative(unit: 1s, nonNegative: true)` \u2192 `aggregateWindow(mean)` \u2192 \u00d7 512\n\n**Not this panel:** `read_time_ms` \u2192 **Disk I/O Busy Time** \u00b7 `.reads` IOPS \u2192 **Disk writes and reads**\n\n---\n\n### How to read\n\n- One line per device \u00b7 legend **`{device} read`**\n- Sustained high reads \u2192 cold data / cache misses \u00b7 working set vs RAM\n- High B/s + low read IOPS \u2192 large sequential reads \u00b7 compare **Disk writes and reads**\n\n---\n\n### Troubleshooting\n\n- High reads + **Page Faults** \u2192 memory pressure\n- High reads + **WiredTiger Cache** full \u2192 increase cache or RAM\n- High reads + **CPU Usage** I/O wait \u2192 disk bandwidth \u00b7 **Disk I/O**, **Disk I/O Busy Time**\n- Mongod-specific reads \u2192 **WiredTiger Block Manager** (bytes read)\n\nCorrelate with **Disk I/O** (combined read/write B/s).\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)\n- Linux [diskstats](https://www.kernel.org/doc/html/latest/admin-guide/iostats.html)", "fieldConfig": { "defaults": { "color": { @@ -3308,7 +4278,9 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3325,6 +4297,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -3334,53 +4307,57 @@ "mode": "off" } }, + "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": 0 } ] }, - "unit": "deckbytes" + "unit": "Bps" }, "overrides": [] }, "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 69 + "h": 8, + "w": 12, + "x": 0, + "y": 157 }, - "id": 40, + "id": 18, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"systemMetrics.memory.MemAvailable_kb\" or r[\"_field\"] == \"systemMetrics.memory.MemFree_kb\")\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> yield(name: \"mean\")", + "query": "import \"strings\"\r\n\r\nfrom(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..+\\.read_sectors$/)\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({\r\n r with\r\n _value: float(v: r._value) * 512.0,\r\n _field:\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".read_sectors\",\r\n ) + \" read\",\r\n }))", "refId": "A" } ], - "title": "Memory", + "title": "Disk Reads", "type": "timeseries" }, { @@ -3388,7 +4365,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "description": "", + "description": "### Disk writes\n\nPer-device **write throughput** from FTDC **systemMetrics** (`/proc/diskstats` cumulative `write_sectors` counter).\n\n**Formula:** `derivative(write_sectors, 1s) \u00d7 512` \u2192 **bytes/sec**\n\n**Unit:** B/s (`Bps`) \u2014 Linux diskstats sectors are **512 bytes** each\n\n**Not mongod-only:** OS view of all processes on the host (not `serverStatus`).\n\n---\n\n### Metrics\n\n**Pattern (device names vary by host \u2014 regex filter):**\n\n`systemMetrics.disks..write_sectors`\n\nExample: `systemMetrics.disks.xvda.write_sectors`\n\nIngested via exporter disk allowlist (`isDiskMetric`).\n\n**Query type:** cumulative counter \u2192 `derivative(unit: 1s, nonNegative: true)` \u2192 `aggregateWindow(mean)` \u2192 \u00d7 512\n\n**Not this panel:** `write_time_ms` \u2192 **Disk I/O Busy Time** \u00b7 `.writes` IOPS \u2192 **Disk writes and reads**\n\n---\n\n### How to read\n\n- One line per device \u00b7 legend **`{device} write`**\n- Spikes every ~**60s** \u2192 WiredTiger **checkpoints** on data volume (normal)\n- Sustained high writes between spikes \u2192 heavy write load or eviction\n\n---\n\n### Troubleshooting\n\n- Write spikes + latency \u2192 checkpoint I/O \u00b7 **Checkpoint Write Throughput**, **Checkpoint Duration**\n- Sustained high writes \u2192 **Dirty Cache Bytes**, **Pages Evicted**, **Operations**\n- High write B/s + high **CPU Usage** I/O wait \u2192 disk bandwidth \u00b7 **Disk I/O**, **Disk I/O Busy Time**\n\nCorrelate with **Disk I/O** (combined read/write B/s) and **WiredTiger Block Manager**.\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)\n- Linux [diskstats](https://www.kernel.org/doc/html/latest/admin-guide/iostats.html)", "fieldConfig": { "defaults": { "color": { @@ -3402,6 +4379,7 @@ "axisPlacement": "auto", "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3418,6 +4396,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -3427,61 +4406,57 @@ "mode": "off" } }, + "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": 0 } ] - } + }, + "unit": "Bps" }, "overrides": [] }, "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 74 + "h": 8, + "w": 12, + "x": 12, + "y": 157 }, - "id": 47, + "id": 46, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..*\\.io_in_progress$/ or r._field =~ /^systemMetrics\\.disks\\..*\\.io_queued_ms$/ or r._field =~ /^systemMetrics\\.disks\\..*\\.io_time_ms$/)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> derivative(unit: v.windowPeriod)", + "query": "import \"strings\"\r\n\r\nfrom(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..+\\.write_sectors$/)\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({\r\n r with\r\n _value: float(v: r._value) * 512.0,\r\n _field:\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".write_sectors\",\r\n ) + \" write\",\r\n }))", "refId": "A" - }, - { - "datasource": { - "type": "influxdb", - "uid": "P3C6603E967DC8568" - }, - "hide": false, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..*\\.io_in_progress$/ )\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n\r\n\r\n\r\n", - "refId": "B" } ], - "title": "Disk I/O", + "title": "Disk Writes", "type": "timeseries" }, { @@ -3489,6 +4464,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "### Disk queue time\n\nPer-device **I/O queue wait time rate** from FTDC **systemMetrics** (`/proc/diskstats` cumulative `io_queued_ms` counter).\n\n**Formula:** `derivative(io_queued_ms, 1s)` \u2192 **ms/s** (milliseconds of queue wait accrued per wall-clock second)\n\n**Unit:** ms/s \u2014 **not** in-flight I/O count \u00b7 **not** `read_time_ms` / `write_time_ms` (**Disk I/O Busy Time**)\n\n**Not mongod-only:** OS view of all processes on the host.\n\n**Saturation hint:** sustained values toward **~1000 ms/s** \u2192 persistent queueing on that device.\n\n---\n\n### Metrics\n\n**Pattern (device names vary by host \u2014 regex filter):**\n\n`systemMetrics.disks..io_queued_ms`\n\nExample: `systemMetrics.disks.xvda.io_queued_ms`\n\nIngested via exporter disk allowlist (`isDiskMetric`).\n\n**Query type:** cumulative counter \u2192 `derivative(unit: 1s, nonNegative: true)` \u2192 `aggregateWindow(mean)`\n\n**In-flight I/O count:** `systemMetrics.disks..io_in_progress` (gauge) \u2014 not plotted here.\n\n---\n\n### How to read\n\n- One line per device \u00b7 **`{device} queue time`** (ms/s)\n- Rising queue time + flat throughput \u2192 device falling behind\n- Spikes with checkpoint windows \u2192 normal on data volume if brief\n\n---\n\n### Troubleshooting\n\n- Sustained high ms/s \u2192 disk saturation \u00b7 **Disk I/O Busy Time**, **Disk I/O**, **CPU Usage** I/O wait\n- Queue time up + **Average Operation Latency** up \u2192 storage bottleneck\n- Queue time without high B/s \u2192 random/small I/O or slow device \u00b7 **Disk writes and reads**\n\nCorrelate with **Disk I/O Busy Time** and **Disk I/O**.\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)\n- Linux [diskstats](https://www.kernel.org/doc/html/latest/admin-guide/iostats.html)", "fieldConfig": { "defaults": { "color": { @@ -3498,10 +4474,10 @@ "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "", + "axisLabel": "ms/s", "axisPlacement": "auto", - "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3518,6 +4494,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -3527,61 +4504,57 @@ "mode": "off" } }, + "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": 0 } ] - } + }, + "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, - "w": 8, - "x": 8, - "y": 76 + "w": 12, + "x": 0, + "y": 165 }, - "id": 46, + "id": 51, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..*\\.write_sectors$/ or r._field =~ /^systemMetrics\\.disks\\..*\\.write_time_ms$/ or r._field =~ /^systemMetrics\\.disks\\..*\\.writes_merged$/)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> derivative(unit: v.windowPeriod)", + "query": "import \"strings\"\r\n\r\nfrom(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..+\\.io_queued_ms$/)\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({\r\n r with\r\n _value: float(v: r._value),\r\n _field:\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".io_queued_ms\",\r\n ) + \" queue time\",\r\n }))", "refId": "A" - }, - { - "datasource": { - "type": "influxdb", - "uid": "P3C6603E967DC8568" - }, - "hide": false, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..*\\.io_in_progress$/ )\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n\r\n\r\n\r\n", - "refId": "B" } ], - "title": "Disk writes", + "title": "Disk Queue Time", "type": "timeseries" }, { @@ -3589,6 +4562,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "### Disk I/O busy time\n\nPer-device **I/O service time rate** from FTDC **systemMetrics** (`/proc/diskstats` cumulative `read_time_ms` / `write_time_ms` counters).\n\n**Formula:** `derivative(read_time_ms|write_time_ms, 1s)` \u2192 **ms/s** (milliseconds of I/O time accrued per wall-clock second)\n\n**Unit:** ms/s \u2014 **not** average latency per operation \u00b7 **not** `serverStatus.opLatencies`\n\n**Not mongod-only:** OS view of all processes on the host.\n\n**Saturation hint:** values approaching **~1000 ms/s** on a line \u2192 device spending ~100% of wall time in that I/O direction.\n\n---\n\n### Metrics\n\n**Pattern (device names vary by host \u2014 regex filter):**\n\n`systemMetrics.disks..read_time_ms`\n`systemMetrics.disks..write_time_ms`\n\nExample: `systemMetrics.disks.xvda.read_time_ms` \u00b7 `systemMetrics.disks.sdb.write_time_ms`\n\nIngested via exporter disk allowlist (`isDiskMetric`).\n\n**Query type:** cumulative counter \u2192 `derivative(unit: 1s, nonNegative: true)` \u2192 `aggregateWindow(mean)`\n\n**Not this panel:** queue wait \u2192 **Disk Queue Time** (`io_queued_ms`) \u00b7 throughput \u2192 **Disk I/O** \u00b7 IOPS \u2192 **Disk writes and reads**\n\n---\n\n### How to read\n\n- One line per device \u00b7 **read time** / **write time** (ms/s)\n- Write spikes ~every **60s** \u2192 checkpoint I/O on data volume (normal)\n- High ms/s + high **Disk I/O** B/s \u2192 bandwidth-heavy workload\n- High ms/s + high IOPS \u2192 random/small I/O or saturation\n\n---\n\n### Troubleshooting\n\n- Sustained high ms/s \u2192 disk busy \u00b7 **Disk Queue Time**, **CPU Usage** I/O wait\n- High ms/s + rising **Average Operation Latency** \u2192 storage bottleneck\n- High read time + **Page Faults** \u2192 cache / memory pressure\n- High write time + **Checkpoint Write Throughput** \u2192 checkpoint / eviction pressure\n\nCorrelate with **Disk I/O**, **Disk writes and reads**, and **Disk Queue Time**.\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)\n- Linux [diskstats](https://www.kernel.org/doc/html/latest/admin-guide/iostats.html)", "fieldConfig": { "defaults": { "color": { @@ -3598,9 +4572,11 @@ "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "", + "axisLabel": "ms/s", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3617,6 +4593,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -3626,52 +4603,57 @@ "mode": "off" } }, + "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": 0 } ] - } + }, + "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, - "w": 8, - "x": 16, - "y": 76 + "w": 12, + "x": 12, + "y": 165 }, - "id": 39, + "id": 50, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"systemMetrics.memory.SwapFree_kb\" or r[\"_field\"] == \"systemMetrics.memory.SwapTotal_kb\")\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> yield(name: \"mean\")", + "query": "import \"strings\"\r\n\r\nfrom(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r._field =~ /^systemMetrics\\.disks\\..+\\.read_time_ms$/ or\r\n r._field =~ /^systemMetrics\\.disks\\..+\\.write_time_ms$/)\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({\r\n r with\r\n _value: float(v: r._value),\r\n _field:\r\n if strings.hasSuffix(v: r._field, suffix: \".read_time_ms\") then\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".read_time_ms\",\r\n ) + \" read time\"\r\n else\r\n strings.trimSuffix(\r\n v: strings.trimPrefix(v: r._field, prefix: \"systemMetrics.disks.\"),\r\n suffix: \".write_time_ms\",\r\n ) + \" write time\",\r\n }))", "refId": "A" } ], - "title": "Swap Memory", + "title": "Disk I/O Busy Time", "type": "timeseries" }, { @@ -3679,6 +4661,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "### TCP segments in / out\n\nPer-second rates of **Tcp:InSegs** and **Tcp:OutSegs** from FTDC **systemMetrics** (`/proc/net/snmp`).\n\n**Unit:** segments/s (`ops`)\n\n**Not mongod-only:** whole-host TCP stack.\n\n---\n\n### Metrics\n\n**Fields:**\n\n`systemMetrics.netstat.Tcp:InSegs`\n`systemMetrics.netstat.Tcp:OutSegs`\n\n**Query:** `derivative(unit: 1s, nonNegative: true)` \u2192 `aggregateWindow(mean)`\n\n---\n\n### How to read\n\n- **TCP segments in** / **TCP segments out** \u2014 raw TCP-level volume\n- High In/Out ratio on a secondary \u2192 receives more replication than it sends\n\n---\n\n### Troubleshooting\n\n- Sudden drops \u2192 network outage\n- Compare with **Network In / Out** (application bytes) and **Replica Members Ping**\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)", "fieldConfig": { "defaults": { "color": { @@ -3692,6 +4675,7 @@ "axisPlacement": "auto", "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3708,6 +4692,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -3717,19 +4702,18 @@ "mode": "off" } }, + "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": 0 } ] - } + }, + "unit": "ops" }, "overrides": [] }, @@ -3737,41 +4721,38 @@ "h": 7, "w": 8, "x": 0, - "y": 81 + "y": 172 }, - "id": 18, + "id": 43, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..*\\.read_sectors$/ or r._field =~ /^systemMetrics\\.disks\\..*\\.read_time_ms$/ or r._field =~ /^systemMetrics\\.disks\\..*\\.reads_merged$/)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> derivative(unit: v.windowPeriod)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:InSegs\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:OutSegs\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"systemMetrics.netstat.Tcp:InSegs\" then \"TCP segments in\" else if r._field == \"systemMetrics.netstat.Tcp:OutSegs\" then \"TCP segments out\" else r._field\r\n }))", "refId": "A" - }, - { - "datasource": { - "type": "influxdb", - "uid": "P3C6603E967DC8568" - }, - "hide": false, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r._field =~ /^systemMetrics\\.disks\\..*\\.io_in_progress$/ )\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n\r\n\r\n\r\n", - "refId": "B" } ], - "title": "Disk reads", + "title": "TCP segments in / out", "type": "timeseries" }, { @@ -3779,6 +4760,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "### Network In / Out\n\nPer-second **logical** network throughput for **mongod** \u2014 cumulative **`serverStatus.network.bytesIn`** / **`bytesOut`**.\n\n**Unit:** B/s (`Bps`) \u2014 `derivative(unit: 1s)` on byte counters\n\n**Logical bytes:** BSON/document size as counted by mongod (not necessarily on-wire size). Compare with **Network Physical In / Out** for compression/wire bytes.\n\n---\n\n### Metrics\n\n**Fields:**\n\n`serverStatus.network.bytesIn`\n`serverStatus.network.bytesOut`\n\n**Query type:** cumulative counter \u2192 `derivative(unit: 1s, nonNegative: true)` \u2192 `aggregateWindow(mean)`\n\n---\n\n### How to read\n\n- **Logical bytes in** \u2014 data received by mongod\n- **Logical bytes out** \u2014 data sent to clients / other nodes\n- Secondary: often **in \u2265 out** during replication catch-up\n- Primary: **out** rises with large query results or replication fan-out\n\n---\n\n### Troubleshooting\n\n- High **bytes out** + high **Documents** returned \u2192 large result sets \u00b7 check projections/`limit`\n- High bytes + low **Operations** \u2192 big payloads per op\n- Logical \u226b physical on **Network Physical In / Out** \u2192 compression effective (normal)\n- Logical \u2248 physical \u2192 little compression \u00b7 replication or already-compressed payloads\n- Bytes flat but lag \u2192 **Replica Members Lag**, **TCP InSegs / OutSegs**, **Replica Members Ping**\n\nCorrelate with **Network requests** and **Operations**.\n\n---\n\n### Docs\n\n- [serverStatus \u2014 network](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.network)", "fieldConfig": { "defaults": { "color": { @@ -3790,7 +4772,9 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3807,6 +4791,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -3816,20 +4801,18 @@ "mode": "off" } }, + "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": 0 } ] }, - "unit": "none" + "unit": "Bps" }, "overrides": [] }, @@ -3837,32 +4820,38 @@ "h": 7, "w": 8, "x": 8, - "y": 83 + "y": 172 }, - "id": 41, + "id": 4, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"systemMetrics.netstat.Tcp:ActiveOpens\" or r[\"_field\"] == \"systemMetrics.netstat.Tcp:AttemptFails\" or r[\"_field\"] == \"systemMetrics.netstat.Tcp:EstabResets\")\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> derivative(unit: v.windowPeriod)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.network.bytesIn\" or\r\n r[\"_field\"] == \"serverStatus.network.bytesOut\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.network.bytesIn\" then \"Logical bytes in\" else if r._field == \"serverStatus.network.bytesOut\" then \"Logical bytes out\" else r._field\r\n }))", "refId": "A" } ], - "title": "TCP ActiveOpens / AttemptFails / EstabResets", + "title": "Network In / Out", "type": "timeseries" }, { @@ -3870,6 +4859,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "### Network In / Out Physical\n\nPer-second **physical (on-the-wire)** network throughput for **mongod** \u2014 cumulative **`serverStatus.network.physicalBytesIn`** / **`physicalBytesOut`**.\n\n**Unit:** B/s (`Bps`) \u2014 `derivative(unit: 1s)` on byte counters\n\n**Physical vs logical:** **Physical** = bytes sent/received on network connections (wire size, after compression where applicable). **Logical** = document/BSON bytes as counted by mongod \u00b7 compare **Network In / Out**.\n\n**Not disk I/O:** network bytes, not storage \u00b7 see **Disk I/O** for block device throughput.\n\n---\n\n### Metrics\n\n**Fields:**\n\n`serverStatus.network.physicalBytesIn`\n`serverStatus.network.physicalBytesOut`\n\n**Query type:** cumulative counter \u2192 `derivative(unit: 1s, nonNegative: true)` \u2192 `aggregateWindow(mean)`\n\n---\n\n### How to read\n\n- **Physical bytes in** \u2014 wire bytes received by mongod\n- **Physical bytes out** \u2014 wire bytes sent by mongod\n- **Logical > physical** on paired panels \u2192 compression saving bandwidth (common on replication)\n- **Logical \u2248 physical** \u2192 little compression benefit on the wire\n\n---\n\n### Troubleshooting\n\n- High physical **out** on primary \u2192 replication + client traffic \u00b7 **Replica Members Lag**, **TCP InSegs / OutSegs**\n- High physical **in** on secondary \u2192 replication catch-up\n- Physical throughput near link capacity \u2192 timeouts, lag \u00b7 **Replica Members Ping**\n- High physical + low **Operations** \u2192 large compressed payloads or oplog batches\n\nCorrelate with **Network In / Out** (logical) and **Network requests**.\n\n---\n\n### Docs\n\n- [serverStatus \u2014 network](https://www.mongodb.com/docs/manual/reference/command/serverStatus/#mongodb-serverstatus-serverstatus.network)", "fieldConfig": { "defaults": { "color": { @@ -3881,7 +4871,9 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3898,6 +4890,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -3907,53 +4900,57 @@ "mode": "off" } }, + "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": 0 } ] }, - "unit": "none" + "unit": "Bps" }, "overrides": [] }, "gridPos": { - "h": 5, + "h": 7, "w": 8, "x": 16, - "y": 83 + "y": 172 }, - "id": 43, + "id": 11, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"systemMetrics.netstat.Tcp:InSegs\" or r[\"_field\"] == \"systemMetrics.netstat.Tcp:OutSegs\")\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> derivative(unit: v.windowPeriod)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"serverStatus.network.physicalBytesIn\" or\r\n r[\"_field\"] == \"serverStatus.network.physicalBytesOut\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"serverStatus.network.physicalBytesIn\" then \"Physical bytes in\" else if r._field == \"serverStatus.network.physicalBytesOut\" then \"Physical bytes out\" else r._field\r\n }))", "refId": "A" } ], - "title": "TCP: InSegs / OutSegs", + "title": "Network In / Out Physical", "type": "timeseries" }, { @@ -3961,6 +4958,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "### TCP connection lifecycle\n\nPer-second rates of host **TCP connection lifecycle counters** from FTDC **systemMetrics** (`/proc/net/snmp`).\n\n**Unit:** events/sec \u2014 `derivative(unit: 1s, nonNegative: true)` on cumulative counters\n\n**Not mongod-only:** whole-host TCP stack.\n\n---\n\n### Metrics\n\n**Fields:**\n\n`systemMetrics.netstat.Tcp:ActiveOpens`\n`systemMetrics.netstat.Tcp:AttemptFails`\n`systemMetrics.netstat.Tcp:EstabResets`\n`systemMetrics.netstat.Tcp:OutRsts`\n\n**Query type:** cumulative counter \u2192 `derivative(unit: 1s, nonNegative: true)` \u2192 `aggregateWindow(mean)`\n\n**Not this panel:** `Tcp:CurrEstab` (open connection **count**, gauge \u2014 use `last`, not derivative)\n\n---\n\n### How to read\n\n- **TCP active opens** \u2014 successful new connections initiated (normally **> 0** on a busy host)\n- **TCP connect failures** \u2014 failed connection attempts (expect **~0**)\n- **TCP established resets** \u2014 established connections reset (expect **~0**)\n- **TCP RST sent** \u2014 segments with RST flag (expect **~0**)\n- Brief spikes during deploys/restarts are normal\n\n---\n\n### Troubleshooting\n\n- Sustained **AttemptFails** \u2192 refused connections \u00b7 firewall, `ulimit`, mongod down, wrong port\n- **EstabResets** / **RST sent** \u2192 mid-connection drops \u00b7 **Replica Members Health**, node restarts\n- High **ActiveOpens** with errors \u2192 churny clients or replication mesh \u00b7 compare **TCP InSegs / OutSegs**\n- With **TCP Packet Loss or Corruption** \u2192 loss before reset\n- With lag \u2192 **Replica Members Lag**, **Replica Members Ping**\n\nCorrelate with **TCP InSegs / OutSegs** and **Replica Members Health**.\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)\n- Linux [SNMP-TCP-MIB / netstat](https://www.kernel.org/doc/html/latest/networking/snmp_counter.html)", "fieldConfig": { "defaults": { "color": { @@ -3970,9 +4968,11 @@ "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "", + "axisLabel": "events/s", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3989,6 +4989,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -3998,16 +4999,15 @@ "mode": "off" } }, + "decimals": 2, "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": 0 } ] }, @@ -4016,35 +5016,41 @@ "overrides": [] }, "gridPos": { - "h": 5, + "h": 7, "w": 8, "x": 0, - "y": 88 + "y": 179 }, - "id": 44, + "id": 45, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"systemMetrics.netstat.Ip:OutNoRoutes\" )\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> derivative(unit: v.windowPeriod)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:ActiveOpens\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:AttemptFails\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:EstabResets\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:OutRsts\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"systemMetrics.netstat.Tcp:ActiveOpens\" then \"TCP active opens\" else if r._field == \"systemMetrics.netstat.Tcp:AttemptFails\" then \"TCP connect failures\" else if r._field == \"systemMetrics.netstat.Tcp:EstabResets\" then \"TCP established resets\" else if r._field == \"systemMetrics.netstat.Tcp:OutRsts\" then \"TCP RST sent\" else r._field\r\n }))", "refId": "A" } ], - "title": "TCP Routing Issues", + "title": "TCP Connection lifecycle", "type": "timeseries" }, { @@ -4052,6 +5058,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "### TCP routing issues\n\nPer-second rates of host **IP routing and fragmentation error counters** from FTDC **systemMetrics** (`/proc/net/snmp`).\n\n**Unit:** events/sec \u2014 `derivative(unit: 1s)` on cumulative counters\n\n**Layer note:** these are **IP** counters (title says \u201cTCP routing\u201d historically). **Not mongod-only.**\n\n**Expectation:** normally **flat at zero**; any sustained rate \u2192 routing, MTU, or path problems.\n\n---\n\n### Metrics\n\n**Fields:**\n\n`systemMetrics.netstat.Ip:OutNoRoutes`\n`systemMetrics.netstat.Ip:FragFails`\n`systemMetrics.netstat.Ip:ReasmFails`\n\n**Query type:** cumulative counter \u2192 `derivative(unit: 1s, nonNegative: true)` \u2192 `aggregateWindow(mean)`\n\n**Not ingested (optional FTDC `IpExt` paths):** `IpExt:InNoRoutes` \u00b7 `IpExt:InTruncatedPkts` \u2014 add to `metrics_to_get.txt` if present in your `metrics.txt`\n\n**Not this panel:** `Ip:InDiscards` / `Ip:OutDiscards` \u2192 **TCP Packet Loss or Corruption**\n\n---\n\n### How to read\n\n- **IP no route (out)** \u2014 packets dropped with no route to destination\n- **IP fragmentation failures** \u2014 could not fragment (often **MTU / PMTU** issues)\n- **IP reassembly failures** \u2014 fragmented packets could not be reassembled\n\n---\n\n### Troubleshooting\n\n- **OutNoRoutes** \u2192 wrong/missing routes, asymmetric routing, firewall blackholes\n- **FragFails** / **ReasmFails** \u2192 **MTU mismatch** (VPN, cloud network, jumbo frames)\n- With **TCP Packet Loss or Corruption** \u2192 broader stack problems\n- Intermittent app errors \u2192 **Replica Members Ping**, **TCP Connection Issues**\n\nCorrelate with **TCP InSegs / OutSegs** and **Replica Members Health**.\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)\n- Linux [SNMP / netstat counters](https://www.kernel.org/doc/html/latest/networking/snmp_counter.html)", "fieldConfig": { "defaults": { "color": { @@ -4061,9 +5068,11 @@ "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "", + "axisLabel": "events/s", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -4080,6 +5089,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -4089,17 +5099,14 @@ "mode": "off" } }, + "decimals": 2, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": 0 } ] }, @@ -4111,32 +5118,38 @@ "h": 7, "w": 8, "x": 8, - "y": 90 + "y": 179 }, - "id": 45, + "id": 44, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"systemMetrics.netstat.Tcp:CurrEstab\" or r[\"_field\"] == \"systemMetrics.netstat.Tcp:EstabResets\" or r[\"_field\"] == \"systemMetrics.netstat.Tcp:OutRsts\")\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> derivative(unit: v.windowPeriod)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"systemMetrics.netstat.Ip:OutNoRoutes\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Ip:FragFails\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Ip:ReasmFails\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"systemMetrics.netstat.Ip:OutNoRoutes\" then \"IP no route (out)\" else if r._field == \"systemMetrics.netstat.Ip:FragFails\" then \"IP fragmentation failures\" else if r._field == \"systemMetrics.netstat.Ip:ReasmFails\" then \"IP reassembly failures\" else r._field\r\n }))", "refId": "A" } ], - "title": "TCP Connection Issues", + "title": "TCP Routing Issues", "type": "timeseries" }, { @@ -4144,6 +5157,7 @@ "type": "influxdb", "uid": "P3C6603E967DC8568" }, + "description": "### TCP packet loss or corruption\n\nPer-second rates of host **TCP/IP error and discard counters** from FTDC **systemMetrics** (`/proc/net/snmp`).\n\n**Unit:** events/sec \u2014 `derivative(unit: 1s)` on cumulative counters\n\n**Not mongod-only:** whole-host TCP/IP stack (all processes).\n\n**Expectation:** normally **flat at zero**; any sustained non-zero rate warrants investigation.\n\n---\n\n### Metrics\n\n**Fields:**\n\n`systemMetrics.netstat.Tcp:InCsumErrors`\n`systemMetrics.netstat.Tcp:InErrs`\n`systemMetrics.netstat.Tcp:RetransSegs`\n`systemMetrics.netstat.Ip:InDiscards`\n`systemMetrics.netstat.Ip:OutDiscards`\n\n**Query type:** cumulative counter \u2192 `derivative(unit: 1s, nonNegative: true)` \u2192 `aggregateWindow(mean)`\n\n**Optional (in `metrics_to_get.txt`, not plotted):** `Ip:FragFails` \u00b7 `Ip:ReasmFails` (fragmentation/reassembly failures)\n\n---\n\n### How to read\n\n- **TCP retransmissions** \u2014 packet loss / timeout on the wire (most common \u201closs\u201d signal)\n- **TCP checksum errors** \u2014 corrupted frames received\n- **TCP input errors** \u2014 other inbound TCP errors\n- **IP discards** \u2014 kernel dropped packets (buffers, policy, congestion)\n\n---\n\n### Troubleshooting\n\n- **RetransSegs** sustained \u2192 loss/congestion \u00b7 **Replica Members Ping**, **Average Operation Latency**\n- **InCsumErrors** \u2192 bad cable/NIC/driver, offload bugs\n- **InDiscards** / **OutDiscards** \u2192 buffer pressure \u00b7 NIC ring, `netdev_max_backlog`, flood traffic\n- Errors + high **TCP InSegs / OutSegs** \u2192 saturated or failing path\n- Errors on one member only \u2192 host/NIC/path to that node \u00b7 **Replica Members Health**\n\nCorrelate with **TCP InSegs / OutSegs** and **TCP Connection Issues**.\n\n---\n\n### Docs\n\n- FTDC **systemMetrics** (OS-level; not `serverStatus`)\n- Linux [SNMP / netstat counters](https://www.kernel.org/doc/html/latest/networking/snmp_counter.html)", "fieldConfig": { "defaults": { "color": { @@ -4153,9 +5167,11 @@ "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "", + "axisLabel": "events/s", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -4172,6 +5188,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -4181,16 +5198,14 @@ "mode": "off" } }, + "decimals": 2, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": 0 } ] }, @@ -4199,31 +5214,37 @@ "overrides": [] }, "gridPos": { - "h": 5, + "h": 7, "w": 8, - "x": 0, - "y": 93 + "x": 16, + "y": 179 }, "id": 42, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.3.2", "targets": [ { "datasource": { "type": "influxdb", "uid": "P3C6603E967DC8568" }, - "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) => r[\"_field\"] == \"systemMetrics.netstat.Tcp:InCsumErrors\" \r\n or r[\"_field\"] == \"systemMetrics.netstat.Tcp:InErrs\"\r\n or r[\"_field\"] == \"systemMetrics.netstat.Tcp:RetransSegs\"\r\n or r[\"_field\"] == \"systemMetrics.netstat.Ip:InDiscards\"\r\n or r[\"_field\"] == \"systemMetrics.netstat.Ip:OutDiscards\")\r\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\r\n |> derivative(unit: v.windowPeriod)", + "query": "from(bucket: v.defaultBucket)\r\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\r\n |> filter(fn: (r) => r[\"_measurement\"] == \"ftdc\")\r\n |> filter(fn: (r) => r[\"hostname\"] =~ /^${hostname:regex}$/)\r\n |> filter(fn: (r) => r[\"version\"] =~ /^${version:regex}$/)\r\n |> filter(fn: (r) =>\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:InCsumErrors\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:InErrs\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Tcp:RetransSegs\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Ip:InDiscards\" or\r\n r[\"_field\"] == \"systemMetrics.netstat.Ip:OutDiscards\")\r\n |> drop(columns: [\"version\"])\r\n |> sort(columns: [\"_time\"])\r\n |> derivative(unit: 1s, nonNegative: true)\r\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\r\n |> map(fn: (r) => ({ r with\r\n _value: float(v: r._value),\r\n _field: if r._field == \"systemMetrics.netstat.Tcp:InCsumErrors\" then \"TCP checksum errors\" else if r._field == \"systemMetrics.netstat.Tcp:InErrs\" then \"TCP input errors\" else if r._field == \"systemMetrics.netstat.Tcp:RetransSegs\" then \"TCP retransmissions\" else if r._field == \"systemMetrics.netstat.Ip:InDiscards\" then \"IP inbound discards\" else if r._field == \"systemMetrics.netstat.Ip:OutDiscards\" then \"IP outbound discards\" else r._field\r\n }))", "refId": "A" } ], @@ -4238,48 +5259,56 @@ "templating": { "list": [ { + "allValue": ".*", "allowCustomValue": false, - "definition": "import \"influxdata/influxdb/schema\"\n\nschema.tagValues(\n bucket: v.bucket,\n tag: \"hostname\"\n)", + "definition": "import \"influxdata/influxdb/schema\"\n\nschema.tagValues(\n bucket: v.defaultBucket,\n tag: \"hostname\",\n start: v.timeRangeStart,\n stop: v.timeRangeStop\n)", "description": "", + "includeAll": true, "label": "hostname", "multi": true, - "includeAll": true, - "allValue": ".*", "name": "hostname", "options": [], "query": { - "query": "import \"influxdata/influxdb/schema\"\n\nschema.tagValues(\n bucket: v.bucket,\n tag: \"hostname\"\n)" + "query": "import \"influxdata/influxdb/schema\"\n\nschema.tagValues(\n bucket: v.defaultBucket,\n tag: \"hostname\",\n start: v.timeRangeStart,\n stop: v.timeRangeStop\n)" }, "refresh": 1, "regex": "", - "type": "query" + "type": "query", + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + } }, { + "allValue": ".*", "allowCustomValue": false, - "definition": "import \"influxdata/influxdb/schema\"\n\nschema.tagValues(\n bucket: v.bucket,\n tag: \"version\"\n)", + "definition": "import \"influxdata/influxdb/schema\"\n\nschema.tagValues(\n bucket: v.defaultBucket,\n tag: \"version\",\n start: v.timeRangeStart,\n stop: v.timeRangeStop\n)", "description": "version", + "includeAll": true, "label": "version", "multi": true, - "includeAll": true, - "allValue": ".*", "name": "version", "options": [], "query": { - "query": "import \"influxdata/influxdb/schema\"\n\nschema.tagValues(\n bucket: v.bucket,\n tag: \"version\"\n)" + "query": "import \"influxdata/influxdb/schema\"\n\nschema.tagValues(\n bucket: v.defaultBucket,\n tag: \"version\",\n start: v.timeRangeStart,\n stop: v.timeRangeStop\n)" }, "refresh": 1, "regex": "", - "type": "query" + "type": "query", + "datasource": { + "type": "influxdb", + "uid": "P3C6603E967DC8568" + } } ] }, "time": { - "from": "2024-05-25T19:23:09.607Z", - "to": "2031-05-26T00:01:28.240Z" + "from": "now-6h", + "to": "now" }, "timepicker": {}, - "timezone": "browser", + "timezone": "UTC", "title": "ftdc_dashboard", "uid": "ddnw277huiv40ae", "version": 1 -} \ No newline at end of file +} diff --git a/grafana/datasources.yaml b/grafana/datasources.yaml index 00092ff..002a2a2 100644 --- a/grafana/datasources.yaml +++ b/grafana/datasources.yaml @@ -10,5 +10,6 @@ datasources: organization: ${INFLUX_ORG} defaultBucket: ${INFLUX_BUCKET} tlsSkipVerify: true + timeout: 60 secureJsonData: token: ${INFLUX_API_TOKEN} diff --git a/grafana/main.yaml b/grafana/main.yaml index fc2534e..590cf66 100644 --- a/grafana/main.yaml +++ b/grafana/main.yaml @@ -4,5 +4,9 @@ providers: - name: "ftdc" orgId: 1 type: file + disableDeletion: false + updateIntervalSeconds: 10 + allowUiUpdates: false options: path: /etc/grafana/provisioning/dashboards/ + foldersFromFilesStructure: false diff --git a/metrics_to_get.txt b/metrics_to_get.txt index 6265f78..72e37d9 100644 --- a/metrics_to_get.txt +++ b/metrics_to_get.txt @@ -35,6 +35,9 @@ serverStatus.flowControl.targetRateLimit serverStatus.wiredTiger.block-manager.bytes read serverStatus.wiredTiger.block-manager.bytes written serverStatus.wiredTiger.block-manager.bytes written for checkpoint +serverStatus.wiredTiger.log.log bytes written +serverStatus.wiredTiger.transaction.transaction checkpoint most recent time (msecs) +serverStatus.wiredTiger.checkpoint.most recent time (msecs) serverStatus.wiredTiger.cache.bytes currently in the cache serverStatus.wiredTiger.cache.maximum bytes configured serverStatus.wiredTiger.cache.modified pages evicted @@ -45,6 +48,15 @@ serverStatus.wiredTiger.cache.unmodified pages evicted serverStatus.wiredTiger.data-handle.connection data handles currently active serverStatus.wiredTiger.concurrentTransactions.read.available serverStatus.wiredTiger.concurrentTransactions.write.available +serverStatus.wiredTiger.concurrentTransactions.monitor.available +serverStatus.wiredTiger.concurrentTransactions.monitor.totalTickets +serverStatus.wiredTiger.concurrentTransactions.monitor.queueLength +serverStatus.queues.execution.read.available +serverStatus.queues.execution.read.totalTickets +serverStatus.queues.execution.read.normalPriority.queueLength +serverStatus.queues.execution.write.available +serverStatus.queues.execution.write.totalTickets +serverStatus.queues.execution.write.normalPriority.queueLength serverStatus.flowControl.enabled serverStatus.flowControl.isLagged serverStatus.flowControl.isLaggedCount @@ -138,3 +150,13 @@ replSetGetStatus.members.1.lastAppliedWallTime replSetGetStatus.members.2.lastAppliedWallTime replSetGetStatus.members.3.lastAppliedWallTime replSetGetStatus.members.4.lastAppliedWallTime +replSetGetStatus.members.0.state +replSetGetStatus.members.1.state +replSetGetStatus.members.2.state +replSetGetStatus.members.3.state +replSetGetStatus.members.4.state +replSetGetStatus.members.0.name +replSetGetStatus.members.1.name +replSetGetStatus.members.2.name +replSetGetStatus.members.3.name +replSetGetStatus.members.4.name \ No newline at end of file diff --git a/run.sh b/run.sh index 682d2ea..66a5821 100755 --- a/run.sh +++ b/run.sh @@ -70,7 +70,7 @@ esac # Function to handle Ctrl-C (SIGINT) cleanup() { echo "Stopping Docker containers..." - $ENVS docker-compose down -v + $ENVS docker compose down -v echo "Docker containers stopped." exit 0 } @@ -80,13 +80,13 @@ trap cleanup SIGINT echo "Checking for image changes and building if needed..." -docker-compose build +docker compose build # remove any running container -docker-compose down -v +docker compose down -v # Start Docker containers in detached mode -docker-compose up -d +docker compose up -d echo "🔐 Credentials:" echo "--------------------------"