Improve FTDC Grafana dashboard and metrics

.gitignore

@@ -3,4 +3,10 @@ diagnostic.data/
 .idea
 .DS_Store
 influxdb_data/
-./ftdc_exporter/vendor
+./ftdc_exporter/vendor
+.cursor/
+mongodb_ftdc_decoder/
+grafana/backups/
+grafana/DASHBOARD.md
+grafana/dashboards/bkp.json
+grafana/dashboards/my.jsonscripts/

ftdc_exporter/ftdc/iterator.go

@@ -1,7 +1,7 @@
 package ftdc
 
 func (it *FTDCDataIterator) NormalisedDocument(includedPatterns map[string]struct{}) map[string]interface{} {
-	return normalizeDocument(it.doc, includedPatterns)
+	return normalizeMetricsDocument(it.doc, includedPatterns)
 }
 
 func (it *FTDCDataIterator) Close() {

ftdc_exporter/ftdc/normalize.go

@@ -1,6 +1,8 @@
 package ftdc
 
 import (
+	"strconv"
+
 	"github.com/evergreen-ci/birch"
 	"github.com/evergreen-ci/birch/bsontype"
 	"strings"
@@ -60,24 +62,156 @@ func isIncluded(key string, includedPatterns map[string]struct{}) bool {
 	return false
 }
 
+func joinMetricKey(prefix, key string) string {
+	if prefix == "" {
+		return key
+	}
+	return prefix + "." + key
+}
+
+// isTopLevelRoleWrapper reports MongoDB 8+ FTDC role document keys that must not
+// appear in the flattened metric path. Only matches at the metrics document root
+// so nested fields named "shard" on older versions are left alone.
+func isTopLevelRoleWrapper(prefix, key string) bool {
+	if prefix != "" {
+		return false
+	}
+	switch key {
+	case "shard", "router", "common":
+		return true
+	default:
+		return false
+	}
+}
+
+// stripRoleKeyPrefix removes a leading role prefix from flattened metric keys.
+// github.com/mongodb/ftdc often emits already-dotted keys such as
+// "shard.replSetGetStatus.members.0.state" at the document root.
+// Nested keys (prefix != "") are left unchanged except for the historical
+// "common." trim applied by the caller.
+func stripRoleKeyPrefix(prefix, key string) string {
+	if prefix != "" {
+		return key
+	}
+	for _, p := range []string{"shard.", "router.", "common."} {
+		if strings.HasPrefix(key, p) {
+			return strings.TrimPrefix(key, p)
+		}
+	}
+	return key
+}
+
+func setIfAbsent(dst map[string]interface{}, key string, value interface{}) {
+	if _, exists := dst[key]; exists {
+		return
+	}
+	dst[key] = value
+}
+
+// normalizeDocument preserves nested maps (used for FTDC metadata).
 func normalizeDocument(document *birch.Document, includedPatterns map[string]struct{}) map[string]interface{} {
 	normalized := make(map[string]interface{})
 	iter := document.Iterator()
 
 	for iter.Next() {
 		elem := iter.Element()
 		key := elem.Key()
-		// on some versions, the metrics are starting with a common. prefix
-		// I decided to get rid of it so that we do not have to change anything in the grafana dashboards.
 		key = strings.TrimPrefix(key, "common.")
 		val := elem.Value()
 		if isIncluded(key, includedPatterns) {
-			normalized[key] = normalizeValue(val, includedPatterns)
+			setIfAbsent(normalized, key, normalizeValue(val, includedPatterns))
+		}
+	}
+	return normalized
+}
+
+// normalizeMetricsDocument flattens nested FTDC samples into dotted field names for InfluxDB.
+func normalizeMetricsDocument(document *birch.Document, includedPatterns map[string]struct{}) map[string]interface{} {
+	return flattenDocument(document, "", includedPatterns)
+}
+
+func flattenDocument(document *birch.Document, prefix string, includedPatterns map[string]struct{}) map[string]interface{} {
+	normalized := make(map[string]interface{})
+	iter := document.Iterator()
+
+	for iter.Next() {
+		elem := iter.Element()
+		key := elem.Key()
+		key = strings.TrimPrefix(key, "common.")
+		key = stripRoleKeyPrefix(prefix, key)
+		val := elem.Value()
+
+		// MongoDB 8+ sharded FTDC wraps mongod/mongos collectors under shard/router/common.
+		if isTopLevelRoleWrapper(prefix, key) && val.Type() == bsontype.EmbeddedDocument {
+			for k, v := range flattenDocument(val.MutableDocument(), prefix, includedPatterns) {
+				setIfAbsent(normalized, k, v)
+			}
+			continue
+		}
+
+		fullKey := joinMetricKey(prefix, key)
+
+		switch val.Type() {
+		case bsontype.EmbeddedDocument:
+			for k, v := range flattenDocument(val.MutableDocument(), fullKey, includedPatterns) {
+				setIfAbsent(normalized, k, v)
+			}
+		case bsontype.Array:
+			flattenArray(val.MutableArray(), fullKey, includedPatterns, normalized)
+		default:
+			if isIncluded(fullKey, includedPatterns) {
+				setIfAbsent(normalized, fullKey, normalizeScalar(val))
+			}
 		}
 	}
 	return normalized
 }
 
+func flattenArray(arr *birch.Array, prefix string, includedPatterns map[string]struct{}, out map[string]interface{}) {
+	it := arr.Iterator()
+	i := 0
+	for it.Next() {
+		indexKey := joinMetricKey(prefix, strconv.Itoa(i))
+		val := it.Value()
+		switch val.Type() {
+		case bsontype.EmbeddedDocument:
+			for k, v := range flattenDocument(val.MutableDocument(), indexKey, includedPatterns) {
+				setIfAbsent(out, k, v)
+			}
+		case bsontype.Array:
+			flattenArray(val.MutableArray(), indexKey, includedPatterns, out)
+		default:
+			if isIncluded(indexKey, includedPatterns) {
+				setIfAbsent(out, indexKey, normalizeScalar(val))
+			}
+		}
+		i++
+	}
+}
+
+func normalizeScalar(val *birch.Value) interface{} {
+	switch val.Type() {
+	case bsontype.Double:
+		return val.Double()
+	case bsontype.String:
+		return val.StringValue()
+	case bsontype.Boolean:
+		return val.Boolean()
+	case bsontype.Int32:
+		return val.Int32()
+	case bsontype.Int64:
+		return val.Int64()
+	case bsontype.Null:
+		return -1
+	case bsontype.ObjectID:
+		return val.ObjectID().Hex()
+	case bsontype.DateTime:
+		return time.UnixMilli(val.DateTime()).UnixMilli()
+	default:
+		return val.Interface()
+	}
+}
+
 func normalizeValue(val *birch.Value, includedPatterns map[string]struct{}) interface{} {
 	switch val.Type() {
 	case bsontype.Double:
@@ -99,16 +233,13 @@ func normalizeValue(val *birch.Value, includedPatterns map[string]struct{}) inte
 	case bsontype.Array:
 		out := []interface{}{}
 		it := val.MutableArray().Iterator()
-		i := 0
 		for it.Next() {
 			out = append(out, normalizeValue(it.Value(), includedPatterns))
-			i++
 		}
 		return out
 	case bsontype.DateTime:
 		return time.UnixMilli(val.DateTime()).UnixMilli()
 	default:
-		// Handle unsupported types as raw or string, or skip
-		return val.Interface() // fallback
+		return val.Interface()
 	}
 }