From 0dd664dece3f8319f6bec5a221acd2c7ad13a23d Mon Sep 17 00:00:00 2001 From: Ben Kochie Date: Tue, 7 Apr 2026 17:44:04 +0200 Subject: [PATCH 1/3] Cut v1.11.1 (#3616) * Fix kernel_hung for no data (#3613) Return an ErrNoData for the kernel_hung collector if the file does not exist. Fixes: https://github.com/prometheus/node_exporter/issues/3612 Signed-off-by: Ben Kochie * Release v1.11.1 (#3615) * [BUGFIX] Fix kernel_hung for no data #3613 Signed-off-by: Ben Kochie --------- Signed-off-by: Ben Kochie --- CHANGELOG.md | 4 ++++ VERSION | 2 +- collector/kernel_hung_linux.go | 6 ++++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a46db9efb..f5e2ef1cb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ * [ENHANCEMENT] * [BUGFIX] +## 1.11.1 / 2026-04-07 + +* [BUGFIX] Fix kernel_hung for no data #3613 + ## 1.11.0 / 2026-04-04 * [CHANGE] Adds distroless container image. #3592 diff --git a/VERSION b/VERSION index 1cac385c6c..720c7384c6 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.11.0 +1.11.1 diff --git a/collector/kernel_hung_linux.go b/collector/kernel_hung_linux.go index 8403977ded..d77c504b6c 100644 --- a/collector/kernel_hung_linux.go +++ b/collector/kernel_hung_linux.go @@ -16,8 +16,10 @@ package collector import ( + "errors" "fmt" "log/slog" + "os" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/procfs" @@ -54,6 +56,10 @@ var ( func (c *kernelHungCollector) Update(ch chan<- prometheus.Metric) error { kernelHung, err := c.fs.KernelHung() if err != nil { + if errors.Is(err, os.ErrNotExist) { + c.logger.Debug("hung_task_detect_count does not exist") + return ErrNoData + } return err } From ae6218dba83364fd95c5ded85d2d1c65db2b7c75 Mon Sep 17 00:00:00 2001 From: Allen Xie Date: Sun, 21 Jun 2026 13:42:38 +0000 Subject: [PATCH 2/3] collector: add ebsnvme collector for Amazon EBS NVMe performance stats Add a new (disabled-by-default) Linux collector, ebsnvme, that exposes the Amazon EBS detailed performance statistics vended by Nitro-based EC2 instances through the EBS NVMe device log page (log page 0xD0). The collector reads the EBS statistics log page from each EBS-backed NVMe device via an NVMe admin ioctl, parses the binary structure, and exposes read/write ops, bytes, time, IOPS/throughput exceeded counters, queue length, and read/write latency histograms. Metrics are labelled by volume_id, device, and mount_path. The log-page parsing logic is derived from the Amazon EBS CSI Driver (pkg/metrics/nvme.go), Apache-2.0, Copyright The Kubernetes Authors. Statistic names and semantics follow the Amazon EBS User Guide: https://docs.aws.amazon.com/ebs/latest/userguide/nvme-detailed-performance-stats.html Signed-off-by: Allen Xie --- README.md | 1 + collector/ebsnvme_linux.go | 371 ++++++++++++++++++++++++++++++++ collector/ebsnvme_linux_test.go | 138 ++++++++++++ 3 files changed, 510 insertions(+) create mode 100644 collector/ebsnvme_linux.go create mode 100644 collector/ebsnvme_linux_test.go diff --git a/README.md b/README.md index 3b8c2d811e..8e20478359 100644 --- a/README.md +++ b/README.md @@ -194,6 +194,7 @@ cpu\_vulnerabilities | Exposes CPU vulnerability information from sysfs. | Linux devstat | Exposes device statistics | Dragonfly, FreeBSD drm | Expose GPU metrics using sysfs / DRM, `amdgpu` is the only driver which exposes this information through DRM | Linux drbd | Exposes Distributed Replicated Block Device statistics (to version 8.4) | Linux +ebsnvme | Exposes [Amazon EBS detailed performance statistics](https://docs.aws.amazon.com/ebs/latest/userguide/nvme-detailed-performance-stats.html) read from the EBS NVMe device log page (IOPS, throughput, latency histograms, queue length), labelled by `volume_id`, `device`, and `mount_path`. Requires running on a Nitro-based EC2 instance. | Linux ethtool | Exposes network interface information and network driver statistics equivalent to `ethtool`, `ethtool -S`, and `ethtool -i`. | Linux interrupts | Exposes detailed interrupts statistics. | Linux, OpenBSD ksmd | Exposes kernel and system statistics from `/sys/kernel/mm/ksm`. | Linux diff --git a/collector/ebsnvme_linux.go b/collector/ebsnvme_linux.go new file mode 100644 index 0000000000..1d55179016 --- /dev/null +++ b/collector/ebsnvme_linux.go @@ -0,0 +1,371 @@ +// Copyright 2024 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// The EBS NVMe log-page parsing logic in this file is derived from the Amazon +// EBS CSI Driver, which is also licensed under the Apache License, Version 2.0: +// +// Copyright 2024 The Kubernetes Authors +// https://github.com/kubernetes-sigs/aws-ebs-csi-driver +// (pkg/metrics/nvme.go) +// +// Author of this node_exporter collector: +// +// Allen Xie , Storage Solution Architect + +//go:build linux && !noebsnvme +// +build linux,!noebsnvme + +// Package collector's ebsnvme collector exposes the Amazon EBS detailed +// performance statistics that Nitro-based EC2 instances vend through the NVMe +// device log page. The statistics, their names, and their semantics are +// documented in the Amazon EBS User Guide: +// +// https://docs.aws.amazon.com/ebs/latest/userguide/nvme-detailed-performance-stats.html +// +// The collector reads NVMe log page 0xD0 from each EBS-backed NVMe device via +// an ioctl, parses the binary EBS statistics structure, and exposes the values +// as Prometheus metrics labelled by volume_id, device, and mount_path. + +package collector + +import ( + "bytes" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "log/slog" + "math" + "os" + "os/exec" + "strings" + "unsafe" + + "github.com/prometheus/client_golang/prometheus" + "golang.org/x/sys/unix" +) + +const ( + ebsNVMeSubsystem = "ebs" + + // ebsMagic identifies a valid EBS NVMe statistics log page. + ebsMagic = 0x3C23B510 + + // ebsLogPageID is the NVMe log page identifier for EBS statistics. + ebsLogPageID = 0xD0 + + // nvmeAdminGetLogPage is the NVMe admin opcode for "Get Log Page". + nvmeAdminGetLogPage = 0x02 + + // nvmeLogPageSize is the length, in bytes, of the EBS log page. + nvmeLogPageSize = 4096 + + // nvmeIoctlAdminCmd is the ioctl request code for NVME_IOCTL_ADMIN_CMD. + nvmeIoctlAdminCmd = 0xC0484E41 + + // microsecondsInSeconds converts microsecond counters to seconds. + microsecondsInSeconds = 1e6 + + // notMounted is the mount_path label value used for NVMe devices that are + // not mounted, or whose top-level device has no direct mount point (for + // example a disk that is only mounted through one of its partitions). + notMounted = "NotMounted" +) + +var ( + errInvalidEBSMagic = errors.New("invalid EBS magic number") + errParseLogPage = errors.New("failed to parse log page") +) + +// ebsMetrics maps the EBS NVMe log page layout. The field order and the +// reserved area size are dictated by the on-disk EBS statistics structure. +// +// Reference: Amazon EBS User Guide, "Amazon EBS detailed performance statistics" +// https://docs.aws.amazon.com/ebs/latest/userguide/nvme-detailed-performance-stats.html +type ebsMetrics struct { + EBSMagic uint64 + ReadOps uint64 // total_read_ops + WriteOps uint64 // total_write_ops + ReadBytes uint64 // total_read_bytes + WriteBytes uint64 // total_write_bytes + TotalReadTime uint64 // total_read_time (microseconds) + TotalWriteTime uint64 // total_write_time (microseconds) + EBSIOPSExceeded uint64 // ebs_volume_performance_exceeded_iops (microseconds) + EBSThroughputExceeded uint64 // ebs_volume_performance_exceeded_tp (microseconds) + EC2IOPSExceeded uint64 // ec2_instance_ebs_performance_exceeded_iops (microseconds) + EC2ThroughputExceeded uint64 // ec2_instance_ebs_performance_exceeded_tp (microseconds) + QueueLength uint64 // volume_queue_length (point in time) + ReservedArea [416]byte + ReadLatency ebsHistogram // read_io_latency_histogram + WriteLatency ebsHistogram // write_io_latency_histogram +} + +// ebsHistogram is the latency histogram embedded in the EBS log page. +type ebsHistogram struct { + BinCount uint64 + Bins [64]ebsHistogramBin +} + +// ebsHistogramBin is a single latency bucket of an ebsHistogram. Bounds are in +// microseconds. +type ebsHistogramBin struct { + Lower uint64 + Upper uint64 + Count uint64 +} + +// nvmePassthruCommand mirrors struct nvme_passthru_cmd from . +type nvmePassthruCommand struct { + opcode uint8 + flags uint8 + rsvd1 uint16 + nsid uint32 + cdw2 uint32 + cdw3 uint32 + metadata uint64 + addr uint64 + metadataLen uint32 + dataLen uint32 + cdw10 uint32 + cdw11 uint32 + cdw12 uint32 + cdw13 uint32 + cdw14 uint32 + cdw15 uint32 + timeoutMs uint32 + result uint32 +} + +// ebsNVMeCollector exposes Amazon EBS volume performance statistics read from +// the EBS NVMe device log page. +type ebsNVMeCollector struct { + logger *slog.Logger + metrics map[string]*prometheus.Desc +} + +func init() { + registerCollector("ebsnvme", defaultDisabled, NewEBSNVMeCollector) +} + +// NewEBSNVMeCollector returns a new Collector exposing Amazon EBS NVMe +// performance statistics. +func NewEBSNVMeCollector(logger *slog.Logger) (Collector, error) { + labels := []string{"volume_id", "device", "mount_path"} + desc := func(name, help string) *prometheus.Desc { + return prometheus.NewDesc(prometheus.BuildFQName(namespace, ebsNVMeSubsystem, name), help, labels, nil) + } + + return &ebsNVMeCollector{ + logger: logger, + metrics: map[string]*prometheus.Desc{ + "read_ops_total": desc("read_ops_total", "The total number of completed read operations. (EBS statistic: total_read_ops)"), + "write_ops_total": desc("write_ops_total", "The total number of completed write operations. (EBS statistic: total_write_ops)"), + "read_bytes_total": desc("read_bytes_total", "The total number of read bytes transferred. (EBS statistic: total_read_bytes)"), + "write_bytes_total": desc("write_bytes_total", "The total number of write bytes transferred. (EBS statistic: total_write_bytes)"), + "read_seconds_total": desc("read_seconds_total", "The total time spent, in seconds, by all completed read operations. (EBS statistic: total_read_time)"), + "write_seconds_total": desc("write_seconds_total", "The total time spent, in seconds, by all completed write operations. (EBS statistic: total_write_time)"), + "exceeded_iops_seconds_total": desc("exceeded_iops_seconds_total", "The total time, in seconds, that IOPS demand exceeded the volume's provisioned IOPS performance. (EBS statistic: ebs_volume_performance_exceeded_iops)"), + "exceeded_tp_seconds_total": desc("exceeded_tp_seconds_total", "The total time, in seconds, that throughput demand exceeded the volume's provisioned throughput performance. (EBS statistic: ebs_volume_performance_exceeded_tp)"), + "ec2_exceeded_iops_seconds_total": desc("ec2_exceeded_iops_seconds_total", "The total time, in seconds, that the EBS volume exceeded the attached EC2 instance's maximum IOPS performance. (EBS statistic: ec2_instance_ebs_performance_exceeded_iops)"), + "ec2_exceeded_tp_seconds_total": desc("ec2_exceeded_tp_seconds_total", "The total time, in seconds, that the EBS volume exceeded the attached EC2 instance's maximum throughput performance. (EBS statistic: ec2_instance_ebs_performance_exceeded_tp)"), + "volume_queue_length": desc("volume_queue_length", "The number of read and write operations waiting to be completed. (EBS statistic: volume_queue_length)"), + "read_io_latency_seconds": desc("read_io_latency_seconds", "The number of read operations completed within each latency bin, in seconds. (EBS statistic: read_io_latency_histogram)"), + "write_io_latency_seconds": desc("write_io_latency_seconds", "The number of write operations completed within each latency bin, in seconds. (EBS statistic: write_io_latency_histogram)"), + }, + }, nil +} + +// Update reads the EBS log page from every EBS NVMe device and emits the parsed +// metrics. Devices that are not EBS volumes (no valid EBS magic) are skipped. +func (c *ebsNVMeCollector) Update(ch chan<- prometheus.Metric) error { + devices, err := fetchDeviceMapping() + if err != nil { + return fmt.Errorf("error mapping NVMe devices to EBS volumes: %w", err) + } + + found := false + for devicePath, info := range devices { + data, err := readEBSLogPage(devicePath) + if err != nil { + c.logger.Debug("skipping device", "device", devicePath, "err", err) + continue + } + + metrics, err := parseEBSLogPage(data) + if err != nil { + // Not an EBS volume (or an unexpected layout); skip silently at debug. + c.logger.Debug("skipping non-EBS device", "device", devicePath, "err", err) + continue + } + found = true + + // Drop the /dev/ prefix to align with the diskstats collector. + device := strings.TrimPrefix(devicePath, "/dev/") + volumeID := info.volumeID + mountPath := info.mountPath + + emit := func(name string, vt prometheus.ValueType, v float64) { + ch <- prometheus.MustNewConstMetric(c.metrics[name], vt, v, volumeID, device, mountPath) + } + + emit("read_ops_total", prometheus.CounterValue, float64(metrics.ReadOps)) + emit("write_ops_total", prometheus.CounterValue, float64(metrics.WriteOps)) + emit("read_bytes_total", prometheus.CounterValue, float64(metrics.ReadBytes)) + emit("write_bytes_total", prometheus.CounterValue, float64(metrics.WriteBytes)) + emit("read_seconds_total", prometheus.CounterValue, float64(metrics.TotalReadTime)/microsecondsInSeconds) + emit("write_seconds_total", prometheus.CounterValue, float64(metrics.TotalWriteTime)/microsecondsInSeconds) + emit("exceeded_iops_seconds_total", prometheus.CounterValue, float64(metrics.EBSIOPSExceeded)/microsecondsInSeconds) + emit("exceeded_tp_seconds_total", prometheus.CounterValue, float64(metrics.EBSThroughputExceeded)/microsecondsInSeconds) + emit("ec2_exceeded_iops_seconds_total", prometheus.CounterValue, float64(metrics.EC2IOPSExceeded)/microsecondsInSeconds) + emit("ec2_exceeded_tp_seconds_total", prometheus.CounterValue, float64(metrics.EC2ThroughputExceeded)/microsecondsInSeconds) + emit("volume_queue_length", prometheus.GaugeValue, float64(metrics.QueueLength)) + + readCount, readBuckets := convertEBSHistogram(metrics.ReadLatency) + ch <- prometheus.MustNewConstHistogram(c.metrics["read_io_latency_seconds"], readCount, 0, readBuckets, volumeID, device, mountPath) + + writeCount, writeBuckets := convertEBSHistogram(metrics.WriteLatency) + ch <- prometheus.MustNewConstHistogram(c.metrics["write_io_latency_seconds"], writeCount, 0, writeBuckets, volumeID, device, mountPath) + } + + if !found { + return ErrNoData + } + return nil +} + +// convertEBSHistogram converts an ebsHistogram (bounds in microseconds) into the +// cumulative bucket form expected by prometheus.MustNewConstHistogram (seconds). +func convertEBSHistogram(hist ebsHistogram) (uint64, map[float64]uint64) { + var count uint64 + buckets := make(map[float64]uint64) + + for i := uint64(0); i < hist.BinCount && i < uint64(len(hist.Bins)); i++ { + count += hist.Bins[i].Count + buckets[float64(hist.Bins[i].Upper)/microsecondsInSeconds] = count + } + + return count, buckets +} + +// readEBSLogPage reads the EBS statistics log page from the NVMe device at the +// given path. The device is opened read-only: a write handle is not required to +// issue the NVMe admin ioctl on Linux (matching the AWS EBS CSI driver). +func readEBSLogPage(devicePath string) ([]byte, error) { + f, err := os.OpenFile(devicePath, os.O_RDONLY, 0) + if err != nil { + return nil, fmt.Errorf("error opening device %s: %w", devicePath, err) + } + defer f.Close() + + return nvmeReadLogPage(f.Fd(), ebsLogPageID) +} + +// nvmeReadLogPage reads an NVMe log page via an ioctl system call. +func nvmeReadLogPage(fd uintptr, logID uint8) ([]byte, error) { + data := make([]byte, nvmeLogPageSize) + if len(data) > math.MaxUint32 { + return nil, errors.New("nvmeReadLogPage: bufferLen exceeds MaxUint32") + } + + cmd := nvmePassthruCommand{ + opcode: nvmeAdminGetLogPage, + addr: uint64(uintptr(unsafe.Pointer(&data[0]))), + nsid: 1, + dataLen: uint32(len(data)), + cdw10: uint32(logID) | (1024 << 16), + } + + status, _, errno := unix.Syscall(unix.SYS_IOCTL, fd, nvmeIoctlAdminCmd, uintptr(unsafe.Pointer(&cmd))) + if errno != 0 { + return nil, fmt.Errorf("nvmeReadLogPage: ioctl error: %w", errno) + } + if status != 0 { + return nil, fmt.Errorf("nvmeReadLogPage: ioctl command failed with status %d", status) + } + + return data, nil +} + +// parseEBSLogPage parses the binary data from an EBS log page into ebsMetrics. +func parseEBSLogPage(data []byte) (ebsMetrics, error) { + var metrics ebsMetrics + + if err := binary.Read(bytes.NewReader(data), binary.LittleEndian, &metrics); err != nil { + return ebsMetrics{}, fmt.Errorf("%w: %w", errParseLogPage, err) + } + + if metrics.EBSMagic != ebsMagic { + return ebsMetrics{}, fmt.Errorf("%w: %x", errInvalidEBSMagic, metrics.EBSMagic) + } + + return metrics, nil +} + +// blockDevice is a single entry of lsblk JSON output. +type blockDevice struct { + Name string `json:"name"` + Serial string `json:"serial"` + MountPoint *string `json:"mountpoint"` +} + +// lsblkOutput is the top-level lsblk JSON document. +type lsblkOutput struct { + BlockDevices []blockDevice `json:"blockdevices"` +} + +// ebsDeviceInfo holds the per-device attributes exported as metric labels. +type ebsDeviceInfo struct { + volumeID string + mountPath string +} + +// mapDevices parses lsblk output and returns a map of device paths to their EBS +// volume ID and mount path. A device with no mount point (lsblk reports null, +// e.g. an unmounted disk or one mounted only through a partition) is reported +// with the mount path set to notMounted. +func mapDevices(raw []byte) (map[string]ebsDeviceInfo, error) { + var parsed lsblkOutput + if err := json.Unmarshal(raw, &parsed); err != nil { + return nil, fmt.Errorf("mapDevices: error unmarshaling lsblk JSON: %w", err) + } + + m := make(map[string]ebsDeviceInfo, len(parsed.BlockDevices)) + for _, device := range parsed.BlockDevices { + volumeID := device.Serial + // EBS exposes the volume ID as the NVMe serial, e.g. "vol0abc..."; AWS + // tooling renders it as "vol-0abc...". + if strings.HasPrefix(volumeID, "vol") && !strings.HasPrefix(volumeID, "vol-") { + volumeID = "vol-" + volumeID[3:] + } + + mountPath := notMounted + if device.MountPoint != nil && *device.MountPoint != "" { + mountPath = *device.MountPoint + } + + m["/dev/"+device.Name] = ebsDeviceInfo{volumeID: volumeID, mountPath: mountPath} + } + + return m, nil +} + +// fetchDeviceMapping returns a map of device paths to their EBS volume ID and +// mount path, derived from lsblk. +func fetchDeviceMapping() (map[string]ebsDeviceInfo, error) { + output, err := exec.Command("lsblk", "-nd", "--json", "-o", "NAME,SERIAL,MOUNTPOINT").Output() + if err != nil { + return nil, fmt.Errorf("fetchDeviceMapping: error running lsblk: %w", err) + } + + return mapDevices(output) +} diff --git a/collector/ebsnvme_linux_test.go b/collector/ebsnvme_linux_test.go new file mode 100644 index 0000000000..5f9b0934ce --- /dev/null +++ b/collector/ebsnvme_linux_test.go @@ -0,0 +1,138 @@ +// Copyright 2024 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux && !noebsnvme +// +build linux,!noebsnvme + +package collector + +import ( + "bytes" + "encoding/binary" + "errors" + "testing" +) + +func TestMapDevices(t *testing.T) { + // lsblk -nd --json -o NAME,SERIAL,MOUNTPOINT style output, including: + // - a mounted EBS volume (serial without dash) + // - an unmounted EBS volume (mountpoint null) -> NotMounted + // - an empty-string mountpoint -> NotMounted + // - a non-EBS device (serial does not start with "vol") + in := []byte(`{ + "blockdevices": [ + {"name": "nvme1n1", "serial": "vol0e9d0ec0c3c24e723", "mountpoint": "/mysql/undo"}, + {"name": "nvme0n1", "serial": "vol08a138e8d1cfb88df", "mountpoint": null}, + {"name": "nvme2n1", "serial": "vol0cd972338c0a5f21b", "mountpoint": ""}, + {"name": "nvme9n1", "serial": "AWS-LOCAL-1234", "mountpoint": "/scratch"} + ] + }`) + + got, err := mapDevices(in) + if err != nil { + t.Fatalf("mapDevices returned error: %v", err) + } + + want := map[string]ebsDeviceInfo{ + "/dev/nvme1n1": {volumeID: "vol-0e9d0ec0c3c24e723", mountPath: "/mysql/undo"}, + "/dev/nvme0n1": {volumeID: "vol-08a138e8d1cfb88df", mountPath: notMounted}, + "/dev/nvme2n1": {volumeID: "vol-0cd972338c0a5f21b", mountPath: notMounted}, + "/dev/nvme9n1": {volumeID: "AWS-LOCAL-1234", mountPath: "/scratch"}, + } + + if len(got) != len(want) { + t.Fatalf("got %d devices, want %d", len(got), len(want)) + } + for path, w := range want { + g, ok := got[path] + if !ok { + t.Errorf("missing device %s", path) + continue + } + if g != w { + t.Errorf("device %s: got %+v, want %+v", path, g, w) + } + } +} + +func TestMapDevicesInvalidJSON(t *testing.T) { + if _, err := mapDevices([]byte("not json")); err == nil { + t.Fatal("expected error for invalid JSON, got nil") + } +} + +func TestParseEBSLogPageInvalidMagic(t *testing.T) { + data := make([]byte, nvmeLogPageSize) + binary.LittleEndian.PutUint64(data[0:8], 0xDEADBEEF) + + _, err := parseEBSLogPage(data) + if !errors.Is(err, errInvalidEBSMagic) { + t.Fatalf("expected errInvalidEBSMagic, got %v", err) + } +} + +func TestParseEBSLogPageValid(t *testing.T) { + var m ebsMetrics + m.EBSMagic = ebsMagic + m.ReadOps = 100 + m.WriteOps = 200 + m.ReadBytes = 4096 * 100 + m.WriteBytes = 16384 * 200 + m.TotalReadTime = 5_000_000 // 5s in microseconds + m.TotalWriteTime = 2_000_000 // 2s in microseconds + m.QueueLength = 3 + + var buf bytes.Buffer + if err := binary.Write(&buf, binary.LittleEndian, &m); err != nil { + t.Fatalf("failed to encode test log page: %v", err) + } + // The EBS metrics struct is smaller than the full NVMe log page; the device + // returns nvmeLogPageSize bytes with the trailing space unused. Pad the test + // buffer to mirror that layout. + page := make([]byte, nvmeLogPageSize) + copy(page, buf.Bytes()) + + got, err := parseEBSLogPage(page) + if err != nil { + t.Fatalf("parseEBSLogPage returned error: %v", err) + } + if got.ReadOps != 100 || got.WriteOps != 200 || got.QueueLength != 3 { + t.Errorf("unexpected parsed metrics: %+v", got) + } +} + +func TestConvertEBSHistogram(t *testing.T) { + var h ebsHistogram + h.BinCount = 3 + // Upper bounds in microseconds; counts are per-bin (non-cumulative). + h.Bins[0] = ebsHistogramBin{Lower: 0, Upper: 100, Count: 10} + h.Bins[1] = ebsHistogramBin{Lower: 100, Upper: 500, Count: 5} + h.Bins[2] = ebsHistogramBin{Lower: 500, Upper: 1000, Count: 2} + + count, buckets := convertEBSHistogram(h) + + if count != 17 { + t.Errorf("total count = %d, want 17", count) + } + // Buckets are cumulative, keyed by upper bound converted to seconds. + wantCumulative := map[float64]uint64{ + 100.0 / microsecondsInSeconds: 10, + 500.0 / microsecondsInSeconds: 15, + 1000.0 / microsecondsInSeconds: 17, + } + for ub, wc := range wantCumulative { + if buckets[ub] != wc { + t.Errorf("bucket le=%g: got %d, want %d", ub, buckets[ub], wc) + } + } +} From 4bd40b1ea0ae15767e8633f037d062175435194d Mon Sep 17 00:00:00 2001 From: Allen Xie Date: Sun, 21 Jun 2026 14:11:48 +0000 Subject: [PATCH 3/3] collector: document ebsnvme CAP_SYS_ADMIN/root requirement Opening the NVMe character device and issuing the admin passthru ioctl requires CAP_SYS_ADMIN (in practice, root). Note this in the package doc comment and the README so users running node_exporter as an unprivileged user understand why no node_ebs_* metrics appear. Signed-off-by: Allen Xie --- README.md | 2 +- collector/ebsnvme_linux.go | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8e20478359..deb9c6df5d 100644 --- a/README.md +++ b/README.md @@ -194,7 +194,7 @@ cpu\_vulnerabilities | Exposes CPU vulnerability information from sysfs. | Linux devstat | Exposes device statistics | Dragonfly, FreeBSD drm | Expose GPU metrics using sysfs / DRM, `amdgpu` is the only driver which exposes this information through DRM | Linux drbd | Exposes Distributed Replicated Block Device statistics (to version 8.4) | Linux -ebsnvme | Exposes [Amazon EBS detailed performance statistics](https://docs.aws.amazon.com/ebs/latest/userguide/nvme-detailed-performance-stats.html) read from the EBS NVMe device log page (IOPS, throughput, latency histograms, queue length), labelled by `volume_id`, `device`, and `mount_path`. Requires running on a Nitro-based EC2 instance. | Linux +ebsnvme | Exposes [Amazon EBS detailed performance statistics](https://docs.aws.amazon.com/ebs/latest/userguide/nvme-detailed-performance-stats.html) read from the EBS NVMe device log page (IOPS, throughput, latency histograms, queue length), labelled by `volume_id`, `device`, and `mount_path`. Requires running on a Nitro-based EC2 instance with `CAP_SYS_ADMIN` (typically as root) to issue the NVMe admin ioctl. | Linux ethtool | Exposes network interface information and network driver statistics equivalent to `ethtool`, `ethtool -S`, and `ethtool -i`. | Linux interrupts | Exposes detailed interrupts statistics. | Linux, OpenBSD ksmd | Exposes kernel and system statistics from `/sys/kernel/mm/ksm`. | Linux diff --git a/collector/ebsnvme_linux.go b/collector/ebsnvme_linux.go index 1d55179016..ef7258ef92 100644 --- a/collector/ebsnvme_linux.go +++ b/collector/ebsnvme_linux.go @@ -35,6 +35,11 @@ // The collector reads NVMe log page 0xD0 from each EBS-backed NVMe device via // an ioctl, parses the binary EBS statistics structure, and exposes the values // as Prometheus metrics labelled by volume_id, device, and mount_path. +// +// Privileges: opening the NVMe character device and issuing the admin +// passthru ioctl requires CAP_SYS_ADMIN (in practice, running as root). When +// node_exporter runs as an unprivileged user, the device cannot be opened and +// the collector returns no metrics. package collector