From f9dc35c88ea8413ed124fcbf483de7538a6050dc Mon Sep 17 00:00:00 2001 From: Vader Yang Date: Mon, 22 Jun 2026 10:22:46 +0800 Subject: [PATCH 1/2] fix(ebpf): detach uprobe links for dead inodes to stop bpf_link fd leak MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The static-target uprobe path attached probes via `program.attach()` and discarded the returned link id. In aya the link (and its `bpf_link` fd) stays owned inside the program's link map until it is explicitly detached, so every attached inode held its fds for the whole process lifetime. `rescan_targets` keeps discovering new inodes as npm-style auto-updates rotate the on-disk binary and as sessions come and go, but nothing ever released probes for inodes whose processes had all exited. The attachments — and their fds — therefore grew monotonically (`ebpf_uprobes_attached` only ever climbed, far past the live working set), eventually crossing the process fd limit and starving the API listener's accept(), so the HTTP port stopped responding. Fix: track the (program, link_id) pairs per attached inode and, on each rescan, detach the probes for any inode no longer backed by an on-disk path or a running target process. Detaching drops the link and closes its fd. The health gauge now falls on detach instead of ratcheting up, and `seen` is pruned to the live set so a reused inode number can be re-attached. Partial-attach failures also unwind their links instead of orphaning fds. --- server/h-capture/src/ebpf/source.rs | 142 +++++++++++++++++++++------- 1 file changed, 109 insertions(+), 33 deletions(-) diff --git a/server/h-capture/src/ebpf/source.rs b/server/h-capture/src/ebpf/source.rs index 21d9c8bf..e3b0571a 100644 --- a/server/h-capture/src/ebpf/source.rs +++ b/server/h-capture/src/ebpf/source.rs @@ -22,6 +22,7 @@ use tokio::io::unix::AsyncFd; use tokio_util::sync::CancellationToken; use aya::maps::RingBuf; +use aya::programs::uprobe::UProbeLinkId; use aya::programs::UProbe; use aya::Ebpf; @@ -37,6 +38,14 @@ use crate::routing::RoutingSender; use crate::source::CaptureSource; use crate::synth::{StreamDir, SynthConfig}; +/// The uprobe links installed on one target inode: the `(program_name, +/// link_id)` pairs returned by each [`UProbe::attach`]. We keep the ids so the +/// links can be **detached** when the inode goes dead — aya owns the link (and +/// its `bpf_link` fd) inside the program's link map until `detach`/`take_link` +/// removes it, so dropping the id at attach time would leak the fd for the +/// process's whole lifetime. +type InodeLinks = Vec<(&'static str, UProbeLinkId)>; + /// eBPF SSL-uprobe capture source. pub struct EbpfSource { source_id: String, @@ -171,9 +180,10 @@ impl CaptureSource for EbpfSource { let libs_count = libs.len(); // `seen` = every inode we've attempted (success or deterministic sig // failure) so we neither re-attach nor re-warn on each rescan; `attached` - // = inodes actually carrying probes (drives the health gauge). + // = inodes actually carrying probes, mapped to the link ids that hold + // their `bpf_link` fds (drives the health gauge and the dead-inode GC). let mut seen_inodes: HashSet<(u64, u64)> = HashSet::new(); - let mut attached_inodes: HashSet<(u64, u64)> = HashSet::new(); + let mut attached_inodes: HashMap<(u64, u64), InodeLinks> = HashMap::new(); rescan_targets( &mut ebpf, &self.targets, @@ -244,18 +254,20 @@ impl CaptureSource for EbpfSource { metrics.counter(Metric::CaptureHeartbeatsEmitted).inc(); } _ = rescan.tick() => { - let newly = rescan_targets( + let (newly, detached) = rescan_targets( &mut ebpf, &self.targets, &mut seen_inodes, &mut attached_inodes, ); - if newly > 0 { + if newly > 0 || detached > 0 { tracing::info!( - "ebpf: rescan attached {newly} new target inode(s) \ - (total target inodes={})", + "ebpf: rescan attached {newly} new, detached {detached} dead \ + target inode(s) (live target inodes={})", attached_inodes.len() ); + // Reflect both attaches and detaches so the gauge tracks + // the live set (it must fall, not ratchet up forever). metrics .counter(Metric::EbpfUprobesAttached) .set((libs_count + attached_inodes.len()) as u64); @@ -420,14 +432,17 @@ fn attach_sym( } /// Attach an already-loaded program to a static binary at a resolved **file -/// offset** (no symbol). The path for the Bun/BoringSSL Phase-3 case. +/// offset** (no symbol). The path for the Bun/BoringSSL Phase-3 case. Returns +/// the [`UProbeLinkId`] so the caller can detach this probe (and free its +/// `bpf_link` fd) once the inode goes dead — aya keeps the link alive in the +/// program's link map until it's explicitly detached. fn attach_offset( ebpf: &mut Ebpf, prog_name: &str, offset: u64, binary: &std::path::Path, ret: bool, -) -> crate::Result<()> { +) -> crate::Result { let program: &mut UProbe = ebpf .program_mut(prog_name) .ok_or_else(|| crate::CaptureError::Other(format!("program {prog_name} missing")))? @@ -440,8 +455,25 @@ fn attach_offset( "attach {prog_name} at offset {offset:#x} in {} (ret={ret}): {e}", binary.display() )) - })?; - Ok(()) + }) +} + +/// Detach a set of uprobe links, freeing each one's `bpf_link` fd. Best-effort: +/// a program that vanished or a link already gone is logged and skipped. Used +/// both for dead-inode GC and to unwind a partially-attached inode on error. +fn detach_links(ebpf: &mut Ebpf, links: InodeLinks) { + for (prog_name, link_id) in links { + let Some(program) = ebpf.program_mut(prog_name) else { + continue; + }; + let uprobe: &mut UProbe = match program.try_into() { + Ok(p) => p, + Err(_) => continue, + }; + if let Err(e) = uprobe.detach(link_id) { + tracing::warn!("ebpf: detach {prog_name} link failed: {e}"); + } + } } /// Built-in BoringSSL prologue signatures for a flavor, using the anchor + window @@ -654,42 +686,63 @@ fn resolve_target_offsets( /// Attach the (already-loaded) SSL uprobes to `attach_path` at the resolved /// offsets. `attach_path` may be the on-disk binary or a `/proc//exe` /// handle to a deleted inode — the kernel resolves both to the same inode and -/// installs the probe on it. Returns `Ok(true)` if at least one probe attached. +/// installs the probe on it. Returns the link ids of the probes attached (empty +/// if neither offset resolved). On a partial failure it detaches whatever did +/// attach before returning the error, so a half-attached inode never orphans a +/// `bpf_link` fd outside the caller's tracking. fn attach_at( ebpf: &mut Ebpf, attach_path: &Path, read_off: Option, write_off: Option, -) -> crate::Result { - let mut any = false; - if let Some(off) = write_off { - attach_offset(ebpf, "ssl_write", off, attach_path, false)?; - any = true; - } - if let Some(off) = read_off { - // Entry probe captures the buffer pointer; the return probe reads the - // bytes SSL_read filled in. Both attach at the function entry. - attach_offset(ebpf, "ssl_read_enter", off, attach_path, false)?; - attach_offset(ebpf, "ssl_read_exit", off, attach_path, true)?; - any = true; +) -> crate::Result { + let mut links: InodeLinks = Vec::new(); + // write first, then the read entry/exit pair — same order as before. The + // entry probe captures the buffer pointer; the return probe reads the bytes + // SSL_read filled in. Both reads attach at the function entry offset. + let probes: [(&'static str, Option, bool); 3] = [ + ("ssl_write", write_off, false), + ("ssl_read_enter", read_off, false), + ("ssl_read_exit", read_off, true), + ]; + for (name, off, ret) in probes { + let Some(off) = off else { continue }; + match attach_offset(ebpf, name, off, attach_path, ret) { + Ok(id) => links.push((name, id)), + Err(e) => { + detach_links(ebpf, std::mem::take(&mut links)); + return Err(e); + } + } } - Ok(any) + Ok(links) } /// Attach SSL uprobes to every distinct inode backing the configured static /// targets, covering both the on-disk install path (future fresh execs of the /// current build) and every running session via `/proc//exe` (already- /// running sessions on a deleted inode after an auto-update). Idempotent across -/// calls: `seen` skips any inode already attempted, `attached` records inodes -/// actually carrying probes. Returns the count of inodes newly attached this -/// pass. Never propagates errors — a bad target must not take down capture. +/// calls: `seen` skips any inode already attempted, `attached` maps each inode +/// carrying probes to its link ids. +/// +/// Also **garbage-collects dead inodes**: an inode that's no longer backed by +/// any on-disk path or running target process has its uprobes detached, freeing +/// their `bpf_link` fds. Without this the attachments — and their fds — pile up +/// monotonically as npm-style auto-updates rotate inodes and sessions come and +/// go, eventually exhausting the process fd limit and starving the API listener. +/// +/// Returns `(newly_attached, detached)` inode counts. Never propagates errors — +/// a bad target must not take down capture. fn rescan_targets( ebpf: &mut Ebpf, targets: &[EbpfTarget], seen: &mut HashSet<(u64, u64)>, - attached: &mut HashSet<(u64, u64)>, -) -> usize { + attached: &mut HashMap<(u64, u64), InodeLinks>, +) -> (usize, usize) { let mut newly = 0; + // Every inode currently backed by an on-disk path or a live target process. + // Anything in `attached`/`seen` but absent here is dead and gets reaped. + let mut live: HashSet<(u64, u64)> = HashSet::new(); for target in targets { if !target_has_source(target) { continue; // warned once at startup @@ -716,6 +769,9 @@ fn rescan_targets( let Some(ino) = inode_of(&path) else { continue; // process exited mid-scan; retry next pass }; + // Resolvable inode ⇒ live this pass; protects it from GC below even + // when it was attached on an earlier pass and is skipped by `seen`. + live.insert(ino); if seen.contains(&ino) { continue; } @@ -732,8 +788,8 @@ fn rescan_targets( seen.insert(ino); let (read_off, write_off) = resolve_target_offsets(&data, target, &target.binary); match attach_at(ebpf, &path, read_off, write_off) { - Ok(true) => { - attached.insert(ino); + Ok(links) if !links.is_empty() => { + attached.insert(ino, links); newly += 1; tracing::info!( "ebpf: attached {} via {} (inode {}:{})", @@ -743,7 +799,7 @@ fn rescan_targets( ino.1 ); } - Ok(false) => tracing::warn!( + Ok(_) => tracing::warn!( "ebpf: {} via {}: no usable SSL offset (wrong signature/flavor?)", target.binary, path.display() @@ -756,7 +812,27 @@ fn rescan_targets( } } } - newly + + // GC dead inodes: detach their uprobes (frees the bpf_link fds) and drop + // them from `seen` so the same inode number reused by a future build is + // re-attached rather than silently skipped. + let dead: Vec<(u64, u64)> = attached + .keys() + .filter(|ino| !live.contains(ino)) + .copied() + .collect(); + let detached = dead.len(); + for ino in dead { + if let Some(links) = attached.remove(&ino) { + tracing::info!("ebpf: detaching dead target inode {}:{}", ino.0, ino.1); + detach_links(ebpf, links); + } + } + // Keep `seen` bounded to the live set (+ still-attached inodes): a dead inode + // won't reappear, and pruning lets a reused inode number be re-attempted. + seen.retain(|ino| live.contains(ino) || attached.contains_key(ino)); + + (newly, detached) } /// Discover `libssl` shared objects on the host. Tries `ldconfig -p` first, then From b49901cb54d964abf1eae1b5d394d5c2ab0e1ed6 Mon Sep 17 00:00:00 2001 From: Vader Yang Date: Mon, 22 Jun 2026 10:23:40 +0800 Subject: [PATCH 2/2] deploy(staging): set LimitNOFILE=65536 as fd-exhaustion headroom The unit inherited systemd's default soft limit of 1024. With the uprobe fd leak fixed the steady-state fd count is small, but a generous ceiling is cheap defense-in-depth: any future fd regression then surfaces as a logged warning with hours of margin rather than a silently dead API port once accept() can no longer get an fd. The prod unit is provisioned on the host (not tracked here) and needs the same LimitNOFILE applied there. --- scripts/staging/heron.service | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/staging/heron.service b/scripts/staging/heron.service index 6852b383..8a90b0d6 100644 --- a/scripts/staging/heron.service +++ b/scripts/staging/heron.service @@ -26,6 +26,13 @@ WorkingDirectory=/opt/heron AmbientCapabilities=CAP_NET_RAW CAP_NET_ADMIN CAP_BPF CAP_PERFMON CAP_SYS_ADMIN CapabilityBoundingSet=CAP_NET_RAW CAP_NET_ADMIN CAP_BPF CAP_PERFMON CAP_SYS_ADMIN NoNewPrivileges=true +# Defense-in-depth fd headroom. The capture pipeline holds a bounded set of +# fds (capture sockets, a few bpf_link/map fds per live target inode, DB +# files, plus the API listener's accepted connections), so the systemd default +# soft limit of 1024 is the floor at which any fd regression starves the API +# listener's accept(). A generous ceiling turns a slow fd leak into a logged +# warning with hours of margin instead of a dead HTTP port. +LimitNOFILE=65536 Restart=on-failure RestartSec=5