diff --git a/manager/manager.go b/manager/manager.go index 57df7e9dd..b972a2c51 100644 --- a/manager/manager.go +++ b/manager/manager.go @@ -1526,7 +1526,7 @@ func cleanupKprobeEvents(pattern *regexp.Regexp, pidMask map[int]procMask) error } // remove the entry - cleanUpErrors = multierror.Append(cleanUpErrors, disableKprobeEvent(match[3])) + cleanUpErrors = multierror.Append(cleanUpErrors, unregisterKprobeEventWithEventName(match[3])) } return cleanUpErrors } @@ -1567,7 +1567,7 @@ func cleanupUprobeEvents(pattern *regexp.Regexp, pidMask map[int]procMask) error } // remove the entry - cleanUpErrors = multierror.Append(cleanUpErrors, disableUprobeEvent(match[3])) + cleanUpErrors = multierror.Append(cleanUpErrors, unregisterUprobeEventWithEventName(match[3])) } return cleanUpErrors } diff --git a/manager/probe.go b/manager/probe.go index a3632deae..699b55898 100644 --- a/manager/probe.go +++ b/manager/probe.go @@ -63,16 +63,17 @@ func (pip ProbeIdentificationPair) Matches(id ProbeIdentificationPair) bool { // Probe - Main eBPF probe wrapper. This structure is used to store the required data to attach a loaded eBPF // program to its hook point. type Probe struct { - manager *Manager - program *ebpf.Program - programSpec *ebpf.ProgramSpec - perfEventFD *internal.FD - state state - stateLock sync.RWMutex - manualLoadNeeded bool - checkPin bool - funcName string - attachPID int + manager *Manager + program *ebpf.Program + programSpec *ebpf.ProgramSpec + perfEventFD *internal.FD + state state + stateLock sync.RWMutex + manualLoadNeeded bool + checkPin bool + funcName string + attachPID int + attachedWithDebugFS bool // lastError - stores the last error that the probe encountered, it is used to surface a more useful error message // when one of the validators (see Options.ActivatedProbes) fails. @@ -532,15 +533,25 @@ func (p *Probe) reset() { p.checkPin = false p.funcName = "" p.attachPID = 0 + p.attachedWithDebugFS = false } // attachKprobe - Attaches the probe to its kprobe func (p *Probe) attachKprobe() error { // Prepare kprobe_events line parameters - var probeType, maxactiveStr string + var probeType, maxactiveStr, sectionPrefix string var err error funcName := p.funcName - if strings.HasPrefix(p.Section, "kretprobe/") { + + dividedSection := strings.Split(p.Section, "/") + if len(dividedSection) >= 1 { + sectionPrefix = dividedSection[0] + } else { + return errors.Wrapf(ErrSectionFormat, "program type unrecognized in section %v", p.Section) + } + + switch sectionPrefix { + case "kretprobe": if funcName == "" { funcName = strings.TrimPrefix(p.Section, "kretprobe/") } @@ -548,30 +559,42 @@ func (p *Probe) attachKprobe() error { maxactiveStr = fmt.Sprintf("%d", p.KProbeMaxActive) } probeType = "r" - } else if strings.HasPrefix(p.Section, "kprobe/") { + case "kprobe": if funcName == "" { funcName = strings.TrimPrefix(p.Section, "kprobe/") } probeType = "p" - } else { + default: // this might actually be a Uprobe return p.attachUprobe() } p.attachPID = os.Getpid() - // Write kprobe_events line to register kprobe - kprobeID, err := EnableKprobeEvent(probeType, funcName, p.UID, maxactiveStr, p.attachPID) - // fallback without KProbeMaxActive - if err == ErrKprobeIDNotExist { - kprobeID, err = EnableKprobeEvent(probeType, funcName, p.UID, "", p.attachPID) - } + // Try to use the perf_event_open API first (e12f03d "perf/core: Implement the 'perf_kprobe' PMU") + perfEventFD, err := perfEventOpenWithProbe(funcName, 0, -1, sectionPrefix, 0) if err != nil { - return errors.Wrapf(err, "couldn't enable kprobe %s", p.Section) + // Try to create the event using debugfs + // Write kprobe_events line to register kprobe + kprobeID, err := registerKprobeEvent(probeType, funcName, p.UID, maxactiveStr, p.attachPID) + // fallback without KProbeMaxActive + if err == ErrKprobeIDNotExist { + kprobeID, err = registerKprobeEvent(probeType, funcName, p.UID, "", p.attachPID) + } + if err != nil { + return errors.Wrapf(err, "couldn't enable kprobe %s", p.Section) + } + p.attachedWithDebugFS = true + + // create perf event FD + perfEventFD, err = perfEventOpenTracingEvent(kprobeID) } - // Activate perf event - p.perfEventFD, err = perfEventOpenTracepoint(kprobeID, p.program.FD()) - return errors.Wrapf(err, "couldn't enable kprobe %s", p.GetIdentificationPair()) + // enable program + if ioctlPerfEventEnable(perfEventFD, p.program.FD()) != nil { + return errors.Wrapf(err, "couldn't enable perf event %s", p.GetIdentificationPair()) + } + p.perfEventFD = internal.NewFD(uint32(perfEventFD)) + return nil } // detachKprobe - Detaches the probe from its kprobe @@ -594,8 +617,13 @@ func (p *Probe) detachKprobe() error { return p.detachUprobe() } + if !p.attachedWithDebugFS { + // nothing to do + return nil + } + // Write kprobe_events line to remove hook point - return DisableKprobeEvent(probeType, funcName, p.UID, p.attachPID) + return unregisterKprobeEvent(probeType, funcName, p.UID, p.attachPID) } // attachTracepoint - Attaches the probe to its tracepoint @@ -615,25 +643,40 @@ func (p *Probe) attachTracepoint() error { } // Hook the eBPF program to the tracepoint - p.perfEventFD, err = perfEventOpenTracepoint(tracepointID, p.program.FD()) - return errors.Wrapf(err, "couldn't enable tracepoint %s", p.GetIdentificationPair()) + perfEventFD, err := perfEventOpenTracingEvent(tracepointID) + if err != nil { + return errors.Wrapf(err, "couldn't enable tracepoint %s", p.GetIdentificationPair()) + } + if ioctlPerfEventEnable(perfEventFD, p.program.FD()) != nil { + return errors.Wrapf(err, "couldn't enable perf event %s", p.GetIdentificationPair()) + } + p.perfEventFD = internal.NewFD(uint32(perfEventFD)) + return nil } // attachUprobe - Attaches the probe to its Uprobe func (p *Probe) attachUprobe() error { - p.attachPID = os.Getpid() + var probeType, funcName, sectionPrefix string + dividedSection := strings.Split(p.Section, "/") + if len(dividedSection) >= 1 { + sectionPrefix = dividedSection[0] + } else { + return errors.Wrapf(ErrSectionFormat, "program type unrecognized in section %v", p.Section) + } + // Prepare uprobe_events line parameters - var probeType, funcName string - if strings.HasPrefix(p.Section, "uretprobe/") { + switch sectionPrefix { + case "uretprobe": funcName = strings.TrimPrefix(p.Section, "uretprobe/") probeType = "r" - } else if strings.HasPrefix(p.Section, "uprobe/") { + case "uprobe": funcName = strings.TrimPrefix(p.Section, "uprobe/") probeType = "p" - } else { + default: // unknown type return errors.Wrapf(ErrSectionFormat, "program type unrecognized in section %v", p.Section) } + p.attachPID = os.Getpid() // compute the offset if it was not provided if p.UprobeOffset == 0 { @@ -657,19 +700,35 @@ func (p *Probe) attachUprobe() error { p.funcName = offsets[0].Name } - // enable uprobe - uprobeID, err := EnableUprobeEvent(probeType, p.funcName, p.BinaryPath, p.UID, p.attachPID, p.UprobeOffset) + // Try to use the perf_event_open API first (e12f03d "perf/core: Implement the 'perf_kprobe' PMU") + perfEventFD, err := perfEventOpenWithProbe(p.BinaryPath, int(p.UprobeOffset), -1, sectionPrefix, 0) if err != nil { - return errors.Wrapf(err, "couldn't enable uprobe %s", p.Section) + // Try to create the event using debugfs + uprobeID, err := registerUprobeEvent(probeType, p.funcName, p.BinaryPath, p.UID, p.attachPID, p.UprobeOffset) + if err != nil { + return errors.Wrapf(err, "couldn't enable uprobe %s", p.Section) + } + + // Activate perf event + perfEventFD, err = perfEventOpenTracingEvent(uprobeID) + p.attachedWithDebugFS = true } - // Activate perf event - p.perfEventFD, err = perfEventOpenTracepoint(uprobeID, p.program.FD()) - return errors.Wrapf(err, "couldn't enable uprobe %s", p.GetIdentificationPair()) + // enable perf event + if ioctlPerfEventEnable(perfEventFD, p.program.FD()) != nil { + return errors.Wrapf(err, "couldn't enable perf event %s", p.GetIdentificationPair()) + } + p.perfEventFD = internal.NewFD(uint32(perfEventFD)) + return nil } // detachUprobe - Detaches the probe from its Uprobe func (p *Probe) detachUprobe() error { + if !p.attachedWithDebugFS { + // nothing to do + return nil + } + // Prepare uprobe_events line parameters var probeType string if strings.HasPrefix(p.Section, "uretprobe/") { @@ -682,7 +741,7 @@ func (p *Probe) detachUprobe() error { } // Write uprobe_events line to remove hook point - return DisableUprobeEvent(probeType, p.funcName, p.UID, p.attachPID) + return unregisterUprobeEvent(probeType, p.funcName, p.UID, p.attachPID) } // attachCGroup - Attaches the probe to a cgroup hook point diff --git a/manager/syscalls.go b/manager/syscalls.go index fb855bb2a..1c3310732 100644 --- a/manager/syscalls.go +++ b/manager/syscalls.go @@ -1,39 +1,94 @@ package manager import ( - "github.com/DataDog/ebpf" - "golang.org/x/sys/unix" "syscall" "unsafe" "github.com/pkg/errors" + "golang.org/x/sys/unix" + "github.com/DataDog/ebpf" "github.com/DataDog/ebpf/internal" ) -func perfEventOpenTracepoint(id int, progFd int) (*internal.FD, error) { +// perfEventOpenWithProbe - Kernel API with e12f03d ("perf/core: Implement the 'perf_kprobe' PMU") allows +// creating [k,u]probe with perf_event_open, which makes it easier to clean up +// the [k,u]probe. This function tries to create pfd with the perf_kprobe PMU. +func perfEventOpenWithProbe(name string, offset, pid int, sectionPrefix string, referenceCounterOffset uint64) (int, error) { + var err error + attr := unix.PerfEventAttr{ + Ext2: uint64(offset), // config2 here is kprobe_addr or probe_offset + } + attr.Size = uint32(unsafe.Sizeof(attr)) + + attr.Type, _ = FindPMUType(sectionPrefix) + //if err != nil { + // return 0, errors.Wrapf(err, "couldn't find PMU type for %s", sectionPrefix) + //} + + var returnBit uint32 + returnBit, _ = FindRetProbeBit(sectionPrefix) + //if err != nil { + // return 0, errors.Wrapf(err, "couldn't find retprobe bit for %s", sectionPrefix) + //} + if returnBit > 0 { + attr.Config = 1 << returnBit + } + if referenceCounterOffset > 0 { + attr.Config |= referenceCounterOffset << 32 + } + + namePtr, err := syscall.BytePtrFromString(name) + if err != nil { + return 0, errors.Wrapf(err, "couldn't create pointer to string %s", name) + } + // config1 here is kprobe_func or uprobe_path + attr.Ext1 = uint64(uintptr(unsafe.Pointer(namePtr))) + + // PID filter is only possible for uprobe events. + if pid < 0 { + pid = -1 + } + // perf_event_open API doesn't allow both pid and cpu to be -1. + // So only set it to -1 when PID is not -1. + // Tracing events do not do CPU filtering in any cases. + var cpu int + if pid != -1 { + cpu = -1 + } + + efd, err := unix.PerfEventOpen(&attr, pid, cpu, -1, unix.PERF_FLAG_FD_CLOEXEC) + if efd < 0 { + return 0, errors.Wrap(err, "perf_event_open error") + } + return efd, nil +} + +func perfEventOpenTracingEvent(probeID int) (int, error) { attr := unix.PerfEventAttr{ Type: unix.PERF_TYPE_TRACEPOINT, Sample_type: unix.PERF_SAMPLE_RAW, Sample: 1, Wakeup: 1, - Config: uint64(id), + Config: uint64(probeID), } attr.Size = uint32(unsafe.Sizeof(attr)) efd, err := unix.PerfEventOpen(&attr, -1, 0, -1, unix.PERF_FLAG_FD_CLOEXEC) if efd < 0 { - return nil, errors.Wrap(err, "perf_event_open error") + return 0, errors.Wrap(err, "perf_event_open error") } + return efd, nil +} - if _, _, err := unix.Syscall(unix.SYS_IOCTL, uintptr(efd), unix.PERF_EVENT_IOC_ENABLE, 0); err != 0 { - return nil, errors.Wrap(err, "error enabling perf event") +func ioctlPerfEventEnable(perfEventOpenFD int, progFD int) error { + if _, _, err := unix.Syscall(unix.SYS_IOCTL, uintptr(perfEventOpenFD), unix.PERF_EVENT_IOC_SET_BPF, uintptr(progFD)); err != 0 { + return errors.Wrap(err, "error attaching bpf program to perf event") } - - if _, _, err := unix.Syscall(unix.SYS_IOCTL, uintptr(efd), unix.PERF_EVENT_IOC_SET_BPF, uintptr(progFd)); err != 0 { - return nil, errors.Wrap(err, "error attaching bpf program to perf event") + if _, _, err := unix.Syscall(unix.SYS_IOCTL, uintptr(perfEventOpenFD), unix.PERF_EVENT_IOC_ENABLE, 0); err != 0 { + return errors.Wrap(err, "error enabling perf event") } - return internal.NewFD(uint32(efd)), nil + return nil } type bpfProgAttachAttr struct { diff --git a/manager/utils.go b/manager/utils.go index f41cc3eee..70002df8a 100644 --- a/manager/utils.go +++ b/manager/utils.go @@ -196,9 +196,9 @@ func ReadKprobeEvents() (string, error) { return string(kprobeEvents), nil } -// EnableKprobeEvent - Writes a new kprobe in kprobe_events with the provided parameters. Call DisableKprobeEvent +// registerKprobeEvent - Writes a new kprobe in kprobe_events with the provided parameters. Call DisableKprobeEvent // to remove the krpobe. -func EnableKprobeEvent(probeType, funcName, UID, maxactiveStr string, kprobeAttachPID int) (int, error) { +func registerKprobeEvent(probeType, funcName, UID, maxactiveStr string, kprobeAttachPID int) (int, error) { // Generate event name eventName, err := GenerateEventName(probeType, funcName, UID, kprobeAttachPID) if err != nil { @@ -233,17 +233,17 @@ func EnableKprobeEvent(probeType, funcName, UID, maxactiveStr string, kprobeAtta return kprobeID, nil } -// DisableKprobeEvent - Removes a kprobe from kprobe_events -func DisableKprobeEvent(probeType, funcName, UID string, kprobeAttachPID int) error { +// unregisterKprobeEvent - Removes a kprobe from kprobe_events +func unregisterKprobeEvent(probeType, funcName, UID string, kprobeAttachPID int) error { // Generate event name eventName, err := GenerateEventName(probeType, funcName, UID, kprobeAttachPID) if err != nil { return err } - return disableKprobeEvent(eventName) + return unregisterKprobeEventWithEventName(eventName) } -func disableKprobeEvent(eventName string) error { +func unregisterKprobeEventWithEventName(eventName string) error { // Write line to kprobe_events kprobeEventsFileName := "/sys/kernel/debug/tracing/kprobe_events" f, err := os.OpenFile(kprobeEventsFileName, os.O_APPEND|os.O_WRONLY, 0) @@ -276,9 +276,9 @@ func ReadUprobeEvents() (string, error) { return string(uprobeEvents), nil } -// EnableUprobeEvent - Writes a new Uprobe in uprobe_events with the provided parameters. Call DisableUprobeEvent +// registerUprobeEvent - Writes a new Uprobe in uprobe_events with the provided parameters. Call DisableUprobeEvent // to remove the krpobe. -func EnableUprobeEvent(probeType string, funcName, path, UID string, uprobeAttachPID int, offset uint64) (int, error) { +func registerUprobeEvent(probeType string, funcName, path, UID string, uprobeAttachPID int, offset uint64) (int, error) { // Generate event name eventName, err := GenerateEventName(probeType, funcName, UID, uprobeAttachPID) if err != nil { @@ -387,17 +387,17 @@ func FindSymbolOffsets(path string, pattern *regexp.Regexp) ([]elf.Symbol, error return matches, nil } -// DisableUprobeEvent - Removes a uprobe from uprobe_events -func DisableUprobeEvent(probeType string, funcName string, UID string, uprobeAttachPID int) error { +// unregisterUprobeEvent - Removes a uprobe from uprobe_events +func unregisterUprobeEvent(probeType string, funcName string, UID string, uprobeAttachPID int) error { // Generate event name eventName, err := GenerateEventName(probeType, funcName, UID, uprobeAttachPID) if err != nil { return err } - return disableUprobeEvent(eventName) + return unregisterUprobeEventWithEventName(eventName) } -func disableUprobeEvent(eventName string) error { +func unregisterUprobeEventWithEventName(eventName string) error { // Write uprobe_events line uprobeEventsFileName := "/sys/kernel/debug/tracing/uprobe_events" f, err := os.OpenFile(uprobeEventsFileName, os.O_APPEND|os.O_WRONLY, 0) @@ -425,3 +425,61 @@ func GetTracepointID(category, name string) (int, error) { } return tracepointID, nil } + +// FindPMUType returns the PMU type of the provided event type +// This function tries to use /sys/bus/event_source/devices/%s/type. +func FindPMUType(eventType string) (uint32, error) { + var defaultType uint32 + switch eventType { + case "kprobe", "kretprobe": + eventType = "kprobe" + defaultType = 6 + case "uprobe", "uretprobe": + eventType = "uprobe" + defaultType = 7 + default: + return 0, errors.Errorf("no PMU type for %s", eventType) + } + + PMUTypeFile := fmt.Sprintf("/sys/bus/event_source/devices/%s/type", eventType) + f, err := os.Open(PMUTypeFile) + if err != nil { + return defaultType, errors.Wrapf(err, "couldn't read %s", PMUTypeFile) + } + + var t uint32 + _, err = fmt.Fscanf(f, "%d\n", &t) + if err != nil { + return defaultType, errors.Wrap(err, "couldn't parse type") + } + return t, nil +} + +// FindRetProbeBit returns the retprobe bit of the provided event type +// This function relies on /sys/bus/event_source/devices/%s/format/retprobe. +func FindRetProbeBit(eventType string) (uint32, error) { + var defaultBit uint32 + switch eventType { + case "kprobe", "kretprobe": + eventType = "kprobe" + defaultBit = 0 + case "uprobe", "uretprobe": + eventType = "uprobe" + defaultBit = 0 + default: + return 0, errors.Errorf("no retprobe bit for %s", eventType) + } + + retProbeFormatFile := fmt.Sprintf("/sys/bus/event_source/devices/%s/format/retprobe", eventType) + f, err := os.Open(retProbeFormatFile) + if err != nil { + return defaultBit, errors.Wrapf(err, "couldn't read %s", retProbeFormatFile) + } + + var bit uint32 + _, err = fmt.Fscanf(f, "config:%d\n", &bit) + if err != nil { + return defaultBit, errors.Wrap(err, "couldn't parse retprobe bit") + } + return bit, nil +}