Implement perf_event_open ABI to attach kprobes and uprobes

manager/manager.go

@@ -1526,7 +1526,7 @@ func cleanupKprobeEvents(pattern *regexp.Regexp, pidMask map[int]procMask) error
 		}
 
 		// remove the entry
-		cleanUpErrors = multierror.Append(cleanUpErrors, disableKprobeEvent(match[3]))
+		cleanUpErrors = multierror.Append(cleanUpErrors, unregisterKprobeEventWithEventName(match[3]))
 	}
 	return cleanUpErrors
 }
@@ -1567,7 +1567,7 @@ func cleanupUprobeEvents(pattern *regexp.Regexp, pidMask map[int]procMask) error
 		}
 
 		// remove the entry
-		cleanUpErrors = multierror.Append(cleanUpErrors, disableUprobeEvent(match[3]))
+		cleanUpErrors = multierror.Append(cleanUpErrors, unregisterUprobeEventWithEventName(match[3]))
 	}
 	return cleanUpErrors
 }

manager/probe.go

@@ -63,16 +63,17 @@ func (pip ProbeIdentificationPair) Matches(id ProbeIdentificationPair) bool {
 // Probe - Main eBPF probe wrapper. This structure is used to store the required data to attach a loaded eBPF
 // program to its hook point.
 type Probe struct {
-	manager          *Manager
-	program          *ebpf.Program
-	programSpec      *ebpf.ProgramSpec
-	perfEventFD      *internal.FD
-	state            state
-	stateLock        sync.RWMutex
-	manualLoadNeeded bool
-	checkPin         bool
-	funcName         string
-	attachPID        int
+	manager             *Manager
+	program             *ebpf.Program
+	programSpec         *ebpf.ProgramSpec
+	perfEventFD         *internal.FD
+	state               state
+	stateLock           sync.RWMutex
+	manualLoadNeeded    bool
+	checkPin            bool
+	funcName            string
+	attachPID           int
+	attachedWithDebugFS bool
 
 	// lastError - stores the last error that the probe encountered, it is used to surface a more useful error message
 	// when one of the validators (see Options.ActivatedProbes) fails.
@@ -532,46 +533,68 @@ func (p *Probe) reset() {
 	p.checkPin = false
 	p.funcName = ""
 	p.attachPID = 0
+	p.attachedWithDebugFS = false
 }
 
 // attachKprobe - Attaches the probe to its kprobe
 func (p *Probe) attachKprobe() error {
 	// Prepare kprobe_events line parameters
-	var probeType, maxactiveStr string
+	var probeType, maxactiveStr, sectionPrefix string
 	var err error
 	funcName := p.funcName
-	if strings.HasPrefix(p.Section, "kretprobe/") {
+
+	dividedSection := strings.Split(p.Section, "/")
+	if len(dividedSection) >= 1 {
+		sectionPrefix = dividedSection[0]
+	} else {
+		return errors.Wrapf(ErrSectionFormat, "program type unrecognized in section %v", p.Section)
+	}
+
+	switch sectionPrefix {
+	case "kretprobe":
 		if funcName == "" {
 			funcName = strings.TrimPrefix(p.Section, "kretprobe/")
 		}
 		if p.KProbeMaxActive > 0 {
 			maxactiveStr = fmt.Sprintf("%d", p.KProbeMaxActive)
 		}
 		probeType = "r"
-	} else if strings.HasPrefix(p.Section, "kprobe/") {
+	case "kprobe":
 		if funcName == "" {
 			funcName = strings.TrimPrefix(p.Section, "kprobe/")
 		}
 		probeType = "p"
-	} else {
+	default:
 		// this might actually be a Uprobe
 		return p.attachUprobe()
 	}
 	p.attachPID = os.Getpid()
 
-	// Write kprobe_events line to register kprobe
-	kprobeID, err := EnableKprobeEvent(probeType, funcName, p.UID, maxactiveStr, p.attachPID)
-	// fallback without KProbeMaxActive
-	if err == ErrKprobeIDNotExist {
-		kprobeID, err = EnableKprobeEvent(probeType, funcName, p.UID, "", p.attachPID)
-	}
+	// Try to use the perf_event_open API first (e12f03d "perf/core: Implement the 'perf_kprobe' PMU")
+	perfEventFD, err := perfEventOpenWithProbe(funcName, 0, -1, sectionPrefix, 0)
 	if err != nil {
-		return errors.Wrapf(err, "couldn't enable kprobe %s", p.Section)
+		// Try to create the event using debugfs
+		// Write kprobe_events line to register kprobe
+		kprobeID, err := registerKprobeEvent(probeType, funcName, p.UID, maxactiveStr, p.attachPID)
+		// fallback without KProbeMaxActive
+		if err == ErrKprobeIDNotExist {
+			kprobeID, err = registerKprobeEvent(probeType, funcName, p.UID, "", p.attachPID)
+		}
+		if err != nil {
+			return errors.Wrapf(err, "couldn't enable kprobe %s", p.Section)
+		}
+		p.attachedWithDebugFS = true
+
+		// create perf event FD
+		perfEventFD, err = perfEventOpenTracingEvent(kprobeID)
 	}
 
-	// Activate perf event
-	p.perfEventFD, err = perfEventOpenTracepoint(kprobeID, p.program.FD())
-	return errors.Wrapf(err, "couldn't enable kprobe %s", p.GetIdentificationPair())
+	// enable program
+	if ioctlPerfEventEnable(perfEventFD, p.program.FD()) != nil {
+		return errors.Wrapf(err, "couldn't enable perf event %s", p.GetIdentificationPair())
+	}
+	p.perfEventFD = internal.NewFD(uint32(perfEventFD))
+	return nil
 }
 
 // detachKprobe - Detaches the probe from its kprobe
@@ -594,8 +617,13 @@ func (p *Probe) detachKprobe() error {
 		return p.detachUprobe()
 	}
 
+	if !p.attachedWithDebugFS {
+		// nothing to do
+		return nil
+	}
+
 	// Write kprobe_events line to remove hook point
-	return DisableKprobeEvent(probeType, funcName, p.UID, p.attachPID)
+	return unregisterKprobeEvent(probeType, funcName, p.UID, p.attachPID)
 }
 
 // attachTracepoint - Attaches the probe to its tracepoint
@@ -615,25 +643,40 @@ func (p *Probe) attachTracepoint() error {
 	}
 
 	// Hook the eBPF program to the tracepoint
-	p.perfEventFD, err = perfEventOpenTracepoint(tracepointID, p.program.FD())
-	return errors.Wrapf(err, "couldn't enable tracepoint %s", p.GetIdentificationPair())
+	perfEventFD, err := perfEventOpenTracingEvent(tracepointID)
+	if err != nil {
+		return errors.Wrapf(err, "couldn't enable tracepoint %s", p.GetIdentificationPair())
+	}
+	if ioctlPerfEventEnable(perfEventFD, p.program.FD()) != nil {
+		return errors.Wrapf(err, "couldn't enable perf event %s", p.GetIdentificationPair())
+	}
+	p.perfEventFD = internal.NewFD(uint32(perfEventFD))
+	return nil
 }
 
 // attachUprobe - Attaches the probe to its Uprobe
 func (p *Probe) attachUprobe() error {
-	p.attachPID = os.Getpid()
+	var probeType, funcName, sectionPrefix string
+	dividedSection := strings.Split(p.Section, "/")
+	if len(dividedSection) >= 1 {
+		sectionPrefix = dividedSection[0]
+	} else {
+		return errors.Wrapf(ErrSectionFormat, "program type unrecognized in section %v", p.Section)
+	}
+
 	// Prepare uprobe_events line parameters
-	var probeType, funcName string
-	if strings.HasPrefix(p.Section, "uretprobe/") {
+	switch sectionPrefix {
+	case "uretprobe":
 		funcName = strings.TrimPrefix(p.Section, "uretprobe/")
 		probeType = "r"
-	} else if strings.HasPrefix(p.Section, "uprobe/") {
+	case "uprobe":
 		funcName = strings.TrimPrefix(p.Section, "uprobe/")
 		probeType = "p"
-	} else {
+	default:
 		// unknown type
 		return errors.Wrapf(ErrSectionFormat, "program type unrecognized in section %v", p.Section)
 	}
+	p.attachPID = os.Getpid()
 
 	// compute the offset if it was not provided
 	if p.UprobeOffset == 0 {
@@ -657,19 +700,35 @@ func (p *Probe) attachUprobe() error {
 		p.funcName = offsets[0].Name
 	}
 
-	// enable uprobe
-	uprobeID, err := EnableUprobeEvent(probeType, p.funcName, p.BinaryPath, p.UID, p.attachPID, p.UprobeOffset)
+	// Try to use the perf_event_open API first (e12f03d "perf/core: Implement the 'perf_kprobe' PMU")
+	perfEventFD, err := perfEventOpenWithProbe(p.BinaryPath, int(p.UprobeOffset), -1, sectionPrefix, 0)
 	if err != nil {
-		return errors.Wrapf(err, "couldn't enable uprobe %s", p.Section)
+		// Try to create the event using debugfs
+		uprobeID, err := registerUprobeEvent(probeType, p.funcName, p.BinaryPath, p.UID, p.attachPID, p.UprobeOffset)
+		if err != nil {
+			return errors.Wrapf(err, "couldn't enable uprobe %s", p.Section)
+		}
+
+		// Activate perf event
+		perfEventFD, err = perfEventOpenTracingEvent(uprobeID)
+		p.attachedWithDebugFS = true
 	}
 
-	// Activate perf event
-	p.perfEventFD, err = perfEventOpenTracepoint(uprobeID, p.program.FD())
-	return errors.Wrapf(err, "couldn't enable uprobe %s", p.GetIdentificationPair())
+	// enable perf event
+	if ioctlPerfEventEnable(perfEventFD, p.program.FD()) != nil {
+		return errors.Wrapf(err, "couldn't enable perf event %s", p.GetIdentificationPair())
+	}
+	p.perfEventFD = internal.NewFD(uint32(perfEventFD))
+	return nil
 }
 
 // detachUprobe - Detaches the probe from its Uprobe
 func (p *Probe) detachUprobe() error {
+	if !p.attachedWithDebugFS {
+		// nothing to do
+		return nil
+	}
+
 	// Prepare uprobe_events line parameters
 	var probeType string
 	if strings.HasPrefix(p.Section, "uretprobe/") {
@@ -682,7 +741,7 @@ func (p *Probe) detachUprobe() error {
 	}
 
 	// Write uprobe_events line to remove hook point
-	return DisableUprobeEvent(probeType, p.funcName, p.UID, p.attachPID)
+	return unregisterUprobeEvent(probeType, p.funcName, p.UID, p.attachPID)
 }
 
 // attachCGroup - Attaches the probe to a cgroup hook point

manager/syscalls.go

@@ -1,39 +1,94 @@
 package manager
 
 import (
-	"github.com/DataDog/ebpf"
-	"golang.org/x/sys/unix"
 	"syscall"
 	"unsafe"
 
 	"github.com/pkg/errors"
+	"golang.org/x/sys/unix"
 
+	"github.com/DataDog/ebpf"
 	"github.com/DataDog/ebpf/internal"
 )
 
-func perfEventOpenTracepoint(id int, progFd int) (*internal.FD, error) {
+// perfEventOpenWithProbe - Kernel API with e12f03d ("perf/core: Implement the 'perf_kprobe' PMU") allows
+// creating [k,u]probe with perf_event_open, which makes it easier to clean up
+// the [k,u]probe. This function tries to create pfd with the perf_kprobe PMU.
+func perfEventOpenWithProbe(name string, offset, pid int, sectionPrefix string, referenceCounterOffset uint64) (int, error) {
+	var err error
+	attr := unix.PerfEventAttr{
+		Ext2: uint64(offset), // config2 here is kprobe_addr or probe_offset
+	}
+	attr.Size = uint32(unsafe.Sizeof(attr))
+
+	attr.Type, _ = FindPMUType(sectionPrefix)
+	//if err != nil {
+	//	return 0, errors.Wrapf(err, "couldn't find PMU type for %s", sectionPrefix)
+	//}
+
+	var returnBit uint32
+	returnBit, _ = FindRetProbeBit(sectionPrefix)
+	//if err != nil {
+	//	return 0, errors.Wrapf(err, "couldn't find retprobe bit for %s", sectionPrefix)
+	//}
+	if returnBit > 0 {
+		attr.Config = 1 << returnBit
+	}
+	if referenceCounterOffset > 0 {
+		attr.Config |= referenceCounterOffset << 32
+	}
+
+	namePtr, err := syscall.BytePtrFromString(name)
+	if err != nil {
+		return 0, errors.Wrapf(err, "couldn't create pointer to string %s", name)
+	}
+	// config1 here is kprobe_func or uprobe_path
+	attr.Ext1 = uint64(uintptr(unsafe.Pointer(namePtr)))
+
+	// PID filter is only possible for uprobe events.
+	if pid < 0 {
+		pid = -1
+	}
+	// perf_event_open API doesn't allow both pid and cpu to be -1.
+	// So only set it to -1 when PID is not -1.
+	// Tracing events do not do CPU filtering in any cases.
+	var cpu int
+	if pid != -1 {
+		cpu = -1
+	}
+
+	efd, err := unix.PerfEventOpen(&attr, pid, cpu, -1, unix.PERF_FLAG_FD_CLOEXEC)
+	if efd < 0 {
+		return 0, errors.Wrap(err, "perf_event_open error")
+	}
+	return efd, nil
+}
+
+func perfEventOpenTracingEvent(probeID int) (int, error) {
 	attr := unix.PerfEventAttr{
 		Type:        unix.PERF_TYPE_TRACEPOINT,
 		Sample_type: unix.PERF_SAMPLE_RAW,
 		Sample:      1,
 		Wakeup:      1,
-		Config:      uint64(id),
+		Config:      uint64(probeID),
 	}
 	attr.Size = uint32(unsafe.Sizeof(attr))
 
 	efd, err := unix.PerfEventOpen(&attr, -1, 0, -1, unix.PERF_FLAG_FD_CLOEXEC)
 	if efd < 0 {
-		return nil, errors.Wrap(err, "perf_event_open error")
+		return 0, errors.Wrap(err, "perf_event_open error")
 	}
+	return efd, nil
+}
 
-	if _, _, err := unix.Syscall(unix.SYS_IOCTL, uintptr(efd), unix.PERF_EVENT_IOC_ENABLE, 0); err != 0 {
-		return nil, errors.Wrap(err, "error enabling perf event")
+func ioctlPerfEventEnable(perfEventOpenFD int, progFD int) error {
+	if _, _, err := unix.Syscall(unix.SYS_IOCTL, uintptr(perfEventOpenFD), unix.PERF_EVENT_IOC_SET_BPF, uintptr(progFD)); err != 0 {
+		return errors.Wrap(err, "error attaching bpf program to perf event")
 	}
-
-	if _, _, err := unix.Syscall(unix.SYS_IOCTL, uintptr(efd), unix.PERF_EVENT_IOC_SET_BPF, uintptr(progFd)); err != 0 {
-		return nil, errors.Wrap(err, "error attaching bpf program to perf event")
+	if _, _, err := unix.Syscall(unix.SYS_IOCTL, uintptr(perfEventOpenFD), unix.PERF_EVENT_IOC_ENABLE, 0); err != 0 {
+		return errors.Wrap(err, "error enabling perf event")
 	}
-	return internal.NewFD(uint32(efd)), nil
+	return nil
 }
 
 type bpfProgAttachAttr struct {