From 81bd8e5fc1f55118db13e0aee346c2085ef71694 Mon Sep 17 00:00:00 2001 From: Vincent Shen Date: Mon, 2 Mar 2026 08:39:29 -0800 Subject: [PATCH] Fix aide-worker memory growth caused by cgroup page cache accumulation AIDE scans the entire host filesystem, and the resulting kernel page cache is charged to the container's cgroup. Without reclamation, reported memory grows toward the resource limit after each scan cycle. Use cgroup v2 memory.reclaim to evict file-backed page cache after each AIDE scan and database initialization. This reduced aide-worker memory from ~570 MiB to ~11 MiB in testing on OCP 4.18.22. Use raw syscalls (syscall.Open/Write/Close) for memory.reclaim instead of os.OpenFile, because Go's runtime registers fds with its epoll poller and the cgroup v2 file's poll support causes the goroutine to hang waiting for write-readiness that never arrives. Additional fixes: - Close leaked file descriptor in getNonEmptyFile when file is empty - Pre-compile regex patterns used in log parsing - Handle AlreadyExists on ConfigMap creation to avoid unnecessary retries - Call runtime.GC and debug.FreeOSMemory after scan to return heap to OS - Update outdated GODEBUG comment (madvdontneed=1 is default since Go 1.16) --- cmd/manager/daemon_util.go | 71 +++++++++++++++++++ cmd/manager/logcollector_util.go | 34 +++++++-- cmd/manager/loops.go | 8 +++ .../fileintegrity/fileintegrity_controller.go | 3 +- 4 files changed, 110 insertions(+), 6 deletions(-) diff --git a/cmd/manager/daemon_util.go b/cmd/manager/daemon_util.go index 77d90dd9b..b97e47725 100644 --- a/cmd/manager/daemon_util.go +++ b/cmd/manager/daemon_util.go @@ -16,6 +16,7 @@ limitations under the License. package manager import ( + "bufio" "context" "fmt" "io" @@ -25,8 +26,11 @@ import ( "os/exec" "path" "path/filepath" + "runtime" + "runtime/debug" "sort" "strings" + "syscall" "time" backoff "github.com/cenkalti/backoff/v4" @@ -38,6 +42,70 @@ import ( "github.com/openshift/file-integrity-operator/pkg/common" ) +// reclaimCgroupPageCache asks the kernel to reclaim file-backed (page cache) +// memory charged to this container's cgroup. AIDE scans the entire host +// filesystem, and the resulting page cache pages are charged to the container's +// cgroup, causing reported memory to grow toward the limit over scan cycles. +// +// We use raw syscalls instead of os.OpenFile because Go's runtime registers +// opened fds with its epoll-based poller. The cgroup v2 memory.reclaim file +// supports poll (via cgroup_file_poll), so Go treats it as a pollable fd and +// waits for write-readiness before issuing the write. That readiness event +// never arrives, hanging the goroutine permanently. +func reclaimCgroupPageCache() { + cgroupPath, err := getOwnCgroupPath() + if err != nil { + LOG("could not determine own cgroup path (page cache not reclaimed): %v", err) + return + } + + reclaimFile := path.Join(cgroupPath, "memory.reclaim") + fd, err := syscall.Open(reclaimFile, syscall.O_WRONLY, 0) + if err != nil { + LOG("memory.reclaim not available at %s (page cache not reclaimed): %v", reclaimFile, err) + return + } + + _, err = syscall.Write(fd, []byte("500M")) + closeErr := syscall.Close(fd) + if err != nil && err != syscall.EAGAIN { + LOG("memory.reclaim write returned (non-fatal): %v", err) + } + if closeErr != nil { + LOG("memory.reclaim close error: %v", closeErr) + } + LOG("reclaimed cgroup page cache after AIDE scan") +} + +// getOwnCgroupPath reads /proc/self/cgroup (cgroup v2 unified format) and +// returns the sysfs path for this process's cgroup. +func getOwnCgroupPath() (string, error) { + f, err := os.Open("/proc/self/cgroup") + if err != nil { + return "", err + } + defer f.Close() + + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := scanner.Text() + // cgroup v2: "0::" + parts := strings.SplitN(line, ":", 3) + if len(parts) == 3 && parts[0] == "0" { + return "/sys/fs/cgroup" + parts[2], nil + } + } + return "", fmt.Errorf("no cgroup v2 entry found in /proc/self/cgroup") +} + +// releaseMemoryAfterScan forces the Go GC to run and returns freed memory to +// the OS. Combined with reclaimCgroupPageCache, this minimizes the container's +// memory footprint between scan cycles. +func releaseMemoryAfterScan() { + runtime.GC() + debug.FreeOSMemory() +} + func aideReadDBPath(c *daemonConfig) string { return path.Join(c.FileDir, aideReadDBFileName) } @@ -345,6 +413,9 @@ func getNonEmptyFile(filename string) *os.File { return file } + if err := file.Close(); err != nil { + LOG("warning: error closing empty/unreadable file %s: %v", cleanFileName, err) + } return nil } diff --git a/cmd/manager/logcollector_util.go b/cmd/manager/logcollector_util.go index 4f543cedc..f8fcadf14 100644 --- a/cmd/manager/logcollector_util.go +++ b/cmd/manager/logcollector_util.go @@ -30,12 +30,19 @@ import ( "github.com/spf13/cobra" corev1 "k8s.io/api/core/v1" + kerr "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "github.com/openshift/file-integrity-operator/pkg/common" ) +var ( + reFilesAdded = regexp.MustCompile(`\s+Added entries:\s+(?P\d+)`) + reFilesChanged = regexp.MustCompile(`\s+Changed entries:\s+(?P\d+)`) + reFilesRemoved = regexp.MustCompile(`\s+Removed entries:\s+(?P\d+)`) +) + const ( crdGroup = "fileintegrity.openshift.io" crdAPIVersion = "v1alpha1" @@ -59,8 +66,7 @@ func getValidStringArg(cmd *cobra.Command, name string) string { return val } -func matchFileChangeRegex(contents string, regex string) string { - re := regexp.MustCompile(regex) +func matchFileChangeRegex(contents string, re *regexp.Regexp) string { match := re.FindStringSubmatch(contents) if len(match) < 2 { return "0" @@ -70,9 +76,9 @@ func matchFileChangeRegex(contents string, regex string) string { } func annotateFileChangeSummary(contents string, annotations map[string]string) { - annotations[common.IntegrityLogFilesAddedAnnotation] = matchFileChangeRegex(contents, `\s+Added entries:\s+(?P\d+)`) - annotations[common.IntegrityLogFilesChangedAnnotation] = matchFileChangeRegex(contents, `\s+Changed entries:\s+(?P\d+)`) - annotations[common.IntegrityLogFilesRemovedAnnotation] = matchFileChangeRegex(contents, `\s+Removed entries:\s+(?P\d+)`) + annotations[common.IntegrityLogFilesAddedAnnotation] = matchFileChangeRegex(contents, reFilesAdded) + annotations[common.IntegrityLogFilesChangedAnnotation] = matchFileChangeRegex(contents, reFilesChanged) + annotations[common.IntegrityLogFilesRemovedAnnotation] = matchFileChangeRegex(contents, reFilesRemoved) DBG("added %s changed %s removed %s", annotations[common.IntegrityLogFilesAddedAnnotation], annotations[common.IntegrityLogFilesChangedAnnotation], @@ -209,6 +215,12 @@ func reportOK(ctx context.Context, conf *daemonConfig, rt *daemonRuntime) error fi := rt.GetFileIntegrityInstance() confMap := newInformationalConfigMap(fi, conf.LogCollectorConfigMapName, conf.LogCollectorNode, nil) _, err := rt.clientset.CoreV1().ConfigMaps(conf.Namespace).Create(ctx, confMap, metav1.CreateOptions{}) + if kerr.IsAlreadyExists(err) { + if delErr := rt.clientset.CoreV1().ConfigMaps(conf.Namespace).Delete(ctx, confMap.Name, metav1.DeleteOptions{}); delErr != nil { + return delErr + } + _, err = rt.clientset.CoreV1().ConfigMaps(conf.Namespace).Create(ctx, confMap, metav1.CreateOptions{}) + } return err }, backoff.WithMaxRetries(backoff.NewExponentialBackOff(), maxRetries)) } @@ -222,6 +234,12 @@ func reportError(ctx context.Context, msg string, conf *daemonConfig, rt *daemon } confMap := newInformationalConfigMap(fi, conf.LogCollectorConfigMapName, conf.LogCollectorNode, annotations) _, err := rt.clientset.CoreV1().ConfigMaps(conf.Namespace).Create(ctx, confMap, metav1.CreateOptions{}) + if kerr.IsAlreadyExists(err) { + if delErr := rt.clientset.CoreV1().ConfigMaps(conf.Namespace).Delete(ctx, confMap.Name, metav1.DeleteOptions{}); delErr != nil { + return delErr + } + _, err = rt.clientset.CoreV1().ConfigMaps(conf.Namespace).Create(ctx, confMap, metav1.CreateOptions{}) + } return err }, backoff.WithMaxRetries(backoff.NewExponentialBackOff(), maxRetries)) } @@ -232,6 +250,12 @@ func uploadLog(ctx context.Context, contents, compressedContents []byte, conf *d fi := rt.GetFileIntegrityInstance() confMap := newLogConfigMap(fi, conf.LogCollectorConfigMapName, common.IntegrityLogContentKey, conf.LogCollectorNode, contents, compressedContents) _, err := rt.clientset.CoreV1().ConfigMaps(conf.Namespace).Create(ctx, confMap, metav1.CreateOptions{}) + if kerr.IsAlreadyExists(err) { + if delErr := rt.clientset.CoreV1().ConfigMaps(conf.Namespace).Delete(ctx, confMap.Name, metav1.DeleteOptions{}); delErr != nil { + return delErr + } + _, err = rt.clientset.CoreV1().ConfigMaps(conf.Namespace).Create(ctx, confMap, metav1.CreateOptions{}) + } return err }, backoff.WithMaxRetries(backoff.NewExponentialBackOff(), maxRetries)) } diff --git a/cmd/manager/loops.go b/cmd/manager/loops.go index 41590d36f..b509a6ade 100644 --- a/cmd/manager/loops.go +++ b/cmd/manager/loops.go @@ -72,6 +72,12 @@ func aideLoop(ctx context.Context, rt *daemonRuntime, conf *daemonConfig, errCha // All done. Send the result. rt.SendResult(aideResult) rt.UnlockAideFiles("aideLoop") + + // AIDE reads the entire host filesystem, and the resulting page + // cache is charged to this container's cgroup. Reclaim it so + // reported memory drops back to the daemon's actual working set. + reclaimCgroupPageCache() + releaseMemoryAfterScan() } time.Sleep(time.Second * time.Duration(conf.Interval)) } @@ -215,6 +221,8 @@ func handleAIDEInit(ctx context.Context, rt *daemonRuntime, conf *daemonConfig, } LOG("initialization finished") + reclaimCgroupPageCache() + releaseMemoryAfterScan() return nil } diff --git a/pkg/controller/fileintegrity/fileintegrity_controller.go b/pkg/controller/fileintegrity/fileintegrity_controller.go index b53971b2f..90baf984e 100644 --- a/pkg/controller/fileintegrity/fileintegrity_controller.go +++ b/pkg/controller/fileintegrity/fileintegrity_controller.go @@ -909,7 +909,8 @@ func aideDaemonset(dsName string, fi *v1alpha1.FileIntegrity, operatorImage stri }, }, { - // Needed for friendlier memory reporting as long as we are on golang < 1.16 + // MADV_DONTNEED is already the default on Go >= 1.16. Kept for + // documentation; harmless no-op on current toolchain (Go 1.22). Name: "GODEBUG", Value: "madvdontneed=1", },