CMP-4007: Fix aide-worker memory growth caused by cgroup page cache accumulation

cmd/manager/daemon_util.go

@@ -16,6 +16,7 @@ limitations under the License.
 package manager
 
 import (
+	"bufio"
 	"context"
 	"fmt"
 	"io"
@@ -25,8 +26,11 @@ import (
 	"os/exec"
 	"path"
 	"path/filepath"
+	"runtime"
+	"runtime/debug"
 	"sort"
 	"strings"
+	"syscall"
 	"time"
 
 	backoff "github.com/cenkalti/backoff/v4"
@@ -38,6 +42,70 @@ import (
 	"github.com/openshift/file-integrity-operator/pkg/common"
 )
 
+// reclaimCgroupPageCache asks the kernel to reclaim file-backed (page cache)
+// memory charged to this container's cgroup. AIDE scans the entire host
+// filesystem, and the resulting page cache pages are charged to the container's
+// cgroup, causing reported memory to grow toward the limit over scan cycles.
+//
+// We use raw syscalls instead of os.OpenFile because Go's runtime registers
+// opened fds with its epoll-based poller. The cgroup v2 memory.reclaim file
+// supports poll (via cgroup_file_poll), so Go treats it as a pollable fd and
+// waits for write-readiness before issuing the write. That readiness event
+// never arrives, hanging the goroutine permanently.
+func reclaimCgroupPageCache() {
+	cgroupPath, err := getOwnCgroupPath()
+	if err != nil {
+		LOG("could not determine own cgroup path (page cache not reclaimed): %v", err)
+		return
+	}
+
+	reclaimFile := path.Join(cgroupPath, "memory.reclaim")
+	fd, err := syscall.Open(reclaimFile, syscall.O_WRONLY, 0)
+	if err != nil {
+		LOG("memory.reclaim not available at %s (page cache not reclaimed): %v", reclaimFile, err)
+		return
+	}
+
+	_, err = syscall.Write(fd, []byte("500M"))
+	closeErr := syscall.Close(fd)
+	if err != nil && err != syscall.EAGAIN {
+		LOG("memory.reclaim write returned (non-fatal): %v", err)
+	}
+	if closeErr != nil {
+		LOG("memory.reclaim close error: %v", closeErr)
+	}
+	LOG("reclaimed cgroup page cache after AIDE scan")
+}
+
+// getOwnCgroupPath reads /proc/self/cgroup (cgroup v2 unified format) and
+// returns the sysfs path for this process's cgroup.
+func getOwnCgroupPath() (string, error) {
+	f, err := os.Open("/proc/self/cgroup")
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := scanner.Text()
+		// cgroup v2: "0::<path>"
+		parts := strings.SplitN(line, ":", 3)
+		if len(parts) == 3 && parts[0] == "0" {
+			return "/sys/fs/cgroup" + parts[2], nil
+		}
+	}
+	return "", fmt.Errorf("no cgroup v2 entry found in /proc/self/cgroup")
+}
+
+// releaseMemoryAfterScan forces the Go GC to run and returns freed memory to
+// the OS. Combined with reclaimCgroupPageCache, this minimizes the container's
+// memory footprint between scan cycles.
+func releaseMemoryAfterScan() {
+	runtime.GC()
+	debug.FreeOSMemory()
+}
+
 func aideReadDBPath(c *daemonConfig) string {
 	return path.Join(c.FileDir, aideReadDBFileName)
 }
@@ -345,6 +413,9 @@ func getNonEmptyFile(filename string) *os.File {
 		return file
 	}
 
+	if err := file.Close(); err != nil {
+		LOG("warning: error closing empty/unreadable file %s: %v", cleanFileName, err)
+	}
 	return nil
 }
 

cmd/manager/logcollector_util.go

@@ -30,12 +30,19 @@ import (
 	"github.com/spf13/cobra"
 
 	corev1 "k8s.io/api/core/v1"
+	kerr "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 
 	"github.com/openshift/file-integrity-operator/pkg/common"
 )
 
+var (
+	reFilesAdded   = regexp.MustCompile(`\s+Added entries:\s+(?P<num_added>\d+)`)
+	reFilesChanged = regexp.MustCompile(`\s+Changed entries:\s+(?P<num_changed>\d+)`)
+	reFilesRemoved = regexp.MustCompile(`\s+Removed entries:\s+(?P<num_removed>\d+)`)
+)
+
 const (
 	crdGroup      = "fileintegrity.openshift.io"
 	crdAPIVersion = "v1alpha1"
@@ -59,8 +66,7 @@ func getValidStringArg(cmd *cobra.Command, name string) string {
 	return val
 }
 
-func matchFileChangeRegex(contents string, regex string) string {
-	re := regexp.MustCompile(regex)
+func matchFileChangeRegex(contents string, re *regexp.Regexp) string {
 	match := re.FindStringSubmatch(contents)
 	if len(match) < 2 {
 		return "0"
@@ -70,9 +76,9 @@ func matchFileChangeRegex(contents string, regex string) string {
 }
 
 func annotateFileChangeSummary(contents string, annotations map[string]string) {
-	annotations[common.IntegrityLogFilesAddedAnnotation] = matchFileChangeRegex(contents, `\s+Added entries:\s+(?P<num_added>\d+)`)
-	annotations[common.IntegrityLogFilesChangedAnnotation] = matchFileChangeRegex(contents, `\s+Changed entries:\s+(?P<num_changed>\d+)`)
-	annotations[common.IntegrityLogFilesRemovedAnnotation] = matchFileChangeRegex(contents, `\s+Removed entries:\s+(?P<num_removed>\d+)`)
+	annotations[common.IntegrityLogFilesAddedAnnotation] = matchFileChangeRegex(contents, reFilesAdded)
+	annotations[common.IntegrityLogFilesChangedAnnotation] = matchFileChangeRegex(contents, reFilesChanged)
+	annotations[common.IntegrityLogFilesRemovedAnnotation] = matchFileChangeRegex(contents, reFilesRemoved)
 	DBG("added %s changed %s removed %s",
 		annotations[common.IntegrityLogFilesAddedAnnotation],
 		annotations[common.IntegrityLogFilesChangedAnnotation],
@@ -209,6 +215,12 @@ func reportOK(ctx context.Context, conf *daemonConfig, rt *daemonRuntime) error
 		fi := rt.GetFileIntegrityInstance()
 		confMap := newInformationalConfigMap(fi, conf.LogCollectorConfigMapName, conf.LogCollectorNode, nil)
 		_, err := rt.clientset.CoreV1().ConfigMaps(conf.Namespace).Create(ctx, confMap, metav1.CreateOptions{})
+		if kerr.IsAlreadyExists(err) {
+			if delErr := rt.clientset.CoreV1().ConfigMaps(conf.Namespace).Delete(ctx, confMap.Name, metav1.DeleteOptions{}); delErr != nil {
+				return delErr
+			}
+			_, err = rt.clientset.CoreV1().ConfigMaps(conf.Namespace).Create(ctx, confMap, metav1.CreateOptions{})
+		}
 		return err
 	}, backoff.WithMaxRetries(backoff.NewExponentialBackOff(), maxRetries))
 }
@@ -222,6 +234,12 @@ func reportError(ctx context.Context, msg string, conf *daemonConfig, rt *daemon
 		}
 		confMap := newInformationalConfigMap(fi, conf.LogCollectorConfigMapName, conf.LogCollectorNode, annotations)
 		_, err := rt.clientset.CoreV1().ConfigMaps(conf.Namespace).Create(ctx, confMap, metav1.CreateOptions{})
+		if kerr.IsAlreadyExists(err) {
+			if delErr := rt.clientset.CoreV1().ConfigMaps(conf.Namespace).Delete(ctx, confMap.Name, metav1.DeleteOptions{}); delErr != nil {
+				return delErr
+			}
+			_, err = rt.clientset.CoreV1().ConfigMaps(conf.Namespace).Create(ctx, confMap, metav1.CreateOptions{})
+		}
 		return err
 	}, backoff.WithMaxRetries(backoff.NewExponentialBackOff(), maxRetries))
 }
@@ -232,6 +250,12 @@ func uploadLog(ctx context.Context, contents, compressedContents []byte, conf *d
 		fi := rt.GetFileIntegrityInstance()
 		confMap := newLogConfigMap(fi, conf.LogCollectorConfigMapName, common.IntegrityLogContentKey, conf.LogCollectorNode, contents, compressedContents)
 		_, err := rt.clientset.CoreV1().ConfigMaps(conf.Namespace).Create(ctx, confMap, metav1.CreateOptions{})
+		if kerr.IsAlreadyExists(err) {
+			if delErr := rt.clientset.CoreV1().ConfigMaps(conf.Namespace).Delete(ctx, confMap.Name, metav1.DeleteOptions{}); delErr != nil {
+				return delErr
+			}
+			_, err = rt.clientset.CoreV1().ConfigMaps(conf.Namespace).Create(ctx, confMap, metav1.CreateOptions{})
+		}
 		return err
 	}, backoff.WithMaxRetries(backoff.NewExponentialBackOff(), maxRetries))
 }

cmd/manager/loops.go

@@ -72,6 +72,12 @@ func aideLoop(ctx context.Context, rt *daemonRuntime, conf *daemonConfig, errCha
 				// All done. Send the result.
 				rt.SendResult(aideResult)
 				rt.UnlockAideFiles("aideLoop")
+
+				// AIDE reads the entire host filesystem, and the resulting page
+				// cache is charged to this container's cgroup. Reclaim it so
+				// reported memory drops back to the daemon's actual working set.
+				reclaimCgroupPageCache()
+				releaseMemoryAfterScan()
 			}
 			time.Sleep(time.Second * time.Duration(conf.Interval))
 		}
@@ -215,6 +221,8 @@ func handleAIDEInit(ctx context.Context, rt *daemonRuntime, conf *daemonConfig,
 	}
 
 	LOG("initialization finished")
+	reclaimCgroupPageCache()
+	releaseMemoryAfterScan()
 	return nil
 }
 

pkg/controller/fileintegrity/fileintegrity_controller.go

@@ -909,7 +909,8 @@ func aideDaemonset(dsName string, fi *v1alpha1.FileIntegrity, operatorImage stri
 									},
 								},
 								{
-									// Needed for friendlier memory reporting as long as we are on golang < 1.16
+									// MADV_DONTNEED is already the default on Go >= 1.16. Kept for
+									// documentation; harmless no-op on current toolchain (Go 1.22).
 									Name:  "GODEBUG",
 									Value: "madvdontneed=1",
 								},