diff --git a/cmd/manager/daemon_util.go b/cmd/manager/daemon_util.go index 77d90dd9b..b97e47725 100644 --- a/cmd/manager/daemon_util.go +++ b/cmd/manager/daemon_util.go @@ -16,6 +16,7 @@ limitations under the License. package manager import ( + "bufio" "context" "fmt" "io" @@ -25,8 +26,11 @@ import ( "os/exec" "path" "path/filepath" + "runtime" + "runtime/debug" "sort" "strings" + "syscall" "time" backoff "github.com/cenkalti/backoff/v4" @@ -38,6 +42,70 @@ import ( "github.com/openshift/file-integrity-operator/pkg/common" ) +// reclaimCgroupPageCache asks the kernel to reclaim file-backed (page cache) +// memory charged to this container's cgroup. AIDE scans the entire host +// filesystem, and the resulting page cache pages are charged to the container's +// cgroup, causing reported memory to grow toward the limit over scan cycles. +// +// We use raw syscalls instead of os.OpenFile because Go's runtime registers +// opened fds with its epoll-based poller. The cgroup v2 memory.reclaim file +// supports poll (via cgroup_file_poll), so Go treats it as a pollable fd and +// waits for write-readiness before issuing the write. That readiness event +// never arrives, hanging the goroutine permanently. +func reclaimCgroupPageCache() { + cgroupPath, err := getOwnCgroupPath() + if err != nil { + LOG("could not determine own cgroup path (page cache not reclaimed): %v", err) + return + } + + reclaimFile := path.Join(cgroupPath, "memory.reclaim") + fd, err := syscall.Open(reclaimFile, syscall.O_WRONLY, 0) + if err != nil { + LOG("memory.reclaim not available at %s (page cache not reclaimed): %v", reclaimFile, err) + return + } + + _, err = syscall.Write(fd, []byte("500M")) + closeErr := syscall.Close(fd) + if err != nil && err != syscall.EAGAIN { + LOG("memory.reclaim write returned (non-fatal): %v", err) + } + if closeErr != nil { + LOG("memory.reclaim close error: %v", closeErr) + } + LOG("reclaimed cgroup page cache after AIDE scan") +} + +// getOwnCgroupPath reads /proc/self/cgroup (cgroup v2 unified format) and +// returns the sysfs path for this process's cgroup. +func getOwnCgroupPath() (string, error) { + f, err := os.Open("/proc/self/cgroup") + if err != nil { + return "", err + } + defer f.Close() + + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := scanner.Text() + // cgroup v2: "0::" + parts := strings.SplitN(line, ":", 3) + if len(parts) == 3 && parts[0] == "0" { + return "/sys/fs/cgroup" + parts[2], nil + } + } + return "", fmt.Errorf("no cgroup v2 entry found in /proc/self/cgroup") +} + +// releaseMemoryAfterScan forces the Go GC to run and returns freed memory to +// the OS. Combined with reclaimCgroupPageCache, this minimizes the container's +// memory footprint between scan cycles. +func releaseMemoryAfterScan() { + runtime.GC() + debug.FreeOSMemory() +} + func aideReadDBPath(c *daemonConfig) string { return path.Join(c.FileDir, aideReadDBFileName) } @@ -345,6 +413,9 @@ func getNonEmptyFile(filename string) *os.File { return file } + if err := file.Close(); err != nil { + LOG("warning: error closing empty/unreadable file %s: %v", cleanFileName, err) + } return nil } diff --git a/cmd/manager/logcollector_util.go b/cmd/manager/logcollector_util.go index 4f543cedc..f8fcadf14 100644 --- a/cmd/manager/logcollector_util.go +++ b/cmd/manager/logcollector_util.go @@ -30,12 +30,19 @@ import ( "github.com/spf13/cobra" corev1 "k8s.io/api/core/v1" + kerr "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "github.com/openshift/file-integrity-operator/pkg/common" ) +var ( + reFilesAdded = regexp.MustCompile(`\s+Added entries:\s+(?P\d+)`) + reFilesChanged = regexp.MustCompile(`\s+Changed entries:\s+(?P\d+)`) + reFilesRemoved = regexp.MustCompile(`\s+Removed entries:\s+(?P\d+)`) +) + const ( crdGroup = "fileintegrity.openshift.io" crdAPIVersion = "v1alpha1" @@ -59,8 +66,7 @@ func getValidStringArg(cmd *cobra.Command, name string) string { return val } -func matchFileChangeRegex(contents string, regex string) string { - re := regexp.MustCompile(regex) +func matchFileChangeRegex(contents string, re *regexp.Regexp) string { match := re.FindStringSubmatch(contents) if len(match) < 2 { return "0" @@ -70,9 +76,9 @@ func matchFileChangeRegex(contents string, regex string) string { } func annotateFileChangeSummary(contents string, annotations map[string]string) { - annotations[common.IntegrityLogFilesAddedAnnotation] = matchFileChangeRegex(contents, `\s+Added entries:\s+(?P\d+)`) - annotations[common.IntegrityLogFilesChangedAnnotation] = matchFileChangeRegex(contents, `\s+Changed entries:\s+(?P\d+)`) - annotations[common.IntegrityLogFilesRemovedAnnotation] = matchFileChangeRegex(contents, `\s+Removed entries:\s+(?P\d+)`) + annotations[common.IntegrityLogFilesAddedAnnotation] = matchFileChangeRegex(contents, reFilesAdded) + annotations[common.IntegrityLogFilesChangedAnnotation] = matchFileChangeRegex(contents, reFilesChanged) + annotations[common.IntegrityLogFilesRemovedAnnotation] = matchFileChangeRegex(contents, reFilesRemoved) DBG("added %s changed %s removed %s", annotations[common.IntegrityLogFilesAddedAnnotation], annotations[common.IntegrityLogFilesChangedAnnotation], @@ -209,6 +215,12 @@ func reportOK(ctx context.Context, conf *daemonConfig, rt *daemonRuntime) error fi := rt.GetFileIntegrityInstance() confMap := newInformationalConfigMap(fi, conf.LogCollectorConfigMapName, conf.LogCollectorNode, nil) _, err := rt.clientset.CoreV1().ConfigMaps(conf.Namespace).Create(ctx, confMap, metav1.CreateOptions{}) + if kerr.IsAlreadyExists(err) { + if delErr := rt.clientset.CoreV1().ConfigMaps(conf.Namespace).Delete(ctx, confMap.Name, metav1.DeleteOptions{}); delErr != nil { + return delErr + } + _, err = rt.clientset.CoreV1().ConfigMaps(conf.Namespace).Create(ctx, confMap, metav1.CreateOptions{}) + } return err }, backoff.WithMaxRetries(backoff.NewExponentialBackOff(), maxRetries)) } @@ -222,6 +234,12 @@ func reportError(ctx context.Context, msg string, conf *daemonConfig, rt *daemon } confMap := newInformationalConfigMap(fi, conf.LogCollectorConfigMapName, conf.LogCollectorNode, annotations) _, err := rt.clientset.CoreV1().ConfigMaps(conf.Namespace).Create(ctx, confMap, metav1.CreateOptions{}) + if kerr.IsAlreadyExists(err) { + if delErr := rt.clientset.CoreV1().ConfigMaps(conf.Namespace).Delete(ctx, confMap.Name, metav1.DeleteOptions{}); delErr != nil { + return delErr + } + _, err = rt.clientset.CoreV1().ConfigMaps(conf.Namespace).Create(ctx, confMap, metav1.CreateOptions{}) + } return err }, backoff.WithMaxRetries(backoff.NewExponentialBackOff(), maxRetries)) } @@ -232,6 +250,12 @@ func uploadLog(ctx context.Context, contents, compressedContents []byte, conf *d fi := rt.GetFileIntegrityInstance() confMap := newLogConfigMap(fi, conf.LogCollectorConfigMapName, common.IntegrityLogContentKey, conf.LogCollectorNode, contents, compressedContents) _, err := rt.clientset.CoreV1().ConfigMaps(conf.Namespace).Create(ctx, confMap, metav1.CreateOptions{}) + if kerr.IsAlreadyExists(err) { + if delErr := rt.clientset.CoreV1().ConfigMaps(conf.Namespace).Delete(ctx, confMap.Name, metav1.DeleteOptions{}); delErr != nil { + return delErr + } + _, err = rt.clientset.CoreV1().ConfigMaps(conf.Namespace).Create(ctx, confMap, metav1.CreateOptions{}) + } return err }, backoff.WithMaxRetries(backoff.NewExponentialBackOff(), maxRetries)) } diff --git a/cmd/manager/loops.go b/cmd/manager/loops.go index 41590d36f..b509a6ade 100644 --- a/cmd/manager/loops.go +++ b/cmd/manager/loops.go @@ -72,6 +72,12 @@ func aideLoop(ctx context.Context, rt *daemonRuntime, conf *daemonConfig, errCha // All done. Send the result. rt.SendResult(aideResult) rt.UnlockAideFiles("aideLoop") + + // AIDE reads the entire host filesystem, and the resulting page + // cache is charged to this container's cgroup. Reclaim it so + // reported memory drops back to the daemon's actual working set. + reclaimCgroupPageCache() + releaseMemoryAfterScan() } time.Sleep(time.Second * time.Duration(conf.Interval)) } @@ -215,6 +221,8 @@ func handleAIDEInit(ctx context.Context, rt *daemonRuntime, conf *daemonConfig, } LOG("initialization finished") + reclaimCgroupPageCache() + releaseMemoryAfterScan() return nil } diff --git a/pkg/controller/fileintegrity/fileintegrity_controller.go b/pkg/controller/fileintegrity/fileintegrity_controller.go index b53971b2f..90baf984e 100644 --- a/pkg/controller/fileintegrity/fileintegrity_controller.go +++ b/pkg/controller/fileintegrity/fileintegrity_controller.go @@ -909,7 +909,8 @@ func aideDaemonset(dsName string, fi *v1alpha1.FileIntegrity, operatorImage stri }, }, { - // Needed for friendlier memory reporting as long as we are on golang < 1.16 + // MADV_DONTNEED is already the default on Go >= 1.16. Kept for + // documentation; harmless no-op on current toolchain (Go 1.22). Name: "GODEBUG", Value: "madvdontneed=1", },