From fb525e3af83c752248eda75101b90e03260f15fe Mon Sep 17 00:00:00 2001 From: Marek Knappe Date: Thu, 21 May 2026 11:05:55 +1000 Subject: [PATCH 1/2] feat(logging): expose file_cache_advise option for file receivers Allow users to configure fluent-bit's file_cache_advise option for file-based logging receivers. This option controls whether POSIX_FADV_DONTNEED is used to reduce kernel file cache usage. Starting with fluent-bit v3.1.6 (Ops Agent 2.50.0), this defaults to true, which causes excessive disk reads in environments with multiple pipelines reading the same files due to cache eviction after each read. Users experiencing high disk I/O can now disable this: logging: receivers: my-logs: type: files include_paths: - /var/log/*.log file_cache_advise: false Fixes #2307 --- confgenerator/logging_receivers.go | 14 ++++++++++++++ confgenerator/testdata/feature/golden.csv | 4 ++++ 2 files changed, 18 insertions(+) diff --git a/confgenerator/logging_receivers.go b/confgenerator/logging_receivers.go index 986f1f7a98..5a5374f389 100644 --- a/confgenerator/logging_receivers.go +++ b/confgenerator/logging_receivers.go @@ -44,6 +44,10 @@ type LoggingReceiverFiles struct { ExcludePaths []string `yaml:"exclude_paths,omitempty"` WildcardRefreshInterval *time.Duration `yaml:"wildcard_refresh_interval,omitempty" validate:"omitempty,min=1s,multipleof_time=1s"` RecordLogFilePath *bool `yaml:"record_log_file_path,omitempty"` + // FileCacheAdvise controls fluent-bit's file_cache_advise option (Linux only). + // When true (fluent-bit's default), uses POSIX_FADV_DONTNEED to reduce kernel file cache usage. + // Set to false to keep files in cache, reducing disk I/O in scenarios with multiple pipelines or high re-read patterns. + FileCacheAdvise *bool `yaml:"file_cache_advise,omitempty"` } func (r LoggingReceiverFiles) Type() string { @@ -56,6 +60,7 @@ func (r LoggingReceiverFiles) mixin() LoggingReceiverFilesMixin { ExcludePaths: r.ExcludePaths, WildcardRefreshInterval: r.WildcardRefreshInterval, RecordLogFilePath: r.RecordLogFilePath, + FileCacheAdvise: r.FileCacheAdvise, } } @@ -78,6 +83,7 @@ type LoggingReceiverFilesMixin struct { MultilineRules []MultilineRule `yaml:"-"` BufferInMemory bool `yaml:"-"` RecordLogFilePath *bool `yaml:"record_log_file_path,omitempty"` + FileCacheAdvise *bool `yaml:"file_cache_advise,omitempty"` // In transformation test mode, the file is read exactly once from the beginning, and then the process exits. TransformationTest bool `yaml:"-" tracking:"-"` } @@ -157,6 +163,14 @@ func (r LoggingReceiverFilesMixin) Components(ctx context.Context, tag string) [ config["Path_Key"] = "agent.googleapis.com/log_file_path" } + if r.FileCacheAdvise != nil { + if *r.FileCacheAdvise { + config["file_cache_advise"] = "true" + } else { + config["file_cache_advise"] = "false" + } + } + if r.BufferInMemory { config["storage.type"] = "memory" } diff --git a/confgenerator/testdata/feature/golden.csv b/confgenerator/testdata/feature/golden.csv index bc8616fa1b..0c4e7fc098 100644 --- a/confgenerator/testdata/feature/golden.csv +++ b/confgenerator/testdata/feature/golden.csv @@ -71,6 +71,7 @@ App,Field,Override, *confgenerator.LoggingProcessorParseJson,confgenerator.ConfigComponent.Type, *confgenerator.LoggingProcessorParseRegex,PreserveKey, *confgenerator.LoggingProcessorParseRegex,confgenerator.ConfigComponent.Type, +*confgenerator.LoggingReceiverFiles,FileCacheAdvise, *confgenerator.LoggingReceiverFiles,RecordLogFilePath, *confgenerator.LoggingReceiverFiles,WildcardRefreshInterval, *confgenerator.LoggingReceiverFiles,confgenerator.ConfigComponent.Type, @@ -203,14 +204,17 @@ App,Field,Override, *confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroZookeeperGeneral]],confgenerator.ConfigComponent.Type, *confgenerator.loggingReceiverMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingReceiverMacroActiveDirectoryDS],confgenerator.ConfigComponent.Type, *confgenerator.loggingReceiverMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingReceiverMacroOracleDBAlert],apps.LoggingReceiverMacroOracleDBAlert.confgenerator.LoggingReceiverFilesMixin.BufferInMemory, +*confgenerator.loggingReceiverMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingReceiverMacroOracleDBAlert],apps.LoggingReceiverMacroOracleDBAlert.confgenerator.LoggingReceiverFilesMixin.FileCacheAdvise, *confgenerator.loggingReceiverMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingReceiverMacroOracleDBAlert],apps.LoggingReceiverMacroOracleDBAlert.confgenerator.LoggingReceiverFilesMixin.RecordLogFilePath, *confgenerator.loggingReceiverMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingReceiverMacroOracleDBAlert],apps.LoggingReceiverMacroOracleDBAlert.confgenerator.LoggingReceiverFilesMixin.WildcardRefreshInterval, *confgenerator.loggingReceiverMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingReceiverMacroOracleDBAlert],confgenerator.ConfigComponent.Type, *confgenerator.loggingReceiverMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingReceiverMacroOracleDBAudit],apps.LoggingReceiverMacroOracleDBAudit.confgenerator.LoggingReceiverFilesMixin.BufferInMemory, +*confgenerator.loggingReceiverMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingReceiverMacroOracleDBAudit],apps.LoggingReceiverMacroOracleDBAudit.confgenerator.LoggingReceiverFilesMixin.FileCacheAdvise, *confgenerator.loggingReceiverMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingReceiverMacroOracleDBAudit],apps.LoggingReceiverMacroOracleDBAudit.confgenerator.LoggingReceiverFilesMixin.RecordLogFilePath, *confgenerator.loggingReceiverMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingReceiverMacroOracleDBAudit],apps.LoggingReceiverMacroOracleDBAudit.confgenerator.LoggingReceiverFilesMixin.WildcardRefreshInterval, *confgenerator.loggingReceiverMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingReceiverMacroOracleDBAudit],confgenerator.ConfigComponent.Type, *confgenerator.loggingReceiverMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingReceiverMacroVaultAuditJson],apps.LoggingReceiverMacroVaultAuditJson.confgenerator.LoggingReceiverFilesMixin.BufferInMemory, +*confgenerator.loggingReceiverMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingReceiverMacroVaultAuditJson],apps.LoggingReceiverMacroVaultAuditJson.confgenerator.LoggingReceiverFilesMixin.FileCacheAdvise, *confgenerator.loggingReceiverMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingReceiverMacroVaultAuditJson],apps.LoggingReceiverMacroVaultAuditJson.confgenerator.LoggingReceiverFilesMixin.RecordLogFilePath, *confgenerator.loggingReceiverMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingReceiverMacroVaultAuditJson],apps.LoggingReceiverMacroVaultAuditJson.confgenerator.LoggingReceiverFilesMixin.WildcardRefreshInterval, *confgenerator.loggingReceiverMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingReceiverMacroVaultAuditJson],confgenerator.ConfigComponent.Type, From 49e7aae752d4cf746a9c7ccff07bd3fcbedb9f4a Mon Sep 17 00:00:00 2001 From: Marek Knappe Date: Thu, 21 May 2026 11:28:14 +1000 Subject: [PATCH 2/2] chore: trigger CLA check