From e90727cca43c013cdc274835aa1067f4635fbf31 Mon Sep 17 00:00:00 2001 From: Raymond Zhao Date: Thu, 12 Jun 2025 17:38:52 +0000 Subject: [PATCH 1/7] Add log file Signed-off-by: Raymond Zhao --- Cargo.lock | 34 ++++++++++++++++ Cargo.toml | 2 + .../src/internal/remote_agent.rs | 19 +++++++-- bin/agent-data-plane/src/main.rs | 10 +++-- lib/saluki-app/Cargo.toml | 2 + lib/saluki-app/src/logging.rs | 39 +++++++++++++++++-- 6 files changed, 96 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6395b448cd..03216c6660 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -605,8 +605,10 @@ checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d" dependencies = [ "android-tzdata", "iana-time-zone", + "js-sys", "num-traits", "serde", + "wasm-bindgen", "windows-link", ] @@ -828,6 +830,15 @@ dependencies = [ "itertools 0.10.5", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -3028,6 +3039,15 @@ dependencies = [ "serde", ] +[[package]] +name = "rolling-file" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8395b4f860856b740f20a296ea2cd4d823e81a2658cf05ef61be22916026a906" +dependencies = [ + "chrono", +] + [[package]] name = "rustc-demangle" version = "0.1.24" @@ -3170,6 +3190,7 @@ dependencies = [ "memory-accounting", "metrics", "rcgen", + "rolling-file", "rustls", "rustls-pemfile", "saluki-api", @@ -3185,6 +3206,7 @@ dependencies = [ "tonic", "tower 0.5.2", "tracing", + "tracing-appender", "tracing-subscriber", ] @@ -4337,6 +4359,18 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-appender" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3566e8ce28cc0a3fe42519fc80e6b4c943cc4c8cef275620eb8dac2d3d4e06cf" +dependencies = [ + "crossbeam-channel", + "thiserror 1.0.69", + "time", + "tracing-subscriber", +] + [[package]] name = "tracing-attributes" version = "0.1.28" diff --git a/Cargo.toml b/Cargo.toml index 5d81bc6fc0..413120bfdb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -179,6 +179,8 @@ fnv = { version = "1", default-features = false } twox-hash = { version = "2", features = ["xxhash64"] } sha3 = { version = "0.10", default-features = false } pyo3 = { version = "0.25", default-features = false } +tracing-appender = { version = "0.2.3", default-features = false } +rolling-file = { version = "0.2.0", default-features = false } [profile.release] lto = "thin" diff --git a/bin/agent-data-plane/src/internal/remote_agent.rs b/bin/agent-data-plane/src/internal/remote_agent.rs index 50af655fa1..a18abe2db5 100644 --- a/bin/agent-data-plane/src/internal/remote_agent.rs +++ b/bin/agent-data-plane/src/internal/remote_agent.rs @@ -204,9 +204,22 @@ impl RemoteAgent for RemoteAgentImpl { async fn get_flare_files( &self, _request: tonic::Request, ) -> Result, tonic::Status> { - let response = GetFlareFilesResponse { - files: HashMap::default(), - }; + let mut files = HashMap::new(); + + let log_file_path = + std::env::var("DD_ADP_LOG_FILE").unwrap_or(saluki_app::logging::DEFAULT_ADP_LOG_FILE.to_string()); + + match tokio::fs::read(&log_file_path).await { + Ok(content) => { + files.insert(log_file_path, content); + } + Err(e) => { + debug!("Failed to read {} log file for flare: {}", log_file_path, e); + } + } + + let response = GetFlareFilesResponse { files }; + Ok(tonic::Response::new(response)) } diff --git a/bin/agent-data-plane/src/main.rs b/bin/agent-data-plane/src/main.rs index 962d7d8e27..fd5ffa2319 100644 --- a/bin/agent-data-plane/src/main.rs +++ b/bin/agent-data-plane/src/main.rs @@ -49,9 +49,13 @@ static ALLOC: memory_accounting::allocator::TrackingAllocator guard, + Err(e) => { + fatal_and_exit(format!("failed to initialize logging: {}", e)); + return; + } + }; if let Err(e) = initialize_metrics("adp").await { fatal_and_exit(format!("failed to initialize metrics: {}", e)); diff --git a/lib/saluki-app/Cargo.toml b/lib/saluki-app/Cargo.toml index 7cab925ea1..e5b20bb84a 100644 --- a/lib/saluki-app/Cargo.toml +++ b/lib/saluki-app/Cargo.toml @@ -41,3 +41,5 @@ tonic = { workspace = true, features = ["transport"] } tower = { workspace = true, features = ["util"], optional = true } tracing = { workspace = true, optional = true } tracing-subscriber = { workspace = true, features = ["ansi", "env-filter", "fmt", "json", "local-time", "registry", "std", "tracing-log"], optional = true } +rolling-file = { workspace = true } +tracing-appender = { workspace = true } diff --git a/lib/saluki-app/src/logging.rs b/lib/saluki-app/src/logging.rs index ab9be555df..a2eabe97b5 100644 --- a/lib/saluki-app/src/logging.rs +++ b/lib/saluki-app/src/logging.rs @@ -20,6 +20,7 @@ use chrono::{ Utc, }; use chrono_tz::Tz; +use rolling_file::{BasicRollingFileAppender, RollingConditionBasic}; use saluki_api::{ extract::{Query, State}, response::IntoResponse, @@ -30,6 +31,7 @@ use saluki_common::task::spawn_traced_named; use serde::Deserialize; use tokio::{select, sync::mpsc, time::sleep}; use tracing::{error, field, info, level_filters::LevelFilter, Event, Subscriber}; +use tracing_appender::non_blocking::WorkerGuard; use tracing_subscriber::{ field::VisitOutput, fmt::{format::Writer, FmtContext, FormatEvent, FormatFields}, @@ -50,6 +52,14 @@ pub fn fatal_and_exit(message: String) { std::process::exit(1); } +#[cfg(target_os = "linux")] +/// The default log file path for ADP on Linux. +pub const DEFAULT_ADP_LOG_FILE: &str = "/var/log/datadog/adp.log"; + +#[cfg(not(target_os = "linux"))] +/// The default log file path for ADP on non-Linux platforms. +pub const DEFAULT_ADP_LOG_FILE: &str = "/opt/datadog-agent/logs/adp.log"; + /// Initializes the logging subsystem for `tracing`. /// /// This function reads the `DD_LOG_LEVEL` environment variable to determine the log level to use. If the environment @@ -61,7 +71,9 @@ pub fn fatal_and_exit(message: String) { /// # Errors /// /// If the logging subsystem was already initialized, an error will be returned. -pub fn initialize_logging(default_level: Option) -> Result<(), Box> { +pub fn initialize_logging( + default_level: Option, +) -> Result> { initialize_logging_inner(default_level, false) } @@ -82,7 +94,7 @@ pub fn initialize_logging(default_level: Option) -> Result<(), Box< /// If the logging subsystem was already initialized, an error will be returned. pub async fn initialize_dynamic_logging( default_level: Option, -) -> Result<(), Box> { +) -> Result> { // We go through this wrapped initialize approach so that we can mark `initialize_dynamic_logging` as `async`, which // ensures we call it in an asynchronous context, thereby all but ensuring we're in a Tokio context when we try to // spawn the background task that handles reloading the filtering layer. @@ -91,7 +103,7 @@ pub async fn initialize_dynamic_logging( fn initialize_logging_inner( default_level: Option, with_reload: bool, -) -> Result<(), Box> { +) -> Result> { let is_json = std::env::var("DD_LOG_FORMAT_JSON") .map(|s| s.trim().to_lowercase()) .map(|s| s == "true" || s == "1") @@ -115,19 +127,38 @@ fn initialize_logging_inner( .replace(LoggingAPIHandler::new(shared_level_filter.clone(), reload_handle)); } + let adp_log_file = std::env::var("DD_ADP_LOG_FILE").unwrap_or(DEFAULT_ADP_LOG_FILE.to_string()); + let file_appender = + BasicRollingFileAppender::new(adp_log_file, RollingConditionBasic::new().max_size(10485760), 1)?; + let (file_nb, guard) = tracing_appender::non_blocking(file_appender); + let file_level_filter = EnvFilter::builder() + .with_default_directive(default_level.unwrap_or(LevelFilter::INFO).into()) + .with_env_var("DD_LOG_LEVEL") + .from_env_lossy(); + if is_json { let json_layer = initialize_tracing_json(); tracing_subscriber::registry() .with(json_layer.with_filter(filter_layer)) + .with( + tracing_subscriber::fmt::Layer::new() + .with_writer(file_nb) + .with_filter(file_level_filter), + ) .try_init()?; } else { let pretty_layer = initialize_tracing_pretty(); tracing_subscriber::registry() .with(pretty_layer.with_filter(filter_layer)) + .with( + tracing_subscriber::fmt::Layer::new() + .with_writer(file_nb) + .with_filter(file_level_filter), + ) .try_init()?; } - Ok(()) + Ok(guard) } fn initialize_tracing_json() -> impl Layer From a6d48d7eae47bbc568be0e8628346cf8a38bd290 Mon Sep 17 00:00:00 2001 From: raymond zhao Date: Tue, 17 Jun 2025 09:35:03 -0400 Subject: [PATCH 2/7] Create a log file to add to flare --- .../src/internal/remote_agent.rs | 21 ++++++++++++------- lib/saluki-app/src/logging.rs | 21 ++++++++++++++++--- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/bin/agent-data-plane/src/internal/remote_agent.rs b/bin/agent-data-plane/src/internal/remote_agent.rs index a18abe2db5..836bfbcd35 100644 --- a/bin/agent-data-plane/src/internal/remote_agent.rs +++ b/bin/agent-data-plane/src/internal/remote_agent.rs @@ -20,6 +20,8 @@ use tokio::time::{interval, MissedTickBehavior}; use tracing::debug; use uuid::Uuid; +use std::path::PathBuf; + use crate::state::metrics::{get_shared_metrics_state, AggregatedMetricsProcessor}; const EVENTS_RECEIVED: &str = "adp.component_events_received_total"; @@ -206,15 +208,18 @@ impl RemoteAgent for RemoteAgentImpl { ) -> Result, tonic::Status> { let mut files = HashMap::new(); - let log_file_path = - std::env::var("DD_ADP_LOG_FILE").unwrap_or(saluki_app::logging::DEFAULT_ADP_LOG_FILE.to_string()); + let log_file_path = PathBuf::from( + std::env::var("DD_ADP_LOG_FILE").unwrap_or(saluki_app::logging::DEFAULT_ADP_LOG_FILE.to_string()), + ); - match tokio::fs::read(&log_file_path).await { - Ok(content) => { - files.insert(log_file_path, content); - } - Err(e) => { - debug!("Failed to read {} log file for flare: {}", log_file_path, e); + if let Some(log_file_name) = log_file_path.file_name() { + match tokio::fs::read(&log_file_path).await { + Ok(content) => { + files.insert(log_file_name.to_string_lossy().to_string(), content); + } + Err(e) => { + debug!("Failed to read {} log file for flare: {}", log_file_path.display(), e); + } } } diff --git a/lib/saluki-app/src/logging.rs b/lib/saluki-app/src/logging.rs index a2eabe97b5..188d04cddb 100644 --- a/lib/saluki-app/src/logging.rs +++ b/lib/saluki-app/src/logging.rs @@ -56,10 +56,16 @@ pub fn fatal_and_exit(message: String) { /// The default log file path for ADP on Linux. pub const DEFAULT_ADP_LOG_FILE: &str = "/var/log/datadog/adp.log"; -#[cfg(not(target_os = "linux"))] +#[cfg(target_os = "macos")] /// The default log file path for ADP on non-Linux platforms. pub const DEFAULT_ADP_LOG_FILE: &str = "/opt/datadog-agent/logs/adp.log"; +#[cfg(target_os = "windows")] +/// The default log file path for ADP on Windows. +pub const DEFAULT_ADP_LOG_FILE: &str = "C:\\ProgramData\\Datadog\\logs\\adp.log"; + +const DEFAULT_LOG_FILE_MAX_SIZE: u64 = 10485760; +const DEFAULT_LOG_FILE_MAX_ROLLS: usize = 1; /// Initializes the logging subsystem for `tracing`. /// /// This function reads the `DD_LOG_LEVEL` environment variable to determine the log level to use. If the environment @@ -128,8 +134,17 @@ fn initialize_logging_inner( } let adp_log_file = std::env::var("DD_ADP_LOG_FILE").unwrap_or(DEFAULT_ADP_LOG_FILE.to_string()); - let file_appender = - BasicRollingFileAppender::new(adp_log_file, RollingConditionBasic::new().max_size(10485760), 1)?; + let log_file_max_size = std::env::var("DD_LOG_FILE_MAX_SIZE") + .map(|s| s.parse::().unwrap_or(DEFAULT_LOG_FILE_MAX_SIZE)) + .unwrap_or(DEFAULT_LOG_FILE_MAX_SIZE); + let log_file_max_rolls = std::env::var("DD_LOG_FILE_MAX_ROLLS") + .map(|s| s.parse::().unwrap_or(DEFAULT_LOG_FILE_MAX_ROLLS)) + .unwrap_or(DEFAULT_LOG_FILE_MAX_ROLLS); + let file_appender = BasicRollingFileAppender::new( + adp_log_file, + RollingConditionBasic::new().max_size(log_file_max_size), + log_file_max_rolls, + )?; let (file_nb, guard) = tracing_appender::non_blocking(file_appender); let file_level_filter = EnvFilter::builder() .with_default_directive(default_level.unwrap_or(LevelFilter::INFO).into()) From 747510d957aec0335a0c19444787748278a0db01 Mon Sep 17 00:00:00 2001 From: raymond zhao Date: Tue, 17 Jun 2025 10:04:30 -0400 Subject: [PATCH 3/7] CI --- LICENSE-3rdparty.csv | 3 +++ bin/agent-data-plane/src/internal/remote_agent.rs | 3 +-- bin/agent-data-plane/src/main.rs | 11 ++++------- lib/saluki-app/Cargo.toml | 4 ++-- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 56a68b34f3..fbd9552f80 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -58,6 +58,7 @@ core-foundation,https://github.com/servo/core-foundation-rs,MIT OR Apache-2.0,Th cpufeatures,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers crc32fast,https://github.com/srijs/rust-crc32fast,MIT OR Apache-2.0,"Sam Rijs , Alex Crichton " criterion-plot,https://github.com/bheisler/criterion.rs,MIT OR Apache-2.0,"Jorge Aparicio , Brook Heisler " +crossbeam-channel,https://github.com/crossbeam-rs/crossbeam,MIT OR Apache-2.0,The crossbeam-channel Authors crossbeam-deque,https://github.com/crossbeam-rs/crossbeam,MIT OR Apache-2.0,The crossbeam-deque Authors crossbeam-epoch,https://github.com/crossbeam-rs/crossbeam,MIT OR Apache-2.0,The crossbeam-epoch Authors crossbeam-queue,https://github.com/crossbeam-rs/crossbeam,MIT OR Apache-2.0,The crossbeam-queue Authors @@ -226,6 +227,7 @@ reqwest,https://github.com/seanmonstar/reqwest,MIT OR Apache-2.0,Sean McArthur < ring,https://github.com/briansmith/ring,Apache-2.0 AND ISC,The ring Authors rmp,https://github.com/3Hren/msgpack-rust,MIT,Evgeny Safronov rmp-serde,https://github.com/3Hren/msgpack-rust,MIT,Evgeny Safronov +rolling-file,https://github.com/Axcient/rolling-file-rs,MIT OR Apache-2.0,Kevin Hoffman rustc-demangle,https://github.com/rust-lang/rustc-demangle,MIT OR Apache-2.0,Alex Crichton rustc-hash,https://github.com/rust-lang-nursery/rustc-hash,Apache-2.0 OR MIT,The Rust Project Developers rustix,https://github.com/bytecodealliance/rustix,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,"Dan Gohman , Jakub Konka " @@ -293,6 +295,7 @@ tonic,https://github.com/hyperium/tonic,MIT,Lucio Franco tower-http,https://github.com/tower-rs/tower-http,MIT,Tower Maintainers tracing,https://github.com/tokio-rs/tracing,MIT,"Eliza Weisman , Tokio Contributors " +tracing-appender,https://github.com/tokio-rs/tracing,MIT,"Zeki Sherif , Tokio Contributors " tracing-attributes,https://github.com/tokio-rs/tracing,MIT,"Tokio Contributors , Eliza Weisman , David Barsky " tracing-core,https://github.com/tokio-rs/tracing,MIT,Tokio Contributors tracing-log,https://github.com/tokio-rs/tracing,MIT,Tokio Contributors diff --git a/bin/agent-data-plane/src/internal/remote_agent.rs b/bin/agent-data-plane/src/internal/remote_agent.rs index 836bfbcd35..ab0adcf9a1 100644 --- a/bin/agent-data-plane/src/internal/remote_agent.rs +++ b/bin/agent-data-plane/src/internal/remote_agent.rs @@ -1,3 +1,4 @@ +use std::path::PathBuf; use std::{collections::hash_map::Entry, time::Duration}; use std::{collections::HashMap, net::SocketAddr}; @@ -20,8 +21,6 @@ use tokio::time::{interval, MissedTickBehavior}; use tracing::debug; use uuid::Uuid; -use std::path::PathBuf; - use crate::state::metrics::{get_shared_metrics_state, AggregatedMetricsProcessor}; const EVENTS_RECEIVED: &str = "adp.component_events_received_total"; diff --git a/bin/agent-data-plane/src/main.rs b/bin/agent-data-plane/src/main.rs index fd5ffa2319..4b1db4843d 100644 --- a/bin/agent-data-plane/src/main.rs +++ b/bin/agent-data-plane/src/main.rs @@ -49,13 +49,10 @@ static ALLOC: memory_accounting::allocator::TrackingAllocator guard, - Err(e) => { - fatal_and_exit(format!("failed to initialize logging: {}", e)); - return; - } - }; + let _guard = initialize_dynamic_logging(None).await.unwrap_or_else(|e| { + fatal_and_exit(format!("failed to initialize logging: {}", e)); + unreachable!() // This will never be reached since fatal_and_exit exits + }); if let Err(e) = initialize_metrics("adp").await { fatal_and_exit(format!("failed to initialize metrics: {}", e)); diff --git a/lib/saluki-app/Cargo.toml b/lib/saluki-app/Cargo.toml index e5b20bb84a..3f48e2eba5 100644 --- a/lib/saluki-app/Cargo.toml +++ b/lib/saluki-app/Cargo.toml @@ -25,6 +25,7 @@ iana-time-zone = { workspace = true, optional = true } memory-accounting = { workspace = true, optional = true } metrics = { workspace = true, optional = true } rcgen = { workspace = true, features = ["crypto", "aws_lc_rs", "pem"] } +rolling-file = { workspace = true } rustls = { workspace = true, features = ["tls12"] } rustls-pemfile = { workspace = true, features = ["std"] } saluki-api = { workspace = true, optional = true } @@ -40,6 +41,5 @@ tokio = { workspace = true, features = ["macros", "sync"], optional = true } tonic = { workspace = true, features = ["transport"] } tower = { workspace = true, features = ["util"], optional = true } tracing = { workspace = true, optional = true } -tracing-subscriber = { workspace = true, features = ["ansi", "env-filter", "fmt", "json", "local-time", "registry", "std", "tracing-log"], optional = true } -rolling-file = { workspace = true } tracing-appender = { workspace = true } +tracing-subscriber = { workspace = true, features = ["ansi", "env-filter", "fmt", "json", "local-time", "registry", "std", "tracing-log"], optional = true } From 84e197f99800fbb2e76ccffdd45c833b8527f5c9 Mon Sep 17 00:00:00 2001 From: raymond zhao Date: Tue, 17 Jun 2025 10:12:28 -0400 Subject: [PATCH 4/7] Clippy --- bin/checks-agent/src/main.rs | 10 ++++------ lib/saluki-app/src/logging.rs | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/bin/checks-agent/src/main.rs b/bin/checks-agent/src/main.rs index 997a89f9c6..295310a732 100644 --- a/bin/checks-agent/src/main.rs +++ b/bin/checks-agent/src/main.rs @@ -38,12 +38,10 @@ static ALLOC: memory_accounting::allocator::TrackingAllocator handler, - Err(e) => { - fatal_and_exit(format!("failed to initialize logging: {}", e)); - } - }; + let _guard = initialize_dynamic_logging(None).await.unwrap_or_else(|e| { + fatal_and_exit(format!("failed to initialize logging: {}", e)); + unreachable!() // This will never be reached since fatal_and_exit exits + }); if let Err(e) = initialize_metrics("checks-agent").await { fatal_and_exit(format!("failed to initialize metrics: {}", e)); diff --git a/lib/saluki-app/src/logging.rs b/lib/saluki-app/src/logging.rs index 188d04cddb..b4600acabb 100644 --- a/lib/saluki-app/src/logging.rs +++ b/lib/saluki-app/src/logging.rs @@ -88,7 +88,7 @@ pub fn initialize_logging( /// /// This function reads the `DD_LOG_LEVEL` environment variable to determine the log level to use. If the environment /// variable is not set, the default log level is `INFO`. Additionally, it reads the `DD_LOG_FORMAT_JSON` environment -/// variable to determine which output format to use. If it is set to `json` (case insensitive), the logs will be +/// variable to determine which output format to use. If it is set to `true` (case insensitive) or `1`, the logs will be /// formatted as JSON. If it is set to any other value, or not set at all, the logs will default to a rich, colored, /// human-readable format. /// From f494f21481580243424ceb10966e4ae30deb2e0f Mon Sep 17 00:00:00 2001 From: raymond zhao Date: Wed, 25 Jun 2025 15:08:43 -0400 Subject: [PATCH 5/7] Add LoggingConfiguration --- .../src/internal/remote_agent.rs | 21 +- bin/agent-data-plane/src/main.rs | 32 ++- bin/checks-agent/src/main.rs | 12 +- bin/correctness/airlock/src/main.rs | 3 +- bin/correctness/ground-truth/src/config.rs | 1 + bin/correctness/ground-truth/src/main.rs | 3 +- bin/correctness/metrics-intake/src/main.rs | 4 +- bin/correctness/millstone/src/main.rs | 3 +- lib/saluki-app/src/logging.rs | 187 +++++++++++++----- 9 files changed, 193 insertions(+), 73 deletions(-) diff --git a/bin/agent-data-plane/src/internal/remote_agent.rs b/bin/agent-data-plane/src/internal/remote_agent.rs index ab0adcf9a1..346099f02c 100644 --- a/bin/agent-data-plane/src/internal/remote_agent.rs +++ b/bin/agent-data-plane/src/internal/remote_agent.rs @@ -1,4 +1,3 @@ -use std::path::PathBuf; use std::{collections::hash_map::Entry, time::Duration}; use std::{collections::HashMap, net::SocketAddr}; @@ -11,6 +10,7 @@ use datadog_protos::agent::{ use http::{Request, Uri}; use http_body_util::BodyExt; use rand::{distributions::Alphanumeric, thread_rng, Rng}; +use saluki_app::logging::LoggingConfiguration; use saluki_common::task::spawn_traced_named; use saluki_config::GenericConfiguration; use saluki_core::state::reflector::Reflector; @@ -45,6 +45,7 @@ pub struct RemoteAgentHelperConfiguration { client: RemoteAgentClient, internal_metrics: Reflector, prometheus_listen_addr: Option, + logging_config: LoggingConfiguration, } impl RemoteAgentHelperConfiguration { @@ -59,6 +60,7 @@ impl RemoteAgentHelperConfiguration { .replace("_", "-") .to_lowercase(); let client = RemoteAgentClient::from_configuration(config).await?; + let logging_config = LoggingConfiguration::try_from_config(config)?; Ok(Self { id: format!("{}-{}", formatted_full_name, Uuid::now_v7()), @@ -67,6 +69,7 @@ impl RemoteAgentHelperConfiguration { client, internal_metrics: get_shared_metrics_state().await, prometheus_listen_addr, + logging_config, }) } @@ -80,6 +83,7 @@ impl RemoteAgentHelperConfiguration { started: Utc::now(), internal_metrics: self.internal_metrics.clone(), prometheus_listen_addr: self.prometheus_listen_addr, + logging_config: self.logging_config, }; let service = RemoteAgentServer::new(service_impl); @@ -129,6 +133,7 @@ pub struct RemoteAgentImpl { started: DateTime, internal_metrics: Reflector, prometheus_listen_addr: Option, + logging_config: LoggingConfiguration, } impl RemoteAgentImpl { @@ -207,17 +212,17 @@ impl RemoteAgent for RemoteAgentImpl { ) -> Result, tonic::Status> { let mut files = HashMap::new(); - let log_file_path = PathBuf::from( - std::env::var("DD_ADP_LOG_FILE").unwrap_or(saluki_app::logging::DEFAULT_ADP_LOG_FILE.to_string()), - ); - - if let Some(log_file_name) = log_file_path.file_name() { - match tokio::fs::read(&log_file_path).await { + if let Some(log_file_name) = self.logging_config.log_file.file_name() { + match tokio::fs::read(&self.logging_config.log_file).await { Ok(content) => { files.insert(log_file_name.to_string_lossy().to_string(), content); } Err(e) => { - debug!("Failed to read {} log file for flare: {}", log_file_path.display(), e); + debug!( + "Failed to read {} log file for flare: {}", + self.logging_config.log_file.display(), + e + ); } } } diff --git a/bin/agent-data-plane/src/main.rs b/bin/agent-data-plane/src/main.rs index fa3104c11b..91cc59b19f 100644 --- a/bin/agent-data-plane/src/main.rs +++ b/bin/agent-data-plane/src/main.rs @@ -5,11 +5,14 @@ #![deny(warnings)] #![deny(missing_docs)] -use std::time::{Duration, Instant}; +use std::{ + path::PathBuf, + time::{Duration, Instant}, +}; use clap::Parser as _; use memory_accounting::{ComponentBounds, ComponentRegistry}; -use saluki_app::prelude::*; +use saluki_app::{logging::LoggingConfiguration, prelude::*}; use saluki_components::{ destinations::{DatadogEventsConfiguration, DatadogMetricsConfiguration, DatadogServiceChecksConfiguration}, sources::DogStatsDConfiguration, @@ -54,7 +57,12 @@ async fn main() { let started = Instant::now(); let cli = Cli::parse(); - let _guard = initialize_dynamic_logging(None).await.unwrap_or_else(|e| { + let configuration = load_configuration(PathBuf::from("/etc/datadog-agent/datadog.yaml")) + .await + .unwrap(); + let logging_config = LoggingConfiguration::try_from_config(&configuration).unwrap(); + + let _guard = initialize_dynamic_logging(&logging_config).await.unwrap_or_else(|e| { fatal_and_exit(format!("failed to initialize logging: {}", e)); unreachable!() // This will never be reached since fatal_and_exit exits }); @@ -99,12 +107,7 @@ async fn run(started: Instant, run_config: RunConfig) -> Result<(), GenericError ); // Load our configuration and create all high-level primitives (health registry, component registry, environment // provider, etc) that are needed to build the topology. - let configuration = ConfigurationLoader::default() - .try_from_yaml(&run_config.config) - .from_environment("DD")? - .with_default_secrets_resolution() - .await? - .into_generic()?; + let configuration = load_configuration(run_config.config).await?; // Set up all of the building blocks for building our topologies and launching internal processes. let component_registry = ComponentRegistry::default(); @@ -296,3 +299,14 @@ fn write_sizing_guide(bounds: ComponentBounds) -> Result<(), GenericError> { Ok(()) } + +async fn load_configuration(config_path: PathBuf) -> Result { + let configuration = ConfigurationLoader::default() + .try_from_yaml(config_path) + .from_environment("DD")? + .with_default_secrets_resolution() + .await? + .into_generic()?; + + Ok(configuration) +} diff --git a/bin/checks-agent/src/main.rs b/bin/checks-agent/src/main.rs index 8c3df044c3..11f877f0e6 100644 --- a/bin/checks-agent/src/main.rs +++ b/bin/checks-agent/src/main.rs @@ -1,7 +1,7 @@ use std::time::{Duration, Instant}; use memory_accounting::ComponentRegistry; -use saluki_app::{api::APIBuilder, metrics::emit_startup_metrics, prelude::*}; +use saluki_app::{api::APIBuilder, logging::LoggingConfiguration, metrics::emit_startup_metrics, prelude::*}; use saluki_components::{ destinations::{DatadogMetricsConfiguration, PrometheusConfiguration}, sources::{ChecksConfiguration, InternalMetricsConfiguration}, @@ -38,10 +38,12 @@ static ALLOC: memory_accounting::allocator::TrackingAllocator LevelFilter { match self.verbose { 0 => LevelFilter::INFO, diff --git a/bin/correctness/ground-truth/src/main.rs b/bin/correctness/ground-truth/src/main.rs index 6f132b6ef7..70aa60b0c7 100644 --- a/bin/correctness/ground-truth/src/main.rs +++ b/bin/correctness/ground-truth/src/main.rs @@ -4,6 +4,7 @@ #![deny(missing_docs)] use clap::Parser as _; +use saluki_app::logging::LoggingConfiguration; use saluki_app::prelude::*; use saluki_error::{ErrorContext as _, GenericError}; use tracing::{error, info}; @@ -22,7 +23,7 @@ mod sync; async fn main() { let cli = Cli::parse(); - if let Err(e) = initialize_logging(Some(cli.log_level())) { + if let Err(e) = initialize_logging(&LoggingConfiguration::default()) { fatal_and_exit(format!("failed to initialize logging: {}", e)); } diff --git a/bin/correctness/metrics-intake/src/main.rs b/bin/correctness/metrics-intake/src/main.rs index dc17eb5bb7..d52c8a1e4f 100644 --- a/bin/correctness/metrics-intake/src/main.rs +++ b/bin/correctness/metrics-intake/src/main.rs @@ -8,7 +8,7 @@ use axum::{ routing::{get, post}, Router, }; -use saluki_app::prelude::*; +use saluki_app::{logging::LoggingConfiguration, prelude::*}; use saluki_error::GenericError; use tokio::{ net::TcpListener, @@ -27,7 +27,7 @@ use self::state::*; #[tokio::main] async fn main() { - if let Err(e) = initialize_logging(None) { + if let Err(e) = initialize_logging(&LoggingConfiguration::default()) { fatal_and_exit(format!("failed to initialize logging: {}", e)); } diff --git a/bin/correctness/millstone/src/main.rs b/bin/correctness/millstone/src/main.rs index b1f137bf06..9c3bfa63b3 100644 --- a/bin/correctness/millstone/src/main.rs +++ b/bin/correctness/millstone/src/main.rs @@ -4,6 +4,7 @@ #![deny(warnings)] #![deny(missing_docs)] +use saluki_app::logging::LoggingConfiguration; use saluki_app::prelude::*; use saluki_error::GenericError; use tracing::{error, info}; @@ -19,7 +20,7 @@ use self::driver::Driver; mod target; fn main() { - if let Err(e) = initialize_logging(None) { + if let Err(e) = initialize_logging(&LoggingConfiguration::default()) { fatal_and_exit(format!("failed to initialize logging: {}", e)); } diff --git a/lib/saluki-app/src/logging.rs b/lib/saluki-app/src/logging.rs index b4600acabb..51e51c6d0e 100644 --- a/lib/saluki-app/src/logging.rs +++ b/lib/saluki-app/src/logging.rs @@ -10,6 +10,7 @@ use std::{ fmt, + path::PathBuf, str::FromStr as _, sync::{Arc, Mutex, OnceLock}, time::Duration, @@ -28,10 +29,12 @@ use saluki_api::{ APIHandler, StatusCode, }; use saluki_common::task::spawn_traced_named; +use saluki_config::GenericConfiguration; +use saluki_error::{ErrorContext as _, GenericError}; use serde::Deserialize; use tokio::{select, sync::mpsc, time::sleep}; use tracing::{error, field, info, level_filters::LevelFilter, Event, Subscriber}; -use tracing_appender::non_blocking::WorkerGuard; +use tracing_appender::non_blocking::{NonBlocking, WorkerGuard}; use tracing_subscriber::{ field::VisitOutput, fmt::{format::Writer, FmtContext, FormatEvent, FormatFields}, @@ -66,32 +69,107 @@ pub const DEFAULT_ADP_LOG_FILE: &str = "C:\\ProgramData\\Datadog\\logs\\adp.log" const DEFAULT_LOG_FILE_MAX_SIZE: u64 = 10485760; const DEFAULT_LOG_FILE_MAX_ROLLS: usize = 1; +const DEFAULT_LOG_LEVEL: &str = "info"; + +/// Configuration for logging. +#[derive(Deserialize, Default, Debug)] +pub struct LoggingConfiguration { + /// Whether to format logs as JSON. + /// + /// Defaults to `false`. + #[serde(default)] + pub log_format_json: bool, + + /// The log level. + /// + /// Defaults to `info`. + #[serde(default = "default_log_level")] + pub log_level: String, + + /// Whether to enable logging to a file. + /// + /// Defaults to `false`. + #[serde(default, rename = "adp_log_file_enabled")] + pub log_file_enabled: bool, + + /// The maximum size of the log file before it is rolled. + /// + /// Defaults to 10MB. + #[serde(default = "default_log_file_max_size")] + pub log_file_max_size: u64, + + /// The maximum number of log files to keep. + /// + /// Defaults to 1. + #[serde(default = "default_log_file_max_rolls")] + pub log_file_max_rolls: usize, + + /// The path to the log file. + /// + /// Defaults to + /// `/var/log/datadog/adp.log` on Linux, + /// `/opt/datadog-agent/logs/adp.log` on macOS, and + /// `C:\\ProgramData\\Datadog\\logs\\adp.log` on Windows. + #[serde(default = "default_log_file", rename = "adp_log_file")] + pub log_file: PathBuf, + + #[serde(skip)] + default_level: Option, +} + +const fn default_log_file_max_size() -> u64 { + DEFAULT_LOG_FILE_MAX_SIZE +} + +const fn default_log_file_max_rolls() -> usize { + DEFAULT_LOG_FILE_MAX_ROLLS +} + +fn default_log_level() -> String { + DEFAULT_LOG_LEVEL.to_owned() +} + +/// The default log file path for ADP. +pub fn default_log_file() -> PathBuf { + PathBuf::from(DEFAULT_ADP_LOG_FILE) +} + +impl LoggingConfiguration { + /// Attempts to read logging configuration from the provided configuration. + /// + /// ## Errors + /// + /// If an error occurs during deserialization, an error will be returned. + pub fn try_from_config(config: &GenericConfiguration) -> Result { + let logging_config = config + .as_typed::() + .error_context("Failed to parse logging configuration.")?; + + Ok(logging_config) + } + + /// Sets the log level used for the default directive. + pub fn with_default_level(self, level: LevelFilter) -> Self { + Self { + default_level: Some(level), + ..self + } + } +} /// Initializes the logging subsystem for `tracing`. /// -/// This function reads the `DD_LOG_LEVEL` environment variable to determine the log level to use. If the environment -/// variable is not set, the default log level is `INFO`. Additionally, it reads the `DD_LOG_FORMAT_JSON` environment -/// variable to determine which output format to use. If it is set to `json` (case insensitive), the logs will be -/// formatted as JSON. If it is set to any other value, or not set at all, the logs will default to a rich, colored, -/// human-readable format. -/// /// # Errors /// /// If the logging subsystem was already initialized, an error will be returned. pub fn initialize_logging( - default_level: Option, + config: &LoggingConfiguration, ) -> Result> { - initialize_logging_inner(default_level, false) + initialize_logging_inner(config, false) } /// Initializes the logging subsystem for `tracing` with the ability to dynamically update the log filtering directives /// at runtime. /// -/// This function reads the `DD_LOG_LEVEL` environment variable to determine the log level to use. If the environment -/// variable is not set, the default log level is `INFO`. Additionally, it reads the `DD_LOG_FORMAT_JSON` environment -/// variable to determine which output format to use. If it is set to `true` (case insensitive) or `1`, the logs will be -/// formatted as JSON. If it is set to any other value, or not set at all, the logs will default to a rich, colored, -/// human-readable format. -/// /// An API handler can be acquired (via [`acquires_logging_api_handler`]) to install the API routes which allow for /// dynamically controlling the logging level filtering. See [`LoggingAPIHandler`] for more information. /// @@ -99,30 +177,24 @@ pub fn initialize_logging( /// /// If the logging subsystem was already initialized, an error will be returned. pub async fn initialize_dynamic_logging( - default_level: Option, + config: &LoggingConfiguration, ) -> Result> { // We go through this wrapped initialize approach so that we can mark `initialize_dynamic_logging` as `async`, which // ensures we call it in an asynchronous context, thereby all but ensuring we're in a Tokio context when we try to // spawn the background task that handles reloading the filtering layer. - initialize_logging_inner(default_level, true) + initialize_logging_inner(config, true) } fn initialize_logging_inner( - default_level: Option, with_reload: bool, + config: &LoggingConfiguration, with_reload: bool, ) -> Result> { - let is_json = std::env::var("DD_LOG_FORMAT_JSON") - .map(|s| s.trim().to_lowercase()) - .map(|s| s == "true" || s == "1") - .unwrap_or(false); - // Load our level filtering directives from the environment, or fallback to INFO if the environment variable is not // specified. // // We also do a little bit of a dance to get the filter into the right shape for use in the dynamic filter layer. let level_filter = EnvFilter::builder() - .with_default_directive(default_level.unwrap_or(LevelFilter::INFO).into()) - .with_env_var("DD_LOG_LEVEL") - .from_env_lossy(); + .with_default_directive(config.default_level.unwrap_or(LevelFilter::INFO).into()) + .parse_lossy(&config.log_level); let shared_level_filter = Arc::new(level_filter); let (filter_layer, reload_handle) = ReloadLayer::new(into_shared_dyn_filter(Arc::clone(&shared_level_filter))); @@ -133,13 +205,9 @@ fn initialize_logging_inner( .replace(LoggingAPIHandler::new(shared_level_filter.clone(), reload_handle)); } - let adp_log_file = std::env::var("DD_ADP_LOG_FILE").unwrap_or(DEFAULT_ADP_LOG_FILE.to_string()); - let log_file_max_size = std::env::var("DD_LOG_FILE_MAX_SIZE") - .map(|s| s.parse::().unwrap_or(DEFAULT_LOG_FILE_MAX_SIZE)) - .unwrap_or(DEFAULT_LOG_FILE_MAX_SIZE); - let log_file_max_rolls = std::env::var("DD_LOG_FILE_MAX_ROLLS") - .map(|s| s.parse::().unwrap_or(DEFAULT_LOG_FILE_MAX_ROLLS)) - .unwrap_or(DEFAULT_LOG_FILE_MAX_ROLLS); + let adp_log_file = &config.log_file; + let log_file_max_size = config.log_file_max_size; + let log_file_max_rolls = config.log_file_max_rolls; let file_appender = BasicRollingFileAppender::new( adp_log_file, RollingConditionBasic::new().max_size(log_file_max_size), @@ -147,29 +215,34 @@ fn initialize_logging_inner( )?; let (file_nb, guard) = tracing_appender::non_blocking(file_appender); let file_level_filter = EnvFilter::builder() - .with_default_directive(default_level.unwrap_or(LevelFilter::INFO).into()) - .with_env_var("DD_LOG_LEVEL") - .from_env_lossy(); + .with_default_directive(config.default_level.unwrap_or(LevelFilter::INFO).into()) + .parse_lossy(&config.log_level); + + let is_json = config.log_format_json; if is_json { let json_layer = initialize_tracing_json(); + let maybe_file_layer = if config.log_file_enabled { + Some(initialize_tracing_json_with_file(file_nb).with_filter(file_level_filter)) + } else { + None + }; + tracing_subscriber::registry() .with(json_layer.with_filter(filter_layer)) - .with( - tracing_subscriber::fmt::Layer::new() - .with_writer(file_nb) - .with_filter(file_level_filter), - ) + .with(maybe_file_layer) .try_init()?; } else { let pretty_layer = initialize_tracing_pretty(); + let maybe_file_layer = if config.log_file_enabled { + Some(initialize_tracing_pretty_with_file(file_nb).with_filter(file_level_filter)) + } else { + None + }; + tracing_subscriber::registry() .with(pretty_layer.with_filter(filter_layer)) - .with( - tracing_subscriber::fmt::Layer::new() - .with_writer(file_nb) - .with_filter(file_level_filter), - ) + .with(maybe_file_layer) .try_init()?; } @@ -195,6 +268,28 @@ where tracing_subscriber::fmt::Layer::new().event_format(AgentLikeFormatter::new()) } +fn initialize_tracing_json_with_file(file_nb: NonBlocking) -> impl Layer +where + S: Subscriber + for<'a> LookupSpan<'a>, +{ + tracing_subscriber::fmt::Layer::new() + .json() + .flatten_event(true) + .with_target(true) + .with_file(true) + .with_line_number(true) + .with_writer(file_nb) +} + +fn initialize_tracing_pretty_with_file(file_nb: NonBlocking) -> impl Layer +where + S: Subscriber + for<'a> LookupSpan<'a>, +{ + tracing_subscriber::fmt::Layer::new() + .event_format(AgentLikeFormatter::new()) + .with_writer(file_nb) +} + /// Acquires the logging API handler. /// /// This function is mutable, and consumes the handler if it's present. This means it should only be called once, and From c4f7b7ffdb2b5675fb74bd6add33064f499239b1 Mon Sep 17 00:00:00 2001 From: raymond zhao Date: Wed, 25 Jun 2025 15:42:32 -0400 Subject: [PATCH 6/7] Fix merge conflicts --- bin/agent-data-plane/src/cli/run.rs | 23 ++++++++++++++++------- bin/agent-data-plane/src/main.rs | 16 +++------------- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/bin/agent-data-plane/src/cli/run.rs b/bin/agent-data-plane/src/cli/run.rs index 086bd04d4f..8da1a44014 100644 --- a/bin/agent-data-plane/src/cli/run.rs +++ b/bin/agent-data-plane/src/cli/run.rs @@ -1,4 +1,7 @@ -use std::time::{Duration, Instant}; +use std::{ + path::PathBuf, + time::{Duration, Instant}, +}; use memory_accounting::{ComponentBounds, ComponentRegistry}; use saluki_app::prelude::*; @@ -34,12 +37,7 @@ pub async fn run(started: Instant, run_config: RunConfig) -> Result<(), GenericE ); // Load our configuration and create all high-level primitives (health registry, component registry, environment // provider, etc) that are needed to build the topology. - let configuration = ConfigurationLoader::default() - .try_from_yaml(&run_config.config) - .from_environment("DD")? - .with_default_secrets_resolution() - .await? - .into_generic()?; + let configuration = load_configuration(run_config.config).await?; // Set up all of the building blocks for building our topologies and launching internal processes. let component_registry = ComponentRegistry::default(); @@ -239,3 +237,14 @@ fn write_sizing_guide(bounds: ComponentBounds) -> Result<(), GenericError> { Ok(()) } + +pub async fn load_configuration(config_path: PathBuf) -> Result { + let configuration = ConfigurationLoader::default() + .try_from_yaml(config_path) + .from_environment("DD")? + .with_default_secrets_resolution() + .await? + .into_generic()?; + + Ok(configuration) +} diff --git a/bin/agent-data-plane/src/main.rs b/bin/agent-data-plane/src/main.rs index d4cc2a5ae5..afb2f7cc9a 100644 --- a/bin/agent-data-plane/src/main.rs +++ b/bin/agent-data-plane/src/main.rs @@ -5,10 +5,11 @@ #![deny(warnings)] #![deny(missing_docs)] +use std::path::PathBuf; use std::time::Instant; use clap::Parser as _; -use saluki_app::prelude::*; +use saluki_app::{logging::LoggingConfiguration, prelude::*}; use tracing::{error, info}; mod components; @@ -40,7 +41,7 @@ async fn main() { let started = Instant::now(); let cli = Cli::parse(); - let configuration = load_configuration(PathBuf::from("/etc/datadog-agent/datadog.yaml")) + let configuration = cli::run::load_configuration(PathBuf::from("/etc/datadog-agent/datadog.yaml")) .await .unwrap(); let logging_config = LoggingConfiguration::try_from_config(&configuration).unwrap(); @@ -88,14 +89,3 @@ async fn main() { } } } - -async fn load_configuration(config_path: PathBuf) -> Result { - let configuration = ConfigurationLoader::default() - .try_from_yaml(config_path) - .from_environment("DD")? - .with_default_secrets_resolution() - .await? - .into_generic()?; - - Ok(configuration) -} From 785439002f237c0373c8c5ad94d90aa6c256b6cb Mon Sep 17 00:00:00 2001 From: raymond zhao Date: Wed, 25 Jun 2025 16:04:58 -0400 Subject: [PATCH 7/7] Move with_reload to the logging configuration --- bin/agent-data-plane/src/main.rs | 14 ++++-- bin/checks-agent/src/main.rs | 2 +- lib/saluki-app/src/logging.rs | 74 +++++++++++++++++++++----------- 3 files changed, 60 insertions(+), 30 deletions(-) diff --git a/bin/agent-data-plane/src/main.rs b/bin/agent-data-plane/src/main.rs index afb2f7cc9a..91908bb556 100644 --- a/bin/agent-data-plane/src/main.rs +++ b/bin/agent-data-plane/src/main.rs @@ -43,12 +43,20 @@ async fn main() { let configuration = cli::run::load_configuration(PathBuf::from("/etc/datadog-agent/datadog.yaml")) .await - .unwrap(); - let logging_config = LoggingConfiguration::try_from_config(&configuration).unwrap(); + .unwrap_or_else(|e| { + fatal_and_exit(format!("failed to load configuration: {}", e)); + unreachable!() + }); + let logging_config = LoggingConfiguration::try_from_config(&configuration) + .unwrap_or_else(|e| { + fatal_and_exit(format!("failed to load logging configuration: {}", e)); + unreachable!() + }) + .with_reload(true); let _guard = initialize_dynamic_logging(&logging_config).await.unwrap_or_else(|e| { fatal_and_exit(format!("failed to initialize logging: {}", e)); - unreachable!() // This will never be reached since fatal_and_exit exits + unreachable!() }); if let Err(e) = initialize_metrics("adp").await { diff --git a/bin/checks-agent/src/main.rs b/bin/checks-agent/src/main.rs index 11f877f0e6..071b8c1867 100644 --- a/bin/checks-agent/src/main.rs +++ b/bin/checks-agent/src/main.rs @@ -38,7 +38,7 @@ static ALLOC: memory_accounting::allocator::TrackingAllocator, } @@ -149,11 +155,15 @@ impl LoggingConfiguration { } /// Sets the log level used for the default directive. - pub fn with_default_level(self, level: LevelFilter) -> Self { - Self { - default_level: Some(level), - ..self - } + pub fn with_default_level(mut self, level: LevelFilter) -> Self { + self.default_level = Some(level); + self + } + + /// Sets whether to enable dynamic reloading of the log level filtering directives. + pub fn with_reload(mut self, reload: bool) -> Self { + self.reload_enabled = reload; + self } } /// Initializes the logging subsystem for `tracing`. @@ -163,8 +173,8 @@ impl LoggingConfiguration { /// If the logging subsystem was already initialized, an error will be returned. pub fn initialize_logging( config: &LoggingConfiguration, -) -> Result> { - initialize_logging_inner(config, false) +) -> Result, Box> { + initialize_logging_inner(config) } /// Initializes the logging subsystem for `tracing` with the ability to dynamically update the log filtering directives @@ -178,16 +188,16 @@ pub fn initialize_logging( /// If the logging subsystem was already initialized, an error will be returned. pub async fn initialize_dynamic_logging( config: &LoggingConfiguration, -) -> Result> { +) -> Result, Box> { // We go through this wrapped initialize approach so that we can mark `initialize_dynamic_logging` as `async`, which // ensures we call it in an asynchronous context, thereby all but ensuring we're in a Tokio context when we try to // spawn the background task that handles reloading the filtering layer. - initialize_logging_inner(config, true) + initialize_logging_inner(config) } fn initialize_logging_inner( - config: &LoggingConfiguration, with_reload: bool, -) -> Result> { + config: &LoggingConfiguration, +) -> Result, Box> { // Load our level filtering directives from the environment, or fallback to INFO if the environment variable is not // specified. // @@ -198,31 +208,22 @@ fn initialize_logging_inner( let shared_level_filter = Arc::new(level_filter); let (filter_layer, reload_handle) = ReloadLayer::new(into_shared_dyn_filter(Arc::clone(&shared_level_filter))); - if with_reload { + if config.reload_enabled { API_HANDLER .lock() .unwrap() .replace(LoggingAPIHandler::new(shared_level_filter.clone(), reload_handle)); } - let adp_log_file = &config.log_file; - let log_file_max_size = config.log_file_max_size; - let log_file_max_rolls = config.log_file_max_rolls; - let file_appender = BasicRollingFileAppender::new( - adp_log_file, - RollingConditionBasic::new().max_size(log_file_max_size), - log_file_max_rolls, - )?; - let (file_nb, guard) = tracing_appender::non_blocking(file_appender); - let file_level_filter = EnvFilter::builder() - .with_default_directive(config.default_level.unwrap_or(LevelFilter::INFO).into()) - .parse_lossy(&config.log_level); - let is_json = config.log_format_json; + let mut worker_guard = None; + if is_json { let json_layer = initialize_tracing_json(); let maybe_file_layer = if config.log_file_enabled { + let (file_nb, guard, file_level_filter) = setup_file_logging(config)?; + worker_guard = Some(guard); Some(initialize_tracing_json_with_file(file_nb).with_filter(file_level_filter)) } else { None @@ -235,6 +236,8 @@ fn initialize_logging_inner( } else { let pretty_layer = initialize_tracing_pretty(); let maybe_file_layer = if config.log_file_enabled { + let (file_nb, guard, file_level_filter) = setup_file_logging(config)?; + worker_guard = Some(guard); Some(initialize_tracing_pretty_with_file(file_nb).with_filter(file_level_filter)) } else { None @@ -246,7 +249,26 @@ fn initialize_logging_inner( .try_init()?; } - Ok(guard) + Ok(worker_guard) +} + +fn setup_file_logging( + config: &LoggingConfiguration, +) -> Result<(NonBlocking, WorkerGuard, EnvFilter), Box> { + let adp_log_file = &config.log_file; + let log_file_max_size = config.log_file_max_size; + let log_file_max_rolls = config.log_file_max_rolls; + let file_appender = BasicRollingFileAppender::new( + adp_log_file, + RollingConditionBasic::new().max_size(log_file_max_size), + log_file_max_rolls, + )?; + let (file_nb, guard) = tracing_appender::non_blocking(file_appender); + let file_level_filter = EnvFilter::builder() + .with_default_directive(config.default_level.unwrap_or(LevelFilter::INFO).into()) + .parse_lossy(&config.log_level); + + Ok((file_nb, guard, file_level_filter)) } fn initialize_tracing_json() -> impl Layer