From c194c7510f07014172eeefe2a95fa4bc26a3aaa7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Fri, 17 Jul 2026 20:45:38 -0700 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20feat(config):=20add=20nested=20avai?= =?UTF-8?q?lability=20config?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the `[availability]` table that selects the none, dc, or ha runtime availability mode. An omitted table and `mode = "none"` resolve alike to single-node operation, so a zero-config node holds no availability state beyond the resolved enum. The dc and ha modes carry a nested `[availability.replication]` role, migrating the former top-level `[replication]` primary/replica shape, and none rejects one. Replace the `Config` replication field with the availability model so the replication runtime, the read-replica startup check, and the backup snapshot read one model. Reject an unknown key or an impossible mode and role pairing with a field-path diagnostic, and document the nested TOML, precedence, validation, and secret handling. Refs #496 --- crates/peryx/src/cli/mod.rs | 2 +- crates/peryx/src/config/load.rs | 2 +- crates/peryx/src/config/merge.rs | 31 ++++++-- crates/peryx/src/config/mod.rs | 14 ++-- crates/peryx/src/config/model.rs | 56 +++++++++++++- crates/peryx/src/config/raw.rs | 17 ++++- crates/peryx/src/operator/snapshot.rs | 51 +++++++++---- crates/peryx/src/replication.rs | 2 +- crates/peryx/src/server.rs | 5 +- .../src/tests/config/availability_tests.rs | 73 +++++++++++++++++++ crates/peryx/src/tests/config/mod.rs | 1 + crates/peryx/src/tests/config/model_tests.rs | 18 +++-- crates/peryx/src/tests/config/raw_tests.rs | 3 +- .../src/tests/config/replication_tests.rs | 38 +++++----- .../peryx/src/tests/operator/backup_tests.rs | 12 +-- crates/peryx/src/tests/replication_tests.rs | 6 +- crates/peryx/src/tests/server_tests.rs | 27 ++++--- site/content/core/configuration.md | 59 ++++++++++++++- site/content/core/high-availability.md | 3 +- 19 files changed, 334 insertions(+), 86 deletions(-) create mode 100644 crates/peryx/src/tests/config/availability_tests.rs diff --git a/crates/peryx/src/cli/mod.rs b/crates/peryx/src/cli/mod.rs index 00eb71e3..c3e07ccb 100644 --- a/crates/peryx/src/cli/mod.rs +++ b/crates/peryx/src/cli/mod.rs @@ -201,7 +201,7 @@ impl RuntimeArgs { }, rate_limit: PartialRateLimitConfig::default(), auth: PartialAuthConfig::default(), - replication: None, + availability: None, jobs: PartialJobsConfig::default(), blob: None, } diff --git a/crates/peryx/src/config/load.rs b/crates/peryx/src/config/load.rs index ecf9b511..412565b9 100644 --- a/crates/peryx/src/config/load.rs +++ b/crates/peryx/src/config/load.rs @@ -51,7 +51,7 @@ pub fn from_env_source(get: impl Fn(&str) -> Option) -> Result Result Ok(BlobStorageConfig::S3(config)) } +/// Resolve the `[availability]` table into a mode and its topology. `none` carries no replication, so +/// pairing it with a role is a configuration error; `dc` and `ha` require the role that carries them. +fn classify_availability(raw: RawAvailability) -> Result { + match (raw.mode.unwrap_or_default(), raw.replication) { + (AvailabilityMode::None, None) => Ok(AvailabilityConfig::None), + (AvailabilityMode::None, Some(_)) => Err(ConfigError::Availability { + reason: "`none` mode configures no replication; select `dc` or `ha` to add a role", + }), + (AvailabilityMode::Dc, Some(role)) => Ok(AvailabilityConfig::Dc(classify_replication(role)?)), + (AvailabilityMode::Ha, Some(role)) => Ok(AvailabilityConfig::Ha(classify_replication(role)?)), + (AvailabilityMode::Dc | AvailabilityMode::Ha, None) => Err(ConfigError::Availability { + reason: "`dc` and `ha` modes need a `[availability.replication]` role", + }), + } +} + fn classify_replication(raw: RawReplication) -> Result { let required_token = |token, token_file| { let token = secret_source(token, token_file).map_err(|reason| ConfigError::Replication { reason })?; diff --git a/crates/peryx/src/config/mod.rs b/crates/peryx/src/config/mod.rs index 712ce903..b6ed98b6 100644 --- a/crates/peryx/src/config/mod.rs +++ b/crates/peryx/src/config/mod.rs @@ -17,15 +17,15 @@ pub use load::{from_env, from_file, from_toml}; #[cfg(test)] pub(crate) use merge::classify_tls; pub use model::{ - AcmeConfig, AuthConfig, BlobStorageConfig, Config, DEFAULT_REPLICA_PAGE_SIZE, DEFAULT_REPLICA_POLL_INTERVAL_SECS, - IndexConfig, IndexKind, JobsConfig, JobsMode, LogConfig, LogFormat, LogSink, PrefetchConfig, PrefetchMode, - ReplicationConfig, S3StorageConfig, SecretSource, TlsConfig, TokenConfig, TrustedPublisherConfig, UpstreamConfig, - UpstreamRoutingConfig, UpstreamTlsConfig, WebhookConfig, WebhookSecret, + AcmeConfig, AuthConfig, AvailabilityConfig, AvailabilityMode, BlobStorageConfig, Config, DEFAULT_REPLICA_PAGE_SIZE, + DEFAULT_REPLICA_POLL_INTERVAL_SECS, IndexConfig, IndexKind, JobsConfig, JobsMode, LogConfig, LogFormat, LogSink, + PrefetchConfig, PrefetchMode, ReplicationConfig, S3StorageConfig, SecretSource, TlsConfig, TokenConfig, + TrustedPublisherConfig, UpstreamConfig, UpstreamRoutingConfig, UpstreamTlsConfig, WebhookConfig, WebhookSecret, }; pub use raw::{ PartialAuthConfig, PartialConfig, PartialJobsConfig, PartialLogConfig, PartialRateLimitConfig, PartialRouteLimit, - RawAcme, RawBlobStorage, RawIndex, RawJobSchedule, RawPolicy, RawPrefetchConfig, RawReplication, RawTls, RawToken, - RawTrustedPublisher, RawUpstream, RawWebhook, + RawAcme, RawAvailability, RawBlobStorage, RawIndex, RawJobSchedule, RawPolicy, RawPrefetchConfig, RawReplication, + RawTls, RawToken, RawTrustedPublisher, RawUpstream, RawWebhook, }; /// An error while assembling configuration. @@ -47,6 +47,8 @@ pub enum ConfigError { Auth { reason: &'static str }, #[error("trusted publisher {id}: {reason}")] TrustedPublisher { id: String, reason: &'static str }, + #[error("availability: {reason}")] + Availability { reason: &'static str }, #[error("replication: {reason}")] Replication { reason: &'static str }, #[error("jobs schedule [{index}]: {reason}")] diff --git a/crates/peryx/src/config/model.rs b/crates/peryx/src/config/model.rs index a5c34da5..c8591fa5 100644 --- a/crates/peryx/src/config/model.rs +++ b/crates/peryx/src/config/model.rs @@ -49,7 +49,7 @@ pub struct Config { pub log: LogConfig, pub rate_limit: RateLimitConfig, pub auth: AuthConfig, - pub replication: Option, + pub availability: AvailabilityConfig, pub jobs: JobsConfig, /// Where blobs are stored: the local filesystem (default) or an S3-compatible object store. pub blob: BlobStorageConfig, @@ -132,6 +132,54 @@ pub enum JobsMode { pub const DEFAULT_REPLICA_PAGE_SIZE: usize = 100; pub const DEFAULT_REPLICA_POLL_INTERVAL_SECS: u64 = 1; +/// The runtime availability contract a node promises for authoritative mutations. +/// +/// The `[availability]` table's `mode` chooses one; each fixes what an acknowledgement guarantees is +/// durable, as the [availability contracts](@/core/availability-contracts.md) page states normatively. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum AvailabilityMode { + /// Single writer, local durability, operator-driven failover: the zero-config default. + #[default] + None, + /// A configured writer and explicit read replicas within one datacenter. + Dc, + /// Metadata durability in a remote datacenter. + Ha, +} + +/// The resolved `[availability]` table: the selected mode and its topology. +/// +/// `dc` and `ha` carry the replication role that fulfills them; `none` holds nothing, so a single-node +/// process allocates no availability state beyond this enum's discriminant. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AvailabilityConfig { + None, + Dc(ReplicationConfig), + Ha(ReplicationConfig), +} + +impl AvailabilityConfig { + /// The selected mode, independent of any topology a stronger mode carries. + #[must_use] + pub const fn mode(&self) -> AvailabilityMode { + match self { + Self::None => AvailabilityMode::None, + Self::Dc(_) => AvailabilityMode::Dc, + Self::Ha(_) => AvailabilityMode::Ha, + } + } + + /// The replication role a `dc` or `ha` node drives, or `None` under single-node `none`. + #[must_use] + pub const fn replication(&self) -> Option<&ReplicationConfig> { + match self { + Self::None => None, + Self::Dc(replication) | Self::Ha(replication) => Some(replication), + } + } +} + /// The process role for replication. #[derive(Debug, Clone, PartialEq, Eq)] pub enum ReplicationConfig { @@ -185,7 +233,9 @@ impl Config { Some(identity) if identity.trim().is_empty() => Err(ConfigError::WriterIdentity { reason: "must not be blank", }), - None if self.read_only || matches!(self.replication, Some(ReplicationConfig::Replica { .. })) => { + None if self.read_only + || matches!(self.availability.replication(), Some(ReplicationConfig::Replica { .. })) => + { Err(ConfigError::WriterIdentity { reason: "required in read replica mode", }) @@ -525,7 +575,7 @@ impl Default for Config { log: LogConfig::default(), rate_limit: RateLimitConfig::default(), auth: AuthConfig::default(), - replication: None, + availability: AvailabilityConfig::None, jobs: JobsConfig::default(), blob: BlobStorageConfig::Filesystem, } diff --git a/crates/peryx/src/config/raw.rs b/crates/peryx/src/config/raw.rs index 318f6426..c4f35358 100644 --- a/crates/peryx/src/config/raw.rs +++ b/crates/peryx/src/config/raw.rs @@ -11,7 +11,7 @@ use toml::Table; use peryx_driver::jobs::ScheduledJob; -use super::model::{JobsMode, LogFormat, LogSink, PrefetchConfig, PrefetchMode}; +use super::model::{AvailabilityMode, JobsMode, LogFormat, LogSink, PrefetchConfig, PrefetchMode}; #[derive(Debug, Default, Clone, PartialEq, Eq, Deserialize)] #[serde(default, deny_unknown_fields)] @@ -75,7 +75,9 @@ pub struct PartialConfig { pub log: PartialLogConfig, pub rate_limit: PartialRateLimitConfig, pub auth: PartialAuthConfig, - pub replication: Option, + /// The `[availability]` table: the runtime availability mode and the replication topology a + /// stronger mode carries. Absent, like `mode = "none"`, selects single-node operation. + pub availability: Option, pub jobs: PartialJobsConfig, /// A `[blob]` table selecting the blob storage backend. pub blob: Option, @@ -101,6 +103,17 @@ pub enum RawBlobStorage { }, } +/// The raw `[availability]` table: the mode selector and its replication role. +/// +/// A `dc` or `ha` node carries a role; an omitted table, and an explicit `mode = "none"`, both resolve +/// to single-node operation with no replication. +#[derive(Debug, Default, Clone, PartialEq, Eq, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct RawAvailability { + pub mode: Option, + pub replication: Option, +} + /// The `[jobs]` half of [`PartialConfig`]. #[derive(Debug, Default, Clone, PartialEq, Eq, Deserialize)] #[serde(default, deny_unknown_fields)] diff --git a/crates/peryx/src/operator/snapshot.rs b/crates/peryx/src/operator/snapshot.rs index 90a619f0..5c15671f 100644 --- a/crates/peryx/src/operator/snapshot.rs +++ b/crates/peryx/src/operator/snapshot.rs @@ -14,9 +14,9 @@ use time::format_description::well_known::Rfc3339; use toml::{Table, Value}; use crate::config::{ - AcmeConfig, AuthConfig, BlobStorageConfig, Config, IndexConfig, IndexKind, JobsConfig, JobsMode, LogConfig, - LogFormat, LogSink, PrefetchConfig, PrefetchMode, ReplicationConfig, SecretSource, TlsConfig, TokenConfig, - WebhookConfig, WebhookSecret, + AcmeConfig, AuthConfig, AvailabilityConfig, BlobStorageConfig, Config, IndexConfig, IndexKind, JobsConfig, + JobsMode, LogConfig, LogFormat, LogSink, PrefetchConfig, PrefetchMode, ReplicationConfig, SecretSource, TlsConfig, + TokenConfig, WebhookConfig, WebhookSecret, }; #[derive(Serialize)] @@ -44,7 +44,7 @@ struct SnapshotConfig<'a> { rate_limit: SnapshotRateLimit<'a>, auth: SnapshotAuth<'a>, #[serde(skip_serializing_if = "Option::is_none")] - replication: Option>, + availability: Option>, #[serde(skip_serializing_if = "Option::is_none")] jobs: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -289,6 +289,13 @@ struct SnapshotTrustedPublisher<'a> { claims: &'a std::collections::BTreeMap, } +#[derive(Serialize)] +struct SnapshotAvailability<'a> { + mode: &'static str, + #[serde(skip_serializing_if = "Option::is_none")] + replication: Option>, +} + #[derive(Serialize)] #[serde(tag = "role", rename_all = "lowercase")] enum SnapshotReplication<'a> { @@ -328,7 +335,7 @@ pub(super) fn config_snapshot(config: &Config) -> anyhow::Result { log, rate_limit, auth, - replication, + availability, jobs, blob, } = config; @@ -387,7 +394,7 @@ pub(super) fn config_snapshot(config: &Config) -> anyhow::Result { }) .collect(), }, - replication: snapshot_replication(replication.as_ref()), + availability: snapshot_availability(availability), jobs: snapshot_jobs(jobs), blob: snapshot_blob(blob), }; @@ -438,32 +445,46 @@ fn snapshot_blob(blob: &BlobStorageConfig) -> Option> { }) } -fn snapshot_replication(replication: Option<&ReplicationConfig>) -> Option> { +/// A snapshot carries the `[availability]` table only for a `dc` or `ha` node, so a single-node `none` +/// backup omits it and restores to the same default. The nested `[availability.replication]` role +/// round-trips the configured topology. +fn snapshot_availability(availability: &AvailabilityConfig) -> Option> { + let (mode, replication) = match availability { + AvailabilityConfig::None => return None, + AvailabilityConfig::Dc(replication) => ("dc", replication), + AvailabilityConfig::Ha(replication) => ("ha", replication), + }; + Some(SnapshotAvailability { + mode, + replication: Some(snapshot_replication(replication)), + }) +} + +fn snapshot_replication(replication: &ReplicationConfig) -> SnapshotReplication<'_> { match replication { - Some(ReplicationConfig::Primary { source, token }) => { + ReplicationConfig::Primary { source, token } => { let (token, token_file, _) = secret_parts(Some(token)); - Some(SnapshotReplication::Primary { + SnapshotReplication::Primary { source, token, token_file, - }) + } } - Some(ReplicationConfig::Replica { + ReplicationConfig::Replica { upstream, token, poll_interval, page_size, - }) => { + } => { let (token, token_file, _) = secret_parts(Some(token)); - Some(SnapshotReplication::Replica { + SnapshotReplication::Replica { upstream, token, token_file, poll_interval_secs: poll_interval.as_secs(), page_size: page_size.get(), - }) + } } - None => None, } } diff --git a/crates/peryx/src/replication.rs b/crates/peryx/src/replication.rs index ab6d4571..80a48144 100644 --- a/crates/peryx/src/replication.rs +++ b/crates/peryx/src/replication.rs @@ -220,7 +220,7 @@ impl ReplicationRuntime { /// Returns an error if a secret cannot be read, the upstream URL is invalid, or the primary /// router rejects its identity or token. pub fn new(config: &Config, state: &Arc) -> anyhow::Result { - let (primary, replica) = match &config.replication { + let (primary, replica) = match config.availability.replication() { None => (None, None), Some(ReplicationConfig::Primary { source, token }) => { let token = token.read().context("read the primary replication token")?; diff --git a/crates/peryx/src/server.rs b/crates/peryx/src/server.rs index 3cdce541..3c54f423 100644 --- a/crates/peryx/src/server.rs +++ b/crates/peryx/src/server.rs @@ -73,7 +73,10 @@ pub fn build_state(config: &Config) -> anyhow::Result> { .with_context(|| format!("create data directory {}", config.data_dir.display()))?; let meta_path = config.data_dir.join("peryx.redb"); let meta = MetaStore::open(&meta_path).with_context(|| format!("open metadata store {}", meta_path.display()))?; - let configured_replica = matches!(config.replication, Some(ReplicationConfig::Replica { .. })); + let configured_replica = matches!( + config.availability.replication(), + Some(ReplicationConfig::Replica { .. }) + ); let read_only = config.read_only || configured_replica; if read_only { let active = meta.writer_identity().context("read metadata store writer identity")?; diff --git a/crates/peryx/src/tests/config/availability_tests.rs b/crates/peryx/src/tests/config/availability_tests.rs new file mode 100644 index 00000000..e8a0cec0 --- /dev/null +++ b/crates/peryx/src/tests/config/availability_tests.rs @@ -0,0 +1,73 @@ +use std::num::NonZeroUsize; +use std::time::Duration; + +use rstest::rstest; + +use super::toml_config; +use crate::config::{self, AvailabilityConfig, AvailabilityMode, Config, ReplicationConfig, SecretSource}; + +#[test] +fn test_omitted_table_and_explicit_none_resolve_alike() { + let omitted = Config::default().availability; + let explicit = toml_config("[availability]\nmode = \"none\"\n").availability; + + assert_eq!(omitted, AvailabilityConfig::None); + assert_eq!(explicit, AvailabilityConfig::None); +} + +#[test] +fn test_empty_table_selects_none() { + assert_eq!(toml_config("[availability]\n").availability, AvailabilityConfig::None); +} + +#[rstest] +#[case::none(AvailabilityConfig::None, AvailabilityMode::None, false)] +#[case::dc(AvailabilityConfig::Dc(primary()), AvailabilityMode::Dc, true)] +#[case::ha(AvailabilityConfig::Ha(primary()), AvailabilityMode::Ha, true)] +fn test_availability_accessors_report_mode_and_topology( + #[case] availability: AvailabilityConfig, + #[case] mode: AvailabilityMode, + #[case] carries_role: bool, +) { + assert_eq!(availability.mode(), mode); + assert_eq!(availability.replication().is_some(), carries_role); +} + +#[rstest] +#[case::none_with_role( + "[availability]\nmode = \"none\"\n[availability.replication]\nrole = \"primary\"\nsource = \"a\"\ntoken = \"b\"\n", + "`none` mode configures no replication" +)] +#[case::role_without_mode( + "[availability]\n[availability.replication]\nrole = \"primary\"\nsource = \"a\"\ntoken = \"b\"\n", + "`none` mode configures no replication" +)] +#[case::dc_without_role("[availability]\nmode = \"dc\"\n", "`dc` and `ha` modes need")] +#[case::ha_without_role("[availability]\nmode = \"ha\"\n", "`dc` and `ha` modes need")] +#[case::unknown_mode("[availability]\nmode = \"quorum\"\n", "unknown variant")] +fn test_availability_rejects_impossible_combinations(#[case] text: &str, #[case] expected: &str) { + let error = config::from_toml("x.toml".into(), text) + .and_then(|partial| Config::default().apply(partial)) + .unwrap_err(); + + assert!(error.to_string().contains(expected), "{error}"); +} + +fn primary() -> ReplicationConfig { + ReplicationConfig::Primary { + source: "primary-a".to_owned(), + token: SecretSource::Literal("secret".to_owned()), + } +} + +#[test] +fn test_dc_and_ha_carry_distinct_topology() { + let replica = || ReplicationConfig::Replica { + upstream: "https://primary.example/".to_owned(), + token: SecretSource::Literal("secret".to_owned()), + poll_interval: Duration::from_secs(1), + page_size: NonZeroUsize::MIN, + }; + + assert_ne!(AvailabilityConfig::Dc(replica()), AvailabilityConfig::Ha(replica())); +} diff --git a/crates/peryx/src/tests/config/mod.rs b/crates/peryx/src/tests/config/mod.rs index c0517112..c84e358d 100644 --- a/crates/peryx/src/tests/config/mod.rs +++ b/crates/peryx/src/tests/config/mod.rs @@ -3,6 +3,7 @@ use std::path::PathBuf; use crate::config::{self, Config, PartialConfig}; mod auth_tests; +mod availability_tests; mod blob_tests; mod integration_tests; mod jobs_tests; diff --git a/crates/peryx/src/tests/config/model_tests.rs b/crates/peryx/src/tests/config/model_tests.rs index f02c5f09..56d71eaf 100644 --- a/crates/peryx/src/tests/config/model_tests.rs +++ b/crates/peryx/src/tests/config/model_tests.rs @@ -3,7 +3,7 @@ use std::path::PathBuf; use peryx_driver::rate_limit::RateLimitConfig; -use crate::config::{Config, IndexKind, LogConfig, ReplicationConfig, SecretSource}; +use crate::config::{AvailabilityConfig, Config, IndexKind, LogConfig, ReplicationConfig, SecretSource}; #[test] fn test_secret_source_file_returns_trimmed_contents() { @@ -113,12 +113,16 @@ fn test_config_rejects_a_blank_writer_identity() { fn test_config_requires_a_writer_identity_in_replica_mode(#[case] configured_replication: bool) { let config = Config { read_only: !configured_replication, - replication: configured_replication.then(|| ReplicationConfig::Replica { - upstream: "https://writer.example/".to_owned(), - token: SecretSource::Literal("secret".to_owned()), - poll_interval: std::time::Duration::from_secs(1), - page_size: std::num::NonZeroUsize::MIN, - }), + availability: if configured_replication { + AvailabilityConfig::Dc(ReplicationConfig::Replica { + upstream: "https://writer.example/".to_owned(), + token: SecretSource::Literal("secret".to_owned()), + poll_interval: std::time::Duration::from_secs(1), + page_size: std::num::NonZeroUsize::MIN, + }) + } else { + AvailabilityConfig::None + }, ..Config::default() }; diff --git a/crates/peryx/src/tests/config/raw_tests.rs b/crates/peryx/src/tests/config/raw_tests.rs index 6f239efb..3d14f79b 100644 --- a/crates/peryx/src/tests/config/raw_tests.rs +++ b/crates/peryx/src/tests/config/raw_tests.rs @@ -114,9 +114,10 @@ max_file_size_bytes = 1048576\nmax_project_size_bytes = 10485760\n"; )] #[case::unknown_log_key("x.toml", "[log]\nbogus = 1\n", None)] #[case::unknown_rate_limit_key("x.toml", "[rate_limit]\nbogus = 1\n", None)] +#[case::unknown_availability_key("x.toml", "[availability]\nmode = \"none\"\nbogus = 1\n", None)] #[case::unknown_replication_key( "x.toml", - "[replication]\nrole = \"primary\"\nsource = \"a\"\ntoken = \"b\"\nbogus = 1\n", + "[availability]\nmode = \"dc\"\n[availability.replication]\nrole = \"primary\"\nsource = \"a\"\ntoken = \"b\"\nbogus = 1\n", None )] #[case::invalid_trusted_proxy("x.toml", "[rate_limit]\ntrusted_proxies = [\"invalid\"]\n", Some("trusted_proxies"))] diff --git a/crates/peryx/src/tests/config/replication_tests.rs b/crates/peryx/src/tests/config/replication_tests.rs index 25c3f34b..fb515d19 100644 --- a/crates/peryx/src/tests/config/replication_tests.rs +++ b/crates/peryx/src/tests/config/replication_tests.rs @@ -5,17 +5,18 @@ use std::time::Duration; use rstest::rstest; use super::toml_config; -use crate::config::{self, Config, ReplicationConfig, SecretSource}; +use crate::config::{self, AvailabilityConfig, Config, ReplicationConfig, SecretSource}; #[test] -fn test_primary_replication_from_toml() { +fn test_dc_primary_replication_from_toml() { let config = toml_config( - "[replication]\nrole = \"primary\"\nsource = \"primary-a\"\ntoken_file = \"/run/secrets/replica\"\n", + "[availability]\nmode = \"dc\"\n[availability.replication]\nrole = \"primary\"\nsource = \"primary-a\"\n\ + token_file = \"/run/secrets/replica\"\n", ); assert_eq!( - config.replication, - Some(ReplicationConfig::Primary { + config.availability, + AvailabilityConfig::Dc(ReplicationConfig::Primary { source: "primary-a".to_owned(), token: SecretSource::File(PathBuf::from("/run/secrets/replica")), }) @@ -23,13 +24,15 @@ fn test_primary_replication_from_toml() { } #[test] -fn test_replica_replication_from_toml_uses_defaults() { - let config = - toml_config("[replication]\nrole = \"replica\"\nupstream = \"https://primary.example/\"\ntoken = \"secret\"\n"); +fn test_ha_replica_replication_from_toml_uses_defaults() { + let config = toml_config( + "[availability]\nmode = \"ha\"\n[availability.replication]\nrole = \"replica\"\n\ + upstream = \"https://primary.example/\"\ntoken = \"secret\"\n", + ); assert_eq!( - config.replication, - Some(ReplicationConfig::Replica { + config.availability, + AvailabilityConfig::Ha(ReplicationConfig::Replica { upstream: "https://primary.example/".to_owned(), token: SecretSource::Literal("secret".to_owned()), poll_interval: Duration::from_secs(1), @@ -41,17 +44,17 @@ fn test_replica_replication_from_toml_uses_defaults() { #[test] fn test_replica_replication_from_toml_accepts_runtime_bounds() { let config = toml_config( - "[replication]\nrole = \"replica\"\nupstream = \"https://primary.example/\"\ntoken = \"secret\"\n\ - poll_interval_secs = 30\npage_size = 250\n", + "[availability]\nmode = \"dc\"\n[availability.replication]\nrole = \"replica\"\n\ + upstream = \"https://primary.example/\"\ntoken = \"secret\"\npoll_interval_secs = 30\npage_size = 250\n", ); - let Some(ReplicationConfig::Replica { + let AvailabilityConfig::Dc(ReplicationConfig::Replica { poll_interval, page_size, .. - }) = config.replication + }) = config.availability else { - panic!("expected replica configuration"); + panic!("expected a dc replica configuration"); }; assert_eq!(poll_interval, Duration::from_secs(30)); assert_eq!(page_size, NonZeroUsize::new(250).unwrap()); @@ -81,8 +84,9 @@ fn test_replica_replication_from_toml_accepts_runtime_bounds() { "role = \"replica\"\nupstream = \"https://primary.example\"\ntoken = \"secret\"\npage_size = 1001", "exceeds the primary limit" )] -fn test_replication_rejects_invalid_configuration(#[case] body: &str, #[case] expected: &str) { - let partial = config::from_toml(PathBuf::from("x.toml"), &format!("[replication]\n{body}\n")).unwrap(); +fn test_replication_rejects_invalid_configuration(#[case] role: &str, #[case] expected: &str) { + let text = format!("[availability]\nmode = \"dc\"\n[availability.replication]\n{role}\n"); + let partial = config::from_toml(PathBuf::from("x.toml"), &text).unwrap(); let error = Config::default().apply(partial).unwrap_err(); diff --git a/crates/peryx/src/tests/operator/backup_tests.rs b/crates/peryx/src/tests/operator/backup_tests.rs index 99e0c2ac..7342d6e3 100644 --- a/crates/peryx/src/tests/operator/backup_tests.rs +++ b/crates/peryx/src/tests/operator/backup_tests.rs @@ -58,15 +58,15 @@ fn test_backup_create_rejects_tampered_source_blob() { } #[rstest] -#[case::manual_primary( +#[case::dc_primary( "[tls]\ncert = \"/etc/peryx/tls.crt\"\nkey = \"/etc/peryx/tls.key\"", - "[replication]\nrole = \"primary\"\nsource = \"primary-a\"\ntoken = \"replication-token\"" + "[availability]\nmode = \"dc\"\n[availability.replication]\nrole = \"primary\"\nsource = \"primary-a\"\ntoken = \"replication-token\"" )] -#[case::acme_replica( +#[case::ha_replica( "[acme]\ndomains = [\"packages.example.com\"]\ncontact = \"ops@example.com\"\ncache-dir = \"/var/cache/peryx/acme\"\nstaging = true", - "[replication]\nrole = \"replica\"\nupstream = \"https://primary.example/\"\ntoken_file = \"/run/secrets/replication-token\"\npoll_interval_secs = 30\npage_size = 250" + "[availability]\nmode = \"ha\"\n[availability.replication]\nrole = \"replica\"\nupstream = \"https://primary.example/\"\ntoken_file = \"/run/secrets/replication-token\"\npoll_interval_secs = 30\npage_size = 250" )] -fn test_backup_config_round_trips_effective_settings(#[case] tls: &str, #[case] replication: &str) { +fn test_backup_config_round_trips_effective_settings(#[case] tls: &str, #[case] availability: &str) { let root = tempfile::tempdir().unwrap(); let data_dir = root.path().join("data"); std::fs::create_dir(&data_dir).unwrap(); @@ -85,7 +85,7 @@ max_stale_secs = 321 {tls} -{replication} +{availability} [log] level = "peryx=debug" diff --git a/crates/peryx/src/tests/replication_tests.rs b/crates/peryx/src/tests/replication_tests.rs index 8e1bac76..6475e822 100644 --- a/crates/peryx/src/tests/replication_tests.rs +++ b/crates/peryx/src/tests/replication_tests.rs @@ -15,8 +15,8 @@ use rstest::rstest; use tower::ServiceExt as _; use crate::config::{ - Config, IndexKind, ReplicationConfig, SecretSource, TokenConfig, UpstreamConfig, UpstreamRoutingConfig, - WebhookConfig, WebhookSecret, + AvailabilityConfig, Config, IndexKind, ReplicationConfig, SecretSource, TokenConfig, UpstreamConfig, + UpstreamRoutingConfig, WebhookConfig, WebhookSecret, }; use crate::replication::ReplicationRuntime; use crate::server::{build_router, build_state, router_for}; @@ -60,7 +60,7 @@ fn config(dir: &tempfile::TempDir, replication: Option) -> Co Config { data_dir: dir.path().to_path_buf(), writer_identity: replica.then(|| WRITER_IDENTITY.to_owned()), - replication, + availability: replication.map_or(AvailabilityConfig::None, AvailabilityConfig::Dc), ..Config::default() } } diff --git a/crates/peryx/src/tests/server_tests.rs b/crates/peryx/src/tests/server_tests.rs index ec0575fd..290bbac2 100644 --- a/crates/peryx/src/tests/server_tests.rs +++ b/crates/peryx/src/tests/server_tests.rs @@ -19,8 +19,9 @@ use peryx_ecosystem_oci::LibraryPrefix; use peryx_storage::blob::S3Credentials; use crate::config::{ - AuthConfig, BlobStorageConfig, Config, IndexConfig, IndexKind, ReplicationConfig, S3StorageConfig, SecretSource, - TrustedPublisherConfig, UpstreamConfig, UpstreamRoutingConfig, WebhookConfig, WebhookSecret, + AuthConfig, AvailabilityConfig, BlobStorageConfig, Config, IndexConfig, IndexKind, ReplicationConfig, + S3StorageConfig, SecretSource, TrustedPublisherConfig, UpstreamConfig, UpstreamRoutingConfig, WebhookConfig, + WebhookSecret, }; use crate::server::{ build_blob_storage, build_index_settings, build_indexes, build_router, build_state, upstream_auth, @@ -163,12 +164,16 @@ fn write_netrc(path: &Path, contents: &str) { } } -fn replication_replica() -> ReplicationConfig { - ReplicationConfig::Replica { - upstream: "https://writer.example/".to_owned(), - token: SecretSource::Literal("secret".to_owned()), - poll_interval: Duration::from_secs(1), - page_size: NonZeroUsize::MIN, +fn availability_replica(configured: bool) -> AvailabilityConfig { + if configured { + AvailabilityConfig::Dc(ReplicationConfig::Replica { + upstream: "https://writer.example/".to_owned(), + token: SecretSource::Literal("secret".to_owned()), + poll_interval: Duration::from_secs(1), + page_size: NonZeroUsize::MIN, + }) + } else { + AvailabilityConfig::None } } @@ -731,7 +736,7 @@ fn test_build_state_rejects_a_replica_without_writer_identity(#[case] configured let config = Config { data_dir: dir.path().to_path_buf(), read_only: !configured_replication, - replication: configured_replication.then(replication_replica), + availability: availability_replica(configured_replication), ..Config::default() }; @@ -757,7 +762,7 @@ fn test_build_state_replica_does_not_claim_writer_identity(#[case] configured_re data_dir: dir.path().to_path_buf(), writer_identity: Some("writer-a".to_owned()), read_only: !configured_replication, - replication: configured_replication.then(replication_replica), + availability: availability_replica(configured_replication), ..Config::default() }) .unwrap(); @@ -782,7 +787,7 @@ fn test_build_state_rejects_a_replica_with_an_unmatched_writer( data_dir: dir.path().to_path_buf(), writer_identity: Some("writer-a".to_owned()), read_only: !configured_replication, - replication: configured_replication.then(replication_replica), + availability: availability_replica(configured_replication), ..Config::default() }; diff --git a/site/content/core/configuration.md b/site/content/core/configuration.md index b4d140b5..052220c5 100644 --- a/site/content/core/configuration.md +++ b/site/content/core/configuration.md @@ -26,11 +26,13 @@ or `PERYX_*` environment variables, which override the file. Precedence is `defa | Indexes | (file only) | (n/a) | `[[index]]` | (see below) | | Rate limits | (file only) | (n/a) | `[rate_limit]` | (see below) | | Background jobs | (file only) | (n/a) | `[jobs]` | (see below) | +| Availability mode | (file only) | (n/a) | `[availability]` | `none` | Environment variables sit between the file and flags: a `PERYX_*` value overrides the TOML file, and a flag overrides -the variable. Only scalar settings are environment-configurable. The `[[index]]` topology and `[rate_limit]` block stay -file-only, since neither maps to a flat variable. An empty variable is treated as unset. The `[log]` block also reads -variables (`PERYX_LOG_LEVEL`, `PERYX_LOG_FORMAT`, `PERYX_LOG_SINK`, `PERYX_LOG_FILE`); see [`[log]`](#log). +the variable. Only scalar settings are environment-configurable. The `[[index]]` topology, `[rate_limit]` block, and +`[availability]` table stay file-only, since none maps to a flat variable. An empty variable is treated as unset. The +`[log]` block also reads variables (`PERYX_LOG_LEVEL`, `PERYX_LOG_FORMAT`, `PERYX_LOG_SINK`, `PERYX_LOG_FILE`); see +[`[log]`](#log). `cache_ttl_secs` is both a fallback and a ceiling. When an upstream response carries a usable `Cache-Control` lifetime (`s-maxage` or `max-age`) that is **shorter**, that lifetime governs the page; a longer one is clamped to @@ -683,6 +685,57 @@ object — harmless and overwritten byte-for-byte by any later write of the same `[blob]` selection (never the credentials) so a restore points at the same bucket; the objects themselves are not copied into the archive. Bucket-level versioning or replication, if you need it, is configured on the object store, not here. +## `[availability]` + +The `[availability]` table picks the runtime availability contract this node promises for authoritative mutations. Its +`mode` selects one of `none`, `dc`, or `ha`, whose acknowledgement guarantees the +[availability contracts](@/core/availability-contracts.md) fix. An omitted table, and an explicit `mode = "none"`, +resolve to the same single-node configuration, so a zero-config deployment carries no availability state at all. + +```toml +[availability] +mode = "none" +``` + +`none` is one writer with local durability and operator-driven [failover](@/core/high-availability.md): peryx opens no +replication client, route, or task. `dc` and `ha` name the stronger promises later work fulfills; each needs a +`[availability.replication]` role that carries it, so peryx rejects a `dc` or `ha` mode with no role, and rejects a +`[availability.replication]` role under `none`, naming the `availability` field. Selecting `dc` or `ha` replaces the +former top-level `[replication]` table, which no longer parses. + +```toml +[availability] +mode = "dc" + +[availability.replication] +role = "primary" +source = "writer-a" +token_file = "/run/secrets/replication-token" +``` + +| Key | Meaning | Default | +| ------ | --------------------- | ------- | +| `mode` | `none`, `dc`, or `ha` | `none` | + +The nested `[availability.replication]` table declares this node's replication role. `role = "primary"` serves the +replication journal other nodes copy; `role = "replica"` follows a primary and, like `read_only`, refuses client +mutations. peryx rejects an unknown key in either table, naming the offending field. + +| Key | Role | Meaning | Default | +| -------------------- | ------- | --------------------------------------------------------------- | ---------- | +| `role` | both | `primary` or `replica` | (required) | +| `source` | primary | This writer's stable name in the replication journal | (required) | +| `upstream` | replica | URL of the primary this replica follows | (required) | +| `token` | both | Shared replication credential, inline | (none) | +| `token_file` | both | Path to read `token` from instead of inlining it | (none) | +| `poll_interval_secs` | replica | Seconds between change-journal polls, must be positive | `1` | +| `page_size` | replica | Changes fetched per poll, positive and within the primary limit | `100` | + +A role needs exactly one of `token` or `token_file`; setting both, or neither, is rejected. Keep the credential out of +the config file with `token_file`, the path to a mounted Docker or Kubernetes secret or a systemd credential, which +peryx reads at startup and never logs. A configuration snapshot (`peryx backup`) preserves a `token_file` as its path +and never resolves the secret behind it into the manifest. + ## `[auth]` The `[auth]` table holds the access settings every index shares: the signing key of peryx's token realm, the lifetime of diff --git a/site/content/core/high-availability.md b/site/content/core/high-availability.md index 2929d75b..ec87e3d4 100644 --- a/site/content/core/high-availability.md +++ b/site/content/core/high-availability.md @@ -36,7 +36,8 @@ peryx serve --config peryx.toml --read-only The environment variable and command-line flag provide the same setting. Replica mode does not claim the configured writer identity, so a restored configuration may retain the source writer's identity. It disables upstream cache fills, -webhook delivery, and background maintenance. A configured replication replica enforces the same writer-identity check. +webhook delivery, and background maintenance. A node that follows a primary through the +[`[availability]`](@/core/configuration.md#availability) table's `replica` role enforces the same writer-identity check. Populate each replica's data directory from a verified backup or an external replication system before routing traffic to it. Copy the metadata store and referenced blobs from the same point in time. peryx does not copy data between nodes