Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion crates/peryx/src/cli/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -201,7 +201,7 @@ impl RuntimeArgs {
},
rate_limit: PartialRateLimitConfig::default(),
auth: PartialAuthConfig::default(),
replication: None,
availability: None,
jobs: PartialJobsConfig::default(),
blob: None,
}
Expand Down
2 changes: 1 addition & 1 deletion crates/peryx/src/config/load.rs
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ pub fn from_env_source(get: impl Fn(&str) -> Option<String>) -> Result<PartialCo
},
rate_limit: PartialRateLimitConfig::default(),
auth: PartialAuthConfig::default(),
replication: None,
availability: None,
jobs: PartialJobsConfig::default(),
blob: None,
})
Expand Down
31 changes: 24 additions & 7 deletions crates/peryx/src/config/merge.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,15 @@ use std::collections::HashSet;

use super::ConfigError;
use super::model::{
AcmeConfig, AuthConfig, BlobStorageConfig, Config, DEFAULT_REPLICA_PAGE_SIZE, DEFAULT_REPLICA_POLL_INTERVAL_SECS,
IndexConfig, IndexKind, JobsConfig, LogConfig, ReplicationConfig, S3StorageConfig, SecretSource, TlsConfig,
TokenConfig, TrustedPublisherConfig, UpstreamConfig, UpstreamRoutingConfig, UpstreamTlsConfig, WebhookConfig,
WebhookSecret,
AcmeConfig, AuthConfig, AvailabilityConfig, AvailabilityMode, BlobStorageConfig, Config, DEFAULT_REPLICA_PAGE_SIZE,
DEFAULT_REPLICA_POLL_INTERVAL_SECS, IndexConfig, IndexKind, JobsConfig, LogConfig, ReplicationConfig,
S3StorageConfig, SecretSource, TlsConfig, TokenConfig, TrustedPublisherConfig, UpstreamConfig,
UpstreamRoutingConfig, UpstreamTlsConfig, WebhookConfig, WebhookSecret,
};
use super::raw::{
PartialAuthConfig, PartialConfig, PartialJobsConfig, PartialLogConfig, PartialRateLimitConfig, PartialRouteLimit,
RawAcme, RawBlobStorage, RawIndex, RawJobSchedule, RawReplication, RawTls, RawToken, RawUpstream, RawWebhook,
RawAcme, RawAvailability, RawBlobStorage, RawIndex, RawJobSchedule, RawReplication, RawTls, RawToken, RawUpstream,
RawWebhook,
};

impl Config {
Expand Down Expand Up @@ -73,8 +74,8 @@ impl Config {
self.log = self.log.apply(partial.log);
self.rate_limit = apply_rate_limit(self.rate_limit, partial.rate_limit);
self.auth = self.auth.apply(partial.auth)?;
if let Some(replication) = partial.replication {
self.replication = Some(classify_replication(replication)?);
if let Some(availability) = partial.availability {
self.availability = classify_availability(availability)?;
}
if let Some(blob) = partial.blob {
self.blob = classify_blob(blob)?;
Expand Down Expand Up @@ -148,6 +149,22 @@ fn classify_blob(raw: RawBlobStorage) -> Result<BlobStorageConfig, ConfigError>
Ok(BlobStorageConfig::S3(config))
}

/// Resolve the `[availability]` table into a mode and its topology. `none` carries no replication, so
/// pairing it with a role is a configuration error; `dc` and `ha` require the role that carries them.
fn classify_availability(raw: RawAvailability) -> Result<AvailabilityConfig, ConfigError> {
match (raw.mode.unwrap_or_default(), raw.replication) {
(AvailabilityMode::None, None) => Ok(AvailabilityConfig::None),
(AvailabilityMode::None, Some(_)) => Err(ConfigError::Availability {
reason: "`none` mode configures no replication; select `dc` or `ha` to add a role",
}),
(AvailabilityMode::Dc, Some(role)) => Ok(AvailabilityConfig::Dc(classify_replication(role)?)),
(AvailabilityMode::Ha, Some(role)) => Ok(AvailabilityConfig::Ha(classify_replication(role)?)),
(AvailabilityMode::Dc | AvailabilityMode::Ha, None) => Err(ConfigError::Availability {
reason: "`dc` and `ha` modes need a `[availability.replication]` role",
}),
}
}

fn classify_replication(raw: RawReplication) -> Result<ReplicationConfig, ConfigError> {
let required_token = |token, token_file| {
let token = secret_source(token, token_file).map_err(|reason| ConfigError::Replication { reason })?;
Expand Down
14 changes: 8 additions & 6 deletions crates/peryx/src/config/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,15 @@ pub use load::{from_env, from_file, from_toml};
#[cfg(test)]
pub(crate) use merge::classify_tls;
pub use model::{
AcmeConfig, AuthConfig, BlobStorageConfig, Config, DEFAULT_REPLICA_PAGE_SIZE, DEFAULT_REPLICA_POLL_INTERVAL_SECS,
IndexConfig, IndexKind, JobsConfig, JobsMode, LogConfig, LogFormat, LogSink, PrefetchConfig, PrefetchMode,
ReplicationConfig, S3StorageConfig, SecretSource, TlsConfig, TokenConfig, TrustedPublisherConfig, UpstreamConfig,
UpstreamRoutingConfig, UpstreamTlsConfig, WebhookConfig, WebhookSecret,
AcmeConfig, AuthConfig, AvailabilityConfig, AvailabilityMode, BlobStorageConfig, Config, DEFAULT_REPLICA_PAGE_SIZE,
DEFAULT_REPLICA_POLL_INTERVAL_SECS, IndexConfig, IndexKind, JobsConfig, JobsMode, LogConfig, LogFormat, LogSink,
PrefetchConfig, PrefetchMode, ReplicationConfig, S3StorageConfig, SecretSource, TlsConfig, TokenConfig,
TrustedPublisherConfig, UpstreamConfig, UpstreamRoutingConfig, UpstreamTlsConfig, WebhookConfig, WebhookSecret,
};
pub use raw::{
PartialAuthConfig, PartialConfig, PartialJobsConfig, PartialLogConfig, PartialRateLimitConfig, PartialRouteLimit,
RawAcme, RawBlobStorage, RawIndex, RawJobSchedule, RawPolicy, RawPrefetchConfig, RawReplication, RawTls, RawToken,
RawTrustedPublisher, RawUpstream, RawWebhook,
RawAcme, RawAvailability, RawBlobStorage, RawIndex, RawJobSchedule, RawPolicy, RawPrefetchConfig, RawReplication,
RawTls, RawToken, RawTrustedPublisher, RawUpstream, RawWebhook,
};

/// An error while assembling configuration.
Expand All @@ -47,6 +47,8 @@ pub enum ConfigError {
Auth { reason: &'static str },
#[error("trusted publisher {id}: {reason}")]
TrustedPublisher { id: String, reason: &'static str },
#[error("availability: {reason}")]
Availability { reason: &'static str },
#[error("replication: {reason}")]
Replication { reason: &'static str },
#[error("jobs schedule [{index}]: {reason}")]
Expand Down
56 changes: 53 additions & 3 deletions crates/peryx/src/config/model.rs
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ pub struct Config {
pub log: LogConfig,
pub rate_limit: RateLimitConfig,
pub auth: AuthConfig,
pub replication: Option<ReplicationConfig>,
pub availability: AvailabilityConfig,
pub jobs: JobsConfig,
/// Where blobs are stored: the local filesystem (default) or an S3-compatible object store.
pub blob: BlobStorageConfig,
Expand Down Expand Up @@ -132,6 +132,54 @@ pub enum JobsMode {
pub const DEFAULT_REPLICA_PAGE_SIZE: usize = 100;
pub const DEFAULT_REPLICA_POLL_INTERVAL_SECS: u64 = 1;

/// The runtime availability contract a node promises for authoritative mutations.
///
/// The `[availability]` table's `mode` chooses one; each fixes what an acknowledgement guarantees is
/// durable, as the [availability contracts](@/core/availability-contracts.md) page states normatively.
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum AvailabilityMode {
/// Single writer, local durability, operator-driven failover: the zero-config default.
#[default]
None,
/// A configured writer and explicit read replicas within one datacenter.
Dc,
/// Metadata durability in a remote datacenter.
Ha,
}

/// The resolved `[availability]` table: the selected mode and its topology.
///
/// `dc` and `ha` carry the replication role that fulfills them; `none` holds nothing, so a single-node
/// process allocates no availability state beyond this enum's discriminant.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum AvailabilityConfig {
None,
Dc(ReplicationConfig),
Ha(ReplicationConfig),
}

impl AvailabilityConfig {
/// The selected mode, independent of any topology a stronger mode carries.
#[must_use]
pub const fn mode(&self) -> AvailabilityMode {
match self {
Self::None => AvailabilityMode::None,
Self::Dc(_) => AvailabilityMode::Dc,
Self::Ha(_) => AvailabilityMode::Ha,
}
}

/// The replication role a `dc` or `ha` node drives, or `None` under single-node `none`.
#[must_use]
pub const fn replication(&self) -> Option<&ReplicationConfig> {
match self {
Self::None => None,
Self::Dc(replication) | Self::Ha(replication) => Some(replication),
}
}
}

/// The process role for replication.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ReplicationConfig {
Expand Down Expand Up @@ -185,7 +233,9 @@ impl Config {
Some(identity) if identity.trim().is_empty() => Err(ConfigError::WriterIdentity {
reason: "must not be blank",
}),
None if self.read_only || matches!(self.replication, Some(ReplicationConfig::Replica { .. })) => {
None if self.read_only
|| matches!(self.availability.replication(), Some(ReplicationConfig::Replica { .. })) =>
{
Err(ConfigError::WriterIdentity {
reason: "required in read replica mode",
})
Expand Down Expand Up @@ -525,7 +575,7 @@ impl Default for Config {
log: LogConfig::default(),
rate_limit: RateLimitConfig::default(),
auth: AuthConfig::default(),
replication: None,
availability: AvailabilityConfig::None,
jobs: JobsConfig::default(),
blob: BlobStorageConfig::Filesystem,
}
Expand Down
17 changes: 15 additions & 2 deletions crates/peryx/src/config/raw.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ use toml::Table;

use peryx_driver::jobs::ScheduledJob;

use super::model::{JobsMode, LogFormat, LogSink, PrefetchConfig, PrefetchMode};
use super::model::{AvailabilityMode, JobsMode, LogFormat, LogSink, PrefetchConfig, PrefetchMode};

#[derive(Debug, Default, Clone, PartialEq, Eq, Deserialize)]
#[serde(default, deny_unknown_fields)]
Expand Down Expand Up @@ -75,7 +75,9 @@ pub struct PartialConfig {
pub log: PartialLogConfig,
pub rate_limit: PartialRateLimitConfig,
pub auth: PartialAuthConfig,
pub replication: Option<RawReplication>,
/// The `[availability]` table: the runtime availability mode and the replication topology a
/// stronger mode carries. Absent, like `mode = "none"`, selects single-node operation.
pub availability: Option<RawAvailability>,
pub jobs: PartialJobsConfig,
/// A `[blob]` table selecting the blob storage backend.
pub blob: Option<RawBlobStorage>,
Expand All @@ -101,6 +103,17 @@ pub enum RawBlobStorage {
},
}

/// The raw `[availability]` table: the mode selector and its replication role.
///
/// A `dc` or `ha` node carries a role; an omitted table, and an explicit `mode = "none"`, both resolve
/// to single-node operation with no replication.
#[derive(Debug, Default, Clone, PartialEq, Eq, Deserialize)]
#[serde(default, deny_unknown_fields)]
pub struct RawAvailability {
pub mode: Option<AvailabilityMode>,
pub replication: Option<RawReplication>,
}

/// The `[jobs]` half of [`PartialConfig`].
#[derive(Debug, Default, Clone, PartialEq, Eq, Deserialize)]
#[serde(default, deny_unknown_fields)]
Expand Down
51 changes: 36 additions & 15 deletions crates/peryx/src/operator/snapshot.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@ use time::format_description::well_known::Rfc3339;
use toml::{Table, Value};

use crate::config::{
AcmeConfig, AuthConfig, BlobStorageConfig, Config, IndexConfig, IndexKind, JobsConfig, JobsMode, LogConfig,
LogFormat, LogSink, PrefetchConfig, PrefetchMode, ReplicationConfig, SecretSource, TlsConfig, TokenConfig,
WebhookConfig, WebhookSecret,
AcmeConfig, AuthConfig, AvailabilityConfig, BlobStorageConfig, Config, IndexConfig, IndexKind, JobsConfig,
JobsMode, LogConfig, LogFormat, LogSink, PrefetchConfig, PrefetchMode, ReplicationConfig, SecretSource, TlsConfig,
TokenConfig, WebhookConfig, WebhookSecret,
};

#[derive(Serialize)]
Expand Down Expand Up @@ -44,7 +44,7 @@ struct SnapshotConfig<'a> {
rate_limit: SnapshotRateLimit<'a>,
auth: SnapshotAuth<'a>,
#[serde(skip_serializing_if = "Option::is_none")]
replication: Option<SnapshotReplication<'a>>,
availability: Option<SnapshotAvailability<'a>>,
#[serde(skip_serializing_if = "Option::is_none")]
jobs: Option<SnapshotJobs>,
#[serde(skip_serializing_if = "Option::is_none")]
Expand Down Expand Up @@ -289,6 +289,13 @@ struct SnapshotTrustedPublisher<'a> {
claims: &'a std::collections::BTreeMap<String, String>,
}

#[derive(Serialize)]
struct SnapshotAvailability<'a> {
mode: &'static str,
#[serde(skip_serializing_if = "Option::is_none")]
replication: Option<SnapshotReplication<'a>>,
}

#[derive(Serialize)]
#[serde(tag = "role", rename_all = "lowercase")]
enum SnapshotReplication<'a> {
Expand Down Expand Up @@ -328,7 +335,7 @@ pub(super) fn config_snapshot(config: &Config) -> anyhow::Result<String> {
log,
rate_limit,
auth,
replication,
availability,
jobs,
blob,
} = config;
Expand Down Expand Up @@ -387,7 +394,7 @@ pub(super) fn config_snapshot(config: &Config) -> anyhow::Result<String> {
})
.collect(),
},
replication: snapshot_replication(replication.as_ref()),
availability: snapshot_availability(availability),
jobs: snapshot_jobs(jobs),
blob: snapshot_blob(blob),
};
Expand Down Expand Up @@ -438,32 +445,46 @@ fn snapshot_blob(blob: &BlobStorageConfig) -> Option<SnapshotBlob<'_>> {
})
}

fn snapshot_replication(replication: Option<&ReplicationConfig>) -> Option<SnapshotReplication<'_>> {
/// A snapshot carries the `[availability]` table only for a `dc` or `ha` node, so a single-node `none`
/// backup omits it and restores to the same default. The nested `[availability.replication]` role
/// round-trips the configured topology.
fn snapshot_availability(availability: &AvailabilityConfig) -> Option<SnapshotAvailability<'_>> {
let (mode, replication) = match availability {
AvailabilityConfig::None => return None,
AvailabilityConfig::Dc(replication) => ("dc", replication),
AvailabilityConfig::Ha(replication) => ("ha", replication),
};
Some(SnapshotAvailability {
mode,
replication: Some(snapshot_replication(replication)),
})
}

fn snapshot_replication(replication: &ReplicationConfig) -> SnapshotReplication<'_> {
match replication {
Some(ReplicationConfig::Primary { source, token }) => {
ReplicationConfig::Primary { source, token } => {
let (token, token_file, _) = secret_parts(Some(token));
Some(SnapshotReplication::Primary {
SnapshotReplication::Primary {
source,
token,
token_file,
})
}
}
Some(ReplicationConfig::Replica {
ReplicationConfig::Replica {
upstream,
token,
poll_interval,
page_size,
}) => {
} => {
let (token, token_file, _) = secret_parts(Some(token));
Some(SnapshotReplication::Replica {
SnapshotReplication::Replica {
upstream,
token,
token_file,
poll_interval_secs: poll_interval.as_secs(),
page_size: page_size.get(),
})
}
}
None => None,
}
}

Expand Down
2 changes: 1 addition & 1 deletion crates/peryx/src/replication.rs
Original file line number Diff line number Diff line change
Expand Up @@ -220,7 +220,7 @@ impl ReplicationRuntime {
/// Returns an error if a secret cannot be read, the upstream URL is invalid, or the primary
/// router rejects its identity or token.
pub fn new(config: &Config, state: &Arc<AppState>) -> anyhow::Result<Self> {
let (primary, replica) = match &config.replication {
let (primary, replica) = match config.availability.replication() {
None => (None, None),
Some(ReplicationConfig::Primary { source, token }) => {
let token = token.read().context("read the primary replication token")?;
Expand Down
5 changes: 4 additions & 1 deletion crates/peryx/src/server.rs
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,10 @@ pub fn build_state(config: &Config) -> anyhow::Result<Arc<AppState>> {
.with_context(|| format!("create data directory {}", config.data_dir.display()))?;
let meta_path = config.data_dir.join("peryx.redb");
let meta = MetaStore::open(&meta_path).with_context(|| format!("open metadata store {}", meta_path.display()))?;
let configured_replica = matches!(config.replication, Some(ReplicationConfig::Replica { .. }));
let configured_replica = matches!(
config.availability.replication(),
Some(ReplicationConfig::Replica { .. })
);
let read_only = config.read_only || configured_replica;
if read_only {
let active = meta.writer_identity().context("read metadata store writer identity")?;
Expand Down
Loading
Loading