diff --git a/crates/core/src/oi/handler.rs b/crates/core/src/oi/handler.rs index bcf4da99..9e7681e0 100644 --- a/crates/core/src/oi/handler.rs +++ b/crates/core/src/oi/handler.rs @@ -21,6 +21,7 @@ mod ingresses; mod key_mgmt; mod params; mod registries; +mod restarts; mod services; mod status; mod templates; @@ -196,6 +197,12 @@ fn parse_and_dispatch(state: &Arc, buf: &[u8], ctx: &RequestCtx) -> Han "/faults/list" => faults::list_faults(state, parse_params(req.params)?), // i[fault.clear-app] "/faults/clear" => faults::clear_app_faults(state, parse_params(req.params)?), + // i[restart.list] + "/restarts/list" => restarts::list_restarts(state, parse_params(req.params)?), + // i[restart.settings] + "/restarts/settings/get" => restarts::get_settings(state), + // i[restart.settings] + "/restarts/settings/set" => restarts::set_settings(state, parse_params(req.params)?), // i[canopy.status] "/canopy/status" => canopy::status(state), // i[canopy.settings] diff --git a/crates/core/src/oi/handler/apps.rs b/crates/core/src/oi/handler/apps.rs index e1785d25..a8706680 100644 --- a/crates/core/src/oi/handler/apps.rs +++ b/crates/core/src/oi/handler/apps.rs @@ -20,7 +20,7 @@ use crate::{ history::{find_instances_for_group, query_observations}, identity::{InstanceId, InstanceVariant, ResourceInstance}, lifecycle::LifecycleState, - restart_gens, scaling, + restart_gens, restarts, scaling, stopped::{self, kind_str, parse_kind}, transition_phase, }, @@ -651,6 +651,10 @@ pub(crate) fn describe_app(state: &OiState, params: AppParams) -> HandlerResult let all_faults_clone = all_faults_for_app.clone(); let stopped_set_clone = stopped_set.clone(); let resources_json: Vec = state.db.call(move |db| { + // i[impl app.describe] + // Read the rate settings once per describe: every instance summary + // reports its recent count over the same window. + let restart_settings = restarts::settings(db).ok(); resource_infos .into_iter() .map(|info| { @@ -662,6 +666,14 @@ pub(crate) fn describe_app(state: &OiState, params: AppParams) -> HandlerResult let observations = query_observations(db, inst).unwrap_or_default(); let (lifecycle, transition_time) = derive_state_with_transition_time(inst, &observations); + // i[impl app.describe] + // Omitted entirely for an instance with no + // restart history, so a resource that has never + // restarted does not read as one that restarted + // zero times just now. + let restart_summary = restart_settings.and_then(|s| { + restarts::summary(db, &inst.id.to_hex(), s).ok().flatten() + }); json!({ "id": inst.id.to_hex(), "display_name": inst.display_name, @@ -669,6 +681,7 @@ pub(crate) fn describe_app(state: &OiState, params: AppParams) -> HandlerResult "transition_time": transition_time.and_then(|t| { jiff::Timestamp::try_from(t).ok().map(|ts| ts.to_string()) }), + "restarts": restart_summary, }) }) .collect() diff --git a/crates/core/src/oi/handler/restarts.rs b/crates/core/src/oi/handler/restarts.rs new file mode 100644 index 00000000..05593387 --- /dev/null +++ b/crates/core/src/oi/handler/restarts.rs @@ -0,0 +1,87 @@ +use seedling_protocol::error::{ErrorCode, OiError}; +use seedling_protocol::names::AppName; +use serde::Deserialize; +use serde_json::json; + +use super::HandlerResult; +use crate::{oi::state::OiState, runtime::restarts}; + +/// Records returned when the caller does not ask for a specific number, and +/// the ceiling on what it may ask for. The cap keeps a single request from +/// pulling the whole retained history of a busy host over the wire. +const DEFAULT_LIMIT: usize = 100; +const MAX_LIMIT: usize = 1000; + +#[derive(Deserialize)] +pub(crate) struct ListRestartsParams { + pub app: Option, + pub instance: Option, + pub limit: Option, +} + +// i[impl restart.list] +pub(crate) fn list_restarts(state: &OiState, params: ListRestartsParams) -> HandlerResult { + let ListRestartsParams { + app, + instance, + limit, + } = params; + let limit = limit.unwrap_or(DEFAULT_LIMIT).clamp(1, MAX_LIMIT); + let records = state + .db + .call(move |db| restarts::list(db, app.as_ref(), instance.as_deref(), limit)) + .map_err(|e| OiError::new(ErrorCode::NotFound, format!("db query: {e}")))?; + + // i[impl restart.record] + let result: Vec = records + .into_iter() + .map(|r| { + json!({ + "id": r.id, + "app": r.app, + "instance_id": r.instance_id, + "resource_type": r.resource_type, + "resource_name": r.resource_name, + "generation": r.generation, + "timestamp": r.timestamp.to_string(), + "cause": r.cause, + "exit_code": r.exit_code, + "exit_kind": r.exit_kind, + }) + }) + .collect(); + Ok(json!(result)) +} + +// i[impl restart.settings] +pub(crate) fn get_settings(state: &OiState) -> HandlerResult { + let s = state + .db + .call(restarts::settings) + .map_err(|e| OiError::new(ErrorCode::NotFound, format!("db query: {e}")))?; + Ok(json!({ "threshold": s.threshold, "window_secs": s.window_secs })) +} + +#[derive(Deserialize)] +pub(crate) struct SetSettingsParams { + #[serde(default)] + pub threshold: Option, + #[serde(default)] + pub window_secs: Option, +} + +// i[impl restart.settings] +pub(crate) fn set_settings(state: &OiState, params: SetSettingsParams) -> HandlerResult { + let SetSettingsParams { + threshold, + window_secs, + } = params; + let s = state + .db + .call(move |db| restarts::set_settings(db, threshold, window_secs)) + .map_err(|e| OiError::new(ErrorCode::RequirementsInvalid, e.to_string()))?; + Ok(json!({ "threshold": s.threshold, "window_secs": s.window_secs })) +} + +#[cfg(test)] +mod tests; diff --git a/crates/core/src/oi/handler/restarts/tests.rs b/crates/core/src/oi/handler/restarts/tests.rs new file mode 100644 index 00000000..d59e2cf7 --- /dev/null +++ b/crates/core/src/oi/handler/restarts/tests.rs @@ -0,0 +1,129 @@ +use serde_json::json; + +use crate::{ + oi::test_support::TestOi, + runtime::restarts::{Cause, ExitKind, ExitStatus, RestartSubject, record}, +}; +use seedling_protocol::names::AppName; + +fn seed(oi: &TestOi, app: &str, instance: &str, cause: Cause, exit: Option) { + let subject = RestartSubject { + app: AppName::new(app).unwrap(), + instance_id: instance.to_owned(), + resource_type: Some("deployment".to_owned()), + resource_name: Some("web".to_owned()), + generation: Some(1), + }; + let at = jiff::Timestamp::now().as_millisecond(); + oi.state + .db + .call(move |db| record(db, &subject, cause, exit, at)) + .expect("record"); +} + +// i[verify restart.list] +// i[verify restart.record] +#[test] +fn list_returns_records_with_their_exit_and_cause() { + let oi = TestOi::new(); + seed( + &oi, + "demo", + "aa", + Cause::Recovery, + Some(ExitStatus { + kind: ExitKind::Signalled, + code: 9, + }), + ); + + let rows = oi.call("/restarts/list", json!({})).unwrap(); + let rows = rows.as_array().unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0]["app"], "demo"); + assert_eq!(rows[0]["instance_id"], "aa"); + assert_eq!(rows[0]["resource_type"], "deployment"); + assert_eq!(rows[0]["resource_name"], "web"); + assert_eq!(rows[0]["generation"], 1); + assert_eq!(rows[0]["cause"], "recovery"); + assert_eq!(rows[0]["exit_code"], 9); + assert_eq!(rows[0]["exit_kind"], "signalled"); + assert!(!rows[0]["timestamp"].as_str().unwrap().is_empty()); +} + +// i[verify restart.record] +#[test] +fn an_unknown_exit_is_reported_as_null_rather_than_invented() { + let oi = TestOi::new(); + seed(&oi, "demo", "aa", Cause::Deliberate, None); + + let rows = oi.call("/restarts/list", json!({})).unwrap(); + let rows = rows.as_array().unwrap(); + assert_eq!(rows[0]["cause"], "deliberate"); + assert!(rows[0]["exit_code"].is_null()); + assert!(rows[0]["exit_kind"].is_null()); +} + +// i[verify restart.list] +#[test] +fn list_filters_by_app_and_instance_and_honours_limit() { + let oi = TestOi::new(); + seed(&oi, "demo", "aa", Cause::Recovery, None); + seed(&oi, "demo", "aa", Cause::Recovery, None); + seed(&oi, "other", "bb", Cause::Recovery, None); + + let by_app = oi.call("/restarts/list", json!({ "app": "demo" })).unwrap(); + assert_eq!(by_app.as_array().unwrap().len(), 2); + + let by_instance = oi + .call("/restarts/list", json!({ "instance": "bb" })) + .unwrap(); + let by_instance = by_instance.as_array().unwrap(); + assert_eq!(by_instance.len(), 1); + assert_eq!(by_instance[0]["app"], "other"); + + let both = oi + .call("/restarts/list", json!({ "app": "demo", "instance": "bb" })) + .unwrap(); + assert!(both.as_array().unwrap().is_empty()); + + let limited = oi.call("/restarts/list", json!({ "limit": 1 })).unwrap(); + assert_eq!(limited.as_array().unwrap().len(), 1); +} + +// i[verify restart.settings] +#[test] +fn settings_round_trip_and_reject_out_of_bounds() { + let oi = TestOi::new(); + + let s = oi.call("/restarts/settings/get", json!({})).unwrap(); + assert_eq!(s["threshold"], 5); + assert_eq!(s["window_secs"], 1800); + + let s = oi + .call("/restarts/settings/set", json!({ "threshold": 3 })) + .unwrap(); + assert_eq!(s["threshold"], 3); + assert_eq!(s["window_secs"], 1800); + + let s = oi + .call("/restarts/settings/set", json!({ "window_secs": 600 })) + .unwrap(); + assert_eq!(s["threshold"], 3); + assert_eq!(s["window_secs"], 600); + + let err = oi + .call("/restarts/settings/set", json!({ "threshold": 1 })) + .unwrap_err(); + assert!(err.1.contains("at least"), "{}", err.1); + + let err = oi + .call("/restarts/settings/set", json!({ "window_secs": 10 })) + .unwrap_err(); + assert!(err.1.contains("at least"), "{}", err.1); + + // A rejected update leaves the stored settings alone. + let s = oi.call("/restarts/settings/get", json!({})).unwrap(); + assert_eq!(s["threshold"], 3); + assert_eq!(s["window_secs"], 600); +} diff --git a/crates/core/src/runtime.rs b/crates/core/src/runtime.rs index d34609a8..1a065359 100644 --- a/crates/core/src/runtime.rs +++ b/crates/core/src/runtime.rs @@ -27,6 +27,7 @@ pub mod probe; pub mod registries; pub mod registry; pub mod restart_gens; +pub mod restarts; pub mod scaling; pub mod scheduler; pub mod schedules; diff --git a/crates/core/src/runtime/barrier/replay.rs b/crates/core/src/runtime/barrier/replay.rs index 2f27e4a7..fb268bb6 100644 --- a/crates/core/src/runtime/barrier/replay.rs +++ b/crates/core/src/runtime/barrier/replay.rs @@ -326,6 +326,7 @@ pub fn run_operation( // call_index 0 each pass; without this marker the duplicated // breadcrumbs after a barrier wake up look like the script ran // twice. + // r[impl actuate.breadcrumb.replay] if !committed.is_empty() { crate::system::breadcrumb::Breadcrumb { app: Some(&app.def.load().name), diff --git a/crates/core/src/runtime/db.rs b/crates/core/src/runtime/db.rs index a167e999..5436644c 100644 --- a/crates/core/src/runtime/db.rs +++ b/crates/core/src/runtime/db.rs @@ -129,6 +129,9 @@ const SQL_V52: &str = include_str!("db/migrations/v52.sql"); // r[impl canopy.settings.enabled] // r[impl canopy.report.identity] const SQL_V53: &str = include_str!("db/migrations/v53.sql"); +// r[impl autonomous.restart.record] +// r[impl autonomous.restart.rate.settings] +const SQL_V54: &str = include_str!("db/migrations/v54.sql"); const MIGRATIONS: &[Migration] = &[ Migration { @@ -391,6 +394,11 @@ const MIGRATIONS: &[Migration] = &[ sql: SQL_V53, custom_run: None, }, + Migration { + version: 54, + sql: SQL_V54, + custom_run: None, + }, ]; fn migration_hash(sql: &str) -> String { diff --git a/crates/core/src/runtime/db/migrations/v54.sql b/crates/core/src/runtime/db/migrations/v54.sql new file mode 100644 index 00000000..1577ecfe --- /dev/null +++ b/crates/core/src/runtime/db/migrations/v54.sql @@ -0,0 +1,61 @@ +-- r[impl autonomous.restart.record] +-- One row per observed or performed restart of a container instance. +-- +-- `cause` is 'recovery' when the restart followed an unexpected exit, and +-- 'deliberate' when the runtime restarted the workload on purpose (rolling +-- update, health check replacement, operator-requested restart). Only recovery +-- rows count towards the crash-loop rate. +-- +-- The split is on why, not on who: on Linux systemd actions recovery restarts +-- and seedling actions deliberate ones, but on a platform with no service +-- supervisor seedling actions both, and a column recording the actor would +-- classify every restart there identically. +-- +-- `exit_kind` is 'exited', 'signalled' or 'dumped'; `exit_code` is the exit +-- status for 'exited' and the signal number otherwise. Both are NULL when the +-- platform did not report an exit for the run that ended. +CREATE TABLE IF NOT EXISTS instance_restarts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + instance_id TEXT NOT NULL, + app TEXT NOT NULL, + resource_type TEXT, + resource_name TEXT, + generation INTEGER, + recorded_at INTEGER NOT NULL, + cause TEXT NOT NULL, + exit_code INTEGER, + exit_kind TEXT +); + +CREATE INDEX IF NOT EXISTS idx_instance_restarts_instance + ON instance_restarts (instance_id, recorded_at); +CREATE INDEX IF NOT EXISTS idx_instance_restarts_app + ON instance_restarts (app, recorded_at); + +-- Last restart counter read from the supervisor for an instance's unit. The +-- counter is monotonic per unit but resets when the unit is recreated or its +-- failed state is cleared, so a decrease is a reset to re-baseline against, +-- not a negative delta. +CREATE TABLE IF NOT EXISTS instance_restart_counters ( + instance_id TEXT PRIMARY KEY, + counter INTEGER NOT NULL, + updated_at INTEGER NOT NULL +); + +-- r[impl autonomous.restart.rate.settings] +-- Crash-loop rate threshold and window. One row, enforced via the singleton +-- primary key. +-- +-- The default of five supervisor-actioned restarts within thirty minutes is +-- loose enough that a container taking seconds to crash gets several chances +-- across a deploy, and tight enough that persistent flapping surfaces within +-- an operator's working session rather than a day later. +CREATE TABLE IF NOT EXISTS restart_settings ( + singleton INTEGER PRIMARY KEY DEFAULT 1 CHECK (singleton = 1), + threshold INTEGER NOT NULL DEFAULT 5, + window_secs INTEGER NOT NULL DEFAULT 1800, + updated_at INTEGER NOT NULL DEFAULT 0 +); + +INSERT OR IGNORE INTO restart_settings (singleton, threshold, window_secs, updated_at) + VALUES (1, 5, 1800, 0); diff --git a/crates/core/src/runtime/db/tests.rs b/crates/core/src/runtime/db/tests.rs index d6e3c2d8..e9b4c9ec 100644 --- a/crates/core/src/runtime/db/tests.rs +++ b/crates/core/src/runtime/db/tests.rs @@ -13,7 +13,7 @@ fn open_in_memory_succeeds() { |r| r.get(0), ) .expect("schema_version should exist"); - assert_eq!(version, 53); + assert_eq!(version, 54); } // r[verify history.persistence] @@ -45,7 +45,7 @@ fn params_table_exists() { |r| r.get(0), ) .expect("schema_version should exist"); - assert_eq!(version, 53); + assert_eq!(version, 54); } // i[verify app.persist] diff --git a/crates/core/src/runtime/gc.rs b/crates/core/src/runtime/gc.rs index e69cd325..5a83ba49 100644 --- a/crates/core/src/runtime/gc.rs +++ b/crates/core/src/runtime/gc.rs @@ -66,6 +66,12 @@ fn run_gc_cycle(db: &Db, config: &GcConfig) { Err(e) => error!(error = %e, "gc: unscheduled instances cleanup failed"), _ => {} } + // r[impl gc.restarts] + match crate::system::reconcile::gc_restart_records(db) { + Ok(n) if n > 0 => debug!(rows = n, "gc: pruned restart records"), + Err(e) => error!(error = %e, "gc: restart records cleanup failed"), + _ => {} + } } fn now_ms() -> i64 { diff --git a/crates/core/src/runtime/restarts.rs b/crates/core/src/runtime/restarts.rs new file mode 100644 index 00000000..bdb371f5 --- /dev/null +++ b/crates/core/src/runtime/restarts.rs @@ -0,0 +1,429 @@ +//! Restart accounting: the durable record of every container restart, and the +//! rate derived from it. +//! +//! Seedling keeps the books even where it does not action the restart. On +//! Linux systemd restarts the unit and seedling reads its counter; on a +//! platform without a supervisor the runtime restarts the workload itself and +//! records the attempt firsthand. Either way the recorded rate — not the +//! supervisor's internal accounting — is what decides a crash loop. + +use jiff::Timestamp; +use rusqlite::OptionalExtension; +use seedling_protocol::names::AppName; +use serde::Serialize; + +use crate::runtime::db::Db; + +/// Most-recent restart records kept per instance. +/// +/// The bound is per instance rather than global, and applied on write rather +/// than by rate-limiting what gets recorded: a hard crash loop produces rows +/// fastest exactly when the per-attempt exit codes are the diagnostic. +// r[impl gc.restarts] +pub const RETAIN_PER_INSTANCE: usize = 50; + +/// Why a restart happened. +/// +/// Deliberately not "who performed it". On Linux systemd actions recovery +/// restarts and seedling actions deliberate ones, so the two splits coincide — +/// but only because of how this platform is put together. Where there is no +/// service supervisor the runtime performs both kinds, and a field recording +/// the actor would put every restart in one bucket and leave the crash-loop +/// rate permanently at zero. +// r[impl autonomous.restart.record] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum Cause { + /// The workload exited unexpectedly and was brought back. Counts towards + /// the crash-loop rate. + Recovery, + /// The runtime restarted the workload on purpose: a rolling update, a + /// health-check replacement, an operator-requested restart. Recorded but + /// excluded from the rate. + Deliberate, +} + +impl Cause { + pub fn as_str(self) -> &'static str { + match self { + Self::Recovery => "recovery", + Self::Deliberate => "deliberate", + } + } +} + +/// How the previous run ended, as far as the platform reports it. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum ExitKind { + /// Exited of its own accord; the code is its exit status. + Exited, + /// Killed by a signal; the code is the signal number. + Signalled, + /// Killed by a signal and dumped core; the code is the signal number. + Dumped, +} + +impl ExitKind { + pub fn as_str(self) -> &'static str { + match self { + Self::Exited => "exited", + Self::Signalled => "signalled", + Self::Dumped => "dumped", + } + } + + fn from_str(s: &str) -> Option { + match s { + "exited" => Some(Self::Exited), + "signalled" => Some(Self::Signalled), + "dumped" => Some(Self::Dumped), + _ => None, + } + } +} + +/// The exit status of the run that ended, where the platform reports one. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ExitStatus { + pub kind: ExitKind, + pub code: i32, +} + +// i[impl restart.record] +#[derive(Debug, Clone, Serialize)] +pub struct RestartRecord { + pub id: i64, + pub app: AppName, + pub instance_id: String, + pub resource_type: Option, + pub resource_name: Option, + pub generation: Option, + pub timestamp: Timestamp, + pub cause: Cause, + pub exit_code: Option, + pub exit_kind: Option, +} + +/// Crash-loop rate parameters. +// r[impl autonomous.restart.rate.settings] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +pub struct RestartSettings { + pub threshold: i64, + pub window_secs: i64, +} + +/// Lower bounds on the settings. A threshold of one would file a crash loop on +/// the first restart, which every container that has ever been rescheduled +/// would trip; a window under a minute is shorter than the pacing the +/// supervisor already applies between attempts. +pub const MIN_THRESHOLD: i64 = 2; +pub const MIN_WINDOW_SECS: i64 = 60; + +/// What the instance's restart history looks like right now, for the app +/// description surface. +// i[impl app.describe] +#[derive(Debug, Clone, Serialize)] +pub struct RestartSummary { + /// Recovery restarts inside the current rate window. + pub recent: i64, + pub window_secs: i64, + /// All retained records for the instance, of either cause. + pub total: i64, + pub last_at: Option, + pub last_exit_code: Option, + pub last_exit_kind: Option, +} + +fn now_ms() -> i64 { + Timestamp::now().as_millisecond() +} + +// --------------------------------------------------------------------------- +// Recording +// --------------------------------------------------------------------------- + +/// Identity carried on every record. Kept as one argument so callers do not +/// thread five positional strings through the reconciler. +#[derive(Debug, Clone)] +pub struct RestartSubject { + pub app: AppName, + pub instance_id: String, + pub resource_type: Option, + pub resource_name: Option, + pub generation: Option, +} + +// r[impl autonomous.restart.record] +/// Record one restart. `at_ms` lets a caller recording a burst of counter +/// deltas stamp them apart rather than collapsing them onto one instant. +pub fn record( + db: &Db, + subject: &RestartSubject, + cause: Cause, + exit: Option, + at_ms: i64, +) -> rusqlite::Result { + db.conn.execute( + "INSERT INTO instance_restarts + (instance_id, app, resource_type, resource_name, generation, + recorded_at, cause, exit_code, exit_kind) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)", + rusqlite::params![ + subject.instance_id, + subject.app, + subject.resource_type, + subject.resource_name, + subject.generation, + at_ms, + cause.as_str(), + exit.map(|e| e.code), + exit.map(|e| e.kind.as_str()), + ], + )?; + let id = db.conn.last_insert_rowid(); + prune_instance(db, &subject.instance_id, RETAIN_PER_INSTANCE)?; + Ok(id) +} + +// r[impl gc.restarts] +/// Drop all but the `retain` most recent records for one instance. +pub fn prune_instance(db: &Db, instance_id: &str, retain: usize) -> rusqlite::Result { + db.conn.execute( + "DELETE FROM instance_restarts + WHERE instance_id = ?1 + AND id NOT IN ( + SELECT id FROM instance_restarts + WHERE instance_id = ?1 + ORDER BY recorded_at DESC, id DESC + LIMIT ?2 + )", + rusqlite::params![instance_id, retain as i64], + ) +} + +// --------------------------------------------------------------------------- +// Queries +// --------------------------------------------------------------------------- + +fn row_to_record(row: &rusqlite::Row<'_>) -> rusqlite::Result { + let recorded_at: i64 = row.get(6)?; + let cause: String = row.get(7)?; + let exit_kind: Option = row.get(9)?; + Ok(RestartRecord { + id: row.get(0)?, + instance_id: row.get(1)?, + app: row.get(2)?, + resource_type: row.get(3)?, + resource_name: row.get(4)?, + generation: row.get(5)?, + timestamp: Timestamp::from_millisecond(recorded_at).unwrap_or_default(), + cause: if cause == "deliberate" { + Cause::Deliberate + } else { + Cause::Recovery + }, + exit_code: row.get(8)?, + exit_kind: exit_kind.as_deref().and_then(ExitKind::from_str), + }) +} + +const SELECT_COLS: &str = "id, instance_id, app, resource_type, resource_name, generation, \ + recorded_at, cause, exit_code, exit_kind"; + +// i[impl restart.list] +/// Restart records, most recent first, optionally narrowed to one app and/or +/// one instance. +pub fn list( + db: &Db, + app: Option<&AppName>, + instance_id: Option<&str>, + limit: usize, +) -> rusqlite::Result> { + let sql = format!( + "SELECT {SELECT_COLS} FROM instance_restarts + WHERE (?1 IS NULL OR app = ?1) + AND (?2 IS NULL OR instance_id = ?2) + ORDER BY recorded_at DESC, id DESC + LIMIT ?3" + ); + let mut stmt = db.conn.prepare(&sql)?; + let rows = stmt.query_map( + rusqlite::params![app, instance_id, limit as i64], + row_to_record, + )?; + rows.collect() +} + +// r[impl autonomous.restart.rate] +/// Recovery restarts recorded for an instance within the last `window_secs`. +/// Deliberate restarts are excluded: a rolling update must not read as a crash +/// burst. +pub fn recent_recovery_count( + db: &Db, + instance_id: &str, + window_secs: i64, +) -> rusqlite::Result { + let cutoff = now_ms() - window_secs * 1000; + db.conn.query_row( + "SELECT COUNT(*) FROM instance_restarts + WHERE instance_id = ?1 AND cause = 'recovery' AND recorded_at >= ?2", + rusqlite::params![instance_id, cutoff], + |r| r.get(0), + ) +} + +/// Per-instance summary for the app description surface. Returns `None` when +/// the instance has no records at all, so callers can omit the field entirely +/// rather than reporting a zeroed summary for a resource that never restarts. +pub fn summary( + db: &Db, + instance_id: &str, + settings: RestartSettings, +) -> rusqlite::Result> { + let total: i64 = db.conn.query_row( + "SELECT COUNT(*) FROM instance_restarts WHERE instance_id = ?1", + rusqlite::params![instance_id], + |r| r.get(0), + )?; + if total == 0 { + return Ok(None); + } + let recent = recent_recovery_count(db, instance_id, settings.window_secs)?; + let last: Option<(i64, Option, Option)> = db + .conn + .query_row( + "SELECT recorded_at, exit_code, exit_kind FROM instance_restarts + WHERE instance_id = ?1 + ORDER BY recorded_at DESC, id DESC + LIMIT 1", + rusqlite::params![instance_id], + |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)), + ) + .optional()?; + let (last_at, last_exit_code, last_exit_kind) = match last { + Some((at, code, kind)) => ( + Timestamp::from_millisecond(at).ok().map(|t| t.to_string()), + code, + kind.as_deref().and_then(ExitKind::from_str), + ), + None => (None, None, None), + }; + Ok(Some(RestartSummary { + recent, + window_secs: settings.window_secs, + total, + last_at, + last_exit_code, + last_exit_kind, + })) +} + +// --------------------------------------------------------------------------- +// Counter baselines +// --------------------------------------------------------------------------- + +/// The last restart counter seen for an instance's unit, if any. +pub fn baseline(db: &Db, instance_id: &str) -> rusqlite::Result> { + db.conn + .query_row( + "SELECT counter FROM instance_restart_counters WHERE instance_id = ?1", + rusqlite::params![instance_id], + |r| r.get(0), + ) + .optional() +} + +pub fn set_baseline(db: &Db, instance_id: &str, counter: i64) -> rusqlite::Result<()> { + db.conn.execute( + "INSERT INTO instance_restart_counters (instance_id, counter, updated_at) + VALUES (?1, ?2, ?3) + ON CONFLICT (instance_id) DO UPDATE + SET counter = excluded.counter, updated_at = excluded.updated_at", + rusqlite::params![instance_id, counter, now_ms()], + )?; + Ok(()) +} + +pub fn clear_baseline(db: &Db, instance_id: &str) -> rusqlite::Result<()> { + db.conn.execute( + "DELETE FROM instance_restart_counters WHERE instance_id = ?1", + rusqlite::params![instance_id], + )?; + Ok(()) +} + +// --------------------------------------------------------------------------- +// Settings +// --------------------------------------------------------------------------- + +// r[impl autonomous.restart.rate.settings] +// i[impl restart.settings] +pub fn settings(db: &Db) -> rusqlite::Result { + db.conn.query_row( + "SELECT threshold, window_secs FROM restart_settings WHERE singleton = 1", + [], + |r| { + Ok(RestartSettings { + threshold: r.get(0)?, + window_secs: r.get(1)?, + }) + }, + ) +} + +/// Rejected values, so the caller can turn them into an interface error. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SettingsError { + ThresholdTooLow, + WindowTooShort, +} + +impl std::fmt::Display for SettingsError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::ThresholdTooLow => write!(f, "threshold must be at least {MIN_THRESHOLD}"), + Self::WindowTooShort => write!(f, "window_secs must be at least {MIN_WINDOW_SECS}"), + } + } +} + +// r[impl autonomous.restart.rate.settings] +// i[impl restart.settings] +/// Update either or both settings. Omitted fields are left as they are. The +/// reconciler reads the settings on each tick, so a change takes effect on the +/// next one without restarting the runtime. +pub fn set_settings( + db: &Db, + threshold: Option, + window_secs: Option, +) -> Result { + if let Some(t) = threshold + && t < MIN_THRESHOLD + { + return Err(SettingsError::ThresholdTooLow); + } + if let Some(w) = window_secs + && w < MIN_WINDOW_SECS + { + return Err(SettingsError::WindowTooShort); + } + let current = settings(db).unwrap_or(RestartSettings { + threshold: 5, + window_secs: 1800, + }); + let next = RestartSettings { + threshold: threshold.unwrap_or(current.threshold), + window_secs: window_secs.unwrap_or(current.window_secs), + }; + let _ = db.conn.execute( + "UPDATE restart_settings + SET threshold = ?1, window_secs = ?2, updated_at = ?3 + WHERE singleton = 1", + rusqlite::params![next.threshold, next.window_secs, now_ms()], + ); + Ok(next) +} + +#[cfg(test)] +mod tests; diff --git a/crates/core/src/runtime/restarts/tests.rs b/crates/core/src/runtime/restarts/tests.rs new file mode 100644 index 00000000..6363b6ac --- /dev/null +++ b/crates/core/src/runtime/restarts/tests.rs @@ -0,0 +1,184 @@ +use super::*; +use crate::runtime::db::Db; + +fn app(s: &str) -> AppName { + AppName::new(s).unwrap() +} + +fn subject(instance: &str) -> RestartSubject { + RestartSubject { + app: app("myapp"), + instance_id: instance.to_owned(), + resource_type: Some("deployment".to_owned()), + resource_name: Some("web".to_owned()), + generation: Some(3), + } +} + +fn exited(code: i32) -> Option { + Some(ExitStatus { + kind: ExitKind::Exited, + code, + }) +} + +// r[verify autonomous.restart.record] +// i[verify restart.record] +#[test] +fn records_carry_identity_exit_and_cause() { + let db = Db::open_in_memory().expect("open"); + let now = now_ms(); + record(&db, &subject("aa"), Cause::Recovery, exited(137), now).expect("record"); + + let rows = list(&db, None, None, 10).expect("list"); + assert_eq!(rows.len(), 1); + let r = &rows[0]; + assert_eq!(r.app, "myapp"); + assert_eq!(r.instance_id, "aa"); + assert_eq!(r.resource_type.as_deref(), Some("deployment")); + assert_eq!(r.resource_name.as_deref(), Some("web")); + assert_eq!(r.generation, Some(3)); + assert_eq!(r.cause, Cause::Recovery); + assert_eq!(r.exit_code, Some(137)); + assert_eq!(r.exit_kind, Some(ExitKind::Exited)); +} + +// i[verify restart.list] +#[test] +fn list_is_most_recent_first_and_filters() { + let db = Db::open_in_memory().expect("open"); + let now = now_ms(); + record(&db, &subject("aa"), Cause::Recovery, None, now - 2000).expect("record"); + record(&db, &subject("aa"), Cause::Recovery, None, now - 1000).expect("record"); + record(&db, &subject("bb"), Cause::Deliberate, None, now).expect("record"); + + let all = list(&db, None, None, 10).expect("list"); + assert_eq!(all.len(), 3); + assert_eq!(all[0].instance_id, "bb"); + + let only_aa = list(&db, None, Some("aa"), 10).expect("list"); + assert_eq!(only_aa.len(), 2); + + let other_app = list(&db, Some(&app("elsewhere")), None, 10).expect("list"); + assert!(other_app.is_empty()); + + let limited = list(&db, None, None, 1).expect("list"); + assert_eq!(limited.len(), 1); +} + +// r[verify autonomous.restart.rate] +#[test] +fn deliberate_restarts_are_excluded_from_the_rate() { + let db = Db::open_in_memory().expect("open"); + let now = now_ms(); + for i in 0..4 { + record(&db, &subject("aa"), Cause::Deliberate, None, now - i * 100).expect("record"); + } + assert_eq!(recent_recovery_count(&db, "aa", 1800).expect("count"), 0); + + record(&db, &subject("aa"), Cause::Recovery, None, now).expect("record"); + assert_eq!(recent_recovery_count(&db, "aa", 1800).expect("count"), 1); +} + +// r[verify autonomous.restart.rate] +#[test] +fn restarts_outside_the_window_do_not_count() { + let db = Db::open_in_memory().expect("open"); + let now = now_ms(); + record(&db, &subject("aa"), Cause::Recovery, None, now - 3_600_000).expect("record"); + record(&db, &subject("aa"), Cause::Recovery, None, now).expect("record"); + + assert_eq!(recent_recovery_count(&db, "aa", 1800).expect("count"), 1); + assert_eq!(recent_recovery_count(&db, "aa", 7200).expect("count"), 2); +} + +// r[verify gc.restarts] +#[test] +fn per_instance_cap_holds_under_a_sustained_crash_loop() { + let db = Db::open_in_memory().expect("open"); + let now = now_ms(); + for i in 0..(RETAIN_PER_INSTANCE as i64 * 3) { + record(&db, &subject("aa"), Cause::Recovery, exited(1), now + i).expect("record"); + } + // A second instance's history must not be pruned by the first's churn. + record(&db, &subject("bb"), Cause::Recovery, None, now).expect("record"); + + let kept = list(&db, None, Some("aa"), 1000).expect("list"); + assert_eq!(kept.len(), RETAIN_PER_INSTANCE); + // The cap keeps the most recent records, which are the diagnostic ones. + assert_eq!(kept[0].timestamp.as_millisecond(), now + 149); + assert_eq!(list(&db, None, Some("bb"), 1000).expect("list").len(), 1); +} + +#[test] +fn summary_is_absent_until_there_is_history() { + let db = Db::open_in_memory().expect("open"); + let settings = RestartSettings { + threshold: 5, + window_secs: 1800, + }; + assert!(summary(&db, "aa", settings).expect("summary").is_none()); + + let now = now_ms(); + record(&db, &subject("aa"), Cause::Deliberate, None, now - 1000).expect("record"); + record(&db, &subject("aa"), Cause::Recovery, exited(2), now).expect("record"); + + let s = summary(&db, "aa", settings) + .expect("summary") + .expect("some"); + assert_eq!(s.total, 2); + assert_eq!(s.recent, 1); + assert_eq!(s.window_secs, 1800); + assert_eq!(s.last_exit_code, Some(2)); + assert_eq!(s.last_exit_kind, Some(ExitKind::Exited)); + assert!(s.last_at.is_some()); +} + +#[test] +fn baselines_round_trip_and_clear() { + let db = Db::open_in_memory().expect("open"); + assert_eq!(baseline(&db, "aa").expect("baseline"), None); + set_baseline(&db, "aa", 4).expect("set"); + assert_eq!(baseline(&db, "aa").expect("baseline"), Some(4)); + set_baseline(&db, "aa", 0).expect("set"); + assert_eq!(baseline(&db, "aa").expect("baseline"), Some(0)); + clear_baseline(&db, "aa").expect("clear"); + assert_eq!(baseline(&db, "aa").expect("baseline"), None); +} + +// r[verify autonomous.restart.rate.settings] +// i[verify restart.settings] +#[test] +fn settings_default_and_update_partially() { + let db = Db::open_in_memory().expect("open"); + let s = settings(&db).expect("settings"); + assert_eq!(s.threshold, 5); + assert_eq!(s.window_secs, 1800); + + let s = set_settings(&db, Some(3), None).expect("set"); + assert_eq!(s.threshold, 3); + assert_eq!(s.window_secs, 1800); + assert_eq!(settings(&db).expect("settings"), s); + + let s = set_settings(&db, None, Some(600)).expect("set"); + assert_eq!(s.threshold, 3); + assert_eq!(s.window_secs, 600); +} + +// i[verify restart.settings] +#[test] +fn settings_reject_out_of_bounds_values() { + let db = Db::open_in_memory().expect("open"); + assert_eq!( + set_settings(&db, Some(1), None), + Err(SettingsError::ThresholdTooLow) + ); + assert_eq!( + set_settings(&db, None, Some(30)), + Err(SettingsError::WindowTooShort) + ); + // A rejected update leaves the stored settings untouched. + let s = settings(&db).expect("settings"); + assert_eq!(s.threshold, 5); + assert_eq!(s.window_secs, 1800); +} diff --git a/crates/core/src/system/breadcrumb.rs b/crates/core/src/system/breadcrumb.rs index bff7c419..e5ee5719 100644 --- a/crates/core/src/system/breadcrumb.rs +++ b/crates/core/src/system/breadcrumb.rs @@ -101,6 +101,7 @@ pub struct Breadcrumb<'a> { impl Breadcrumb<'_> { /// Send the breadcrumb to journald. Silently no-ops if journald is /// unavailable (dev runs outside systemd). + // r[impl actuate.breadcrumb] pub fn emit(&self) { // Build the per-target record set. Each target produces one // journal entry with its SEEDLING_RESOURCE / SEEDLING_INSTANCE diff --git a/crates/core/src/system/observer.rs b/crates/core/src/system/observer.rs index 43d129b9..686a2a27 100644 --- a/crates/core/src/system/observer.rs +++ b/crates/core/src/system/observer.rs @@ -82,8 +82,7 @@ impl Observer { match resource { Resource::Deployment(_) | Resource::Job(_) => { - self.observe_pod_instance(instance, resource, now, &mut facts) - .await?; + self.observe_pod_instance(instance, now, &mut facts).await?; } Resource::Volume(vol) => { // r[impl observe.volume] @@ -146,7 +145,6 @@ impl Observer { async fn observe_pod_instance( &self, instance: &ResourceInstance, - _resource: &Resource, now: SystemTime, facts: &mut Vec<(ObservationFact, SystemTime)>, ) -> Result<(), ObserveError> { @@ -244,6 +242,127 @@ impl Observer { }; facts.push((unit_fact, now)); + // r[impl autonomous.restart.record] + // The counter is reported separately from the unit's state because it + // is the only signal that survives a restart the observer never saw: + // a unit that went down and came back inside one observe interval + // looks `active` at both ends, but its counter has moved. + if let Some(s) = unit_state.as_ref() + && let Some(count) = s.restarts + { + facts.push(( + ObservationFact::UnitRestartCounter { + count, + exit: s.last_exit, + }, + now, + )); + } + Ok(()) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + defs::resource::ResourceKind, + system::{ + ContainerRuntime, ProcessManager, + stub::{StubContainerRuntime, StubDataPlane, StubNetworkProxy, StubProcessManager}, + types::{TransientRestart, TransientUnitSpec, UnitExit, UnitExitKind}, + volume_store::VolumeStore, + }, + }; + use seedling_protocol::names::AppName; + + fn unit_spec(name: &str) -> TransientUnitSpec { + TransientUnitSpec { + name: name.to_owned(), + description: String::new(), + exec_start: vec![ + "podman".to_owned(), + "run".to_owned(), + "img:latest".to_owned(), + ], + restart: TransientRestart::Always, + log_extra_fields: vec![], + kill_signal: None, + timeout_stop_secs: None, + restart_sec: None, + start_limit_interval_sec: None, + start_limit_burst: None, + } + } + + /// A stubbed system whose process manager stays reachable, so a test can + /// move the restart counter behind the observer's back — which is exactly + /// what a restart completing between two ticks looks like. + fn stubbed() -> (Arc, Arc, tempfile::TempDir) { + let dir = tempfile::tempdir().expect("tempdir"); + let volumes = dir.path().join("stub-volumes"); + std::fs::create_dir_all(&volumes).expect("volumes dir"); + let container = Arc::new(StubContainerRuntime::new(volumes)); + let process = Arc::new(StubProcessManager::new(Arc::clone(&container))); + let system = Arc::new(System { + container: Arc::clone(&container) as Arc, + process: Arc::clone(&process) as Arc, + proxy: Arc::new(StubNetworkProxy), + data_plane: Arc::new(StubDataPlane), + volume_store: VolumeStore::new(dir.path(), false).expect("volume store"), + degraded: None, + }); + (system, process, dir) + } + + async fn observed_counter( + observer: &Observer, + instance: &ResourceInstance, + ) -> Option<(u32, Option)> { + let mut facts = Vec::new(); + observer + .observe_pod_instance(instance, SystemTime::now(), &mut facts) + .await + .expect("observe"); + facts.into_iter().find_map(|(f, _)| match f { + ObservationFact::UnitRestartCounter { count, exit } => Some((count, exit)), + _ => None, + }) + } + + // r[verify autonomous.restart.record] + #[tokio::test] + async fn observes_the_restart_counter_and_last_exit() { + let (system, process, _dir) = stubbed(); + let instance = ResourceInstance::new_singleton( + AppName::new("myapp").unwrap(), + ResourceKind::Deployment, + "web", + ); + let unit = unit_name(&instance); + process + .start_transient(unit_spec(&unit)) + .await + .expect("start"); + + let observer = Observer::new(Arc::clone(&system)); + assert_eq!( + observed_counter(&observer, &instance).await, + Some((0, None)) + ); + + // The unit restarts twice and is running again by the time the next + // observation lands; only the counter carries the evidence. + let exit = UnitExit { + kind: UnitExitKind::Exited, + code: 1, + }; + process.simulate_restarts(&unit, 2, Some(exit)); + + assert_eq!( + observed_counter(&observer, &instance).await, + Some((2, Some(exit))) + ); + } +} diff --git a/crates/core/src/system/reconcile.rs b/crates/core/src/system/reconcile.rs index 6820f80d..941b0fc4 100644 --- a/crates/core/src/system/reconcile.rs +++ b/crates/core/src/system/reconcile.rs @@ -32,9 +32,12 @@ use crate::{ }, }; +pub use restarts::gc as gc_restart_records; + mod faults; mod images; mod phases; +mod restarts; mod site_proxy; mod state; @@ -307,6 +310,11 @@ pub struct Reconciler { /// Job instance IDs known to have completed during this process lifetime. /// If these appear running on a subsequent tick they are stopped immediately. completed_jobs: HashSet, + /// Instances with an active `crash_loop` fault. Re-derived from the fault + /// table at the top of each tick so the suppression survives a daemon + /// restart and lifts as soon as an operator clears the fault. + // r[impl fault.crash-loop] + crash_looped: HashSet, event_tx: EventSender, shells: Arc, /// Previous tick's lifecycle states, keyed by (app, instance_id_hex). @@ -451,6 +459,7 @@ impl Reconciler { written_obs, started_jobs: HashSet::new(), completed_jobs: HashSet::new(), + crash_looped: HashSet::new(), event_tx, prev_states: BTreeMap::new(), rolling_updates: HashSet::new(), @@ -699,6 +708,27 @@ impl Reconciler { } } + // r[impl fault.crash-loop] + // The auto-restart suppression is in-memory but its truth lives in the + // fault table, so re-derive it each tick: the suppression then + // survives a daemon restart, and an operator clearing the fault takes + // effect on the very next tick. + self.crash_looped = self.db.call(|db| { + let mut out = HashSet::new(); + let Ok(active) = crate::runtime::faults::list_active_faults(db, None) else { + return out; + }; + for f in active { + if f.kind == "crash_loop" + && let Some(hex) = f.instance_id.as_deref() + && let Some(id) = InstanceId::from_hex(hex) + { + out.insert(id); + } + } + out + }); + // r[impl reconciliation.liveness] // --- Concurrent phase: pods ∥ volumes ∥ caddy ∥ resolver --- let (pod_updates, vol_observations, caddy_result, resolver_result) = tokio::join!( @@ -716,6 +746,7 @@ impl Reconciler { &self.written_obs, &self.started_jobs, &self.completed_jobs, + &self.crash_looped, ), phases::run_volumes_phase(&self.observer, &self.actuator, &self.db, &apps), tokio::time::timeout( @@ -1196,11 +1227,16 @@ impl Reconciler { // Rebuild rolling_updates from scratch each tick so that completed // rollouts are automatically cleared. self.rolling_updates.clear(); - for (app_name, pod_update) in pod_updates { + for (app_name, mut pod_update) in pod_updates { // r[fault.image-pull] self.file_image_pull_faults(&app_name, &pod_update); // r[fault.container-start] self.file_unit_failure_faults(&app_name, &pod_update); + // r[autonomous.restart.record] + // r[autonomous.restart.rate] + // Restart bookkeeping runs first: it appends rate-derived crash + // loops to the update so both triggers file through one path. + self.record_restarts(&app_name, &mut pod_update); // r[fault.crash-loop] self.file_crash_loop_faults(&app_name, &pod_update); // r[fault.healthcheck] diff --git a/crates/core/src/system/reconcile/faults.rs b/crates/core/src/system/reconcile/faults.rs index 4fbf5f53..9178f216 100644 --- a/crates/core/src/system/reconcile/faults.rs +++ b/crates/core/src/system/reconcile/faults.rs @@ -1,7 +1,7 @@ use seedling_protocol::names::AppName; use super::{Reconciler, pods, volumes}; -use crate::runtime::{faults, identity::ResourceInstance}; +use crate::runtime::{db::Db, faults, identity::ResourceInstance}; impl Reconciler { /// File a fault scoped to a specific resource instance, if no active fault @@ -371,63 +371,17 @@ impl Reconciler { } // r[impl fault.crash-loop] - /// File a `crash_loop` fault for each instance whose backing systemd unit - /// reached `failed/start-limit-hit`. This is a hard fault: the runtime - /// stops auto-recovering until the operator clears it (typically by - /// fixing config and reinstalling, which generates a new instance ID and - /// therefore a fresh fault scope). + /// File a `crash_loop` fault for each instance the restart bookkeeping or + /// the start-limit observation has flagged. This is a hard fault: the + /// runtime stops auto-recovering until the operator clears it (typically + /// by fixing config and reinstalling, which generates a new instance ID + /// and therefore a fresh fault scope). pub(super) fn file_crash_loop_faults(&self, app: &AppName, update: &pods::PodActuationUpdate) { let app = app.clone(); - let crash_loops: Vec = update.crash_loops.to_vec(); + let crash_loops: Vec = update.crash_loops.to_vec(); let unit_healthy: Vec = update.unit_healthy.to_vec(); - self.db.call(move |db| { - for instance in &crash_loops { - let inst_hex = instance.id.to_hex(); - let kind_str = format!("{:?}", instance.kind).to_lowercase(); - let already_filed = faults::list_active_faults(db, Some(&app)) - .unwrap_or_default() - .iter() - .any(|f| { - f.kind == "crash_loop" && f.instance_id.as_deref() == Some(&inst_hex) - }); - if !already_filed { - let desc = format!( - "systemd hit start-limit for {}: too many restarts in window. \ - Auto-recovery is paused until this fault is cleared.", - instance.display_name - ); - if let Err(e) = faults::file_fault( - db, - &app, - Some(&kind_str), - instance.name.as_deref(), - Some(&inst_hex), - "crash_loop", - &desc, - ) { - tracing::warn!(app = %app, instance = %inst_hex, "failed to file crash_loop fault: {e}"); - } - } - } - // Once the unit is back up healthy, clear any prior crash_loop - // fault — operator clearing the fault and the unit recovering - // are both valid paths out of this state. - for instance in &unit_healthy { - let inst_hex = instance.id.to_hex(); - let cleared: Vec<_> = faults::list_active_faults(db, Some(&app)) - .unwrap_or_default() - .into_iter() - .filter(|f| { - f.kind == "crash_loop" && f.instance_id.as_deref() == Some(&inst_hex) - }) - .collect(); - for f in cleared { - if let Err(e) = faults::clear_fault(db, &f.id, &app) { - tracing::warn!(app = %app, fault_id = %f.id, "failed to clear crash_loop fault: {e}"); - } - } - } - }); + self.db + .call(move |db| apply_crash_loop_faults(db, &app, &crash_loops, &unit_healthy)); } // r[impl fault.external-volume-unmapped] @@ -1130,3 +1084,72 @@ impl Reconciler { } } } + +// r[impl fault.crash-loop] +/// The database side of [`Reconciler::file_crash_loop_faults`], split out so +/// the filing and clearing policy can be exercised against a database without +/// standing up a reconciliation tick. +pub(super) fn apply_crash_loop_faults( + db: &Db, + app: &AppName, + crash_loops: &[pods::CrashLoop], + unit_healthy: &[ResourceInstance], +) { + for crash_loop in crash_loops { + let instance = &crash_loop.instance; + let inst_hex = instance.id.to_hex(); + let kind_str = format!("{:?}", instance.kind).to_lowercase(); + let already_filed = faults::list_active_faults(db, Some(app)) + .unwrap_or_default() + .iter() + .any(|f| f.kind == "crash_loop" && f.instance_id.as_deref() == Some(&inst_hex)); + if !already_filed { + // The description names which trigger fired: a rate-derived crash + // loop and one systemd has already given up on need different + // operator responses. + let desc = match crash_loop.cause { + pods::CrashLoopCause::RestartRate { count, window_secs } => format!( + "{} restarted {count} times in the last {} minutes. \ + Auto-recovery is paused until this fault is cleared.", + instance.display_name, + window_secs / 60, + ), + pods::CrashLoopCause::StartLimitHit => format!( + "systemd hit start-limit for {}: too many restarts in window. \ + Auto-recovery is paused until this fault is cleared.", + instance.display_name + ), + }; + if let Err(e) = faults::file_fault( + db, + app, + Some(&kind_str), + instance.name.as_deref(), + Some(&inst_hex), + "crash_loop", + &desc, + ) { + tracing::warn!(app = %app, instance = %inst_hex, "failed to file crash_loop fault: {e}"); + } + } + } + // Once the unit is back up healthy, clear any prior crash_loop fault — + // operator clearing the fault and the unit recovering are both valid paths + // out of this state. + for instance in unit_healthy { + let inst_hex = instance.id.to_hex(); + let cleared: Vec<_> = faults::list_active_faults(db, Some(app)) + .unwrap_or_default() + .into_iter() + .filter(|f| f.kind == "crash_loop" && f.instance_id.as_deref() == Some(&inst_hex)) + .collect(); + for f in cleared { + if let Err(e) = faults::clear_fault(db, &f.id, app) { + tracing::warn!(app = %app, fault_id = %f.id, "failed to clear crash_loop fault: {e}"); + } + } + } +} + +#[cfg(test)] +mod crash_loop_tests; diff --git a/crates/core/src/system/reconcile/faults/crash_loop_tests.rs b/crates/core/src/system/reconcile/faults/crash_loop_tests.rs new file mode 100644 index 00000000..6db3d55f --- /dev/null +++ b/crates/core/src/system/reconcile/faults/crash_loop_tests.rs @@ -0,0 +1,115 @@ +use super::*; +use crate::{ + defs::resource::ResourceKind, + system::reconcile::pods::{CrashLoop, CrashLoopCause}, +}; + +fn app() -> AppName { + AppName::new("myapp").unwrap() +} + +fn instance() -> ResourceInstance { + ResourceInstance::new_singleton(app(), ResourceKind::Deployment, "web") +} + +fn active(db: &Db) -> Vec { + faults::list_active_faults(db, Some(&app())) + .expect("list") + .into_iter() + .filter(|f| f.kind == "crash_loop") + .collect() +} + +fn rate_loop(inst: &ResourceInstance, count: i64) -> CrashLoop { + CrashLoop { + instance: inst.clone(), + cause: CrashLoopCause::RestartRate { + count, + window_secs: 1800, + }, + } +} + +fn start_limit_loop(inst: &ResourceInstance) -> CrashLoop { + CrashLoop { + instance: inst.clone(), + cause: CrashLoopCause::StartLimitHit, + } +} + +// r[verify fault.crash-loop] +// r[verify autonomous.restart.rate] +#[test] +fn the_rate_trigger_files_the_fault_and_observed_healthy_clears_it() { + let db = Db::open_in_memory().expect("open"); + let inst = instance(); + + apply_crash_loop_faults(&db, &app(), &[rate_loop(&inst, 5)], &[]); + + let filed = active(&db); + assert_eq!(filed.len(), 1); + assert_eq!(filed[0].instance_id.as_deref(), Some(&*inst.id.to_hex())); + assert_eq!(filed[0].resource_type.as_deref(), Some("deployment")); + // The description says which trigger fired, and in the operator's units. + assert!( + filed[0] + .description + .contains("restarted 5 times in the last 30 minutes"), + "{}", + filed[0].description + ); + + // The instance comes back healthy on a later tick. + apply_crash_loop_faults(&db, &app(), &[], std::slice::from_ref(&inst)); + assert!(active(&db).is_empty()); +} + +// r[verify fault.crash-loop] +// r[verify autonomous.restart.start-limit-hit] +#[test] +fn start_limit_hit_files_the_fault_below_the_rate_threshold() { + let db = Db::open_in_memory().expect("open"); + let inst = instance(); + + // No rate-derived loop is present at all: systemd gave up on its own + // accounting before the recorded rate reached the threshold. + apply_crash_loop_faults(&db, &app(), &[start_limit_loop(&inst)], &[]); + + let filed = active(&db); + assert_eq!(filed.len(), 1); + assert!( + filed[0].description.contains("start-limit"), + "{}", + filed[0].description + ); +} + +// r[verify fault.crash-loop] +#[test] +fn a_persisting_crash_loop_is_not_filed_twice() { + let db = Db::open_in_memory().expect("open"); + let inst = instance(); + + apply_crash_loop_faults(&db, &app(), &[rate_loop(&inst, 5)], &[]); + apply_crash_loop_faults(&db, &app(), &[rate_loop(&inst, 6)], &[]); + apply_crash_loop_faults(&db, &app(), &[start_limit_loop(&inst)], &[]); + + assert_eq!(active(&db).len(), 1); +} + +// r[verify fault.crash-loop] +#[test] +fn clearing_is_scoped_to_the_instance_that_recovered() { + let db = Db::open_in_memory().expect("open"); + let one = instance(); + let two = instance(); + + apply_crash_loop_faults(&db, &app(), &[rate_loop(&one, 5), rate_loop(&two, 5)], &[]); + assert_eq!(active(&db).len(), 2); + + apply_crash_loop_faults(&db, &app(), &[], std::slice::from_ref(&one)); + + let remaining = active(&db); + assert_eq!(remaining.len(), 1); + assert_eq!(remaining[0].instance_id.as_deref(), Some(&*two.id.to_hex())); +} diff --git a/crates/core/src/system/reconcile/phases.rs b/crates/core/src/system/reconcile/phases.rs index 125ae94c..7ebe29e0 100644 --- a/crates/core/src/system/reconcile/phases.rs +++ b/crates/core/src/system/reconcile/phases.rs @@ -35,6 +35,7 @@ pub(super) async fn run_pods_phase( written_obs: &HashSet<(InstanceId, &'static str)>, started_jobs: &HashSet, completed_jobs: &HashSet, + crash_looped: &HashSet, ) -> Vec<(AppName, pods::PodActuationUpdate)> { let futures: Vec<_> = apps .iter() @@ -49,6 +50,7 @@ pub(super) async fn run_pods_phase( written_obs, started_jobs, completed_jobs, + crash_looped, ) .await; (app.name.clone(), update) diff --git a/crates/core/src/system/reconcile/pods.rs b/crates/core/src/system/reconcile/pods.rs index 9ed99263..8d2a7ed6 100644 --- a/crates/core/src/system/reconcile/pods.rs +++ b/crates/core/src/system/reconcile/pods.rs @@ -35,9 +35,14 @@ pub(super) struct PodActuationUpdate { pub unit_healthy: Vec, /// Instances whose backing unit reached `failed/start-limit-hit` — /// systemd has given up restarting and the runtime treats this as a - /// hard fault rather than auto-recovering. + /// hard fault rather than auto-recovering. The rate-derived crash loops + /// are added to this list by the restart bookkeeping step. // r[impl autonomous.restart.start-limit-hit] - pub crash_loops: Vec, + pub crash_loops: Vec, + /// The supervisor's restart counter as observed this tick, per instance. + /// Reconciled against the stored baseline to derive restart records. + // r[impl autonomous.restart.record] + pub restart_counters: Vec<(ResourceInstance, RestartCounter)>, /// Instances whose declared healthcheck was observed as failing this tick. pub health_check_failures: Vec, /// Instances whose healthcheck was observed as passing this tick. @@ -64,6 +69,35 @@ pub(super) struct PodActuationUpdate { pub completed_job_instances: Vec, } +/// Why an instance is considered to be crash-looping. +// r[impl fault.crash-loop] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum CrashLoopCause { + /// The recorded restart rate reached the configured threshold. The + /// primary trigger: it catches flapping the supervisor's own start limit + /// never notices. + // r[impl autonomous.restart.rate] + RestartRate { count: i64, window_secs: i64 }, + /// systemd refused to restart the unit any further. Secondary: it fires + /// even when the rate has not reached the threshold. + // r[impl autonomous.restart.start-limit-hit] + StartLimitHit, +} + +#[derive(Debug, Clone)] +pub(super) struct CrashLoop { + pub instance: ResourceInstance, + pub cause: CrashLoopCause, +} + +/// The supervisor's restart counter for an instance's unit, as read this tick. +// r[impl autonomous.restart.record] +#[derive(Debug, Clone, Copy)] +pub(super) struct RestartCounter { + pub count: u32, + pub exit: Option, +} + struct PodInstanceResult { running: Option, observations: Vec<(ResourceInstance, &'static str, serde_json::Value)>, @@ -72,7 +106,9 @@ struct PodInstanceResult { unit_failure: Option, unit_healthy: Option, // r[impl autonomous.restart.start-limit-hit] - crash_loop: Option, + crash_loop: Option, + // r[impl autonomous.restart.record] + restart_counter: Option<(ResourceInstance, RestartCounter)>, health_check_failure: Option, health_check_pass: Option, registry_failure: Option, @@ -126,6 +162,7 @@ async fn observe_one_pod<'a>( unit_failure: None, unit_healthy: None, crash_loop: None, + restart_counter: None, health_check_failure: None, health_check_pass: None, registry_failure: None, @@ -234,6 +271,20 @@ async fn observe_one_pod<'a>( let network_exists = facts .iter() .any(|(f, _)| matches!(f, ObservationFact::NetworkPresent)); + // r[impl autonomous.restart.record] + result.restart_counter = facts.iter().find_map(|(f, _)| { + if let ObservationFact::UnitRestartCounter { count, exit } = f { + Some(( + dr.instance.clone(), + RestartCounter { + count: *count, + exit: *exit, + }, + )) + } else { + None + } + }); // Collect running pods from the pre-actuation observation. // @@ -292,6 +343,10 @@ async fn observe_one_pod<'a>( // r[fault.non-blocking] // r[fault.container-start] // r[impl autonomous.job-terminal] +#[expect( + clippy::too_many_arguments, + reason = "per-instance actuation reads the same shared tick state as the phase" +)] async fn actuate_one_pod( actuator: &Actuator, db: &DbHandle, @@ -300,6 +355,7 @@ async fn actuate_one_pod( written_obs: &HashSet<(InstanceId, &'static str)>, started_jobs: &HashSet, completed_jobs: &HashSet, + crash_looped: &HashSet, ) -> Option { let dr = obs.dr; let result = &mut obs.result; @@ -381,7 +437,10 @@ async fn actuate_one_pod( // Surface as a hard fault separate from the routine // container_start_failed signal: the operator needs to know // that systemd has stopped trying. - result.crash_loop = Some(dr.instance.clone()); + result.crash_loop = Some(CrashLoop { + instance: dr.instance.clone(), + cause: CrashLoopCause::StartLimitHit, + }); } else if obs.unit_failed || (obs.unit_active && !obs.is_running) { result.unit_failure = Some(dr.instance.clone()); } @@ -398,7 +457,13 @@ async fn actuate_one_pod( // Skip the auto-recovery path but allow the desired=Unscheduled branch // below to run so a stuck unit can still be torn down on uninstall / // resource removal. - if obs.unit_start_limit_hit && dr.desired == LifecycleState::Ready { + // r[impl fault.crash-loop] + // The same suppression applies to a crash loop derived from the recorded + // restart rate: an instance under an active crash_loop fault is not + // auto-restarted, whichever trigger filed the fault. + if (obs.unit_start_limit_hit || crash_looped.contains(&dr.instance.id)) + && dr.desired == LifecycleState::Ready + { return Some(obs.result); } @@ -686,6 +751,7 @@ pub(super) async fn observe_and_actuate( written_obs: &HashSet<(InstanceId, &'static str)>, started_jobs: &HashSet, completed_jobs: &HashSet, + crash_looped: &HashSet, ) -> PodActuationUpdate { // Phase 1: observe all instances concurrently. let pod_resources: Vec<&DesiredResource> = desired @@ -759,6 +825,7 @@ pub(super) async fn observe_and_actuate( written_obs, started_jobs, completed_jobs, + crash_looped, )); } @@ -772,6 +839,7 @@ pub(super) async fn observe_and_actuate( unit_failures: Vec::new(), unit_healthy: Vec::new(), crash_loops: Vec::new(), + restart_counters: Vec::new(), health_check_failures: Vec::new(), health_check_passes: Vec::new(), registry_failures: Vec::new(), @@ -804,6 +872,10 @@ pub(super) async fn observe_and_actuate( if let Some(c) = result.crash_loop { update.crash_loops.push(c); } + // r[impl autonomous.restart.record] + if let Some(rc) = result.restart_counter { + update.restart_counters.push(rc); + } if let Some(f) = result.health_check_failure { update.health_check_failures.push(f); } diff --git a/crates/core/src/system/reconcile/restarts.rs b/crates/core/src/system/reconcile/restarts.rs new file mode 100644 index 00000000..9ccf1b07 --- /dev/null +++ b/crates/core/src/system/reconcile/restarts.rs @@ -0,0 +1,244 @@ +//! Turning the supervisor's restart counter into seedling's restart records, +//! and the records into a crash-loop verdict. +//! +//! The counter is monotonic while a unit lives and zero on a fresh one, so the +//! reconciler keeps a baseline per instance and records the difference. That +//! is what makes recording independent of the observe interval: a container +//! that goes down and comes back between two ticks moves the counter even +//! though it was never seen down. + +use jiff::Timestamp; +use seedling_protocol::names::AppName; +use tracing::warn; + +use crate::{ + runtime::{ + db::Db, + generations, + identity::ResourceInstance, + restarts::{ + self, Cause, ExitKind, ExitStatus, RETAIN_PER_INSTANCE, RestartSettings, RestartSubject, + }, + }, + system::types::{UnitExit, UnitExitKind}, +}; + +use super::pods::{self, CrashLoop, CrashLoopCause}; + +fn subject(instance: &ResourceInstance, generation: Option) -> RestartSubject { + RestartSubject { + app: instance.app.clone(), + instance_id: instance.id.to_hex(), + resource_type: Some(format!("{:?}", instance.kind).to_lowercase()), + resource_name: instance.name.clone(), + generation, + } +} + +fn exit_status(exit: UnitExit) -> ExitStatus { + ExitStatus { + kind: match exit.kind { + UnitExitKind::Exited => ExitKind::Exited, + UnitExitKind::Signalled => ExitKind::Signalled, + UnitExitKind::Dumped => ExitKind::Dumped, + }, + code: exit.code, + } +} + +/// Reconcile one app's observed restart counters against the stored +/// baselines, record what moved, and return the instances whose rate has +/// reached the threshold. +/// +/// Split from the `Reconciler` method so the counter arithmetic — deltas, +/// resets, and the runtime-initiated exclusion — can be exercised directly +/// against a database without standing up a reconciliation tick. +// r[impl autonomous.restart.record] +// r[impl autonomous.restart.rate] +pub(super) fn reconcile_counters( + db: &Db, + app: &AppName, + counters: &[(ResourceInstance, pods::RestartCounter)], + started: &[ResourceInstance], +) -> Vec { + let settings = restarts::settings(db).unwrap_or(RestartSettings { + threshold: 5, + window_secs: 1800, + }); + let generation = generations::current(db, app) + .ok() + .flatten() + .map(|g| g as i64); + + // r[impl autonomous.restart.record] + // A start the reconciler issued for an instance it has already run is a + // deliberate restart. The fresh transient unit's counter begins at zero, so + // re-baseline here rather than reading the drop as a reset next tick. + for instance in started { + let hex = instance.id.to_hex(); + match restarts::baseline(db, &hex) { + Ok(Some(_)) => { + if let Err(e) = restarts::record( + db, + &subject(instance, generation), + Cause::Deliberate, + None, + Timestamp::now().as_millisecond(), + ) { + warn!(app = %app, instance = %hex, "failed to record deliberate restart: {e}"); + } + } + // No baseline: the reconciler has never seen this instance's unit, + // so this is a first start, not a restart. + Ok(None) => continue, + Err(e) => { + warn!(app = %app, instance = %hex, "failed to read restart baseline: {e}"); + continue; + } + } + if let Err(e) = restarts::set_baseline(db, &hex, 0) { + warn!(app = %app, instance = %hex, "failed to re-baseline restart counter: {e}"); + } + } + + let mut rate_loops = Vec::new(); + for (instance, counter) in counters { + let hex = instance.id.to_hex(); + if started.iter().any(|s| s.id == instance.id) { + // The counter was read before this tick's actuation; the start + // above already re-baselined it. + continue; + } + let observed = i64::from(counter.count); + let previous = match restarts::baseline(db, &hex) { + Ok(v) => v, + Err(e) => { + warn!(app = %app, instance = %hex, "failed to read restart baseline: {e}"); + continue; + } + }; + + let new_restarts = match previous { + // First sighting of this unit. Adopt the counter as the baseline + // without recording: the restarts it already holds happened at + // times seedling cannot know, and inventing timestamps for them + // would corrupt the rate. + None => 0, + // The counter went backwards, so it was reset — the unit was + // recreated or its failed state cleared. Whatever it holds now + // accrued after the reset. + Some(prev) if observed < prev => observed, + Some(prev) => observed - prev, + }; + + if previous != Some(observed) + && let Err(e) = restarts::set_baseline(db, &hex, observed) + { + warn!(app = %app, instance = %hex, "failed to store restart baseline: {e}"); + } + + if new_restarts <= 0 { + continue; + } + + // Only the most recent run's exit is known, so it goes on the last of + // the batch; earlier ones are recorded without one. + let now = Timestamp::now().as_millisecond(); + let subject = subject(instance, generation); + for n in 0..new_restarts { + let last = n == new_restarts - 1; + let exit = if last { + counter.exit.map(exit_status) + } else { + None + }; + // Stamp a burst apart so its ordering survives. + let at = now - (new_restarts - 1 - n); + if let Err(e) = restarts::record(db, &subject, Cause::Recovery, exit, at) { + warn!(app = %app, instance = %hex, "failed to record restart: {e}"); + } + } + + // r[impl autonomous.restart.rate] + match restarts::recent_recovery_count(db, &hex, settings.window_secs) { + Ok(count) if count >= settings.threshold => rate_loops.push(CrashLoop { + instance: instance.clone(), + cause: CrashLoopCause::RestartRate { + count, + window_secs: settings.window_secs, + }, + }), + Ok(_) => {} + Err(e) => { + warn!(app = %app, instance = %hex, "failed to count recent restarts: {e}"); + } + } + } + rate_loops +} + +impl super::Reconciler { + // r[impl autonomous.restart.record] + // r[impl autonomous.restart.rate] + /// Run the restart bookkeeping for one app's pod update, appending any + /// rate-derived crash loops to the update's list. + /// + /// Runs before the fault filing step so that a rate-derived crash loop and + /// a start-limit-hit one are filed through the same path. + pub(super) fn record_restarts(&self, app: &AppName, update: &mut pods::PodActuationUpdate) { + let app_owned = app.clone(); + let counters = update.restart_counters.clone(); + let started: Vec = update.started_instances.to_vec(); + let rate_loops = self + .db + .call(move |db| reconcile_counters(db, &app_owned, &counters, &started)); + + for loop_ in rate_loops { + // A unit that also hit the start limit is already listed; one + // crash_loop fault per instance is what the operator needs. + if update + .crash_loops + .iter() + .any(|c| c.instance.id == loop_.instance.id) + { + continue; + } + update.crash_loops.push(loop_); + } + } +} + +// r[impl gc.restarts] +/// Drop restart bookkeeping for instances that no longer exist, and re-apply +/// the per-instance cap. Recording already enforces the cap on write; this +/// catches rows left by an older build or a partial write. +pub fn gc(db: &Db) -> rusqlite::Result { + let orphaned_records = db.conn.execute( + "DELETE FROM instance_restarts + WHERE instance_id NOT IN (SELECT id FROM resource_instances)", + [], + )?; + let orphaned_counters = db.conn.execute( + "DELETE FROM instance_restart_counters + WHERE instance_id NOT IN (SELECT id FROM resource_instances)", + [], + )?; + + let over_cap: Vec = { + let mut stmt = db.conn.prepare( + "SELECT instance_id FROM instance_restarts + GROUP BY instance_id HAVING COUNT(*) > ?1", + )?; + let rows = stmt.query_map([RETAIN_PER_INSTANCE as i64], |r| r.get(0))?; + rows.collect::>()? + }; + let mut pruned = 0; + for instance_id in over_cap { + pruned += restarts::prune_instance(db, &instance_id, RETAIN_PER_INSTANCE)?; + } + + Ok(orphaned_records + orphaned_counters + pruned) +} + +#[cfg(test)] +mod tests; diff --git a/crates/core/src/system/reconcile/restarts/tests.rs b/crates/core/src/system/reconcile/restarts/tests.rs new file mode 100644 index 00000000..e0028d2a --- /dev/null +++ b/crates/core/src/system/reconcile/restarts/tests.rs @@ -0,0 +1,286 @@ +use super::*; +use crate::{ + defs::resource::ResourceKind, + runtime::restarts::{self, Cause}, + system::types::{UnitExit, UnitExitKind}, +}; + +fn app() -> AppName { + AppName::new("myapp").unwrap() +} + +fn instance() -> ResourceInstance { + ResourceInstance::new_singleton(app(), ResourceKind::Deployment, "web") +} + +fn counter(count: u32, exit: Option) -> pods::RestartCounter { + pods::RestartCounter { count, exit } +} + +/// One observe tick: the counter the supervisor reports for `inst`. +fn tick(db: &Db, inst: &ResourceInstance, count: u32, exit: Option) -> Vec { + reconcile_counters(db, &app(), &[(inst.clone(), counter(count, exit))], &[]) +} + +fn records(db: &Db, inst: &ResourceInstance) -> Vec { + restarts::list(db, None, Some(&inst.id.to_hex()), 1000).expect("list") +} + +// r[verify autonomous.restart.record] +#[test] +fn first_sighting_baselines_without_recording() { + let db = Db::open_in_memory().expect("open"); + let inst = instance(); + + // The daemon has just come up against a unit that has already restarted + // three times. Those happened at times seedling cannot know, so they are + // adopted as the baseline rather than invented into the history. + tick(&db, &inst, 3, None); + assert!(records(&db, &inst).is_empty()); + assert_eq!( + restarts::baseline(&db, &inst.id.to_hex()).expect("baseline"), + Some(3) + ); +} + +// r[verify autonomous.restart.record] +#[test] +fn counter_delta_across_a_restart_is_recorded_with_its_exit() { + let db = Db::open_in_memory().expect("open"); + let inst = instance(); + tick(&db, &inst, 0, None); + + // The container restarted and came back between two ticks: the unit looks + // active at both ends and only the counter moved. + tick( + &db, + &inst, + 1, + Some(UnitExit { + kind: UnitExitKind::Exited, + code: 137, + }), + ); + + let rows = records(&db, &inst); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].cause, Cause::Recovery); + assert_eq!(rows[0].exit_code, Some(137)); + assert_eq!(rows[0].exit_kind, Some(restarts::ExitKind::Exited)); + assert_eq!(rows[0].resource_name.as_deref(), Some("web")); +} + +// r[verify autonomous.restart.record] +#[test] +fn a_burst_between_ticks_records_every_restart() { + let db = Db::open_in_memory().expect("open"); + let inst = instance(); + tick(&db, &inst, 0, None); + tick( + &db, + &inst, + 4, + Some(UnitExit { + kind: UnitExitKind::Signalled, + code: 9, + }), + ); + + let rows = records(&db, &inst); + assert_eq!(rows.len(), 4); + // Only the most recent run's exit is known. + assert_eq!(rows[0].exit_code, Some(9)); + assert_eq!(rows[0].exit_kind, Some(restarts::ExitKind::Signalled)); + assert!(rows[1..].iter().all(|r| r.exit_code.is_none())); +} + +// r[verify autonomous.restart.record] +#[test] +fn an_unmoved_counter_records_nothing() { + let db = Db::open_in_memory().expect("open"); + let inst = instance(); + tick(&db, &inst, 0, None); + tick(&db, &inst, 2, None); + tick(&db, &inst, 2, None); + tick(&db, &inst, 2, None); + + assert_eq!(records(&db, &inst).len(), 2); +} + +// r[verify autonomous.restart.record] +#[test] +fn a_counter_reset_rebaselines_instead_of_recording_a_negative() { + let db = Db::open_in_memory().expect("open"); + let inst = instance(); + tick(&db, &inst, 0, None); + tick(&db, &inst, 5, None); + assert_eq!(records(&db, &inst).len(), 5); + + // `systemctl reset-failed` (or a recreated unit) zeroes the counter. The + // drop is a reset, not five restarts unhappening. + tick(&db, &inst, 0, None); + assert_eq!(records(&db, &inst).len(), 5); + assert_eq!( + restarts::baseline(&db, &inst.id.to_hex()).expect("baseline"), + Some(0) + ); + + // Counting resumes from the new baseline. + tick(&db, &inst, 1, None); + assert_eq!(records(&db, &inst).len(), 6); +} + +// r[verify autonomous.restart.record] +#[test] +fn restarts_after_a_reset_are_recorded_not_dropped() { + let db = Db::open_in_memory().expect("open"); + let inst = instance(); + tick(&db, &inst, 0, None); + tick(&db, &inst, 5, None); + + // The reset and two further restarts both land between the same pair of + // ticks, so the counter comes back lower than the baseline but non-zero. + // Those two restarts really happened. + tick(&db, &inst, 2, None); + assert_eq!(records(&db, &inst).len(), 7); +} + +// r[verify autonomous.restart.record] +#[test] +fn a_runtime_start_is_recorded_as_deliberate_and_rebaselines() { + let db = Db::open_in_memory().expect("open"); + let inst = instance(); + tick(&db, &inst, 0, None); + tick(&db, &inst, 3, None); + + // The reconciler tore the unit down and started a fresh one. systemd's + // counter for the new transient unit begins at zero. + reconcile_counters(&db, &app(), &[], std::slice::from_ref(&inst)); + + let rows = records(&db, &inst); + assert_eq!(rows.len(), 4); + assert_eq!(rows[0].cause, Cause::Deliberate); + assert_eq!( + restarts::baseline(&db, &inst.id.to_hex()).expect("baseline"), + Some(0) + ); + + // The zeroed counter on the next tick is the expected state, not a reset + // to record against. + tick(&db, &inst, 0, None); + assert_eq!(records(&db, &inst).len(), 4); +} + +// r[verify autonomous.restart.record] +#[test] +fn a_first_start_is_not_a_restart() { + let db = Db::open_in_memory().expect("open"); + let inst = instance(); + + // No baseline yet: the reconciler has never seen this instance's unit. + reconcile_counters(&db, &app(), &[], std::slice::from_ref(&inst)); + assert!(records(&db, &inst).is_empty()); +} + +// r[verify autonomous.restart.rate] +#[test] +fn crossing_the_rate_threshold_reports_a_crash_loop() { + let db = Db::open_in_memory().expect("open"); + let inst = instance(); + tick(&db, &inst, 0, None); + + for n in 1..5 { + assert!( + tick(&db, &inst, n, None).is_empty(), + "{n} restarts is below the default threshold of 5" + ); + } + + let loops = tick(&db, &inst, 5, None); + assert_eq!(loops.len(), 1); + assert_eq!(loops[0].instance.id, inst.id); + assert_eq!( + loops[0].cause, + CrashLoopCause::RestartRate { + count: 5, + window_secs: 1800 + } + ); +} + +// r[verify autonomous.restart.rate] +// r[verify autonomous.restart.rate.settings] +#[test] +fn the_threshold_follows_the_operator_setting() { + let db = Db::open_in_memory().expect("open"); + let inst = instance(); + restarts::set_settings(&db, Some(3), None).expect("set"); + tick(&db, &inst, 0, None); + + assert!(tick(&db, &inst, 2, None).is_empty()); + let loops = tick(&db, &inst, 3, None); + assert_eq!(loops.len(), 1); +} + +// r[verify autonomous.restart.rate] +#[test] +fn a_rolling_update_does_not_read_as_a_crash_burst() { + let db = Db::open_in_memory().expect("open"); + let inst = instance(); + tick(&db, &inst, 0, None); + + // Ten reconciler-driven restarts in a row — well past the threshold if + // they counted. They are recorded, but not against the rate. + for _ in 0..10 { + reconcile_counters(&db, &app(), &[], std::slice::from_ref(&inst)); + assert!(tick(&db, &inst, 0, None).is_empty()); + } + + assert_eq!(records(&db, &inst).len(), 10); + assert_eq!( + restarts::recent_recovery_count(&db, &inst.id.to_hex(), 1800).expect("count"), + 0 + ); +} + +// r[verify autonomous.restart.rate] +#[test] +fn instances_are_accounted_for_independently() { + let db = Db::open_in_memory().expect("open"); + let one = instance(); + let two = instance(); + let counters = |a: u32, b: u32| { + vec![ + (one.clone(), counter(a, None)), + (two.clone(), counter(b, None)), + ] + }; + + reconcile_counters(&db, &app(), &counters(0, 0), &[]); + let loops = reconcile_counters(&db, &app(), &counters(6, 1), &[]); + + assert_eq!(loops.len(), 1); + assert_eq!(loops[0].instance.id, one.id); + assert_eq!(records(&db, &one).len(), 6); + assert_eq!(records(&db, &two).len(), 1); +} + +// r[verify gc.restarts] +#[test] +fn gc_drops_bookkeeping_for_instances_that_no_longer_exist() { + let db = Db::open_in_memory().expect("open"); + let inst = instance(); + tick(&db, &inst, 0, None); + tick(&db, &inst, 2, None); + assert_eq!(records(&db, &inst).len(), 2); + + // The instance was never written to the registry, so from GC's point of + // view it has been retired. + let removed = gc(&db).expect("gc"); + assert!(removed >= 2); + assert!(records(&db, &inst).is_empty()); + assert_eq!( + restarts::baseline(&db, &inst.id.to_hex()).expect("baseline"), + None + ); +} diff --git a/crates/core/src/system/stub.rs b/crates/core/src/system/stub.rs index fb5f906b..eae7b0be 100644 --- a/crates/core/src/system/stub.rs +++ b/crates/core/src/system/stub.rs @@ -23,7 +23,8 @@ use super::{ BoxError, BoxFuture, ContainerFilter, ContainerHealth, ContainerRuntime, ContainerSpec, ContainerState, ContainerStatus, ContainerSummary, DataPlane, DataPlaneRules, ExecHandle, ImageSummary, NetworkProxy, NetworkSummary, ProcessManager, ProxyConfig, ServiceRoute, - TransientUnitSpec, UnitState, UnitSummary, types::ActiveState, + TransientUnitSpec, UnitState, UnitSummary, + types::{ActiveState, UnitExit}, }; /// Stub `ContainerRuntime`. Pretends every started container is healthy and @@ -437,6 +438,10 @@ struct UnitState_ { struct UnitRecord { state: ActiveState, sub: String, + /// Mirrors systemd's `NRestarts`: monotonic while the unit lives, reset + /// when the unit is recreated. Driven directly by tests. + restarts: u32, + last_exit: Option, } impl StubProcessManager { @@ -446,6 +451,25 @@ impl StubProcessManager { container, } } + + /// Simulate the supervisor restarting a unit `times` times, the last run + /// having ended with `exit`. The container keeps running throughout, as it + /// does on a real host when the restart completes between two observe + /// ticks — the only trace is the counter. + pub fn simulate_restarts(&self, unit: &str, times: u32, exit: Option) { + if let Some(u) = self.state.lock().units.get_mut(unit) { + u.restarts += times; + u.last_exit = exit; + } + } + + /// Set the counter directly, for the reset case: systemd zeroes + /// `NRestarts` when a unit is recreated or its failed state is cleared. + pub fn set_restart_counter(&self, unit: &str, count: u32) { + if let Some(u) = self.state.lock().units.get_mut(unit) { + u.restarts = count; + } + } } impl ProcessManager for StubProcessManager { @@ -514,11 +538,15 @@ impl ProcessManager for StubProcessManager { ); } + // A fresh transient unit starts its restart counter at zero, + // exactly as systemd does. self.state.lock().units.insert( spec.name.clone(), UnitRecord { state: ActiveState::Active, sub: "running".to_owned(), + restarts: 0, + last_exit: None, }, ); drop(spec); @@ -551,6 +579,9 @@ impl ProcessManager for StubProcessManager { { u.state = ActiveState::Inactive; u.sub = "dead".to_owned(); + // `systemctl reset-failed` zeroes NRestarts along with the + // failed state. + u.restarts = 0; } Ok(()) } @@ -565,6 +596,8 @@ impl ProcessManager for StubProcessManager { Ok(self.state.lock().units.get(name).map(|u| UnitState { active: u.state, sub: u.sub.clone(), + restarts: Some(u.restarts), + last_exit: u.last_exit, })) } .boxed() @@ -586,6 +619,7 @@ impl ProcessManager for StubProcessManager { state: UnitState { active: u.state, sub: u.sub.clone(), + ..Default::default() }, }) .collect()) @@ -626,6 +660,8 @@ impl ProcessManager for StubProcessManager { s.units.entry(name.to_owned()).or_insert(UnitRecord { state: ActiveState::Active, sub: "running".to_owned(), + restarts: 0, + last_exit: None, }); Ok(()) } diff --git a/crates/core/src/system/systemd.rs b/crates/core/src/system/systemd.rs index b993c0ff..d85cb586 100644 --- a/crates/core/src/system/systemd.rs +++ b/crates/core/src/system/systemd.rs @@ -8,7 +8,10 @@ use zbus::{ use crate::system::{ BoxError, BoxFuture, ProcessManager, - types::{ActiveState, TransientRestart, TransientUnitSpec, UnitState, UnitSummary}, + types::{ + ActiveState, TransientRestart, TransientUnitSpec, UnitExit, UnitExitKind, UnitState, + UnitSummary, + }, }; const UNIT_DIR: &str = "/etc/systemd/system"; @@ -171,6 +174,49 @@ trait Systemd1Unit { fn sub_state(&self) -> zbus::Result; } +// --------------------------------------------------------------------------- +// D-Bus proxy — systemd Service interface (restart accounting) +// --------------------------------------------------------------------------- + +/// Restart accounting lives on the `Service` interface, not the `Unit` +/// interface above, so it needs its own proxy against the same object path. +// r[impl autonomous.restart.record] +#[zbus::proxy( + interface = "org.freedesktop.systemd1.Service", + default_service = "org.freedesktop.systemd1" +)] +trait Systemd1Service { + #[zbus(property, name = "NRestarts")] + fn n_restarts(&self) -> zbus::Result; + + /// The main process's exit status, or the signal number that killed it, + /// depending on `ExecMainCode`. + #[zbus(property, name = "ExecMainStatus")] + fn exec_main_status(&self) -> zbus::Result; + + /// A `siginfo_t` `si_code`: `CLD_EXITED` (1), `CLD_KILLED` (2) or + /// `CLD_DUMPED` (3). Zero while the main process has not exited. + #[zbus(property, name = "ExecMainCode")] + fn exec_main_code(&self) -> zbus::Result; +} + +/// `si_code` values reported by systemd for a unit's main process. +const CLD_EXITED: i32 = 1; +const CLD_KILLED: i32 = 2; +const CLD_DUMPED: i32 = 3; + +fn parse_exec_main(code: i32, status: i32) -> Option { + let kind = match code { + CLD_EXITED => UnitExitKind::Exited, + CLD_KILLED => UnitExitKind::Signalled, + CLD_DUMPED => UnitExitKind::Dumped, + // Zero means the main process has not exited; anything else is a + // si_code systemd does not use for this property. + _ => return None, + }; + Some(UnitExit { kind, code: status }) +} + // --------------------------------------------------------------------------- // SystemdManager // --------------------------------------------------------------------------- @@ -383,7 +429,7 @@ impl SystemdManager { let unit_proxy = Systemd1UnitProxy::builder(&self.conn) .destination("org.freedesktop.systemd1") .context(DBusSnafu)? - .path(unit_path) + .path(unit_path.clone()) .context(DBusSnafu)? .build() .await @@ -392,12 +438,52 @@ impl SystemdManager { let active = unit_proxy.active_state().await.context(DBusSnafu)?; let sub = unit_proxy.sub_state().await.context(DBusSnafu)?; + // r[impl autonomous.restart.record] + // Two extra property reads on a path already fetching ActiveState and + // SubState per instance. Failures are not fatal: the Service interface + // only exists on .service units, and a unit that vanished between the + // GetUnit call and here should still report the state we did read. + let (restarts, last_exit) = match self.service_accounting(&unit_path).await { + Ok(v) => v, + Err(e) => { + tracing::debug!(unit = %name, error = %e, "systemd: no restart accounting for unit"); + (None, None) + } + }; + Ok(Some(UnitState { active: parse_active_state(&active), sub, + restarts, + last_exit, })) } + /// Read `NRestarts` and the last exit from the `Service` interface. + async fn service_accounting( + &self, + unit_path: &OwnedObjectPath, + ) -> Result<(Option, Option), SystemdError> { + let proxy = Systemd1ServiceProxy::builder(&self.conn) + .destination("org.freedesktop.systemd1") + .context(DBusSnafu)? + .path(unit_path.clone()) + .context(DBusSnafu)? + .build() + .await + .context(DBusSnafu)?; + + let restarts = proxy.n_restarts().await.context(DBusSnafu)?; + // The exit properties are best-effort on top of the counter: a unit + // whose main process has not exited reports ExecMainCode 0, which + // parse_exec_main turns into None. + let last_exit = match (proxy.exec_main_code().await, proxy.exec_main_status().await) { + (Ok(code), Ok(status)) => parse_exec_main(code, status), + _ => None, + }; + Ok((Some(restarts), last_exit)) + } + async fn list_units_impl(&self, prefix: &str) -> Result, SystemdError> { let proxy = Systemd1ManagerProxy::new(&self.conn) .await @@ -413,6 +499,9 @@ impl SystemdManager { state: UnitState { active: parse_active_state(&u.active_state), sub: u.sub_state, + // ListUnits carries no per-unit properties; callers that + // need restart accounting go through unit_state. + ..Default::default() }, }) .collect(); @@ -653,7 +742,39 @@ impl ProcessManager for SystemdManager { #[cfg(test)] mod tests { - use super::validate_unit_name; + use super::{UnitExit, UnitExitKind, parse_exec_main, validate_unit_name}; + + // r[verify autonomous.restart.record] + #[test] + fn exec_main_maps_si_codes_to_exit_kinds() { + assert_eq!( + parse_exec_main(1, 137), + Some(UnitExit { + kind: UnitExitKind::Exited, + code: 137 + }) + ); + assert_eq!( + parse_exec_main(2, 9), + Some(UnitExit { + kind: UnitExitKind::Signalled, + code: 9 + }) + ); + assert_eq!( + parse_exec_main(3, 11), + Some(UnitExit { + kind: UnitExitKind::Dumped, + code: 11 + }) + ); + } + + // r[verify autonomous.restart.record] + #[test] + fn exec_main_is_absent_while_the_main_process_lives() { + assert_eq!(parse_exec_main(0, 0), None); + } #[test] fn accepts_valid_service() { diff --git a/crates/core/src/system/types.rs b/crates/core/src/system/types.rs index 6e3c2c77..f901264a 100644 --- a/crates/core/src/system/types.rs +++ b/crates/core/src/system/types.rs @@ -249,10 +249,39 @@ pub enum TransientRestart { // --------------------------------------------------------------------------- /// `unit_state` returns `None` when the unit does not exist or is masked. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Default)] pub struct UnitState { pub active: ActiveState, pub sub: String, + /// The supervisor's own count of how many times it has restarted this + /// unit. Monotonic while the unit lives, but reset when the unit is + /// recreated or its failed state is cleared, so a decrease is a reset + /// rather than a negative delta. + /// + /// `None` when the supervisor does not report one (a non-service unit, or + /// a listing that does not carry per-unit properties). + // r[impl autonomous.restart.record] + pub restarts: Option, + /// How the unit's main process last exited, where the supervisor reports + /// it. This is what makes a restart record diagnostic rather than a tally. + // r[impl autonomous.restart.record] + pub last_exit: Option, +} + +/// The exit status of a unit's main process on its most recent run. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct UnitExit { + pub kind: UnitExitKind, + /// The exit status for [`UnitExitKind::Exited`], the signal number + /// otherwise. + pub code: i32, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum UnitExitKind { + Exited, + Signalled, + Dumped, } #[derive(Debug, Clone)] @@ -261,11 +290,12 @@ pub struct UnitSummary { pub state: UnitState, } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub enum ActiveState { Active, Activating, Deactivating, + #[default] Inactive, Failed, } @@ -504,6 +534,16 @@ pub enum ObservationFact { UnitStartLimitHit, /// The unit is not loaded by systemd at all (unit_state returned None). UnitGone, + /// The supervisor's restart counter for this unit, plus how the main + /// process last exited. The reconciler diffs the counter against the + /// baseline it recorded on the previous tick, so a restart that completed + /// between two observations is still counted — polling container state + /// alone would miss it entirely. + // r[impl autonomous.restart.record] + UnitRestartCounter { + count: u32, + exit: Option, + }, // Proxy ProxyReachable, @@ -572,7 +612,10 @@ impl ObservationFact { | ObservationFact::RouteAbsent { .. } | ObservationFact::UnitActive | ObservationFact::UnitInactive - | ObservationFact::UnitGone => vec![], + | ObservationFact::UnitGone + // Restart counters are reconciled against a stored baseline and + // written to instance_restarts, not to the observation oracle. + | ObservationFact::UnitRestartCounter { .. } => vec![], } } } diff --git a/crates/ctl/src/main.rs b/crates/ctl/src/main.rs index 3126c4f3..11524228 100644 --- a/crates/ctl/src/main.rs +++ b/crates/ctl/src/main.rs @@ -17,6 +17,7 @@ mod ingresses; mod known_hosts; mod logs; mod op; +mod restarts; mod services; mod shell; mod subscribe; @@ -152,6 +153,11 @@ enum Command { }, /// Clear all active faults for an app ClearFaults { app: String }, + /// Container restart history and the crash-loop rate derived from it + Restarts { + #[command(subcommand)] + command: restarts::RestartsCommand, + }, /// Subscribe to event feed (streams JSON to stdout) Events, /// Client info (fingerprint) @@ -413,6 +419,7 @@ async fn main() { .await, ); } + Command::Restarts { command } => restarts::dispatch(&client, command).await, Command::Events => { op::dispatch_events( endpoint_addr, diff --git a/crates/ctl/src/restarts.rs b/crates/ctl/src/restarts.rs new file mode 100644 index 00000000..7158410a --- /dev/null +++ b/crates/ctl/src/restarts.rs @@ -0,0 +1,81 @@ +//! Container restart history and the crash-loop rate that is derived from it. +//! +//! The rate — not systemd's own start limit — is what files a `crash_loop` +//! fault, so the threshold and window are operator business and live here +//! rather than being an implementation constant. + +use clap::Subcommand; +use seedling_protocol::client::OiClient; +use serde_json::json; + +use super::print_result; + +#[derive(Subcommand)] +pub(super) enum RestartsCommand { + /// List recorded container restarts, most recent first + List { + /// Only restarts for this app + #[arg(long)] + app: Option, + /// Only restarts for this instance id + #[arg(long)] + instance: Option, + /// Maximum records to return (default 100, max 1000) + #[arg(long)] + limit: Option, + }, + /// Show the crash-loop rate threshold and window + Settings, + /// Change the crash-loop rate threshold and/or window. + /// + /// A `crash_loop` fault is filed once an instance records this many + /// recovery restarts inside the window. Restarts seedling performs + /// deliberately (rolling updates, replacements) do not count. + SetSettings { + /// Restarts within the window that file the fault (minimum 2) + #[arg(long)] + threshold: Option, + /// Width of the window in seconds (minimum 60) + #[arg(long)] + window_secs: Option, + }, +} + +pub(super) async fn dispatch(client: &OiClient, cmd: RestartsCommand) { + match cmd { + RestartsCommand::List { + app, + instance, + limit, + } => { + print_result( + client + .request( + "/restarts/list", + json!({ "app": app, "instance": instance, "limit": limit }), + ) + .await, + ); + } + RestartsCommand::Settings => { + print_result(client.request("/restarts/settings/get", json!({})).await); + } + RestartsCommand::SetSettings { + threshold, + window_secs, + } => { + if threshold.is_none() && window_secs.is_none() { + eprintln!("error: pass at least one of --threshold or --window-secs"); + std::process::exit(1); + } + print_result( + client + .request( + "/restarts/settings/set", + json!({ "threshold": threshold, "window_secs": window_secs }), + ) + .await, + ); + } + } +} diff --git a/crates/web/frontend/src/App.tsx b/crates/web/frontend/src/App.tsx index ef75e02e..4bbd1a18 100644 --- a/crates/web/frontend/src/App.tsx +++ b/crates/web/frontend/src/App.tsx @@ -19,6 +19,7 @@ import Keys from "./routes/Keys"; import Login from "./routes/Login"; import Logs from "./routes/Logs"; import Registries from "./routes/Registries"; +import Restarts from "./routes/Restarts"; import Services from "./routes/Services"; import Shell from "./routes/Shell"; import TemplateDetail from "./routes/TemplateDetail"; @@ -39,6 +40,7 @@ const router = createBrowserRouter([ { path: "templates/:name", element: }, { path: "templates/:name/edit", element: }, { path: "faults", element: }, + { path: "restarts", element: }, { path: "volumes", element: }, { path: "services", element: }, { path: "ingresses", element: }, diff --git a/crates/web/frontend/src/components/Navbar.tsx b/crates/web/frontend/src/components/Navbar.tsx index 1146c011..9d73dae8 100644 --- a/crates/web/frontend/src/components/Navbar.tsx +++ b/crates/web/frontend/src/components/Navbar.tsx @@ -9,6 +9,7 @@ import InventoryIcon from "@mui/icons-material/Inventory2"; import KeyIcon from "@mui/icons-material/Key"; import ParkIcon from "@mui/icons-material/Park"; import PeopleAltIcon from "@mui/icons-material/PeopleAlt"; +import RestartAltIcon from "@mui/icons-material/RestartAlt"; import StorageIcon from "@mui/icons-material/Storage"; import { AppBar, Badge, Box, Chip, IconButton, Toolbar, Tooltip, Typography } from "@mui/material"; import { useCallback, useEffect, useMemo } from "react"; @@ -288,6 +289,16 @@ export function Navbar() { + + + + + {faultCount > 0 && ( diff --git a/crates/web/frontend/src/hooks/useOi.ts b/crates/web/frontend/src/hooks/useOi.ts index fe0c2bd6..38258b83 100644 --- a/crates/web/frontend/src/hooks/useOi.ts +++ b/crates/web/frontend/src/hooks/useOi.ts @@ -98,6 +98,12 @@ export function useOiQuery( ): OiQueryState { const { session } = useContext(SessionContext); const cacheMs = options?.cacheMs ?? 0; + // Params are compared by value, not by identity: callers pass a fresh object + // literal on every render, so keying the effect on the object itself would + // refetch forever. Keying on the serialisation means a view that narrows its + // query (a filter selection, say) refetches, while a constant param object + // does not. + const paramsKey = stableStringify(params); const key = cacheMs > 0 ? cacheKey(method, params) : null; // Seed state from cache synchronously so reopening a cached view doesn't @@ -167,7 +173,7 @@ export function useOiQuery( cancelled = true; }; // eslint-disable-next-line react-hooks/exhaustive-deps - }, [session, method, key, force]); + }, [session, method, key, paramsKey, force]); return { data, loading, error, refetch, cachedAt }; } diff --git a/crates/web/frontend/src/lib/types.ts b/crates/web/frontend/src/lib/types.ts index c8669006..b46274d0 100644 --- a/crates/web/frontend/src/lib/types.ts +++ b/crates/web/frontend/src/lib/types.ts @@ -33,6 +33,38 @@ export interface ResourceInstance { display_name: string; lifecycle: string; transition_time?: string; + restarts?: RestartSummary | null; +} + +export type RestartCause = "recovery" | "deliberate"; + +export type RestartExitKind = "exited" | "signalled" | "dumped"; + +export interface RestartRecord { + id: number; + app: string; + instance_id: string; + resource_type?: string | null; + resource_name?: string | null; + generation?: number | null; + timestamp: string; + cause: RestartCause; + exit_code?: number | null; + exit_kind?: RestartExitKind | null; +} + +export interface RestartSettings { + threshold: number; + window_secs: number; +} + +export interface RestartSummary { + recent: number; + window_secs: number; + total: number; + last_at?: string | null; + last_exit_code?: number | null; + last_exit_kind?: RestartExitKind | null; } export interface ScaleBounds { diff --git a/crates/web/frontend/src/routes/AppDetail.resources.test.tsx b/crates/web/frontend/src/routes/AppDetail.resources.test.tsx index d6fa98d6..71d58e76 100644 --- a/crates/web/frontend/src/routes/AppDetail.resources.test.tsx +++ b/crates/web/frontend/src/routes/AppDetail.resources.test.tsx @@ -105,6 +105,46 @@ describe("AppDetail params", () => { }); }); +describe("AppDetail restarts", () => { + // w[verify routes.restarts] + it("shows the instance's restart count and links to its history", async () => { + mount( + baseFixtures( + makeDetail({ + resources: [ + makeWebDeployment({ + instances: [ + { + id: "inst-web-0", + display_name: "web-0", + lifecycle: "ready", + restarts: { + recent: 3, + window_secs: 1800, + total: 7, + last_at: "2026-07-09T10:00:00Z", + last_exit_code: 137, + last_exit_kind: "exited", + }, + }, + ], + }), + ], + }), + ), + ); + const chip = await screen.findByRole("link", { name: /3/ }); + expect(chip.getAttribute("href")).toBe("/restarts?instance=inst-web-0"); + }); + + // w[verify routes.restarts] + it("shows no restart chip for an instance with no history", async () => { + mount(baseFixtures(makeDetail())); + expect(await screen.findByText("web-0")).toBeTruthy(); + expect(screen.queryByRole("link", { name: /restarts/ })).toBeNull(); + }); +}); + describe("AppDetail resources", () => { // w[verify routes.apps] it("scales a deployment up and down", async () => { diff --git a/crates/web/frontend/src/routes/AppDetail.tsx b/crates/web/frontend/src/routes/AppDetail.tsx index 1bed92fc..ca7eee1a 100644 --- a/crates/web/frontend/src/routes/AppDetail.tsx +++ b/crates/web/frontend/src/routes/AppDetail.tsx @@ -10,6 +10,7 @@ import PauseIcon from "@mui/icons-material/Pause"; import PlayArrowIcon from "@mui/icons-material/PlayArrow"; import RefreshIcon from "@mui/icons-material/Refresh"; import RemoveIcon from "@mui/icons-material/Remove"; +import RestartAltIcon from "@mui/icons-material/RestartAlt"; import RestoreIcon from "@mui/icons-material/Restore"; import TerminalIcon from "@mui/icons-material/Terminal"; import VisibilityIcon from "@mui/icons-material/Visibility"; @@ -89,10 +90,45 @@ import type { ImageSummary, InstallRequirement, ResourceDef, + RestartSummary, SeedlingEvent, SiteVolume, } from "../lib/types"; +/** Restart count for an instance, linking through to its full history. The + * count shown is the one the crash-loop rate is measured against, so it goes + * warning-coloured as soon as any recovery restart lands in the window. */ +// w[impl routes.restarts] +function RestartIndicator({ + instanceId, + restarts, +}: { + instanceId: string; + restarts: RestartSummary; +}) { + const title = + `${restarts.recent} recovery restart${restarts.recent === 1 ? "" : "s"} ` + + `in the last ${restarts.window_secs / 60} minutes · ` + + `${restarts.total} recorded in total` + + (restarts.last_at + ? ` · last ${new Date(restarts.last_at).toLocaleString()}` + : ""); + return ( + + } + label={restarts.recent} + size="small" + variant="outlined" + color={restarts.recent > 0 ? "warning" : "default"} + component={Link} + to={`/restarts?instance=${instanceId}`} + clickable + /> + + ); +} + function lifecycleColor( state: string, ): "success" | "warning" | "error" | "default" { @@ -727,6 +763,12 @@ function ResourcesSection({ faults={r.faults} /> )} + {inst.restarts && ( + + )} { + it("renders the empty state", async () => { + renderWithSession(, { + fixtures: { "/restarts/list": [], "/restarts/settings/get": settings }, + }); + expect(await screen.findByText("No restarts recorded.")).toBeTruthy(); + }); + + // w[verify routes.restarts] + it("lists records with their exit status and app link", async () => { + renderWithSession(, { + fixtures: { + "/restarts/list": [recovery, deliberate], + "/restarts/settings/get": settings, + }, + }); + const link = await screen.findAllByRole("link", { name: "shop" }); + expect(link[0].getAttribute("href")).toBe("/apps/shop"); + expect(screen.getAllByText("deployment/web").length).toBe(2); + expect(screen.getByText("exit 137")).toBeTruthy(); + // An unrecorded exit says so rather than showing a fabricated code. + expect(screen.getByText("unknown")).toBeTruthy(); + }); + + // w[verify routes.restarts] + it("distinguishes deliberate restarts from recovery ones", async () => { + renderWithSession(, { + fixtures: { + "/restarts/list": [recovery, deliberate], + "/restarts/settings/get": settings, + }, + }); + expect(await screen.findByText("recovery")).toBeTruthy(); + expect(screen.getByText("deliberate")).toBeTruthy(); + }); + + // w[verify routes.restarts] + it("shows the crash-loop threshold and window", async () => { + renderWithSession(, { + fixtures: { "/restarts/list": [], "/restarts/settings/get": settings }, + }); + expect( + await screen.findByText( + /5 recovery restarts within 30 minutes/, + ), + ).toBeTruthy(); + }); + + // w[verify routes.restarts] + it("sends the window in seconds when the operator saves it in minutes", async () => { + const setSettings = vi.fn(() => settings); + renderWithSession(, { + safetyMode: "write", + fixtures: { + "/restarts/list": [], + "/restarts/settings/get": settings, + "/restarts/settings/set": setSettings, + }, + }); + + fireEvent.change(await screen.findByLabelText("Threshold"), { + target: { value: "3" }, + }); + fireEvent.change(screen.getByLabelText("Window"), { + target: { value: "10" }, + }); + fireEvent.click(screen.getByRole("button", { name: "Save" })); + + await waitFor(() => + expect(setSettings).toHaveBeenCalledWith({ + threshold: 3, + window_secs: 600, + }), + ); + }); + + // w[verify routes.restarts] + it("filters by app", async () => { + const list = vi.fn(() => []); + renderWithSession(, { + fixtures: { + "/restarts/list": list, + "/restarts/settings/get": settings, + "/apps/list": [{ name: "shop", status: "running" }], + }, + }); + + fireEvent.mouseDown(await screen.findByRole("combobox", { name: "App" })); + const options = await screen.findByRole("listbox"); + fireEvent.click(within(options).getByRole("option", { name: "shop" })); + + await waitFor(() => expect(list).toHaveBeenCalledWith({ app: "shop" })); + }); + + it("shows an error alert when the query fails", async () => { + renderWithSession(, { + fixtures: { + "/restarts/list": { + ok: false, + error: { code: "internal", message: "db exploded" }, + }, + "/restarts/settings/get": settings, + }, + }); + expect(await screen.findByText(/db exploded/)).toBeTruthy(); + }); +}); diff --git a/crates/web/frontend/src/routes/Restarts.tsx b/crates/web/frontend/src/routes/Restarts.tsx new file mode 100644 index 00000000..4fcefe93 --- /dev/null +++ b/crates/web/frontend/src/routes/Restarts.tsx @@ -0,0 +1,248 @@ +import RefreshIcon from "@mui/icons-material/Refresh"; +import { + Box, + Chip, + CircularProgress, + MenuItem, + Paper, + Stack, + Table, + TableBody, + TableCell, + TableHead, + TableRow, + TextField, + Typography, +} from "@mui/material"; +import { useMemo, useState } from "react"; +import { Link, useSearchParams } from "react-router-dom"; +import { + IconActionButton, + SolidActionButton, +} from "../components/ActionButton"; +import { OiErrorAlert } from "../components/OiErrorAlert"; +import { useOiQuery } from "../hooks/useOi"; +import { useOiAction } from "../hooks/useOiAction"; +import type { AppSummary, RestartRecord, RestartSettings } from "../lib/types"; + +/** How the previous run ended, in the terms an operator thinks in. */ +function exitLabel(r: RestartRecord): string { + if (r.exit_code === null || r.exit_code === undefined) return "unknown"; + switch (r.exit_kind) { + case "signalled": + return `signal ${r.exit_code}`; + case "dumped": + return `signal ${r.exit_code} (core dumped)`; + default: + return `exit ${r.exit_code}`; + } +} + +// w[impl routes.restarts] +export default function Restarts() { + const [app, setApp] = useState(""); + // An instance filter only ever arrives by link — from the restart chip on an + // app's resource table — so it lives in the URL rather than in a control. + const [search, setSearch] = useSearchParams(); + const instance = search.get("instance") ?? ""; + const params = useMemo( + () => ({ + ...(app ? { app } : {}), + ...(instance ? { instance } : {}), + }), + [app, instance], + ); + + const { data, loading, error, refetch } = useOiQuery( + "/restarts/list", + params, + ); + const { data: apps } = useOiQuery("/apps/list", {}); + const { + data: settings, + error: settingsError, + refetch: refetchSettings, + } = useOiQuery("/restarts/settings/get", {}); + const { + execute, + loading: mutating, + error: mutateError, + } = useOiAction(); + + const [threshold, setThreshold] = useState(""); + const [windowMins, setWindowMins] = useState(""); + + const saveSettings = async () => { + const body: Record = {}; + if (threshold !== "") body.threshold = Number(threshold); + if (windowMins !== "") body.window_secs = Number(windowMins) * 60; + if (Object.keys(body).length === 0) return; + if ((await execute("/restarts/settings/set", body)) === null) return; + setThreshold(""); + setWindowMins(""); + refetchSettings(); + }; + + return ( + + + + Restarts + + + + + + + Every container restart Seedling observes or performs. Recovery from an + unexpected exit counts towards the crash-loop rate; restarts Seedling + performed deliberately — rolling updates, replacements — are recorded + but do not, so a rollout never reads as a crash burst. + + + {settingsError && } + {mutateError && } + + + Crash-loop rate + + {settings + ? `A crash_loop fault is filed once an instance records ${settings.threshold} recovery restarts within ${settings.window_secs / 60} minutes.` + : "Loading…"} + + + setThreshold(e.target.value)} + placeholder={settings ? String(settings.threshold) : ""} + helperText="restarts (min 2)" + sx={{ width: 160 }} + /> + setWindowMins(e.target.value)} + placeholder={settings ? String(settings.window_secs / 60) : ""} + helperText="minutes (min 1)" + sx={{ width: 160 }} + /> + + Save + + + + + + setApp(e.target.value)} + sx={{ minWidth: 220 }} + > + All apps + {(apps ?? []).map((a) => ( + + {a.name} + + ))} + + {instance && ( + setSearch({})} + sx={{ fontFamily: "monospace" }} + /> + )} + + + {error && } + {loading && !data && ( + + + + )} + {data && data.length === 0 && ( + + No restarts recorded. + + )} + {data && data.length > 0 && ( + + + + When + App + Resource + Instance + Cause + Exit + Gen + + + + {data.map((r) => ( + + + {new Date(r.timestamp).toLocaleString()} + + + + {r.app} + + + + {r.resource_name + ? `${r.resource_type}/${r.resource_name}` + : (r.resource_type ?? "—")} + + + {r.instance_id.slice(0, 12)} + + + {/* The distinction is the whole point of recording the + cause: only recovery rows move the rate. */} + + + + {exitLabel(r)} + + {r.generation ?? "—"} + + ))} + +
+ )} + + ); +} diff --git a/docs/runtime-overview.md b/docs/runtime-overview.md index ba24426c..bd8174a6 100644 --- a/docs/runtime-overview.md +++ b/docs/runtime-overview.md @@ -22,9 +22,9 @@ Steps 1–3 update the runtime's model of the world. Step 4 advances scripted orchestration. Steps 5–8 change the world. -## Three Histories +## Four Histories -The runtime maintains three distinct categories of persistent records: +The runtime maintains four distinct categories of persistent records: ### World Observation History @@ -45,12 +45,27 @@ Examples: - A container exited and `OnTerminate=Recreate`, so a replacement was started. - Scale requires 2 replicas but only 1 was observed running, so another was started. - Caddy became unreachable, so its entire configuration was rebuilt. -- A container has crash-looped 5 times in 60 seconds, so the runtime is backing off. +- A crash-looping container reached the start limit, so the runtime stopped auto-recovering it. This log enables: - **Auditability**: operators can review what the runtime did autonomously. - **Rate limiting and backoff**: the runtime can detect repeated failures and avoid tight restart loops. -- **Fault detection**: patterns like crash-looping or persistent convergence failures are derived from this log, and result in faults filed for external intervention. +- **Fault detection**: persistent convergence failures are derived from this log, and result in faults filed for external intervention. + +### Restart History + +A record of every container restart, one row per attempt: which instance, when, the exit status of the run that ended where the platform reports one, and the cause — recovery from an unexpected exit, or a restart the runtime performed deliberately. + +The cause records why, not who. On Linux systemd actions recovery restarts and seedling actions deliberate ones, so the two coincide; on a platform with no service supervisor seedling actions both, and a field naming the actor would classify every restart there identically. + +Restarts cannot be counted by watching container state. A container that goes down and comes back between two observation ticks looks running at both ends. On Linux the runtime instead reads systemd's own restart counter and records the difference, so what is recorded does not depend on how often the runtime looks. + +This log enables: +- **Crash-loop detection**: a `crash_loop` fault is filed once an instance's recovery restarts within the configured window reach the configured threshold. That threshold and window are operator-settable, because the judgement of what counts as flapping is an operational one. +- **Seeing sub-threshold flapping**: a container that crashes twice a day forever never exhausts systemd's own start limit, so before this history existed it was silent — no fault, no record, nothing to query. +- **Diagnosis**: the per-attempt exit statuses say whether a workload is being OOM-killed, exiting on a config error, or dying on a signal. + +Deliberate restarts — rolling updates, health-check replacements — are recorded but excluded from the rate, so a rollout never reads as a crash burst. Records are bounded per instance rather than globally: a hard crash loop produces rows fastest exactly when they are most wanted. ### Action Execution Log @@ -149,7 +164,7 @@ They are surfaced to human or agentic operators through the operator interface ( Examples of faults: - A barrier deadline expires: the action closure expected a resource to reach a state within N seconds, and it didn't. -- Crash-looping: a container repeatedly exits shortly after starting, and backoff has been exhausted. +- Crash-looping: an instance's recorded restart rate reached its threshold, or the supervisor gave up restarting it. - Permanent divergence: a resource cannot be created (e.g. image doesn't exist, port is occupied by an external process). ## Resource Identity diff --git a/docs/spec/interface.md b/docs/spec/interface.md index e1b5424d..33ef63e1 100644 --- a/docs/spec/interface.md +++ b/docs/spec/interface.md @@ -250,7 +250,8 @@ Absent specification bugs, anything that is not defined here is either defined i > - `status`: the app's current status as defined in [app.status](#i--app.status). > - `faults`: array of app-level [fault records](#i--fault.record) not associated with a specific resource instance (e.g. script evaluation errors). Empty when there are no active app-level faults. > - `resources`: array of objects with fields `name`, `type`, `instances`, `faults`, `def`, and for Deployment resources, `scale`. -> Each instance has fields `id`, `display_name`, `lifecycle`, and `transition_time` (RFC 3339, optional). +> Each instance has fields `id`, `display_name`, `lifecycle`, `transition_time` (RFC 3339, optional), and `restarts`. +> `restarts` summarises the instance's [restart history](#i--restart.record): `{ recent, window_secs, total, last_at, last_exit_code, last_exit_kind }`, where `recent` counts recovery restarts within the current rate window, `total` counts all retained records for the instance, and the `last_*` fields describe the most recent record (null when there is none). It is omitted for resource kinds that have no backing container. > Each fault entry is a [fault record](#i--fault.record). > `def` is an object describing the resource's configuration. The shape varies by `type`: > for `ingress`: `{ hostname, port, tls, dtls, http_terminate, redirect }`; @@ -696,6 +697,21 @@ Absent specification bugs, anything that is not defined here is either defined i > Faults derived from observable conditions (e.g. `image_pull_failed`, `health_check_failed`) will be re-filed on the next reconciliation tick if the underlying condition still holds. Hard faults that require operator action (e.g. `health_check_replace_failed`, `script_error`) are cleared definitively until the underlying issue recurs. > The endpoint is intended for operators stuck behind a fault that the runtime cannot itself resolve, including the case where a not-installed app's faults are blocking a script update. +# Restart Surface + +> i[restart.record] +> A restart record contains the following fields: `id` (monotonically increasing integer), `app`, `instance_id`, `resource_type`, `resource_name`, `generation` (integer, null when the app had no current generation at the time), `timestamp` (RFC 3339), `cause` (`"recovery"` or `"deliberate"`), `exit_code` (integer, null when unknown), and `exit_kind` (`"exited"`, `"signalled"`, `"dumped"`, or null when unknown). +> `cause` distinguishes recovery from an unexpected exit from a restart the runtime performed on purpose. It describes why the restart happened, not who performed it: which component actions a restart is a platform detail, and on a platform with no service supervisor the runtime performs both kinds. +> For `exit_kind: "exited"`, `exit_code` is the process's exit status; for `"signalled"` and `"dumped"` it is the signal number that terminated it. + +> i[restart.list] +> `/restarts/list { app?, instance?, limit? }` returns an array of [restart records](#i--restart.record), most recent first. +> `app` restricts the result to one app and `instance` to one instance id; both may be given. `limit` caps the number of records returned, defaulting to 100 and capped at 1000. + +> i[restart.settings] +> `/restarts/settings/get` returns `{ threshold, window_secs }` — the number of recovery restarts within `window_secs` seconds that files a `crash_loop` fault (see [autonomous.restart.rate](runtime.md#r--autonomous.restart.rate)). +> `/restarts/settings/set { threshold?, window_secs? }` updates either or both and returns the full settings object. Omitted fields are left unchanged. `threshold` must be at least 2 and `window_secs` at least 60; values outside those bounds are rejected. + # Event Feed > i[event.subscribe] diff --git a/docs/spec/runtime.md b/docs/spec/runtime.md index c136ec0b..51912101 100644 --- a/docs/spec/runtime.md +++ b/docs/spec/runtime.md @@ -377,6 +377,11 @@ Absent specification bugs, anything that is not defined here is either defined i > r[gc.autonomous-operations] > Completed autonomous operation records must be deleted after a configurable retention period (default: 7 days). +> r[gc.restarts] +> [Restart records](#r--autonomous.restart.record) must be bounded per instance: only a configurable number of the most recent records for an instance are retained (default: 50), and older records for that instance are deleted. +> The bound is per instance rather than global, and applies to storage rather than to recording: a crash loop produces records fastest exactly when the per-attempt exit statuses are the diagnostic, so no restart may go unrecorded merely because the instance is restarting quickly. +> Restart records whose instance identity no longer appears in the resource instance registry must be deleted. + > r[gc.instances] > Resource instance records that have remained in the Unscheduled lifecycle state for longer than a configurable retention period (default: 10 minutes) must be deleted, along with their associated world observation rows. > Instances that are part of the active desired state (i.e. in the `keep` set of a scaled group or a singleton) must never be deleted regardless of their lifecycle state. @@ -687,18 +692,37 @@ Some internal operations (for example [backup.list](#r--backup.list), [backup.re > r[autonomous.restart] > When a container resource in the desired state reaches the Terminated lifecycle state and its `on_exit` or `on_terminate` policy requires recreation, the reconciler must start a replacement. +> r[autonomous.restart.record] +> The runtime must keep a durable, per-instance record of container restarts. Each record identifies the instance it belongs to, the app generation in force when it was recorded, when the restart happened, the exit status of the run that ended where the platform reports one, and the restart's cause: whether it was recovery from an unexpected exit, or a restart the runtime performed deliberately. +> +> The cause is a statement about why the restart happened, not about which component performed it. Who actions a restart is a platform detail — a platform with a service supervisor leaves recovery to it, and one without leaves the runtime to perform both kinds — so recording the actor would make the record mean different things on different platforms. +> +> Recording must not depend on catching a state transition. A container that restarts and returns to running between two observations must still be recorded, so the count of restarts the runtime holds does not depend on how often it looks. +> +> Rolling updates, [replacements](#r--autonomous.healthcheck-replace) and operator-requested restarts are recorded as deliberate and excluded from the crash-loop rate. Otherwise every rolling update reads as a crash burst. + +> r[autonomous.restart.rate] +> Crash-loop detection is a function of the recorded restart rate: when the number of recovery restarts recorded for an instance within the configured window reaches the configured threshold, the reconciler must file a `crash_loop` fault against that instance (see [fault.crash-loop](#r--fault.crash-loop)). +> +> This is the primary crash-loop trigger. It catches sub-threshold flapping — a container that crashes a few times a day forever, never exhausting the supervisor's own start limit inside its window — which is otherwise invisible to an operator. + +> r[autonomous.restart.rate.settings] +> The restart-rate threshold and window must be operator-visible and operator-settable, and must take effect without restarting the runtime. +> +> The default must be loose enough that a slow-failing container (one that takes seconds to crash) gets several chances, and tight enough to catch flapping on a human-meaningful timescale. + > r[autonomous.restart.backoff] -> Per-unit restarts must be paced so that a crash-looping container does not exhaust systemd's start-rate limit before the reconciler has a chance to detect the problem. Container units must specify: +> Where the platform's supervisor actions restarts, per-unit restarts must be paced so that a crash-looping container does not exhaust the supervisor's start-rate limit before the reconciler has a chance to detect the problem. On Linux, container units must specify: > > - A non-default `RestartSec` (no shorter than several seconds) so the unit does not retry at the systemd default cadence (~100ms). > - A `StartLimitIntervalSec` and `StartLimitBurst` that allow several attempts within a window measured in minutes, not seconds, before systemd gives up. > -> The exact values are an implementation concern — they need only be loose enough that a slow-failing container (one that takes seconds to crash) gets multiple chances, and tight enough that a permanently broken container reaches the start limit on a human-meaningful timescale. +> The exact values are an implementation concern — they need only be loose enough that a slow-failing container gets multiple chances, and tight enough that a permanently broken container reaches the start limit on a human-meaningful timescale. These are pacing parameters, not the definition of a crash loop; that is [autonomous.restart.rate](#r--autonomous.restart.rate). > r[autonomous.restart.start-limit-hit] > When a container unit reaches `failed/start-limit-hit` (systemd has refused further restarts because the unit exhausted [`StartLimitBurst`](#r--autonomous.restart.backoff)) the reconciler must: > -> - File a `crash_loop` fault scoped to the offending instance, distinct from `container_start_failed`. +> - File a `crash_loop` fault scoped to the offending instance, distinct from `container_start_failed`. This is a secondary trigger: a unit the supervisor has given up on must produce the fault even when the recorded rate has not reached its threshold. > - Stop attempting to auto-recover the instance (no `reset_failed_unit` + restart cycle) until the fault is cleared. The expected recovery path is operator intervention — fixing the underlying cause, redeploying with new config, or explicitly clearing the fault. > - Clear the fault automatically if the instance is later observed healthy. @@ -822,6 +846,23 @@ Some internal operations (for example [backup.list](#r--backup.list), [backup.re > journal field that identifies the infrastructure component so that log queries can > target infrastructure logs independently of workload logs. +> r[actuate.breadcrumb] +> The runtime must record its own action breadcrumbs into the same log sink that +> carries container output, tagged with the same app, resource kind, resource, and +> instance fields. A breadcrumb names the `rt.*` primitive it records — or a synthetic +> kind for runtime events such as unit creation and replay boundaries — and, where the +> script surfaced one, the call site. +> +> Sharing the sink and the tagging scheme is the requirement, not an implementation +> convenience: a log query at any granularity must return breadcrumbs and container +> output interleaved in time order, so that an operator reading an app's logs sees the +> closure's call sequence against the output it produced. + +> r[actuate.breadcrumb.replay] +> A breadcrumb is emitted on a call's first fresh execution and not on replays of that +> call, so a barrier-suspended operation does not flood the log with each pass. Each +> replay pass instead surfaces a single boundary breadcrumb. + > r[actuate.ingress.warm-certs] > When an action closure invokes [`rt.warm_certs`](#l--rt.warm-certs) with a selection that contains TLS-terminating ingresses, the runtime must initiate certificate acquisition for those ingresses' hostnames without exposing the ingresses to live traffic. > A typical implementation pushes a partial proxy configuration that requests certificate acquisition while not routing requests to any backend; once the certificate is `valid`, it is served from the proxy's cache when the same ingress is later started for real. @@ -1085,7 +1126,12 @@ Some internal operations (for example [backup.list](#r--backup.list), [backup.re > The fault is cleared automatically when the unit is subsequently observed in an active or activating state. > r[fault.crash-loop] -> When the reconciler observes that a resource instance's backing unit has reached the start-limit-hit terminal state (per [autonomous.restart.start-limit-hit](#r--autonomous.restart.start-limit-hit)), it must file a fault of kind `crash_loop` associated with that instance, distinct from `container_start_failed`. +> The reconciler must file a fault of kind `crash_loop` associated with a resource instance, distinct from `container_start_failed`, when either: +> +> - the instance's recorded restart rate reaches the configured threshold (per [autonomous.restart.rate](#r--autonomous.restart.rate)), or +> - its backing unit has reached the start-limit-hit terminal state (per [autonomous.restart.start-limit-hit](#r--autonomous.restart.start-limit-hit)). +> +> The fault must identify which of the two conditions filed it, so that an operator can tell a rate-derived crash loop from one the supervisor has already given up on. > The fault is cleared automatically when the instance is subsequently observed healthy. While the fault is active, the reconciler must not auto-restart the affected instance. > r[fault.external-volume-unmapped] diff --git a/docs/spec/web.md b/docs/spec/web.md index 99e2b3c7..26a703af 100644 --- a/docs/spec/web.md +++ b/docs/spec/web.md @@ -240,6 +240,11 @@ Absent specification bugs, anything not defined here is either defined in anothe > w[routes.volumes.held-count] > The navbar's held-volumes badge must reflect the current count of held volumes without requiring a page reload, both when new held volumes are created and when the operator confirms their deletion. +> w[routes.restarts] +> The web interface must expose container restart history at `/restarts`, listing [restart records](interface.md#i--restart.record) most recent first with their app, instance, time, initiator, and exit status. +> The list must be filterable by app, and deliberate restarts must be visually distinguishable from recovery ones, since only recovery restarts count towards the crash-loop rate. +> The route must also present the crash-loop rate threshold and window, and allow an operator to change them. + > w[routes.certificates] > The web interface must expose TLS certificate management at `/certificates`, with the following sections: >