feat: restart accounting on the Linux runtime

crates/core/src/oi/handler.rs

@@ -21,6 +21,7 @@ mod ingresses;
 mod key_mgmt;
 mod params;
 mod registries;
+mod restarts;
 mod services;
 mod status;
 mod templates;
@@ -196,6 +197,12 @@ fn parse_and_dispatch(state: &Arc<OiState>, buf: &[u8], ctx: &RequestCtx) -> Han
         "/faults/list" => faults::list_faults(state, parse_params(req.params)?),
         // i[fault.clear-app]
         "/faults/clear" => faults::clear_app_faults(state, parse_params(req.params)?),
+        // i[restart.list]
+        "/restarts/list" => restarts::list_restarts(state, parse_params(req.params)?),
+        // i[restart.settings]
+        "/restarts/settings/get" => restarts::get_settings(state),
+        // i[restart.settings]
+        "/restarts/settings/set" => restarts::set_settings(state, parse_params(req.params)?),
         // i[canopy.status]
         "/canopy/status" => canopy::status(state),
         // i[canopy.settings]

crates/core/src/oi/handler/apps.rs

@@ -20,7 +20,7 @@ use crate::{
         history::{find_instances_for_group, query_observations},
         identity::{InstanceId, InstanceVariant, ResourceInstance},
         lifecycle::LifecycleState,
-        restart_gens, scaling,
+        restart_gens, restarts, scaling,
         stopped::{self, kind_str, parse_kind},
         transition_phase,
     },
@@ -651,6 +651,10 @@ pub(crate) fn describe_app(state: &OiState, params: AppParams) -> HandlerResult
     let all_faults_clone = all_faults_for_app.clone();
     let stopped_set_clone = stopped_set.clone();
     let resources_json: Vec<Value> = state.db.call(move |db| {
+        // i[impl app.describe]
+        // Read the rate settings once per describe: every instance summary
+        // reports its recent count over the same window.
+        let restart_settings = restarts::settings(db).ok();
         resource_infos
             .into_iter()
             .map(|info| {
@@ -662,13 +666,22 @@ pub(crate) fn describe_app(state: &OiState, params: AppParams) -> HandlerResult
                             let observations = query_observations(db, inst).unwrap_or_default();
                             let (lifecycle, transition_time) =
                                 derive_state_with_transition_time(inst, &observations);
+                            // i[impl app.describe]
+                            // Omitted entirely for an instance with no
+                            // restart history, so a resource that has never
+                            // restarted does not read as one that restarted
+                            // zero times just now.
+                            let restart_summary = restart_settings.and_then(|s| {
+                                restarts::summary(db, &inst.id.to_hex(), s).ok().flatten()
+                            });
                             json!({
                                 "id": inst.id.to_hex(),
                                 "display_name": inst.display_name,
                                 "lifecycle": format!("{lifecycle:?}"),
                                 "transition_time": transition_time.and_then(|t| {
                                     jiff::Timestamp::try_from(t).ok().map(|ts| ts.to_string())
                                 }),
+                                "restarts": restart_summary,
                             })
                         })
                         .collect()

crates/core/src/oi/handler/restarts.rs

@@ -0,0 +1,87 @@
+use seedling_protocol::error::{ErrorCode, OiError};
+use seedling_protocol::names::AppName;
+use serde::Deserialize;
+use serde_json::json;
+
+use super::HandlerResult;
+use crate::{oi::state::OiState, runtime::restarts};
+
+/// Records returned when the caller does not ask for a specific number, and
+/// the ceiling on what it may ask for. The cap keeps a single request from
+/// pulling the whole retained history of a busy host over the wire.
+const DEFAULT_LIMIT: usize = 100;
+const MAX_LIMIT: usize = 1000;
+
+#[derive(Deserialize)]
+pub(crate) struct ListRestartsParams {
+    pub app: Option<AppName>,
+    pub instance: Option<String>,
+    pub limit: Option<usize>,
+}
+
+// i[impl restart.list]
+pub(crate) fn list_restarts(state: &OiState, params: ListRestartsParams) -> HandlerResult {
+    let ListRestartsParams {
+        app,
+        instance,
+        limit,
+    } = params;
+    let limit = limit.unwrap_or(DEFAULT_LIMIT).clamp(1, MAX_LIMIT);
+    let records = state
+        .db
+        .call(move |db| restarts::list(db, app.as_ref(), instance.as_deref(), limit))
+        .map_err(|e| OiError::new(ErrorCode::NotFound, format!("db query: {e}")))?;
+
+    // i[impl restart.record]
+    let result: Vec<serde_json::Value> = records
+        .into_iter()
+        .map(|r| {
+            json!({
+                "id": r.id,
+                "app": r.app,
+                "instance_id": r.instance_id,
+                "resource_type": r.resource_type,
+                "resource_name": r.resource_name,
+                "generation": r.generation,
+                "timestamp": r.timestamp.to_string(),
+                "cause": r.cause,
+                "exit_code": r.exit_code,
+                "exit_kind": r.exit_kind,
+            })
+        })
+        .collect();
+    Ok(json!(result))
+}
+
+// i[impl restart.settings]
+pub(crate) fn get_settings(state: &OiState) -> HandlerResult {
+    let s = state
+        .db
+        .call(restarts::settings)
+        .map_err(|e| OiError::new(ErrorCode::NotFound, format!("db query: {e}")))?;
+    Ok(json!({ "threshold": s.threshold, "window_secs": s.window_secs }))
+}
+
+#[derive(Deserialize)]
+pub(crate) struct SetSettingsParams {
+    #[serde(default)]
+    pub threshold: Option<i64>,
+    #[serde(default)]
+    pub window_secs: Option<i64>,
+}
+
+// i[impl restart.settings]
+pub(crate) fn set_settings(state: &OiState, params: SetSettingsParams) -> HandlerResult {
+    let SetSettingsParams {
+        threshold,
+        window_secs,
+    } = params;
+    let s = state
+        .db
+        .call(move |db| restarts::set_settings(db, threshold, window_secs))
+        .map_err(|e| OiError::new(ErrorCode::RequirementsInvalid, e.to_string()))?;
+    Ok(json!({ "threshold": s.threshold, "window_secs": s.window_secs }))
+}
+
+#[cfg(test)]
+mod tests;

crates/core/src/oi/handler/restarts/tests.rs

@@ -0,0 +1,129 @@
+use serde_json::json;
+
+use crate::{
+    oi::test_support::TestOi,
+    runtime::restarts::{Cause, ExitKind, ExitStatus, RestartSubject, record},
+};
+use seedling_protocol::names::AppName;
+
+fn seed(oi: &TestOi, app: &str, instance: &str, cause: Cause, exit: Option<ExitStatus>) {
+    let subject = RestartSubject {
+        app: AppName::new(app).unwrap(),
+        instance_id: instance.to_owned(),
+        resource_type: Some("deployment".to_owned()),
+        resource_name: Some("web".to_owned()),
+        generation: Some(1),
+    };
+    let at = jiff::Timestamp::now().as_millisecond();
+    oi.state
+        .db
+        .call(move |db| record(db, &subject, cause, exit, at))
+        .expect("record");
+}
+
+// i[verify restart.list]
+// i[verify restart.record]
+#[test]
+fn list_returns_records_with_their_exit_and_cause() {
+    let oi = TestOi::new();
+    seed(
+        &oi,
+        "demo",
+        "aa",
+        Cause::Recovery,
+        Some(ExitStatus {
+            kind: ExitKind::Signalled,
+            code: 9,
+        }),
+    );
+
+    let rows = oi.call("/restarts/list", json!({})).unwrap();
+    let rows = rows.as_array().unwrap();
+    assert_eq!(rows.len(), 1);
+    assert_eq!(rows[0]["app"], "demo");
+    assert_eq!(rows[0]["instance_id"], "aa");
+    assert_eq!(rows[0]["resource_type"], "deployment");
+    assert_eq!(rows[0]["resource_name"], "web");
+    assert_eq!(rows[0]["generation"], 1);
+    assert_eq!(rows[0]["cause"], "recovery");
+    assert_eq!(rows[0]["exit_code"], 9);
+    assert_eq!(rows[0]["exit_kind"], "signalled");
+    assert!(!rows[0]["timestamp"].as_str().unwrap().is_empty());
+}
+
+// i[verify restart.record]
+#[test]
+fn an_unknown_exit_is_reported_as_null_rather_than_invented() {
+    let oi = TestOi::new();
+    seed(&oi, "demo", "aa", Cause::Deliberate, None);
+
+    let rows = oi.call("/restarts/list", json!({})).unwrap();
+    let rows = rows.as_array().unwrap();
+    assert_eq!(rows[0]["cause"], "deliberate");
+    assert!(rows[0]["exit_code"].is_null());
+    assert!(rows[0]["exit_kind"].is_null());
+}
+
+// i[verify restart.list]
+#[test]
+fn list_filters_by_app_and_instance_and_honours_limit() {
+    let oi = TestOi::new();
+    seed(&oi, "demo", "aa", Cause::Recovery, None);
+    seed(&oi, "demo", "aa", Cause::Recovery, None);
+    seed(&oi, "other", "bb", Cause::Recovery, None);
+
+    let by_app = oi.call("/restarts/list", json!({ "app": "demo" })).unwrap();
+    assert_eq!(by_app.as_array().unwrap().len(), 2);
+
+    let by_instance = oi
+        .call("/restarts/list", json!({ "instance": "bb" }))
+        .unwrap();
+    let by_instance = by_instance.as_array().unwrap();
+    assert_eq!(by_instance.len(), 1);
+    assert_eq!(by_instance[0]["app"], "other");
+
+    let both = oi
+        .call("/restarts/list", json!({ "app": "demo", "instance": "bb" }))
+        .unwrap();
+    assert!(both.as_array().unwrap().is_empty());
+
+    let limited = oi.call("/restarts/list", json!({ "limit": 1 })).unwrap();
+    assert_eq!(limited.as_array().unwrap().len(), 1);
+}
+
+// i[verify restart.settings]
+#[test]
+fn settings_round_trip_and_reject_out_of_bounds() {
+    let oi = TestOi::new();
+
+    let s = oi.call("/restarts/settings/get", json!({})).unwrap();
+    assert_eq!(s["threshold"], 5);
+    assert_eq!(s["window_secs"], 1800);
+
+    let s = oi
+        .call("/restarts/settings/set", json!({ "threshold": 3 }))
+        .unwrap();
+    assert_eq!(s["threshold"], 3);
+    assert_eq!(s["window_secs"], 1800);
+
+    let s = oi
+        .call("/restarts/settings/set", json!({ "window_secs": 600 }))
+        .unwrap();
+    assert_eq!(s["threshold"], 3);
+    assert_eq!(s["window_secs"], 600);
+
+    let err = oi
+        .call("/restarts/settings/set", json!({ "threshold": 1 }))
+        .unwrap_err();
+    assert!(err.1.contains("at least"), "{}", err.1);
+
+    let err = oi
+        .call("/restarts/settings/set", json!({ "window_secs": 10 }))
+        .unwrap_err();
+    assert!(err.1.contains("at least"), "{}", err.1);
+
+    // A rejected update leaves the stored settings alone.
+    let s = oi.call("/restarts/settings/get", json!({})).unwrap();
+    assert_eq!(s["threshold"], 3);
+    assert_eq!(s["window_secs"], 600);
+}

crates/core/src/runtime.rs

@@ -27,6 +27,7 @@ pub mod probe;
 pub mod registries;
 pub mod registry;
 pub mod restart_gens;
+pub mod restarts;
 pub mod scaling;
 pub mod scheduler;
 pub mod schedules;

crates/core/src/runtime/barrier/replay.rs

@@ -326,6 +326,7 @@ pub fn run_operation<W: WorldStateOracle + 'static>(
     // call_index 0 each pass; without this marker the duplicated
     // breadcrumbs after a barrier wake up look like the script ran
     // twice.
+    // r[impl actuate.breadcrumb.replay]
     if !committed.is_empty() {
         crate::system::breadcrumb::Breadcrumb {
             app: Some(&app.def.load().name),

crates/core/src/runtime/db.rs

@@ -129,6 +129,9 @@ const SQL_V52: &str = include_str!("db/migrations/v52.sql");
 // r[impl canopy.settings.enabled]
 // r[impl canopy.report.identity]
 const SQL_V53: &str = include_str!("db/migrations/v53.sql");
+// r[impl autonomous.restart.record]
+// r[impl autonomous.restart.rate.settings]
+const SQL_V54: &str = include_str!("db/migrations/v54.sql");
 
 const MIGRATIONS: &[Migration] = &[
     Migration {
@@ -391,6 +394,11 @@ const MIGRATIONS: &[Migration] = &[
         sql: SQL_V53,
         custom_run: None,
     },
+    Migration {
+        version: 54,
+        sql: SQL_V54,
+        custom_run: None,
+    },
 ];
 
 fn migration_hash(sql: &str) -> String {

crates/core/src/runtime/db/migrations/v54.sql

@@ -0,0 +1,61 @@
+-- r[impl autonomous.restart.record]
+-- One row per observed or performed restart of a container instance.
+--
+-- `cause` is 'recovery' when the restart followed an unexpected exit, and
+-- 'deliberate' when the runtime restarted the workload on purpose (rolling
+-- update, health check replacement, operator-requested restart). Only recovery
+-- rows count towards the crash-loop rate.
+--
+-- The split is on why, not on who: on Linux systemd actions recovery restarts
+-- and seedling actions deliberate ones, but on a platform with no service
+-- supervisor seedling actions both, and a column recording the actor would
+-- classify every restart there identically.
+--
+-- `exit_kind` is 'exited', 'signalled' or 'dumped'; `exit_code` is the exit
+-- status for 'exited' and the signal number otherwise. Both are NULL when the
+-- platform did not report an exit for the run that ended.
+CREATE TABLE IF NOT EXISTS instance_restarts (
+    id            INTEGER PRIMARY KEY AUTOINCREMENT,
+    instance_id   TEXT    NOT NULL,
+    app           TEXT    NOT NULL,
+    resource_type TEXT,
+    resource_name TEXT,
+    generation    INTEGER,
+    recorded_at   INTEGER NOT NULL,
+    cause         TEXT    NOT NULL,
+    exit_code     INTEGER,
+    exit_kind     TEXT
+);
+
+CREATE INDEX IF NOT EXISTS idx_instance_restarts_instance
+    ON instance_restarts (instance_id, recorded_at);
+CREATE INDEX IF NOT EXISTS idx_instance_restarts_app
+    ON instance_restarts (app, recorded_at);
+
+-- Last restart counter read from the supervisor for an instance's unit. The
+-- counter is monotonic per unit but resets when the unit is recreated or its
+-- failed state is cleared, so a decrease is a reset to re-baseline against,
+-- not a negative delta.
+CREATE TABLE IF NOT EXISTS instance_restart_counters (
+    instance_id TEXT    PRIMARY KEY,
+    counter     INTEGER NOT NULL,
+    updated_at  INTEGER NOT NULL
+);
+
+-- r[impl autonomous.restart.rate.settings]
+-- Crash-loop rate threshold and window. One row, enforced via the singleton
+-- primary key.
+--
+-- The default of five supervisor-actioned restarts within thirty minutes is
+-- loose enough that a container taking seconds to crash gets several chances
+-- across a deploy, and tight enough that persistent flapping surfaces within
+-- an operator's working session rather than a day later.
+CREATE TABLE IF NOT EXISTS restart_settings (
+    singleton   INTEGER PRIMARY KEY DEFAULT 1 CHECK (singleton = 1),
+    threshold   INTEGER NOT NULL DEFAULT 5,
+    window_secs INTEGER NOT NULL DEFAULT 1800,
+    updated_at  INTEGER NOT NULL DEFAULT 0
+);
+
+INSERT OR IGNORE INTO restart_settings (singleton, threshold, window_secs, updated_at)
+    VALUES (1, 5, 1800, 0);

crates/core/src/runtime/db/tests.rs

@@ -13,7 +13,7 @@ fn open_in_memory_succeeds() {
             |r| r.get(0),
         )
         .expect("schema_version should exist");
-    assert_eq!(version, 53);
+    assert_eq!(version, 54);
 }
 
 // r[verify history.persistence]
@@ -45,7 +45,7 @@ fn params_table_exists() {
             |r| r.get(0),
         )
         .expect("schema_version should exist");
-    assert_eq!(version, 53);
+    assert_eq!(version, 54);
 }
 
 // i[verify app.persist]