diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_inbox/store_write.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_inbox/store_write.rs index 41225e0dc8..bef2d4a65c 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_inbox/store_write.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_inbox/store_write.rs @@ -28,7 +28,7 @@ impl AgentInboxStore { /// Atomically persist a recovery/outbox message only while its Agent Org /// run is still Running. This shares the sessions writer lock and an - /// IMMEDIATE transaction with run finality, so a queued watchdog action + /// IMMEDIATE transaction with Team Quiescence, so a queued watchdog action /// cannot insert a new unread row after pause or terminal transition. pub(crate) fn insert_if_run_running( params: InsertInboxParams, diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_plan_approvals/store.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_plan_approvals/store.rs index e8cbaae534..c78a03b3f6 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_plan_approvals/store.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_plan_approvals/store.rs @@ -640,7 +640,7 @@ impl AgentOrgPlanApprovalStore { OR EXISTS ( SELECT 1 FROM agent_org_runs run WHERE run.id=approval.org_run_id - AND run.status IN ('completed','failed','cancelled','abandoned') + AND run.status IN ('failed','archived') ) )", ) @@ -667,7 +667,7 @@ impl AgentOrgPlanApprovalStore { OR EXISTS ( SELECT 1 FROM agent_org_runs run WHERE run.id=agent_org_plan_approvals.org_run_id - AND run.status IN ('completed','failed','cancelled','abandoned') + AND run.status IN ('failed','archived') ) )", params![ diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/helpers.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/helpers.rs index aa266e7e16..a3fbd9bb73 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/helpers.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/helpers.rs @@ -49,14 +49,18 @@ pub(super) fn load_by_id(run_id: &str) -> SqliteResult org_snapshot_json, entry_mode, status, + activation_generation, + has_initial_work, work_item_id, project_slug, routine_fire_id, summary, last_error, + failure_json, + last_activity_outcome, created_at, updated_at, - completed_at + idled_at FROM agent_org_runs WHERE id = ?1 LIMIT 1", @@ -78,14 +82,18 @@ pub(super) fn load_by_root_session( org_snapshot_json, entry_mode, status, + activation_generation, + has_initial_work, work_item_id, project_slug, routine_fire_id, summary, last_error, + failure_json, + last_activity_outcome, created_at, updated_at, - completed_at + idled_at FROM agent_org_runs WHERE root_session_id = ?1 ORDER BY created_at DESC @@ -121,14 +129,18 @@ pub(super) fn row_to_run(row: &rusqlite::Row<'_>) -> SqliteResult(8)? != 0, + work_item_id: row.get(9)?, + project_slug: row.get(10)?, + routine_fire_id: row.get(11)?, + summary: row.get(12)?, + last_error: row.get(13)?, + failure_json: row.get(14)?, + last_activity_outcome: row.get(15)?, + created_at: row.get(16)?, + updated_at: row.get(17)?, + idled_at: row.get(18)?, }) } @@ -204,15 +216,19 @@ pub(super) fn insert_run(conn: &Connection, run: &AgentOrgRunRecord) -> SqliteRe org_snapshot_json, entry_mode, status, + activation_generation, + has_initial_work, work_item_id, project_slug, routine_fire_id, summary, last_error, + failure_json, + last_activity_outcome, created_at, updated_at, - completed_at - ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15)", + idled_at + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18, ?19)", params![ &run.id, &run.org_id, @@ -221,14 +237,18 @@ pub(super) fn insert_run(conn: &Connection, run: &AgentOrgRunRecord) -> SqliteRe run.org_snapshot_json.as_deref(), run.entry_mode.as_str(), run.status.as_str(), + run.activation_generation, + i64::from(run.has_initial_work), run.work_item_id.as_deref(), run.project_slug.as_deref(), run.routine_fire_id.as_deref(), run.summary.as_deref(), run.last_error.as_deref(), + run.failure_json.as_deref(), + run.last_activity_outcome.as_deref(), &run.created_at, &run.updated_at, - run.completed_at.as_deref(), + run.idled_at.as_deref(), ], )?; Ok(()) diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/materialization.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/materialization.rs new file mode 100644 index 0000000000..18c517334a --- /dev/null +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/materialization.rs @@ -0,0 +1,378 @@ +//! Durable certificates for the one-time construction of an Agent Org Team. +//! +//! These rows describe stable identities, not live Provider runtimes. A +//! restart retries the same `(member_id, agent_id, session_id)` intent and can +//! therefore never mint a second identity for the same Team generation. + +use rusqlite::{params, Connection, OptionalExtension}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum AgentOrgMaterializationAuthority { + Starting, + Formal, + UserDirected, +} + +impl AgentOrgMaterializationAuthority { + pub fn as_str(self) -> &'static str { + match self { + Self::Starting => "starting", + Self::Formal => "formal", + Self::UserDirected => "user_directed", + } + } + + fn parse(value: &str) -> Option { + Some(match value { + "starting" => Self::Starting, + "formal" => Self::Formal, + "user_directed" => Self::UserDirected, + _ => return None, + }) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum AgentOrgMaterializationStatus { + Pending, + Succeeded, + Failed, +} + +impl AgentOrgMaterializationStatus { + pub fn as_str(self) -> &'static str { + match self { + Self::Pending => "pending", + Self::Succeeded => "succeeded", + Self::Failed => "failed", + } + } + + fn parse(value: &str) -> Option { + Some(match value { + "pending" => Self::Pending, + "succeeded" => Self::Succeeded, + "failed" => Self::Failed, + _ => return None, + }) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct AgentOrgMaterializationIntent { + pub org_run_id: String, + pub member_id: String, + pub agent_id: String, + pub generation: i64, + pub session_id: String, + pub authority: AgentOrgMaterializationAuthority, + pub status: AgentOrgMaterializationStatus, + pub error_code: Option, + pub error_json: Option, + pub created_at: String, + pub updated_at: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CreateAgentOrgMaterializationIntent { + pub member_id: String, + pub agent_id: String, + pub session_id: String, + pub succeeded: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum AgentOrgInitialInputStatus { + PendingPersistence, + Queued, + Dispatched, +} + +impl AgentOrgInitialInputStatus { + pub fn as_str(self) -> &'static str { + match self { + Self::PendingPersistence => "pending_persistence", + Self::Queued => "queued", + Self::Dispatched => "dispatched", + } + } + + fn parse(value: &str) -> Option { + Some(match value { + "pending_persistence" => Self::PendingPersistence, + "queued" => Self::Queued, + "dispatched" => Self::Dispatched, + _ => return None, + }) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct AgentOrgInitialInput { + pub org_run_id: String, + pub turn_intent_id: String, + pub message_id: String, + pub content: String, + /// Canonical launch-time attachments/context required to replay this exact + /// accepted input after a crash. The launch owner validates and decodes + /// the versioned JSON; the lifecycle store treats it as opaque evidence. + pub payload_json: String, + pub status: AgentOrgInitialInputStatus, + pub created_at: String, + pub updated_at: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CreateAgentOrgInitialInput { + pub turn_intent_id: String, + pub message_id: String, + pub content: String, + pub payload_json: String, +} + +pub(super) fn init_schema(conn: &Connection) -> rusqlite::Result<()> { + conn.execute_batch( + "CREATE TABLE IF NOT EXISTS agent_org_member_materializations ( + org_run_id TEXT NOT NULL, + member_id TEXT NOT NULL, + agent_id TEXT NOT NULL, + generation INTEGER NOT NULL CHECK(generation >= 1), + session_id TEXT NOT NULL, + authority_class TEXT NOT NULL CHECK(authority_class IN ( + 'starting', 'formal', 'user_directed' + )), + status TEXT NOT NULL CHECK(status IN ( + 'pending', 'succeeded', 'failed' + )), + error_code TEXT, + error_json TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY(org_run_id, member_id, generation), + UNIQUE(org_run_id, session_id), + FOREIGN KEY(org_run_id) REFERENCES agent_org_runs(id) ON DELETE CASCADE + ); + CREATE INDEX IF NOT EXISTS idx_agent_org_materializations_pending + ON agent_org_member_materializations(status, org_run_id, generation); + + CREATE TABLE IF NOT EXISTS agent_org_initial_inputs ( + org_run_id TEXT PRIMARY KEY, + turn_intent_id TEXT NOT NULL, + message_id TEXT NOT NULL, + content TEXT NOT NULL, + payload_json TEXT NOT NULL, + status TEXT NOT NULL CHECK(status IN ( + 'pending_persistence', 'queued', 'dispatched' + )), + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + UNIQUE(turn_intent_id), + UNIQUE(message_id), + FOREIGN KEY(org_run_id) REFERENCES agent_org_runs(id) ON DELETE CASCADE + ); + CREATE INDEX IF NOT EXISTS idx_agent_org_initial_inputs_dispatch + ON agent_org_initial_inputs(status, org_run_id);", + ) +} + +pub(super) fn insert_materialization_intent( + conn: &Connection, + org_run_id: &str, + generation: i64, + intent: &CreateAgentOrgMaterializationIntent, + now: &str, +) -> Result<(), String> { + let status = if intent.succeeded { + AgentOrgMaterializationStatus::Succeeded + } else { + AgentOrgMaterializationStatus::Pending + }; + conn.execute( + "INSERT INTO agent_org_member_materializations ( + org_run_id, member_id, agent_id, generation, session_id, + authority_class, status, created_at, updated_at + ) VALUES (?1, ?2, ?3, ?4, ?5, 'starting', ?6, ?7, ?7)", + params![ + org_run_id, + &intent.member_id, + &intent.agent_id, + generation, + &intent.session_id, + status.as_str(), + now, + ], + ) + .map_err(|error| error.to_string())?; + Ok(()) +} + +pub(super) fn insert_initial_input( + conn: &Connection, + org_run_id: &str, + input: &CreateAgentOrgInitialInput, + now: &str, +) -> Result<(), String> { + conn.execute( + "INSERT INTO agent_org_initial_inputs ( + org_run_id, turn_intent_id, message_id, content, payload_json, + status, created_at, updated_at + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?7)", + params![ + org_run_id, + &input.turn_intent_id, + &input.message_id, + &input.content, + &input.payload_json, + AgentOrgInitialInputStatus::PendingPersistence.as_str(), + now, + ], + ) + .map_err(|error| error.to_string())?; + Ok(()) +} + +pub(super) fn list_materializations_with_connection( + conn: &Connection, + org_run_id: &str, +) -> Result, String> { + let mut statement = conn + .prepare( + "SELECT org_run_id, member_id, agent_id, generation, session_id, + authority_class, status, error_code, error_json, + created_at, updated_at + FROM agent_org_member_materializations + WHERE org_run_id=?1 + ORDER BY member_id ASC", + ) + .map_err(|error| error.to_string())?; + let rows = statement + .query_map([org_run_id], row_to_materialization) + .map_err(|error| error.to_string())?; + rows.collect::, _>>() + .map_err(|error| error.to_string()) +} + +pub(super) fn load_initial_input_with_connection( + conn: &Connection, + org_run_id: &str, +) -> Result, String> { + conn.query_row( + "SELECT org_run_id, turn_intent_id, message_id, content, payload_json, + status, created_at, updated_at + FROM agent_org_initial_inputs WHERE org_run_id=?1", + [org_run_id], + row_to_initial_input, + ) + .optional() + .map_err(|error| error.to_string()) +} + +pub(super) fn load_initial_input_by_turn_with_connection( + conn: &Connection, + turn_intent_id: &str, +) -> Result, String> { + conn.query_row( + "SELECT org_run_id, turn_intent_id, message_id, content, payload_json, + status, created_at, updated_at + FROM agent_org_initial_inputs WHERE turn_intent_id=?1", + [turn_intent_id], + row_to_initial_input, + ) + .optional() + .map_err(|error| error.to_string()) +} + +pub(super) fn list_recoverable_initial_inputs_with_connection( + conn: &Connection, + limit: usize, +) -> Result, String> { + if limit == 0 { + return Ok(Vec::new()); + } + let limit = i64::try_from(limit) + .map_err(|_| format!("initial input recovery limit is too large: {limit}"))?; + let mut statement = conn + .prepare( + "SELECT initial.org_run_id, initial.turn_intent_id, + initial.message_id, initial.content, initial.payload_json, + initial.status, + initial.created_at, initial.updated_at + FROM agent_org_initial_inputs initial + JOIN agent_org_runs run ON run.id=initial.org_run_id + JOIN session_turn_intents turn + ON turn.org_run_id=initial.org_run_id + AND turn.turn_intent_id=initial.turn_intent_id + WHERE run.status='running' + AND initial.status IN ('queued', 'dispatched') + AND turn.status='queued' + ORDER BY initial.updated_at ASC, initial.org_run_id ASC + LIMIT ?1", + ) + .map_err(|error| error.to_string())?; + let rows = statement + .query_map([limit], row_to_initial_input) + .map_err(|error| error.to_string())?; + rows.collect::, _>>() + .map_err(|error| error.to_string()) +} + +fn row_to_materialization( + row: &rusqlite::Row<'_>, +) -> rusqlite::Result { + let authority_raw: String = row.get(5)?; + let status_raw: String = row.get(6)?; + let authority = AgentOrgMaterializationAuthority::parse(&authority_raw).ok_or_else(|| { + rusqlite::Error::FromSqlConversionFailure( + 5, + rusqlite::types::Type::Text, + format!("unknown materialization authority: {authority_raw:?}").into(), + ) + })?; + let status = AgentOrgMaterializationStatus::parse(&status_raw).ok_or_else(|| { + rusqlite::Error::FromSqlConversionFailure( + 6, + rusqlite::types::Type::Text, + format!("unknown materialization status: {status_raw:?}").into(), + ) + })?; + Ok(AgentOrgMaterializationIntent { + org_run_id: row.get(0)?, + member_id: row.get(1)?, + agent_id: row.get(2)?, + generation: row.get(3)?, + session_id: row.get(4)?, + authority, + status, + error_code: row.get(7)?, + error_json: row.get(8)?, + created_at: row.get(9)?, + updated_at: row.get(10)?, + }) +} + +fn row_to_initial_input(row: &rusqlite::Row<'_>) -> rusqlite::Result { + let status_raw: String = row.get(5)?; + let status = AgentOrgInitialInputStatus::parse(&status_raw).ok_or_else(|| { + rusqlite::Error::FromSqlConversionFailure( + 5, + rusqlite::types::Type::Text, + format!("unknown initial input status: {status_raw:?}").into(), + ) + })?; + Ok(AgentOrgInitialInput { + org_run_id: row.get(0)?, + turn_intent_id: row.get(1)?, + message_id: row.get(2)?, + content: row.get(3)?, + payload_json: row.get(4)?, + status, + created_at: row.get(6)?, + updated_at: row.get(7)?, + }) +} diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/mod.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/mod.rs index 5e2a0851a5..6ac6ff116f 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/mod.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/mod.rs @@ -3,23 +3,33 @@ //! A run records that an Agent Org launched through the normal Rust session //! stack, while the root session remains the transcript source of truth. -mod finality; mod helpers; +mod materialization; mod progress; +mod quiescence; +mod rollout; mod store; mod worker; #[cfg(test)] mod tests; -pub(crate) use finality::guaranteed_current_turn_effects_with_connection; -pub use finality::{ - AgentOrgFinalityAssessment, AgentOrgFinalityBlocker, AgentOrgFinalityDecision, - AgentOrgFinalityFacts, AgentOrgFinalityProjection, AgentOrgFinalitySessionFact, - AgentOrgGuaranteedTurnEffects, +pub use materialization::{ + AgentOrgInitialInput, AgentOrgInitialInputStatus, AgentOrgMaterializationAuthority, + AgentOrgMaterializationIntent, AgentOrgMaterializationStatus, CreateAgentOrgInitialInput, + CreateAgentOrgMaterializationIntent, }; pub(crate) use progress::bump_work_revision_in_tx; pub use progress::AgentOrgRunProgress; +pub(crate) use quiescence::guaranteed_current_turn_effects_with_connection; +pub use quiescence::{ + AgentOrgGuaranteedTurnEffects, AgentOrgQuiescenceAssessment, AgentOrgQuiescenceBlocker, + AgentOrgQuiescenceDecision, AgentOrgQuiescenceFacts, AgentOrgQuiescenceProjection, + AgentOrgQuiescenceSessionFact, +}; +pub use rollout::{ + is_enabled as agent_org_redesign_enabled, require_enabled as require_agent_org_redesign, +}; pub use store::AgentOrgRunStore; pub(crate) use worker::recovery_dispatch_recipient_is_available; pub use worker::{WorkerSessionInfo, WorkerSessionRuntime}; @@ -62,50 +72,40 @@ impl std::fmt::Display for AgentOrgRunEntryMode { #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] #[serde(rename_all = "snake_case")] pub enum AgentOrgRunStatus { + Starting, Running, - /// User-initiated pause. Non-terminal: the run can be resumed via - /// `AgentOrgRunStore::mark_resumed`. Polling and member switching remain - /// available while paused; the coordinator and members simply stop - /// receiving new dispatch until resumed. + /// Reserved non-terminal user-pause state. PR1 freezes the canonical enum + /// but deliberately does not define Pause/Resume handoff behavior; Paused + /// Teams are not fallback-polled. Paused, - Completed, + Idle, Failed, - Cancelled, - Abandoned, + Archived, } impl AgentOrgRunStatus { pub fn as_str(self) -> &'static str { match self { + Self::Starting => "starting", Self::Running => "running", Self::Paused => "paused", - Self::Completed => "completed", + Self::Idle => "idle", Self::Failed => "failed", - Self::Cancelled => "cancelled", - Self::Abandoned => "abandoned", + Self::Archived => "archived", } } pub fn parse(value: &str) -> Option { match value { + "starting" => Some(Self::Starting), "running" => Some(Self::Running), "paused" => Some(Self::Paused), - "completed" => Some(Self::Completed), + "idle" => Some(Self::Idle), "failed" => Some(Self::Failed), - "cancelled" => Some(Self::Cancelled), - "abandoned" => Some(Self::Abandoned), + "archived" => Some(Self::Archived), _ => None, } } - - /// Whether this status represents a terminal state (no further transitions - /// possible). `Paused` is explicitly non-terminal. - pub fn is_terminal(self) -> bool { - matches!( - self, - Self::Completed | Self::Failed | Self::Cancelled | Self::Abandoned - ) - } } impl std::fmt::Display for AgentOrgRunStatus { @@ -399,14 +399,18 @@ pub struct AgentOrgRunRecord { pub org_snapshot_json: Option, pub entry_mode: AgentOrgRunEntryMode, pub status: AgentOrgRunStatus, + pub activation_generation: i64, + pub has_initial_work: bool, pub work_item_id: Option, pub project_slug: Option, pub routine_fire_id: Option, pub summary: Option, pub last_error: Option, + pub failure_json: Option, + pub last_activity_outcome: Option, pub created_at: String, pub updated_at: String, - pub completed_at: Option, + pub idled_at: Option, } #[derive(Debug, Clone)] @@ -422,6 +426,36 @@ pub struct CreateAgentOrgRunParams { pub routine_fire_id: Option, } +#[derive(Debug, Clone)] +pub struct CreateStartingAgentOrgRunParams { + pub org_id: String, + pub coordinator_agent_id: String, + pub root_session_id: String, + pub org_snapshot: OrgDefinition, + pub entry_mode: AgentOrgRunEntryMode, + pub work_item_id: Option, + pub project_slug: Option, + pub routine_fire_id: Option, + pub materialization_intents: Vec, + pub initial_input: Option, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct AgentOrgStartingFailure { + pub code: String, + pub message: String, +} + +impl AgentOrgStartingFailure { + pub fn new(code: impl Into, message: impl Into) -> Self { + Self { + code: code.into(), + message: message.into(), + } + } +} + /// Initialize runtime Agent Org tables in `sessions.db`. pub fn init_schema(conn: &Connection) -> SqliteResult<()> { conn.execute_batch( @@ -432,15 +466,25 @@ pub fn init_schema(conn: &Connection) -> SqliteResult<()> { root_session_id TEXT, org_snapshot_json TEXT, entry_mode TEXT NOT NULL, - status TEXT NOT NULL, + status TEXT NOT NULL CHECK(status IN ( + 'starting', 'running', 'paused', 'idle', 'failed', 'archived' + )), + activation_generation INTEGER NOT NULL DEFAULT 1 + CHECK(activation_generation >= 1), + has_initial_work INTEGER NOT NULL DEFAULT 0 + CHECK(has_initial_work IN (0, 1)), work_item_id TEXT, project_slug TEXT, routine_fire_id TEXT, summary TEXT, last_error TEXT, + failure_json TEXT, + last_activity_outcome TEXT CHECK(last_activity_outcome IN ( + 'completed', 'failed', 'cancelled' + )), created_at TEXT NOT NULL, updated_at TEXT NOT NULL, - completed_at TEXT + idled_at TEXT ); CREATE INDEX IF NOT EXISTS idx_agent_org_runs_org_updated ON agent_org_runs(org_id, updated_at); @@ -451,6 +495,7 @@ pub fn init_schema(conn: &Connection) -> SqliteResult<()> { CREATE INDEX IF NOT EXISTS idx_agent_org_runs_status ON agent_org_runs(status);", )?; + materialization::init_schema(conn)?; progress::init_schema(conn)?; Ok(()) } diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/progress.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/progress.rs index ed25aad591..933b860817 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/progress.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/progress.rs @@ -1,7 +1,7 @@ //! Monotonic Agent Org work observation and explicit completion intent. //! //! Timestamps are useful for display but are not a safe concurrency token. -//! This table records a small monotonic revision so finality can prove that a +//! This table records a small monotonic revision so Quiescence can prove that a //! coordinator turn was presented with (and successfully observed) the latest //! durable task mutation before announcing completion. diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/finality.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/quiescence.rs similarity index 69% rename from src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/finality.rs rename to src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/quiescence.rs index bce23aa776..2e70a7d234 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/finality.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/quiescence.rs @@ -1,4 +1,4 @@ -//! Canonical Agent Org finality facts and decision policy. +//! Canonical Agent Org formal-work quiescence facts and decision policy. //! //! Every caller (watchdog inspection, lifecycle reconciliation, completion //! snapshots) must reason from this same typed assessment. This prevents a @@ -16,7 +16,7 @@ use super::{AgentOrgRunStatus, AgentOrgRunStore}; #[derive(Debug, Clone, PartialEq, Eq, Serialize)] #[serde(rename_all = "camelCase")] -pub struct AgentOrgFinalitySessionFact { +pub struct AgentOrgQuiescenceSessionFact { pub session_id: String, pub member_id: Option, pub status: SessionStatus, @@ -24,11 +24,12 @@ pub struct AgentOrgFinalitySessionFact { #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "camelCase")] -pub struct AgentOrgFinalityFacts { +pub struct AgentOrgQuiescenceFacts { pub run_status: Option, + pub activation_generation: Option, pub root_session_id: Option, pub root_status: Option, - pub worker_sessions: Vec, + pub worker_sessions: Vec, pub task_count: usize, pub unresolved_task_count: usize, pub corrupt_task_count: usize, @@ -38,11 +39,14 @@ pub struct AgentOrgFinalityFacts { pub unread_inbox_count: usize, pub active_intervention_member_ids: Vec, pub in_flight_turn_intent_count: usize, + pub unknown_turn_intent_count: usize, + pub pending_formal_materialization_count: usize, + pub active_recovery_reservation_count: usize, pub pending_plan_approval_count: usize, pub progress: Option, } -impl AgentOrgFinalityFacts { +impl AgentOrgQuiescenceFacts { /// Canonical set of non-quiescent worker member ids for UI/task /// projections. Keeping the status classification here prevents Run View, /// task_list, and the reconciler from growing subtly different ideas of @@ -51,7 +55,7 @@ impl AgentOrgFinalityFacts { let mut member_ids = self .worker_sessions .iter() - .filter(|session| !session_is_quiescent_for_completed_run(session.status)) + .filter(|session| !session_is_quiescent(session.status)) .map(|session| { session .member_id @@ -67,15 +71,14 @@ impl AgentOrgFinalityFacts { #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] #[serde(rename_all = "snake_case")] -pub enum AgentOrgFinalityDecision { - KeepRunning, - Complete, - Abandon, +pub enum AgentOrgQuiescenceDecision { + KeepWorking, + Quiescent, } #[derive(Debug, Clone, PartialEq, Eq, Serialize)] #[serde(tag = "kind", rename_all = "snake_case")] -pub enum AgentOrgFinalityBlocker { +pub enum AgentOrgQuiescenceBlocker { RunMissing, RunNotRunning { status: AgentOrgRunStatus, @@ -108,6 +111,15 @@ pub enum AgentOrgFinalityBlocker { InFlightTurnIntents { count: usize, }, + UnknownTurnIntents { + count: usize, + }, + PendingFormalMaterializations { + count: usize, + }, + ActiveRecoveryReservations { + count: usize, + }, PendingPlanApprovals { count: usize, }, @@ -116,7 +128,7 @@ pub enum AgentOrgFinalityBlocker { /// retained facts disagree with the invariants that normally gate that /// transition. This is diagnostic state for repair/audit surfaces, not a /// request to mutate the run back to Running. - TerminalStateInconsistent { + QuietStateInconsistent { status: AgentOrgRunStatus, root_session_missing: bool, active_session_count: usize, @@ -125,29 +137,32 @@ pub enum AgentOrgFinalityBlocker { unread_inbox_count: usize, active_intervention_count: usize, in_flight_turn_intent_count: usize, + unknown_turn_intent_count: usize, + pending_formal_materialization_count: usize, + active_recovery_reservation_count: usize, pending_plan_approval_count: usize, }, } #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "camelCase")] -pub struct AgentOrgFinalityAssessment { - pub facts: AgentOrgFinalityFacts, - pub decision: AgentOrgFinalityDecision, - pub blockers: Vec, +pub struct AgentOrgQuiescenceAssessment { + pub facts: AgentOrgQuiescenceFacts, + pub decision: AgentOrgQuiescenceDecision, + pub blockers: Vec, } #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "camelCase")] -pub struct AgentOrgFinalityProjection { - pub decision: AgentOrgFinalityDecision, - pub blockers: Vec, +pub struct AgentOrgQuiescenceProjection { + pub decision: AgentOrgQuiescenceDecision, + pub blockers: Vec, } /// Exact effects that the currently executing coordinator turn will commit /// if (and only if) that turn succeeds. These counts are not caller hints: /// they are revalidated from durable intent/materialization rows inside the -/// same read transaction as the finality snapshot. +/// same read transaction as the quiescence snapshot. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub struct AgentOrgGuaranteedTurnEffects { pub current_coordinator_turn: bool, @@ -155,7 +170,7 @@ pub struct AgentOrgGuaranteedTurnEffects { pub unread_inbox_rows: usize, } -impl AgentOrgFinalityAssessment { +impl AgentOrgQuiescenceAssessment { /// Canonical prospective certificate used by the coordinator inside its /// current turn. It answers one narrow question: "if this coordinator /// turn succeeds now, will the strict reconciler be able to complete?" @@ -164,7 +179,7 @@ impl AgentOrgFinalityAssessment { /// the root session becomes quiescent, and the revision staged into this /// prompt becomes observed. Every worker, task, inbox, approval, /// intervention, corruption, and turn-intent blocker remains unchanged. - pub fn after_successful_coordinator_turn(&self) -> AgentOrgFinalityProjection { + pub fn after_successful_coordinator_turn(&self) -> AgentOrgQuiescenceProjection { self.after_successful_coordinator_turn_with_effects(AgentOrgGuaranteedTurnEffects { current_coordinator_turn: true, ..AgentOrgGuaranteedTurnEffects::default() @@ -174,11 +189,11 @@ impl AgentOrgFinalityAssessment { pub fn after_successful_coordinator_turn_with_effects( &self, effects: AgentOrgGuaranteedTurnEffects, - ) -> AgentOrgFinalityProjection { + ) -> AgentOrgQuiescenceProjection { if self.facts.run_status != Some(AgentOrgRunStatus::Running) || !effects.current_coordinator_turn { - return AgentOrgFinalityProjection { + return AgentOrgQuiescenceProjection { decision: self.decision, blockers: self.blockers.clone(), }; @@ -190,30 +205,30 @@ impl AgentOrgFinalityAssessment { let mut blockers = Vec::new(); for blocker in &self.blockers { match blocker { - AgentOrgFinalityBlocker::SessionsActive { session_ids } => { + AgentOrgQuiescenceBlocker::SessionsActive { session_ids } => { let remaining = session_ids .iter() .filter(|session_id| Some(session_id.as_str()) != root_session_id) .cloned() .collect::>(); if !remaining.is_empty() { - blockers.push(AgentOrgFinalityBlocker::SessionsActive { + blockers.push(AgentOrgQuiescenceBlocker::SessionsActive { session_ids: remaining, }); } } - AgentOrgFinalityBlocker::CoordinatorHasNotObservedLatestWork { .. } + AgentOrgQuiescenceBlocker::CoordinatorHasNotObservedLatestWork { .. } if presented_current_revision => {} - AgentOrgFinalityBlocker::UnreadInbox { count } => { + AgentOrgQuiescenceBlocker::UnreadInbox { count } => { let remaining = count.saturating_sub(effects.unread_inbox_rows); if remaining > 0 { - blockers.push(AgentOrgFinalityBlocker::UnreadInbox { count: remaining }); + blockers.push(AgentOrgQuiescenceBlocker::UnreadInbox { count: remaining }); } } - AgentOrgFinalityBlocker::InFlightTurnIntents { count } => { + AgentOrgQuiescenceBlocker::InFlightTurnIntents { count } => { let remaining = count.saturating_sub(effects.in_flight_turn_intents); if remaining > 0 { - blockers.push(AgentOrgFinalityBlocker::InFlightTurnIntents { + blockers.push(AgentOrgQuiescenceBlocker::InFlightTurnIntents { count: remaining, }); } @@ -221,11 +236,11 @@ impl AgentOrgFinalityAssessment { other => blockers.push(other.clone()), } } - AgentOrgFinalityProjection { + AgentOrgQuiescenceProjection { decision: if blockers.is_empty() { - AgentOrgFinalityDecision::Complete + AgentOrgQuiescenceDecision::Quiescent } else { - AgentOrgFinalityDecision::KeepRunning + AgentOrgQuiescenceDecision::KeepWorking }, blockers, } @@ -311,18 +326,20 @@ pub(crate) fn guaranteed_current_turn_effects_with_connection( pub(super) fn load_and_assess( conn: &Connection, run_id: &str, -) -> Result { - let run_row: Option<(String, Option)> = conn +) -> Result { + let run_row: Option<(String, Option, i64)> = conn .query_row( - "SELECT status, root_session_id FROM agent_org_runs WHERE id=?1", + "SELECT status, root_session_id, activation_generation + FROM agent_org_runs WHERE id=?1", params![run_id], - |row| Ok((row.get(0)?, row.get(1)?)), + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), ) .optional() .map_err(|err| err.to_string())?; - let Some((run_status_raw, root_session_id)) = run_row else { - return Ok(assess(AgentOrgFinalityFacts { + let Some((run_status_raw, root_session_id, activation_generation)) = run_row else { + return Ok(assess_quiescence(AgentOrgQuiescenceFacts { run_status: None, + activation_generation: None, root_session_id: None, root_status: None, worker_sessions: Vec::new(), @@ -335,6 +352,9 @@ pub(super) fn load_and_assess( unread_inbox_count: 0, active_intervention_member_ids: Vec::new(), in_flight_turn_intent_count: 0, + unknown_turn_intent_count: 0, + pending_formal_materialization_count: 0, + active_recovery_reservation_count: 0, pending_plan_approval_count: 0, progress: None, })); @@ -361,12 +381,12 @@ pub(super) fn load_and_assess( // Use the same cross-transport canonical worker projection as Run View // and recovery. Duplicating the Rust/CLI queries here used to let a stale - // session for the same member block finality even though the UI and + // session for the same member block quiescence even though the UI and // watchdog correctly selected the freshest one. let worker_sessions = AgentOrgRunStore::list_descendant_worker_sessions_with_connection(conn, run_id)? .into_iter() - .map(|session| AgentOrgFinalitySessionFact { + .map(|session| AgentOrgQuiescenceSessionFact { session_id: session.session_id, member_id: session.member_id, status: session.status, @@ -478,6 +498,36 @@ pub(super) fn load_and_assess( |row| row.get(0), ) .map_err(|err| err.to_string())?; + let unknown_turn_intent_count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM session_turn_intents + WHERE org_run_id=?1 + AND status NOT IN ( + 'optimistic', 'queued', 'running', 'completed', 'failed', + 'cancelled', 'stale', 'coalesced', 'rejected' + )", + params![run_id], + |row| row.get(0), + ) + .map_err(|err| err.to_string())?; + let pending_formal_materialization_count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM agent_org_member_materializations + WHERE org_run_id=?1 AND generation=?2 + AND authority_class IN ('starting', 'formal') + AND status<>'succeeded'", + params![run_id, activation_generation], + |row| row.get(0), + ) + .map_err(|err| err.to_string())?; + let active_recovery_reservation_count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM agent_org_recovery_attempts + WHERE org_run_id=?1 AND reservation_token IS NOT NULL", + params![run_id], + |row| row.get(0), + ) + .map_err(|err| err.to_string())?; let pending_plan_approval_count: i64 = conn .query_row( "SELECT COUNT(*) FROM agent_org_plan_approvals @@ -487,8 +537,9 @@ pub(super) fn load_and_assess( ) .map_err(|err| err.to_string())?; - Ok(assess(AgentOrgFinalityFacts { + Ok(assess_quiescence(AgentOrgQuiescenceFacts { run_status: Some(run_status), + activation_generation: Some(activation_generation), root_session_id, root_status, worker_sessions, @@ -504,6 +555,18 @@ pub(super) fn load_and_assess( "in-flight turn intent", in_flight_turn_intent_count, )?, + unknown_turn_intent_count: count_to_usize( + "unknown turn intent", + unknown_turn_intent_count, + )?, + pending_formal_materialization_count: count_to_usize( + "pending formal materialization", + pending_formal_materialization_count, + )?, + active_recovery_reservation_count: count_to_usize( + "active recovery reservation", + active_recovery_reservation_count, + )?, pending_plan_approval_count: count_to_usize( "pending plan approval", pending_plan_approval_count, @@ -512,49 +575,42 @@ pub(super) fn load_and_assess( })) } -pub(super) fn assess(facts: AgentOrgFinalityFacts) -> AgentOrgFinalityAssessment { +pub fn assess_quiescence(facts: AgentOrgQuiescenceFacts) -> AgentOrgQuiescenceAssessment { let mut blockers = Vec::new(); let Some(run_status) = facts.run_status else { - blockers.push(AgentOrgFinalityBlocker::RunMissing); - return AgentOrgFinalityAssessment { + blockers.push(AgentOrgQuiescenceBlocker::RunMissing); + return AgentOrgQuiescenceAssessment { + decision: AgentOrgQuiescenceDecision::KeepWorking, facts, - decision: AgentOrgFinalityDecision::KeepRunning, blockers, }; }; - if run_status == AgentOrgRunStatus::Completed { - if let Some(inconsistency) = terminal_state_inconsistency(&facts, run_status) { + if run_status == AgentOrgRunStatus::Idle { + if let Some(inconsistency) = quiet_state_inconsistency(&facts, run_status) { blockers.push(inconsistency); } - return AgentOrgFinalityAssessment { + return AgentOrgQuiescenceAssessment { facts, - decision: AgentOrgFinalityDecision::Complete, - blockers, - }; - } - if run_status == AgentOrgRunStatus::Abandoned { - return AgentOrgFinalityAssessment { - facts, - decision: AgentOrgFinalityDecision::Abandon, + decision: AgentOrgQuiescenceDecision::Quiescent, blockers, }; } if run_status != AgentOrgRunStatus::Running { - blockers.push(AgentOrgFinalityBlocker::RunNotRunning { status: run_status }); - return AgentOrgFinalityAssessment { + blockers.push(AgentOrgQuiescenceBlocker::RunNotRunning { status: run_status }); + return AgentOrgQuiescenceAssessment { facts, - decision: AgentOrgFinalityDecision::KeepRunning, + decision: AgentOrgQuiescenceDecision::KeepWorking, blockers, }; } if facts.root_session_id.is_none() || facts.root_status.is_none() { - blockers.push(AgentOrgFinalityBlocker::RootSessionMissing); + blockers.push(AgentOrgQuiescenceBlocker::RootSessionMissing); } let mut active_session_ids = Vec::new(); if facts .root_status - .is_some_and(|status| !session_is_quiescent_for_completed_run(status)) + .is_some_and(|status| !session_is_quiescent(status)) { if let Some(root_session_id) = facts.root_session_id.as_ref() { active_session_ids.push(root_session_id.clone()); @@ -564,55 +620,71 @@ pub(super) fn assess(facts: AgentOrgFinalityFacts) -> AgentOrgFinalityAssessment facts .worker_sessions .iter() - .filter(|session| !session_is_quiescent_for_completed_run(session.status)) + .filter(|session| !session_is_quiescent(session.status)) .map(|session| session.session_id.clone()), ); if !active_session_ids.is_empty() { - blockers.push(AgentOrgFinalityBlocker::SessionsActive { + blockers.push(AgentOrgQuiescenceBlocker::SessionsActive { session_ids: active_session_ids, }); } if facts.unresolved_task_count > 0 { - blockers.push(AgentOrgFinalityBlocker::OpenTasks { + blockers.push(AgentOrgQuiescenceBlocker::OpenTasks { count: facts.unresolved_task_count, }); } if facts.corrupt_task_count > 0 { - blockers.push(AgentOrgFinalityBlocker::CorruptTaskData { + blockers.push(AgentOrgQuiescenceBlocker::CorruptTaskData { count: facts.corrupt_task_count, }); } if facts.unread_inbox_count > 0 { - blockers.push(AgentOrgFinalityBlocker::UnreadInbox { + blockers.push(AgentOrgQuiescenceBlocker::UnreadInbox { count: facts.unread_inbox_count, }); } if !facts.active_intervention_member_ids.is_empty() { - blockers.push(AgentOrgFinalityBlocker::ActiveInterventions { + blockers.push(AgentOrgQuiescenceBlocker::ActiveInterventions { count: facts.active_intervention_member_ids.len(), }); } if facts.in_flight_turn_intent_count > 0 { - blockers.push(AgentOrgFinalityBlocker::InFlightTurnIntents { + blockers.push(AgentOrgQuiescenceBlocker::InFlightTurnIntents { count: facts.in_flight_turn_intent_count, }); } + if facts.unknown_turn_intent_count > 0 { + blockers.push(AgentOrgQuiescenceBlocker::UnknownTurnIntents { + count: facts.unknown_turn_intent_count, + }); + } + if facts.pending_formal_materialization_count > 0 { + blockers.push(AgentOrgQuiescenceBlocker::PendingFormalMaterializations { + count: facts.pending_formal_materialization_count, + }); + } + if facts.active_recovery_reservation_count > 0 { + blockers.push(AgentOrgQuiescenceBlocker::ActiveRecoveryReservations { + count: facts.active_recovery_reservation_count, + }); + } if facts.pending_plan_approval_count > 0 { - blockers.push(AgentOrgFinalityBlocker::PendingPlanApprovals { + blockers.push(AgentOrgQuiescenceBlocker::PendingPlanApprovals { count: facts.pending_plan_approval_count, }); } match facts.progress.as_ref() { - None => blockers.push(AgentOrgFinalityBlocker::ProgressStateMissing), + None => blockers.push(AgentOrgQuiescenceBlocker::ProgressStateMissing), Some(progress) => { if facts.task_count == 0 { if !progress.completion_requested { - blockers.push(AgentOrgFinalityBlocker::EmptyTaskBoardRequiresCompletionIntent); + blockers + .push(AgentOrgQuiescenceBlocker::EmptyTaskBoardRequiresCompletionIntent); } else if progress.completion_requested_work_revision != Some(progress.work_revision) { - blockers.push(AgentOrgFinalityBlocker::StaleCompletionIntent { + blockers.push(AgentOrgQuiescenceBlocker::StaleCompletionIntent { requested_work_revision: progress.completion_requested_work_revision, current_work_revision: progress.work_revision, }); @@ -620,7 +692,7 @@ pub(super) fn assess(facts: AgentOrgFinalityFacts) -> AgentOrgFinalityAssessment } if progress.coordinator_observed_work_revision < Some(progress.work_revision) { blockers.push( - AgentOrgFinalityBlocker::CoordinatorHasNotObservedLatestWork { + AgentOrgQuiescenceBlocker::CoordinatorHasNotObservedLatestWork { observed_work_revision: progress.coordinator_observed_work_revision, current_work_revision: progress.work_revision, }, @@ -629,41 +701,31 @@ pub(super) fn assess(facts: AgentOrgFinalityFacts) -> AgentOrgFinalityAssessment } } - let coordinator_is_permanently_unavailable = facts.root_status == Some(SessionStatus::Archived); - let every_worker_is_permanently_unavailable = facts - .worker_sessions - .iter() - .all(|session| session.status == SessionStatus::Archived); - let decision = if facts.unresolved_task_count > 0 - && coordinator_is_permanently_unavailable - && every_worker_is_permanently_unavailable - { - AgentOrgFinalityDecision::Abandon - } else if blockers.is_empty() { - AgentOrgFinalityDecision::Complete + let decision = if blockers.is_empty() { + AgentOrgQuiescenceDecision::Quiescent } else { - AgentOrgFinalityDecision::KeepRunning + AgentOrgQuiescenceDecision::KeepWorking }; - AgentOrgFinalityAssessment { + AgentOrgQuiescenceAssessment { facts, decision, blockers, } } -fn terminal_state_inconsistency( - facts: &AgentOrgFinalityFacts, +fn quiet_state_inconsistency( + facts: &AgentOrgQuiescenceFacts, status: AgentOrgRunStatus, -) -> Option { +) -> Option { let root_session_missing = facts.root_session_id.is_none() || facts.root_status.is_none(); let active_session_count = usize::from( facts .root_status - .is_some_and(|session| !session_is_quiescent_for_completed_run(session)), + .is_some_and(|session| !session_is_quiescent(session)), ) + facts .worker_sessions .iter() - .filter(|session| !session_is_quiescent_for_completed_run(session.status)) + .filter(|session| !session_is_quiescent(session.status)) .count(); let inconsistent = root_session_missing || active_session_count > 0 @@ -672,8 +734,11 @@ fn terminal_state_inconsistency( || facts.unread_inbox_count > 0 || !facts.active_intervention_member_ids.is_empty() || facts.in_flight_turn_intent_count > 0 + || facts.unknown_turn_intent_count > 0 + || facts.pending_formal_materialization_count > 0 + || facts.active_recovery_reservation_count > 0 || facts.pending_plan_approval_count > 0; - inconsistent.then_some(AgentOrgFinalityBlocker::TerminalStateInconsistent { + inconsistent.then_some(AgentOrgQuiescenceBlocker::QuietStateInconsistent { status, root_session_missing, active_session_count, @@ -682,11 +747,14 @@ fn terminal_state_inconsistency( unread_inbox_count: facts.unread_inbox_count, active_intervention_count: facts.active_intervention_member_ids.len(), in_flight_turn_intent_count: facts.in_flight_turn_intent_count, + unknown_turn_intent_count: facts.unknown_turn_intent_count, + pending_formal_materialization_count: facts.pending_formal_materialization_count, + active_recovery_reservation_count: facts.active_recovery_reservation_count, pending_plan_approval_count: facts.pending_plan_approval_count, }) } -pub(super) fn session_is_quiescent_for_completed_run(status: SessionStatus) -> bool { +pub(super) fn session_is_quiescent(status: SessionStatus) -> bool { matches!( status, SessionStatus::Idle @@ -710,12 +778,13 @@ mod tests { fn completed_board_facts( presented_revision: Option, worker_status: SessionStatus, - ) -> AgentOrgFinalityFacts { - AgentOrgFinalityFacts { + ) -> AgentOrgQuiescenceFacts { + AgentOrgQuiescenceFacts { run_status: Some(AgentOrgRunStatus::Running), + activation_generation: Some(1), root_session_id: Some("root".to_string()), root_status: Some(SessionStatus::Running), - worker_sessions: vec![AgentOrgFinalitySessionFact { + worker_sessions: vec![AgentOrgQuiescenceSessionFact { session_id: "worker".to_string(), member_id: Some("member".to_string()), status: worker_status, @@ -729,6 +798,9 @@ mod tests { unread_inbox_count: 0, active_intervention_member_ids: Vec::new(), in_flight_turn_intent_count: 0, + unknown_turn_intent_count: 0, + pending_formal_materialization_count: 0, + active_recovery_reservation_count: 0, pending_plan_approval_count: 0, progress: Some(AgentOrgRunProgress { org_run_id: "run".to_string(), @@ -746,51 +818,57 @@ mod tests { #[test] fn prospective_certificate_allows_current_coordinator_turn_only() { - let assessment = assess(completed_board_facts(Some(2), SessionStatus::Idle)); - assert_eq!(assessment.decision, AgentOrgFinalityDecision::KeepRunning); + let assessment = assess_quiescence(completed_board_facts(Some(2), SessionStatus::Idle)); + assert_eq!(assessment.decision, AgentOrgQuiescenceDecision::KeepWorking); let prospective = assessment.after_successful_coordinator_turn(); - assert_eq!(prospective.decision, AgentOrgFinalityDecision::Complete); + assert_eq!(prospective.decision, AgentOrgQuiescenceDecision::Quiescent); assert!(prospective.blockers.is_empty()); } #[test] fn prospective_certificate_rejects_stale_presented_revision() { - let assessment = assess(completed_board_facts(Some(1), SessionStatus::Idle)); + let assessment = assess_quiescence(completed_board_facts(Some(1), SessionStatus::Idle)); let prospective = assessment.after_successful_coordinator_turn(); - assert_eq!(prospective.decision, AgentOrgFinalityDecision::KeepRunning); + assert_eq!( + prospective.decision, + AgentOrgQuiescenceDecision::KeepWorking + ); assert!(prospective.blockers.iter().any(|blocker| matches!( blocker, - AgentOrgFinalityBlocker::CoordinatorHasNotObservedLatestWork { .. } + AgentOrgQuiescenceBlocker::CoordinatorHasNotObservedLatestWork { .. } ))); } #[test] fn prospective_certificate_never_hides_active_worker() { - let assessment = assess(completed_board_facts(Some(2), SessionStatus::Running)); + let assessment = assess_quiescence(completed_board_facts(Some(2), SessionStatus::Running)); let prospective = assessment.after_successful_coordinator_turn(); - assert_eq!(prospective.decision, AgentOrgFinalityDecision::KeepRunning); + assert_eq!( + prospective.decision, + AgentOrgQuiescenceDecision::KeepWorking + ); assert!(prospective.blockers.iter().any(|blocker| matches!( blocker, - AgentOrgFinalityBlocker::SessionsActive { session_ids } + AgentOrgQuiescenceBlocker::SessionsActive { session_ids } if session_ids == &["worker".to_string()] ))); } #[test] - fn completed_run_stays_terminal_but_reports_inconsistent_retained_facts() { + fn idle_run_stays_quiescent_but_reports_inconsistent_retained_facts() { let mut facts = completed_board_facts(Some(2), SessionStatus::Idle); - facts.run_status = Some(AgentOrgRunStatus::Completed); + facts.run_status = Some(AgentOrgRunStatus::Idle); facts.root_status = Some(SessionStatus::Idle); facts.unresolved_task_count = 1; facts.corrupt_task_count = 1; facts.unread_inbox_count = 2; - let assessment = assess(facts); - assert_eq!(assessment.decision, AgentOrgFinalityDecision::Complete); + let assessment = assess_quiescence(facts); + assert_eq!(assessment.decision, AgentOrgQuiescenceDecision::Quiescent); assert!(matches!( assessment.blockers.as_slice(), - [AgentOrgFinalityBlocker::TerminalStateInconsistent { - status: AgentOrgRunStatus::Completed, + [AgentOrgQuiescenceBlocker::QuietStateInconsistent { + status: AgentOrgRunStatus::Idle, open_task_count: 1, corrupt_task_count: 1, unread_inbox_count: 2, @@ -798,4 +876,33 @@ mod tests { }] )); } + + #[test] + fn unknown_turn_materialization_and_reservation_fail_closed() { + let mut facts = completed_board_facts(Some(2), SessionStatus::Idle); + facts.root_status = Some(SessionStatus::Idle); + facts + .progress + .as_mut() + .expect("progress") + .coordinator_observed_work_revision = Some(2); + facts.unknown_turn_intent_count = 1; + facts.pending_formal_materialization_count = 1; + facts.active_recovery_reservation_count = 1; + + let assessment = assess_quiescence(facts); + assert_eq!(assessment.decision, AgentOrgQuiescenceDecision::KeepWorking); + assert!(assessment.blockers.iter().any(|blocker| matches!( + blocker, + AgentOrgQuiescenceBlocker::UnknownTurnIntents { count: 1 } + ))); + assert!(assessment.blockers.iter().any(|blocker| matches!( + blocker, + AgentOrgQuiescenceBlocker::PendingFormalMaterializations { count: 1 } + ))); + assert!(assessment.blockers.iter().any(|blocker| matches!( + blocker, + AgentOrgQuiescenceBlocker::ActiveRecoveryReservations { count: 1 } + ))); + } } diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/rollout.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/rollout.rs new file mode 100644 index 0000000000..9520f6a0ee --- /dev/null +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/rollout.rs @@ -0,0 +1,40 @@ +//! Single internal rollout gate for the long-lived Agent Org redesign. +//! +//! This is deliberately not persisted in Team definitions or exposed to +//! model/tool context. Until the final stack PR changes the default, missing +//! or malformed configuration fails closed. + +const ENABLED_VALUE: &str = "1"; +const ROLLOUT_ENV: &str = "ORGII_AGENT_ORG_REDESIGN"; + +fn configured_enabled(value: Option<&str>, test_build: bool) -> bool { + test_build || value.is_some_and(|value| value.trim() == ENABLED_VALUE) +} + +pub fn is_enabled() -> bool { + let configured = std::env::var(ROLLOUT_ENV).ok(); + configured_enabled(configured.as_deref(), cfg!(test)) +} + +pub fn require_enabled() -> Result<(), String> { + is_enabled().then_some(()).ok_or_else(|| { + "agent_org_redesign_disabled: the long-lived Agent Team lifecycle is not enabled" + .to_string() + }) +} + +#[cfg(test)] +mod tests { + #[test] + fn unit_test_builds_use_the_internal_gate_without_environment_state() { + assert!(super::is_enabled()); + } + + #[test] + fn production_gate_defaults_and_malformed_values_fail_closed() { + assert!(!super::configured_enabled(None, false)); + assert!(!super::configured_enabled(Some("true"), false)); + assert!(!super::configured_enabled(Some("0"), false)); + assert!(super::configured_enabled(Some("1"), false)); + } +} diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/store.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/store.rs index c50a58d3d0..b410c81186 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/store.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/store.rs @@ -9,20 +9,26 @@ use crate::definitions::orgs::AgentOrgsStore; use crate::session::SessionStatus; use database::db::{get_connection, with_sessions_writer}; -use super::finality::load_and_assess; use super::helpers::{ context_for_run_record, flatten_members, insert_run, load_by_id, load_by_root_session, parent_session_id_of, row_to_run, validate_entry_mode, validate_status, }; +use super::materialization::{ + insert_initial_input, insert_materialization_intent, list_materializations_with_connection, + list_recoverable_initial_inputs_with_connection, load_initial_input_by_turn_with_connection, + load_initial_input_with_connection, +}; use super::progress::{ ensure_progress_in_conn, load_progress_with_conn, mark_coordinator_observed_revision_with_conn, record_completion_request_in_tx, stage_coordinator_presented_with_conn, }; +use super::quiescence::load_and_assess; use super::worker::{WorkerSessionInfo, WorkerSessionRuntime}; use super::{ - AgentOrgCompletionRequestOutcome, AgentOrgFinalityAssessment, AgentOrgRunContext, - AgentOrgRunProgress, AgentOrgRunRecord, AgentOrgRunStatus, CreateAgentOrgRunParams, - COORDINATOR_MEMBER_ID, + AgentOrgCompletionRequestOutcome, AgentOrgInitialInput, AgentOrgMaterializationIntent, + AgentOrgQuiescenceAssessment, AgentOrgRunContext, AgentOrgRunProgress, AgentOrgRunRecord, + AgentOrgRunStatus, AgentOrgStartingFailure, CreateAgentOrgRunParams, + CreateStartingAgentOrgRunParams, COORDINATOR_MEMBER_ID, }; pub struct AgentOrgRunStore; @@ -62,14 +68,18 @@ impl AgentOrgRunStore { org_snapshot_json, entry_mode, status, + activation_generation, + has_initial_work, work_item_id, project_slug, routine_fire_id, summary, last_error, + failure_json, + last_activity_outcome, created_at, updated_at, - completed_at + idled_at FROM agent_org_runs WHERE root_session_id IN ({placeholders}) ORDER BY updated_at DESC, id DESC" @@ -99,14 +109,18 @@ impl AgentOrgRunStore { org_snapshot_json: Some(org_snapshot_json), entry_mode, status, + activation_generation: 1, + has_initial_work: false, work_item_id: params.work_item_id, project_slug: params.project_slug, routine_fire_id: params.routine_fire_id, summary: None, last_error: None, + failure_json: None, + last_activity_outcome: None, created_at: now.clone(), updated_at: now, - completed_at: None, + idled_at: None, }; with_sessions_writer(|| -> Result<(), String> { @@ -122,6 +136,457 @@ impl AgentOrgRunStore { Ok(run) } + /// Create the authoritative Team construction envelope and every stable + /// identity/input intent in one IMMEDIATE transaction. + pub fn create_starting( + params: CreateStartingAgentOrgRunParams, + ) -> Result { + let entry_mode = validate_entry_mode(params.entry_mode.as_str())?; + let org_snapshot_json = serde_json::to_string(¶ms.org_snapshot) + .map_err(|error| format!("failed to serialize Agent Org launch snapshot: {error}"))?; + let now = chrono::Utc::now().to_rfc3339(); + let run = AgentOrgRunRecord { + id: format!("agent-org-run-{}", uuid::Uuid::new_v4()), + org_id: params.org_id, + coordinator_agent_id: params.coordinator_agent_id, + root_session_id: Some(params.root_session_id.clone()), + org_snapshot_json: Some(org_snapshot_json), + entry_mode, + status: AgentOrgRunStatus::Starting, + activation_generation: 1, + has_initial_work: params.initial_input.is_some(), + work_item_id: params.work_item_id, + project_slug: params.project_slug, + routine_fire_id: params.routine_fire_id, + summary: None, + last_error: None, + failure_json: None, + last_activity_outcome: None, + created_at: now.clone(), + updated_at: now.clone(), + idled_at: None, + }; + + let mut member_ids = HashSet::new(); + let mut session_ids = HashSet::new(); + let mut expected_roster = flatten_members(¶ms.org_snapshot.children, None) + .into_iter() + .map(|member| (member.member_id, member.agent_id)) + .collect::>(); + expected_roster.insert( + COORDINATOR_MEMBER_ID.to_string(), + run.coordinator_agent_id.clone(), + ); + for intent in ¶ms.materialization_intents { + if intent.member_id.trim().is_empty() + || intent.agent_id.trim().is_empty() + || intent.session_id.trim().is_empty() + { + return Err("Agent Org materialization intent contains an empty identity".into()); + } + if !member_ids.insert(intent.member_id.clone()) { + return Err(format!( + "duplicate Agent Org materialization member_id: {}", + intent.member_id + )); + } + if !session_ids.insert(intent.session_id.clone()) { + return Err(format!( + "duplicate Agent Org materialization session_id: {}", + intent.session_id + )); + } + if expected_roster.get(&intent.member_id) != Some(&intent.agent_id) { + return Err(format!( + "Agent Org materialization roster does not match the launch snapshot for member {}", + intent.member_id + )); + } + if intent.member_id == COORDINATOR_MEMBER_ID { + if intent.session_id != params.root_session_id || !intent.succeeded { + return Err( + "Agent Org coordinator receipt must certify the canonical root Session" + .into(), + ); + } + } else if intent.succeeded { + return Err(format!( + "Agent Org member {} cannot be pre-certified before materialization", + intent.member_id + )); + } + } + if member_ids.len() != expected_roster.len() + || expected_roster + .keys() + .any(|member_id| !member_ids.contains(member_id)) + { + return Err( + "Agent Org materialization roster does not exactly match the launch snapshot" + .into(), + ); + } + + with_sessions_writer(|| -> Result<(), String> { + let mut connection = get_connection().map_err(|error| error.to_string())?; + let transaction = connection + .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate) + .map_err(|error| error.to_string())?; + let coordinator_identity: Option<(Option, Option, Option)> = + transaction + .query_row( + "SELECT agent_definition_id, org_member_id, parent_session_id + FROM agent_sessions WHERE session_id=?1", + [¶ms.root_session_id], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .optional() + .map_err(|error| error.to_string())?; + if coordinator_identity + != Some(( + Some(run.coordinator_agent_id.clone()), + Some(COORDINATOR_MEMBER_ID.to_string()), + None, + )) + { + return Err(format!( + "Agent Org coordinator Session identity is missing or mismatched: {}", + params.root_session_id + )); + } + insert_run(&transaction, &run).map_err(|error| error.to_string())?; + for intent in ¶ms.materialization_intents { + insert_materialization_intent( + &transaction, + &run.id, + run.activation_generation, + intent, + &now, + )?; + } + if let Some(input) = params.initial_input.as_ref() { + insert_initial_input(&transaction, &run.id, input, &now)?; + } + ensure_progress_in_conn(&transaction, &run.id)?; + transaction.commit().map_err(|error| error.to_string()) + })?; + crate::coordination::agent_org_run_events::notify_agent_org_run_changed(&run.id); + Ok(run) + } + + pub fn materializations(run_id: &str) -> Result, String> { + let connection = get_connection().map_err(|error| error.to_string())?; + list_materializations_with_connection(&connection, run_id) + } + + pub fn initial_input(run_id: &str) -> Result, String> { + let connection = get_connection().map_err(|error| error.to_string())?; + load_initial_input_with_connection(&connection, run_id) + } + + pub fn initial_input_for_turn( + turn_intent_id: &str, + ) -> Result, String> { + let connection = get_connection().map_err(|error| error.to_string())?; + load_initial_input_by_turn_with_connection(&connection, turn_intent_id) + } + + pub fn recoverable_initial_inputs(limit: usize) -> Result, String> { + let connection = get_connection().map_err(|error| error.to_string())?; + list_recoverable_initial_inputs_with_connection(&connection, limit) + } + + pub fn load(run_id: &str) -> Result, String> { + load_by_id(run_id).map_err(|error| error.to_string()) + } + + /// Certify one stable member identity after the exact persisted Session + /// row has been read back. A retry of the same receipt is a no-op; a + /// different identity can never satisfy it. + pub fn mark_materialization_succeeded( + run_id: &str, + member_id: &str, + generation: i64, + session_id: &str, + ) -> Result { + with_sessions_writer(|| -> Result { + let mut connection = get_connection().map_err(|error| error.to_string())?; + let transaction = connection + .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate) + .map_err(|error| error.to_string())?; + let receipt: Option<(String, String, String, String)> = transaction + .query_row( + "SELECT materialization.agent_id, materialization.session_id, + run.root_session_id, materialization.status + FROM agent_org_member_materializations materialization + JOIN agent_org_runs run ON run.id=materialization.org_run_id + WHERE materialization.org_run_id=?1 + AND materialization.member_id=?2 + AND materialization.generation=?3 + AND run.status='starting' + AND run.activation_generation=?3", + params![run_id, member_id, generation], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + ) + .optional() + .map_err(|error| error.to_string())?; + let Some((agent_id, expected_session_id, root_session_id, status)) = receipt else { + transaction.commit().map_err(|error| error.to_string())?; + return Ok(false); + }; + if expected_session_id != session_id { + return Err(format!( + "materialization session mismatch for {run_id}/{member_id}: expected {expected_session_id}, got {session_id}" + )); + } + if status == "succeeded" { + transaction.commit().map_err(|error| error.to_string())?; + return Ok(false); + } + let persisted_identity: Option<(Option, Option, Option)> = + transaction + .query_row( + "SELECT agent_definition_id, org_member_id, parent_session_id + FROM agent_sessions WHERE session_id=?1", + [session_id], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .optional() + .map_err(|error| error.to_string())?; + let Some((persisted_agent_id, persisted_member_id, parent_session_id)) = + persisted_identity + else { + return Err(format!( + "materialized Session {session_id} is missing for {run_id}/{member_id}" + )); + }; + let expected_parent = (member_id != COORDINATOR_MEMBER_ID).then_some(root_session_id); + if persisted_agent_id.as_deref() != Some(agent_id.as_str()) + || persisted_member_id.as_deref() != Some(member_id) + || parent_session_id != expected_parent + { + return Err(format!( + "materialized Session identity mismatch for {run_id}/{member_id}" + )); + } + let changed = transaction + .execute( + "UPDATE agent_org_member_materializations + SET status='succeeded', error_code=NULL, error_json=NULL, + updated_at=?5 + WHERE org_run_id=?1 AND member_id=?2 AND generation=?3 + AND session_id=?4 AND status='pending'", + params![ + run_id, + member_id, + generation, + session_id, + chrono::Utc::now().to_rfc3339() + ], + ) + .map_err(|error| error.to_string())?; + transaction.commit().map_err(|error| error.to_string())?; + Ok(changed == 1) + }) + } + + /// Finish Starting only after the stable roster and initial-input + /// certificate are complete. This is the sole Starting completion owner. + pub fn finish_starting( + run_id: &str, + expected_generation: i64, + ) -> Result { + let status = with_sessions_writer(|| -> Result { + let mut connection = get_connection().map_err(|error| error.to_string())?; + let transaction = connection + .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate) + .map_err(|error| error.to_string())?; + let run: Option<(String, String, i64, bool)> = transaction + .query_row( + "SELECT status, root_session_id, activation_generation, has_initial_work + FROM agent_org_runs WHERE id=?1", + [run_id], + |row| { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get::<_, i64>(3)? != 0, + )) + }, + ) + .optional() + .map_err(|error| error.to_string())?; + let Some((status_raw, root_session_id, generation, has_initial_work)) = run else { + return Err(format!("agent_org_run_not_found: {run_id}")); + }; + let current = AgentOrgRunStatus::parse(&status_raw) + .ok_or_else(|| format!("unknown Agent Org run status: {status_raw:?}"))?; + if current != AgentOrgRunStatus::Starting { + transaction.commit().map_err(|error| error.to_string())?; + return Ok(current); + } + if generation != expected_generation { + return Err(format!( + "stale Starting generation for {run_id}: expected {expected_generation}, current {generation}" + )); + } + let invalid_materialized_identities: i64 = transaction + .query_row( + "SELECT COUNT(*) + FROM agent_org_member_materializations materialization + LEFT JOIN agent_sessions session + ON session.session_id=materialization.session_id + WHERE materialization.org_run_id=?1 + AND materialization.generation=?2 + AND materialization.status='succeeded' + AND ( + session.session_id IS NULL + OR session.agent_definition_id IS NULL + OR session.agent_definition_id<>materialization.agent_id + OR session.org_member_id IS NULL + OR session.org_member_id<>materialization.member_id + OR ( + materialization.member_id=?3 + AND ( + materialization.session_id<>?4 + OR session.parent_session_id IS NOT NULL + ) + ) + OR ( + materialization.member_id<>?3 + AND ( + materialization.session_id=?4 + OR session.parent_session_id IS NULL + OR session.parent_session_id<>?4 + ) + ) + )", + params![ + run_id, + expected_generation, + COORDINATOR_MEMBER_ID, + &root_session_id + ], + |row| row.get(0), + ) + .map_err(|error| error.to_string())?; + if invalid_materialized_identities != 0 { + return Err(format!( + "materialization_identity_mismatch: {invalid_materialized_identities} certified Session identity row(s) are invalid for {run_id}" + )); + } + let incomplete_materializations: i64 = transaction + .query_row( + "SELECT COUNT(*) FROM agent_org_member_materializations + WHERE org_run_id=?1 AND generation=?2 AND status<>'succeeded'", + params![run_id, expected_generation], + |row| row.get(0), + ) + .map_err(|error| error.to_string())?; + if incomplete_materializations != 0 { + return Err(format!( + "team_not_materialized: {incomplete_materializations} receipt(s) incomplete for {run_id}" + )); + } + let initial_input = load_initial_input_with_connection(&transaction, run_id)?; + if has_initial_work { + let input = initial_input.as_ref().ok_or_else(|| { + format!("initial input certificate missing for Starting run {run_id}") + })?; + let message_exists: bool = transaction + .query_row( + "SELECT EXISTS(SELECT 1 FROM agent_messages + WHERE id=?1 AND session_id=?2 AND role='user' AND content=?3)", + params![&input.message_id, &root_session_id, &input.content], + |row| row.get(0), + ) + .map_err(|error| error.to_string())?; + let event_id = format!("user-message-{}", input.message_id); + let event_exists: bool = transaction + .query_row( + "SELECT EXISTS(SELECT 1 FROM events WHERE id=?1 AND session_id=?2)", + params![event_id, &root_session_id], + |row| row.get(0), + ) + .map_err(|error| error.to_string())?; + if !message_exists || !event_exists { + return Err(format!( + "initial input is not durably materialized for Starting run {run_id}" + )); + } + crate::foundation::session_bridge::upsert_turn_intent_with_connection( + &transaction, + &root_session_id, + &input.turn_intent_id, + Some(&input.message_id), + Some(run_id), + crate::foundation::session_bridge::TurnIntentBridgeSource::AgentOrg, + crate::foundation::session_bridge::TurnIntentBridgeStatus::Queued, + )?; + transaction + .execute( + "UPDATE agent_org_initial_inputs + SET status='queued', updated_at=?2 + WHERE org_run_id=?1 AND status='pending_persistence'", + params![run_id, chrono::Utc::now().to_rfc3339()], + ) + .map_err(|error| error.to_string())?; + } else if initial_input.is_some() { + return Err(format!( + "unexpected initial input certificate for no-work Starting run {run_id}" + )); + } + + let next = if has_initial_work { + AgentOrgRunStatus::Running + } else { + AgentOrgRunStatus::Idle + }; + let now = chrono::Utc::now().to_rfc3339(); + let changed = transaction + .execute( + "UPDATE agent_org_runs + SET status=?1, updated_at=?2, + idled_at=CASE WHEN ?1='idle' THEN ?2 ELSE NULL END + WHERE id=?3 AND status='starting' + AND activation_generation=?4", + params![next.as_str(), &now, run_id, expected_generation], + ) + .map_err(|error| error.to_string())?; + if changed != 1 { + return Err(format!("Starting transition lost for run {run_id}")); + } + transaction.commit().map_err(|error| error.to_string())?; + Ok(next) + })?; + crate::coordination::agent_org_run_events::notify_agent_org_run_changed(run_id); + Ok(status) + } + + pub fn mark_initial_input_dispatched( + run_id: &str, + turn_intent_id: &str, + ) -> Result { + with_sessions_writer(|| { + let connection = get_connection().map_err(|error| error.to_string())?; + let changed = connection + .execute( + "UPDATE agent_org_initial_inputs + SET status='dispatched', updated_at=?3 + WHERE org_run_id=?1 AND turn_intent_id=?2 + AND status IN ('queued', 'dispatched')", + params![run_id, turn_intent_id, chrono::Utc::now().to_rfc3339()], + ) + .map_err(|error| error.to_string())?; + Ok(changed == 1) + }) + } + + pub fn list_starting_runs(limit: usize) -> Result, String> { + Self::list_runs_by_status(AgentOrgRunStatus::Starting, limit) + } + /// Pause a running run. Only transitions `running → paused`; already /// non-running runs are left unchanged and return `Ok(false)` (idempotent). pub fn mark_paused(run_id: &str) -> Result { @@ -148,42 +613,14 @@ impl AgentOrgRunStore { Ok(changed) } - /// Called once at app startup to pause every org run that was `running` - /// when the previous process exited. The member sessions will have been - /// marked `abandoned` by `mark_stale_running_sessions_abandoned`, but the - /// org run itself should remain accessible and resumable — not auto-terminated - /// by `reconcile_run_finality`. Transitioning to `paused` achieves this: - /// `reconcile_run_finality` is a no-op for non-`running` runs, and the - /// frontend's `TERMINAL_RUN_STATUSES` set excludes `paused`, so the overview - /// panel, member switcher, and task board stay visible. - /// - /// Returns the number of runs transitioned. - pub fn mark_all_running_as_paused_on_startup() -> Result { - let paused = validate_status(AgentOrgRunStatus::Paused.as_str())?; - let running = validate_status(AgentOrgRunStatus::Running.as_str())?; - let now = chrono::Utc::now().to_rfc3339(); - with_sessions_writer(|| -> Result { - let conn = get_connection().map_err(|err| err.to_string())?; - let rows_changed = conn - .execute( - "UPDATE agent_org_runs - SET status = ?1, - updated_at = ?2 - WHERE status = ?3", - params![paused.as_str(), now, running.as_str()], - ) - .map_err(|err| err.to_string())?; - Ok(rows_changed) - }) - } - /// Apply the normal failed-member task disposition after crash recovery - /// has converted stranded Running sessions to Abandoned, but before the - /// parent runs are paused. Tasks with an eligible peer return to the pool; - /// sole-member work stays owned and pending for an explicit retry. + /// has converted stranded Running sessions to Abandoned. Tasks with an + /// eligible peer return to the pool; sole-member work stays owned and + /// pending for an explicit retry. The Team remains Running and is then + /// reconciled from its Quiescence facts. pub fn requeue_abandoned_member_tasks_on_startup() -> Result { let mut changed = 0usize; - for run in Self::list_running_runs(usize::MAX)? { + for run in Self::list_running_runs(100)? { for worker in Self::list_descendant_worker_sessions(&run.id)? { if worker.status != SessionStatus::Abandoned { continue; @@ -198,24 +635,6 @@ impl AgentOrgRunStore { Ok(changed) } - /// Complete already-resolved runs before the generic startup pause sweep. - /// - /// A previous process may have left a run `running` only because an - /// orphaned turn intent incorrectly looked queued. Startup reconciliation - /// closes those intents. Run the canonical atomic finality check for every - /// Running run, including an empty board with an explicit completion - /// intent; only runs that still have blockers fall through to - /// `mark_all_running_as_paused_on_startup`. - pub fn reconcile_resolved_running_runs_on_startup() -> Result { - let mut completed = 0usize; - for run in Self::list_running_runs(usize::MAX)? { - if Self::reconcile_run_finality(&run.id)? == Some(AgentOrgRunStatus::Completed) { - completed += 1; - } - } - Ok(completed) - } - /// Resume a paused run. Only transitions `paused → running`; already /// non-paused runs are left unchanged and return `Ok(false)` (idempotent). pub fn mark_resumed(run_id: &str) -> Result { @@ -242,67 +661,45 @@ impl AgentOrgRunStore { Ok(changed) } - /// Establish the durable fence for a user-requested hierarchy deletion. - /// - /// `paused` remains resumable, so deletion must not use it as the final - /// stop signal. Moving a live run to `cancelled` prevents resume and wake - /// paths from starting new work while the caller drains Rust runtimes. - pub(crate) fn cancel_for_delete_with_connection( - conn: &Connection, + pub fn fail_starting( run_id: &str, + expected_generation: i64, + failure: &AgentOrgStartingFailure, ) -> Result { + let failure_json = serde_json::to_string(failure) + .map_err(|error| format!("failed to serialize Starting failure: {error}"))?; let now = chrono::Utc::now().to_rfc3339(); - let changed = conn - .execute( - "UPDATE agent_org_runs - SET status='cancelled', - updated_at=?2, - completed_at=COALESCE(completed_at, ?2) - WHERE id=?1 - AND status IN ('running', 'paused')", - params![run_id, &now], - ) - .map_err(|err| err.to_string())? - > 0; - conn.execute( - "UPDATE agent_org_plan_approvals - SET status='cancelled', decision_by='system', resolved_at=?2 - WHERE org_run_id=?1 AND status='pending'", - params![run_id, &now], - ) - .map_err(|err| err.to_string())?; - Ok(changed) - } - - pub fn mark_failed(run_id: &str, error_message: &str) -> Result<(), String> { - let status = validate_status(AgentOrgRunStatus::Failed.as_str())?; - let now = chrono::Utc::now().to_rfc3339(); - with_sessions_writer(|| -> Result<(), String> { + let changed = with_sessions_writer(|| -> Result { let mut conn = get_connection().map_err(|err| err.to_string())?; let tx = conn .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate) .map_err(|err| err.to_string())?; - tx.execute( - "UPDATE agent_org_runs - SET status = ?1, + let changed = tx + .execute( + "UPDATE agent_org_runs + SET status = 'failed', last_error = ?2, - updated_at = ?3, - completed_at = ?3 - WHERE id = ?4", - params![status.as_str(), error_message, now, run_id], - ) - .map_err(|err| err.to_string())?; - tx.execute( - "UPDATE agent_org_plan_approvals - SET status='cancelled', decision_by='system', resolved_at=?2 - WHERE org_run_id=?1 AND status='pending'", - params![run_id, &now], - ) - .map_err(|err| err.to_string())?; - tx.commit().map_err(|err| err.to_string()) + failure_json = ?3, + last_activity_outcome = 'failed', + updated_at = ?4 + WHERE id = ?1 AND status='starting' + AND activation_generation=?5", + params![ + run_id, + &failure.message, + &failure_json, + &now, + expected_generation + ], + ) + .map_err(|err| err.to_string())?; + tx.commit().map_err(|err| err.to_string())?; + Ok(changed == 1) })?; - crate::coordination::agent_org_run_events::notify_agent_org_run_changed(run_id); - Ok(()) + if changed { + crate::coordination::agent_org_run_events::notify_agent_org_run_changed(run_id); + } + Ok(changed) } pub fn progress(run_id: &str) -> Result, String> { @@ -363,7 +760,7 @@ impl AgentOrgRunStore { } /// Persist a coordinator-only completion request without forcing the run - /// terminal. Finality still waits for delivery, approvals, interventions, + /// terminal. Quiescence still waits for delivery, approvals, interventions, /// sessions, and work-observation invariants to become safe. pub fn request_completion( run_id: &str, @@ -423,7 +820,7 @@ impl AgentOrgRunStore { Ok(outcome) } - pub fn assess_run_finality(run_id: &str) -> Result { + pub fn assess_run_quiescence(run_id: &str) -> Result { let mut conn = get_connection().map_err(|err| err.to_string())?; let tx = conn .transaction_with_behavior(rusqlite::TransactionBehavior::Deferred) @@ -433,72 +830,70 @@ impl AgentOrgRunStore { Ok(assessment) } - pub fn reconcile_run_finality(run_id: &str) -> Result, String> { - // Finality and every task mutation share the sessions writer lock. The - // canonical typed facts are re-read inside this IMMEDIATE transaction; - // no analyzer snapshot is trusted across the lock boundary. - let (status, changed) = - with_sessions_writer(|| -> Result<(Option, bool), String> { - let mut conn = get_connection().map_err(|err| err.to_string())?; - let tx = conn - .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate) - .map_err(|err| err.to_string())?; - let assessment = load_and_assess(&tx, run_id)?; - let Some(current_status) = assessment.facts.run_status else { - tx.commit().map_err(|err| err.to_string())?; - return Ok((None, false)); - }; - let next_status = match assessment.decision { - super::AgentOrgFinalityDecision::Complete => AgentOrgRunStatus::Completed, - super::AgentOrgFinalityDecision::Abandon => AgentOrgRunStatus::Abandoned, - super::AgentOrgFinalityDecision::KeepRunning => { - tx.commit().map_err(|err| err.to_string())?; - return Ok((Some(current_status), false)); - } - }; - let now = chrono::Utc::now().to_rfc3339(); - let completion_summary = assessment + /// Atomically commit the only automatic lifecycle transition owned by + /// PR 1. Both snapshot certificates are required so a stale finalizer or + /// watchdog pass cannot idle a newer activation or newer work graph. + pub fn try_transition_working_to_idle( + run_id: &str, + expected_generation: i64, + expected_work_revision: i64, + ) -> Result { + let changed = with_sessions_writer(|| -> Result { + let mut conn = get_connection().map_err(|err| err.to_string())?; + let tx = conn + .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate) + .map_err(|err| err.to_string())?; + let assessment = load_and_assess(&tx, run_id)?; + if assessment.facts.run_status != Some(AgentOrgRunStatus::Running) + || assessment.facts.activation_generation != Some(expected_generation) + || assessment .facts .progress .as_ref() - .and_then(|progress| progress.completion_summary.as_deref()); - let changed = tx - .execute( - "UPDATE agent_org_runs - SET status=?1, - summary=COALESCE(?2, summary), - updated_at=?3, - completed_at=?3 - WHERE id=?4 AND status=?5", - params![ - next_status.as_str(), - completion_summary, - &now, - run_id, - AgentOrgRunStatus::Running.as_str(), - ], - ) - .map_err(|err| err.to_string())?; - if changed != 1 { - tx.commit().map_err(|err| err.to_string())?; - return Ok((Self::get_run_status(run_id)?, false)); - } - // Terminal status and cancellation of an otherwise stranded plan - // approval are one atomic state transition. - tx.execute( - "UPDATE agent_org_plan_approvals - SET status='cancelled', decision_by='system', resolved_at=?2 - WHERE org_run_id=?1 AND status='pending'", - params![run_id, &now], + .map(|progress| progress.work_revision) + != Some(expected_work_revision) + || assessment.decision != super::AgentOrgQuiescenceDecision::Quiescent + { + tx.commit().map_err(|err| err.to_string())?; + return Ok(false); + } + let now = chrono::Utc::now().to_rfc3339(); + let completion_summary = assessment + .facts + .progress + .as_ref() + .and_then(|progress| progress.completion_summary.as_deref()); + let changed = tx + .execute( + "UPDATE agent_org_runs + SET status='idle', + summary=COALESCE(?1, summary), + last_activity_outcome='completed', + updated_at=?2, + idled_at=?2 + WHERE id=?3 AND status='running' + AND activation_generation=?4 + AND EXISTS ( + SELECT 1 FROM agent_org_run_progress progress + WHERE progress.org_run_id=agent_org_runs.id + AND progress.work_revision=?5 + )", + params![ + completion_summary, + &now, + run_id, + expected_generation, + expected_work_revision, + ], ) .map_err(|err| err.to_string())?; - tx.commit().map_err(|err| err.to_string())?; - Ok((Some(next_status), true)) - })?; + tx.commit().map_err(|err| err.to_string())?; + Ok(changed == 1) + })?; if changed { crate::coordination::agent_org_run_events::notify_agent_org_run_changed(run_id); } - Ok(status) + Ok(changed) } /// Resolve the org-run context for an arbitrary session — works for @@ -617,14 +1012,18 @@ impl AgentOrgRunStore { org_snapshot_json, entry_mode, status, + activation_generation, + has_initial_work, work_item_id, project_slug, routine_fire_id, summary, last_error, + failure_json, + last_activity_outcome, created_at, updated_at, - completed_at + idled_at FROM agent_org_runs WHERE root_session_id IS NOT NULL ORDER BY updated_at DESC @@ -641,11 +1040,21 @@ impl AgentOrgRunStore { Ok(out) } - /// List runs currently in `running` status, newest-updated first. - /// SQL-side status filter avoids loading terminal runs. Callers that must - /// inspect every running run (the watchdog) pass `usize::MAX`, which is - /// safely clamped to SQLite's `i64` limit. + /// List runs currently in `running` status, oldest-updated first. Periodic + /// callers must pass their explicit bounded batch size. pub fn list_running_runs(limit: usize) -> Result, String> { + Self::list_runs_by_status(AgentOrgRunStatus::Running, limit) + } + + fn list_runs_by_status( + status: AgentOrgRunStatus, + limit: usize, + ) -> Result, String> { + if limit == 0 { + return Ok(Vec::new()); + } + let bounded_limit = i64::try_from(limit) + .map_err(|_| format!("Agent Org run list limit is too large: {limit}"))?; let conn = get_connection().map_err(|err| err.to_string())?; let mut stmt = conn .prepare( @@ -656,29 +1065,27 @@ impl AgentOrgRunStore { org_snapshot_json, entry_mode, status, + activation_generation, + has_initial_work, work_item_id, project_slug, routine_fire_id, summary, last_error, + failure_json, + last_activity_outcome, created_at, updated_at, - completed_at + idled_at FROM agent_org_runs WHERE root_session_id IS NOT NULL AND status = ?1 - ORDER BY updated_at DESC + ORDER BY updated_at ASC, id ASC LIMIT ?2", ) .map_err(|err| err.to_string())?; let rows = stmt - .query_map( - params![ - AgentOrgRunStatus::Running.as_str(), - i64::try_from(limit).unwrap_or(i64::MAX) - ], - row_to_run, - ) + .query_map(params![status.as_str(), bounded_limit], row_to_run) .map_err(|err| err.to_string())?; let mut out = Vec::new(); for row in rows { @@ -708,14 +1115,14 @@ impl AgentOrgRunStore { Ok(status_raw.as_deref().and_then(AgentOrgRunStatus::parse)) } - /// Read the canonical finality facts and decision from an existing + /// Read the canonical quiescence facts and decision from an existing /// connection or read transaction. Run View and task-list projections use /// this to keep all of their independently-shaped rows on one SQLite /// snapshot instead of opening a fresh connection for each block. - pub(crate) fn finality_assessment_with_connection( + pub(crate) fn quiescence_assessment_with_connection( conn: &Connection, run_id: &str, - ) -> Result { + ) -> Result { load_and_assess(conn, run_id) } diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/tests.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/tests.rs index def031c102..1071dc5bc5 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/tests.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/tests.rs @@ -13,11 +13,72 @@ fn enum_values_round_trip() { AgentOrgRunEntryMode::parse(AgentOrgRunEntryMode::StandaloneSession.as_str()), Some(AgentOrgRunEntryMode::StandaloneSession) ); - assert_eq!( - AgentOrgRunStatus::parse(AgentOrgRunStatus::Running.as_str()), - Some(AgentOrgRunStatus::Running) - ); - assert_eq!(AgentOrgRunStatus::parse("idle"), None); + for status in [ + AgentOrgRunStatus::Starting, + AgentOrgRunStatus::Running, + AgentOrgRunStatus::Paused, + AgentOrgRunStatus::Idle, + AgentOrgRunStatus::Failed, + AgentOrgRunStatus::Archived, + ] { + assert_eq!(AgentOrgRunStatus::parse(status.as_str()), Some(status)); + } + for retired in ["completed", "cancelled", "abandoned", "unknown"] { + assert_eq!(AgentOrgRunStatus::parse(retired), None); + } +} + +#[test] +fn canonical_schema_snapshot_contains_only_the_long_lived_run_states() { + let _sandbox = test_helpers::test_env::sandbox(); + ensure_runtime_schemas(); + let conn = database::db::get_connection().expect("test sqlite connection"); + let run_ddl: String = conn + .query_row( + "SELECT sql FROM sqlite_master + WHERE type='table' AND name='agent_org_runs'", + [], + |row| row.get(0), + ) + .expect("canonical Agent Org run DDL"); + let status_ddl = run_ddl + .split_once("status TEXT") + .and_then(|(_, tail)| tail.split_once("activation_generation")) + .map(|(status_ddl, _)| status_ddl) + .expect("isolated run-status CHECK"); + for status in [ + "starting", "running", "paused", "idle", "failed", "archived", + ] { + assert!( + status_ddl.contains(&format!("'{status}'")), + "DDL: {status_ddl}" + ); + } + for retired in ["'abandoned'", "'completed'", "'cancelled'"] { + assert!(!status_ddl.contains(retired), "DDL: {status_ddl}"); + } + + let materialization_ddl: String = conn + .query_row( + "SELECT sql FROM sqlite_master + WHERE type='table' AND name='agent_org_member_materializations'", + [], + |row| row.get(0), + ) + .expect("canonical materialization receipt DDL"); + assert!(materialization_ddl.contains("PRIMARY KEY(org_run_id, member_id, generation)")); + assert!(materialization_ddl.contains("UNIQUE(org_run_id, session_id)")); + + let initial_input_ddl: String = conn + .query_row( + "SELECT sql FROM sqlite_master + WHERE type='table' AND name='agent_org_initial_inputs'", + [], + |row| row.get(0), + ) + .expect("canonical initial-input DDL"); + assert!(initial_input_ddl.contains("UNIQUE(turn_intent_id)")); + assert!(initial_input_ddl.contains("UNIQUE(message_id)")); } /// Build an `AgentOrgsStore` pre-loaded with a single org definition. @@ -50,6 +111,35 @@ fn sample_org() -> OrgDefinition { } } +fn test_upsert_turn_intent_with_connection( + conn: &rusqlite::Connection, + session_id: &str, + turn_intent_id: &str, + client_message_id: Option<&str>, + org_run_id: Option<&str>, + source: crate::foundation::session_bridge::TurnIntentBridgeSource, + status: crate::foundation::session_bridge::TurnIntentBridgeStatus, +) -> Result<(), String> { + let now = chrono::Utc::now().to_rfc3339(); + conn.execute( + "INSERT OR IGNORE INTO session_turn_intents ( + session_id, turn_intent_id, client_message_id, org_run_id, + source, status, created_at, updated_at + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?7)", + params![ + session_id, + turn_intent_id, + client_message_id, + org_run_id, + source.as_str(), + status.as_str(), + now, + ], + ) + .map_err(|error| error.to_string())?; + Ok(()) +} + fn ensure_runtime_schemas() { let conn = database::db::get_connection().expect("test sqlite connection"); crate::foundation::persistence::test_schema::ensure_agent_sessions_schema(&conn); @@ -76,9 +166,16 @@ fn ensure_runtime_schemas() { created_at TEXT NOT NULL, updated_at TEXT NOT NULL, PRIMARY KEY (session_id, turn_intent_id) + ); + CREATE TABLE IF NOT EXISTS events ( + id TEXT PRIMARY KEY, + session_id TEXT NOT NULL );", ) .expect("cli session schema"); + crate::foundation::session_bridge::register_upsert_turn_intent_with_connection( + test_upsert_turn_intent_with_connection, + ); } fn create_run_for_root(org: &OrgDefinition, root_session_id: &str) -> AgentOrgRunRecord { @@ -97,6 +194,237 @@ fn create_run_for_root(org: &OrgDefinition, root_session_id: &str) -> AgentOrgRu .expect("create run") } +/// Exercise the production two-step protocol used by lifecycle owners: read a +/// pure quiescence certificate, then present its exact generation and work +/// revision to the atomic CAS transition. Keeping this helper in tests makes +/// old completion scenarios validate the new protocol instead of recreating the +/// removed one-shot reconciler. +fn reconcile_run_to_idle_for_test(run_id: &str) -> Result { + let assessment = AgentOrgRunStore::assess_run_quiescence(run_id)?; + if assessment.decision == AgentOrgQuiescenceDecision::Quiescent { + let generation = assessment + .facts + .activation_generation + .ok_or_else(|| "missing activation generation".to_string())?; + let work_revision = assessment + .facts + .progress + .as_ref() + .map(|progress| progress.work_revision) + .ok_or_else(|| "missing work revision".to_string())?; + AgentOrgRunStore::try_transition_working_to_idle(run_id, generation, work_revision)?; + } + load_by_id(run_id) + .map_err(|err| err.to_string())? + .map(|run| run.status) + .ok_or_else(|| format!("agent_org_run_not_found: {run_id}")) +} + +fn create_starting_fixture(has_initial_work: bool) -> AgentOrgRunRecord { + ensure_runtime_schemas(); + let org = sample_org(); + upsert_session_row_for_member( + "starting-root", + None, + Some("agent-coord"), + Some(COORDINATOR_MEMBER_ID), + SessionStatus::Idle.as_str(), + ); + AgentOrgRunStore::create_starting(CreateStartingAgentOrgRunParams { + org_id: org.id.clone(), + coordinator_agent_id: org.agent_id.clone(), + root_session_id: "starting-root".to_string(), + org_snapshot: org, + entry_mode: AgentOrgRunEntryMode::StandaloneSession, + work_item_id: None, + project_slug: None, + routine_fire_id: None, + materialization_intents: vec![ + CreateAgentOrgMaterializationIntent { + member_id: COORDINATOR_MEMBER_ID.to_string(), + agent_id: "agent-coord".to_string(), + session_id: "starting-root".to_string(), + succeeded: true, + }, + CreateAgentOrgMaterializationIntent { + member_id: "member-w1".to_string(), + agent_id: "agent-w1".to_string(), + session_id: "starting-member-w1".to_string(), + succeeded: false, + }, + ], + initial_input: has_initial_work.then(|| CreateAgentOrgInitialInput { + turn_intent_id: "starting-turn".to_string(), + message_id: "starting-message".to_string(), + content: "Start the work".to_string(), + payload_json: serde_json::json!({ + "version": 1, + "images": ["image-a"], + "ideContext": null, + "subAgentIds": [], + }) + .to_string(), + }), + }) + .expect("create Starting fixture") +} + +#[test] +fn starting_creation_commits_exact_roster_and_initial_input_receipts() { + let _sandbox = test_helpers::test_env::sandbox(); + let run = create_starting_fixture(true); + + assert_eq!(run.status, AgentOrgRunStatus::Starting); + assert_eq!(run.activation_generation, 1); + assert!(run.has_initial_work); + let receipts = AgentOrgRunStore::materializations(&run.id).expect("load receipts"); + assert_eq!(receipts.len(), 2); + assert_eq!(receipts[0].session_id, "starting-root"); + assert_eq!(receipts[0].status, AgentOrgMaterializationStatus::Succeeded); + assert_eq!(receipts[1].session_id, "starting-member-w1"); + assert_eq!(receipts[1].status, AgentOrgMaterializationStatus::Pending); + let input = AgentOrgRunStore::initial_input(&run.id) + .expect("load initial input") + .expect("initial input exists"); + assert_eq!(input.turn_intent_id, "starting-turn"); + assert!(input.payload_json.contains("image-a")); +} + +#[test] +fn starting_finish_requires_exact_member_and_input_durability_then_is_idempotent() { + let _sandbox = test_helpers::test_env::sandbox(); + let run = create_starting_fixture(true); + assert!(AgentOrgRunStore::finish_starting(&run.id, 1) + .expect_err("pending member must block Starting") + .contains("receipt(s) incomplete")); + + upsert_session_row_for_member( + "starting-member-w1", + Some("starting-root"), + Some("agent-w1"), + Some("member-w1"), + SessionStatus::Idle.as_str(), + ); + assert!(AgentOrgRunStore::mark_materialization_succeeded( + &run.id, + "member-w1", + 1, + "starting-member-w1", + ) + .expect("certify stable member")); + assert!(!AgentOrgRunStore::mark_materialization_succeeded( + &run.id, + "member-w1", + 1, + "starting-member-w1", + ) + .expect("retry same receipt")); + assert!(AgentOrgRunStore::finish_starting(&run.id, 1) + .expect_err("missing initial EventStore row must block Starting") + .contains("not durably materialized")); + + crate::session::persistence::save_user_msg_with_id( + "starting-message", + "starting-root", + "Start the work", + ) + .expect("persist transcript input"); + database::db::get_connection() + .expect("db") + .execute( + "INSERT INTO events (id, session_id) VALUES (?1, ?2)", + params!["user-message-starting-message", "starting-root"], + ) + .expect("persist EventStore proof"); + + assert_eq!( + AgentOrgRunStore::finish_starting(&run.id, 1).expect("finish Starting"), + AgentOrgRunStatus::Running + ); + assert_eq!( + AgentOrgRunStore::finish_starting(&run.id, 1).expect("idempotent finish"), + AgentOrgRunStatus::Running + ); + let conn = database::db::get_connection().expect("db"); + let member_count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM agent_sessions WHERE session_id='starting-member-w1'", + [], + |row| row.get(0), + ) + .expect("count stable member identity"); + assert_eq!(member_count, 1); + let turn_status: String = conn + .query_row( + "SELECT status FROM session_turn_intents + WHERE session_id='starting-root' AND turn_intent_id='starting-turn'", + [], + |row| row.get(0), + ) + .expect("load durable initial Turn Intent"); + assert_eq!(turn_status, "queued"); +} + +#[test] +fn starting_without_initial_work_finishes_idle() { + let _sandbox = test_helpers::test_env::sandbox(); + let run = create_starting_fixture(false); + upsert_session_row_for_member( + "starting-member-w1", + Some("starting-root"), + Some("agent-w1"), + Some("member-w1"), + SessionStatus::Idle.as_str(), + ); + AgentOrgRunStore::mark_materialization_succeeded(&run.id, "member-w1", 1, "starting-member-w1") + .expect("certify stable member"); + + assert_eq!( + AgentOrgRunStore::finish_starting(&run.id, 1).expect("finish no-work Starting"), + AgentOrgRunStatus::Idle + ); + assert!(load_by_id(&run.id) + .expect("load run") + .expect("run exists") + .idled_at + .is_some()); +} + +#[test] +fn starting_finish_revalidates_every_certified_session_identity() { + let _sandbox = test_helpers::test_env::sandbox(); + let run = create_starting_fixture(false); + upsert_session_row_for_member( + "starting-member-w1", + Some("starting-root"), + Some("agent-w1"), + Some("member-w1"), + SessionStatus::Idle.as_str(), + ); + AgentOrgRunStore::mark_materialization_succeeded(&run.id, "member-w1", 1, "starting-member-w1") + .expect("certify stable member"); + database::db::get_connection() + .expect("db") + .execute( + "UPDATE agent_sessions + SET parent_session_id='wrong-root' + WHERE session_id='starting-member-w1'", + [], + ) + .expect("corrupt certified identity after receipt"); + + let error = AgentOrgRunStore::finish_starting(&run.id, 1) + .expect_err("a stale receipt must not authorize Starting completion"); + assert!(error.starts_with("materialization_identity_mismatch:")); + assert_eq!( + AgentOrgRunStore::load(&run.id) + .expect("load Starting run") + .expect("run exists") + .status, + AgentOrgRunStatus::Starting + ); +} + #[test] fn delete_by_id_cascades_all_run_owned_state_and_plan_artifact() { let sandbox = test_helpers::test_env::sandbox(); @@ -253,7 +581,7 @@ fn delete_by_id_cascades_all_run_owned_state_and_plan_artifact() { } #[test] -fn delete_by_id_preserves_nested_run_intents_and_finality_isolation() { +fn delete_by_id_preserves_nested_run_intents_and_quiescence_isolation() { let _sandbox = test_helpers::test_env::sandbox(); let org = sample_org(); let outer = create_run_for_root(&org, "outer-root"); @@ -301,15 +629,15 @@ fn delete_by_id_preserves_nested_run_intents_and_finality_isolation() { .expect("seed independently owned intents"); let outer_assessment = - AgentOrgRunStore::assess_run_finality(&outer.id).expect("assess outer run finality"); + AgentOrgRunStore::assess_run_quiescence(&outer.id).expect("assess outer run quiescence"); assert_eq!( outer_assessment.facts.in_flight_turn_intent_count, 1, - "nested run work must not block outer run finality" + "nested run work must not block outer run quiescence" ); assert_eq!(outer_assessment.facts.worker_sessions.len(), 1); assert_eq!( outer_assessment.facts.worker_sessions[0].session_id, "outer-worker", - "a Running worker owned by a nested run must not block outer finality" + "a Running worker owned by a nested run must not block outer quiescence" ); AgentOrgRunStore::delete_by_id(&outer.id).expect("delete outer run"); @@ -361,7 +689,7 @@ fn recursive_session_queries_terminate_on_parent_cycle() { descendants.len() <= 3, "cycle must not duplicate descendants" ); - AgentOrgRunStore::assess_run_finality(&run.id).expect("cyclic finality scan terminates"); + AgentOrgRunStore::assess_run_quiescence(&run.id).expect("cyclic quiescence scan terminates"); let conn = database::db::get_connection().expect("test sqlite connection"); let now = chrono::Utc::now().to_rfc3339(); @@ -756,7 +1084,7 @@ fn find_worker_session_by_member_id_picks_most_recent_when_multi_instance() { } #[test] -fn cross_transport_duplicate_member_uses_fresh_rust_session_and_does_not_block_finality() { +fn cross_transport_duplicate_member_uses_fresh_rust_session_and_does_not_block_quiescence() { use crate::coordination::agent_org_tasks::{AgentOrgTaskStore, CreateTaskParams, TaskStatus}; let _sandbox = test_helpers::test_env::sandbox(); @@ -837,16 +1165,16 @@ fn cross_transport_duplicate_member_uses_fresh_rust_session_and_does_not_block_f mark_coordinator_observed_current_work(&run.id); stamp_coordinator_terminal_turn("coord-root-cross-transport"); - let assessment = AgentOrgRunStore::assess_run_finality(&run.id).expect("assess finality"); + let assessment = AgentOrgRunStore::assess_run_quiescence(&run.id).expect("assess quiescence"); assert_eq!(assessment.facts.worker_sessions.len(), 1); assert_eq!( assessment.facts.worker_sessions[0].session_id, "rust-worker-current" ); - assert_eq!(assessment.decision, AgentOrgFinalityDecision::Complete); + assert_eq!(assessment.decision, AgentOrgQuiescenceDecision::Quiescent); assert_eq!( - AgentOrgRunStore::reconcile_run_finality(&run.id).expect("reconcile finality"), - Some(AgentOrgRunStatus::Completed), + reconcile_run_to_idle_for_test(&run.id).expect("transition to idle"), + AgentOrgRunStatus::Idle, "the stale CLI Running row must not keep the run falsely active" ); } @@ -923,7 +1251,7 @@ fn coordinator_observation_records_only_the_exact_presented_revision() { } #[test] -fn reconcile_run_finality_completes_run_when_all_tasks_completed() { +fn quiescence_transitions_run_to_idle_when_all_tasks_completed() { use crate::coordination::agent_org_tasks::{AgentOrgTaskStore, CreateTaskParams, TaskStatus}; let _sandbox = test_helpers::test_env::sandbox(); @@ -985,8 +1313,8 @@ fn reconcile_run_finality_completes_run_when_all_tasks_completed() { ) .expect("advance pending turn intent"); assert_eq!( - AgentOrgRunStore::reconcile_run_finality(&run.id).expect("reconcile pending intent"), - Some(AgentOrgRunStatus::Running), + reconcile_run_to_idle_for_test(&run.id).expect("reconcile pending intent"), + AgentOrgRunStatus::Running, "a {pending_status} turn intent must keep the run open" ); } @@ -1004,12 +1332,12 @@ fn reconcile_run_finality_completes_run_when_all_tasks_completed() { ) .expect("set terminal turn intent"); assert_eq!( - AgentOrgRunStore::reconcile_run_finality(&run.id).expect("reconcile terminal intent"), - Some(AgentOrgRunStatus::Completed), + reconcile_run_to_idle_for_test(&run.id).expect("reconcile terminal intent"), + AgentOrgRunStatus::Idle, "a {terminal_status} turn intent must not keep the run open" ); conn.execute( - "UPDATE agent_org_runs SET status='running', completed_at=NULL WHERE id=?1", + "UPDATE agent_org_runs SET status='running', idled_at=NULL WHERE id=?1", params![&run.id], ) .expect("reset run for next terminal status"); @@ -1037,13 +1365,12 @@ fn reconcile_run_finality_completes_run_when_all_tasks_completed() { 1 ); assert_eq!( - AgentOrgRunStore::reconcile_resolved_running_runs_on_startup() - .expect("startup reconcile ok"), - 1 + reconcile_run_to_idle_for_test(&run.id).expect("explicit lifecycle reconcile ok"), + AgentOrgRunStatus::Idle ); let reloaded = load_by_id(&run.id).expect("load run").expect("run exists"); - assert_eq!(reloaded.status, AgentOrgRunStatus::Completed); - assert!(reloaded.completed_at.is_some()); + assert_eq!(reloaded.status, AgentOrgRunStatus::Idle); + assert!(reloaded.idled_at.is_some()); let legacy_cleared_at: Option = conn .query_row( "SELECT cleared_at FROM agent_member_interventions @@ -1056,7 +1383,7 @@ fn reconcile_run_finality_completes_run_when_all_tasks_completed() { } #[test] -fn reconcile_completes_normal_idle_run_only_after_inbox_is_drained() { +fn quiescence_idles_run_only_after_inbox_is_drained() { use crate::coordination::agent_inbox::{ AgentInboxStore, AgentMessage, InsertInboxParams, SYSTEM_SENDER_ID, }; @@ -1113,20 +1440,20 @@ fn reconcile_completes_normal_idle_run_only_after_inbox_is_drained() { stamp_coordinator_terminal_turn("coord-root-idle-complete"); assert_eq!( - AgentOrgRunStore::reconcile_run_finality(&run.id).unwrap(), - Some(AgentOrgRunStatus::Running), - "unread completion facts must be delivered before finality" + reconcile_run_to_idle_for_test(&run.id).unwrap(), + AgentOrgRunStatus::Running, + "unread completion facts must be delivered before quiescence" ); AgentInboxStore::mark_many_read(&[row.id]).unwrap(); assert_eq!( - AgentOrgRunStore::reconcile_run_finality(&run.id).unwrap(), - Some(AgentOrgRunStatus::Completed), - "normal successful members settle to Idle and must still allow run completion" + reconcile_run_to_idle_for_test(&run.id).unwrap(), + AgentOrgRunStatus::Idle, + "normal successful members settle to Idle and must allow the Team to become Idle" ); } #[test] -fn resolved_undeliverable_inbox_stays_unread_but_no_longer_blocks_finality() { +fn resolved_undeliverable_inbox_stays_unread_but_no_longer_blocks_quiescence() { use crate::coordination::agent_inbox::{ AgentInboxDeliveryResolutionKind, AgentInboxStore, AgentMessage, ResolveInboxDeliveryParams, }; @@ -1188,9 +1515,9 @@ fn resolved_undeliverable_inbox_stays_unread_but_no_longer_blocks_finality() { .expect("seed historical orphan row"); let inbox_id = conn.last_insert_rowid(); - let before = AgentOrgRunStore::assess_run_finality(&run.id).expect("assess before repair"); + let before = AgentOrgRunStore::assess_run_quiescence(&run.id).expect("assess before repair"); assert_eq!(before.facts.unread_inbox_count, 1); - assert_eq!(before.decision, AgentOrgFinalityDecision::KeepRunning); + assert_eq!(before.decision, AgentOrgQuiescenceDecision::KeepWorking); AgentInboxStore::resolve_delivery(ResolveInboxDeliveryParams { inbox_id, @@ -1203,12 +1530,12 @@ fn resolved_undeliverable_inbox_stays_unread_but_no_longer_blocks_finality() { }) .expect("resolve undeliverable delivery"); - let after = AgentOrgRunStore::assess_run_finality(&run.id).expect("assess after repair"); + let after = AgentOrgRunStore::assess_run_quiescence(&run.id).expect("assess after repair"); assert_eq!(after.facts.unread_inbox_count, 0); - assert_eq!(after.decision, AgentOrgFinalityDecision::Complete); + assert_eq!(after.decision, AgentOrgQuiescenceDecision::Quiescent); assert_eq!( - AgentOrgRunStore::reconcile_run_finality(&run.id).expect("reconcile repaired run"), - Some(AgentOrgRunStatus::Completed) + reconcile_run_to_idle_for_test(&run.id).expect("transition repaired run"), + AgentOrgRunStatus::Idle ); let evidence = AgentInboxStore::get_by_id_for_run(&run.id, inbox_id) .unwrap() @@ -1220,7 +1547,7 @@ fn resolved_undeliverable_inbox_stays_unread_but_no_longer_blocks_finality() { } #[test] -fn startup_reconcile_completes_empty_board_with_explicit_completion_intent() { +fn explicit_lifecycle_reconcile_idles_empty_board_with_completion_intent() { let _sandbox = test_helpers::test_env::sandbox(); let org = sample_org(); let run = create_run_for_root(&org, "coord-root-empty-complete"); @@ -1235,21 +1562,20 @@ fn startup_reconcile_completes_empty_board_with_explicit_completion_intent() { .expect("record explicit empty-board completion intent"); assert_eq!( - AgentOrgRunStore::reconcile_resolved_running_runs_on_startup() - .expect("startup reconcile empty board"), - 1 + reconcile_run_to_idle_for_test(&run.id).expect("reconcile empty board"), + AgentOrgRunStatus::Idle ); assert_eq!( load_by_id(&run.id) .expect("load run") .expect("run exists") .status, - AgentOrgRunStatus::Completed + AgentOrgRunStatus::Idle ); } #[test] -fn reconcile_run_finality_abandons_run_with_open_work_only_after_all_sessions_archived() { +fn archived_sessions_with_open_work_do_not_auto_archive_the_run() { use crate::coordination::agent_org_tasks::{AgentOrgTaskStore, CreateTaskParams, TaskStatus}; let _sandbox = test_helpers::test_env::sandbox(); @@ -1313,11 +1639,11 @@ fn reconcile_run_finality_abandons_run_with_open_work_only_after_all_sessions_ar }) .expect("create open task"); - let status = AgentOrgRunStore::reconcile_run_finality(&run.id).expect("reconcile ok"); - assert_eq!(status, Some(AgentOrgRunStatus::Abandoned)); + let status = reconcile_run_to_idle_for_test(&run.id).expect("reconcile ok"); + assert_eq!(status, AgentOrgRunStatus::Running); let reloaded = load_by_id(&run.id).expect("load run").expect("run exists"); - assert_eq!(reloaded.status, AgentOrgRunStatus::Abandoned); - assert!(reloaded.completed_at.is_some()); + assert_eq!(reloaded.status, AgentOrgRunStatus::Running); + assert!(reloaded.idled_at.is_none()); } #[test] @@ -1358,32 +1684,32 @@ fn failed_or_cancelled_sessions_do_not_abandon_recoverable_open_work() { .expect("create recoverable task"); assert_eq!( - AgentOrgRunStore::reconcile_run_finality(&run.id).expect("reconcile"), - Some(AgentOrgRunStatus::Running) + reconcile_run_to_idle_for_test(&run.id).expect("reconcile"), + AgentOrgRunStatus::Running ); } #[test] -fn reconcile_and_task_create_have_one_serializable_outcome() { +fn idle_cas_and_task_create_have_one_serializable_outcome() { use std::sync::{Arc, Barrier}; use crate::coordination::agent_org_tasks::{AgentOrgTaskStore, CreateTaskParams, TaskStatus}; let _sandbox = test_helpers::test_env::sandbox(); let org = sample_org(); - let run = create_run_for_root(&org, "coord-root-finality-race"); + let run = create_run_for_root(&org, "coord-root-quiescence-race"); upsert_session_row_full( - "coord-root-finality-race", + "coord-root-quiescence-race", None, Some("agent-coord"), SessionStatus::Completed.as_str(), ); upsert_session(&UnifiedSessionRecord { - session_id: "worker-finality-race".to_string(), + session_id: "worker-quiescence-race".to_string(), name: "worker".to_string(), status: SessionStatus::Completed.as_str().to_string(), session_type: crate::core::session::persistence::session_type::ORG_MEMBER.to_string(), - parent_session_id: Some("coord-root-finality-race".to_string()), + parent_session_id: Some("coord-root-quiescence-race".to_string()), agent_definition_id: Some("agent-w1".to_string()), org_member_id: Some("member-w1".to_string()), created_at: chrono::Utc::now().to_rfc3339(), @@ -1405,14 +1731,14 @@ fn reconcile_and_task_create_have_one_serializable_outcome() { }) .unwrap(); mark_coordinator_observed_current_work(&run.id); - stamp_coordinator_terminal_turn("coord-root-finality-race"); + stamp_coordinator_terminal_turn("coord-root-quiescence-race"); let barrier = Arc::new(Barrier::new(2)); let reconcile_barrier = Arc::clone(&barrier); let reconcile_run_id = run.id.clone(); let reconcile = std::thread::spawn(move || { reconcile_barrier.wait(); - AgentOrgRunStore::reconcile_run_finality(&reconcile_run_id) + reconcile_run_to_idle_for_test(&reconcile_run_id) }); let create_barrier = Arc::clone(&barrier); let create_run_id = run.id.clone(); @@ -1435,17 +1761,17 @@ fn reconcile_and_task_create_have_one_serializable_outcome() { }) }); - let status = reconcile.join().unwrap().unwrap().unwrap(); + let status = reconcile.join().unwrap().unwrap(); let created = create.join().unwrap(); match (status, created) { - (AgentOrgRunStatus::Completed, Err(error)) => { + (AgentOrgRunStatus::Idle, Err(error)) => { assert!(error.contains("agent_org_run_not_mutable"), "got {error}"); } // The create committed first. Reconcile then sees recoverable open // work and correctly leaves the Run Running; this is the other valid // serial order. Abandoning here would lose a newly-created task. (AgentOrgRunStatus::Running, Ok(task)) => assert_eq!(task.id, "racing-task"), - (status, result) => panic!("non-serializable finality result: {status:?}, {result:?}"), + (status, result) => panic!("non-serializable quiescence result: {status:?}, {result:?}"), } } diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_tasks/mod.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_tasks/mod.rs index 5c275ada9a..6f8e309917 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_tasks/mod.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_tasks/mod.rs @@ -39,10 +39,10 @@ pub const TASK_METADATA_REQUIRED_ROLE: &str = "required_role"; pub const TASK_METADATA_OUTPUT: &str = "output"; pub const TASK_METADATA_EXECUTION_MODE: &str = "execution_mode"; -/// SQL predicate shared by finality and watchdog repair discovery. +/// SQL predicate shared by Quiescence and watchdog repair discovery. /// /// Historical/manual SQLite rows can bypass the typed write boundary. Keep -/// this predicate in one place so the finality count and the watchdog's +/// this predicate in one place so the Quiescence count and the watchdog's /// concrete repair identities cannot disagree about whether a row is safe to /// deserialize. The numeric values are interpolated from the same payload /// constants used by new writes. @@ -514,7 +514,7 @@ impl TaskStatus { } } - /// `completed` is treated as resolved for dependency and finality checks. + /// `completed` is treated as resolved for dependency and Quiescence checks. pub fn is_resolved(&self) -> bool { matches!(self, TaskStatus::Completed) } diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_tasks/store/read.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_tasks/store/read.rs index b9b05c54dd..a3016d3156 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_tasks/store/read.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_tasks/store/read.rs @@ -93,7 +93,7 @@ impl AgentOrgTaskStore { } /// Internal projection for a caller that has already run the shared - /// finality/corruption assessment in the same SQLite read snapshot. + /// Quiescence/corruption assessment in the same SQLite read snapshot. /// Keeping this separate avoids evaluating the expensive JSON integrity /// predicate twice per watchdog tick while the public wrapper remains /// fail-closed for every other caller. diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_tasks/tests.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_tasks/tests.rs index 047e5b0b16..1ec49c5754 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_tasks/tests.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_tasks/tests.rs @@ -153,7 +153,7 @@ fn task_mutations_require_running_parent_run() { )) .expect("running run permits create"); conn.execute( - "UPDATE agent_org_runs SET status='completed' WHERE id='guarded-run'", + "UPDATE agent_org_runs SET status='archived' WHERE id='guarded-run'", [], ) .unwrap(); @@ -694,7 +694,7 @@ fn store_rejects_malformed_reserved_dispatch_metadata() { .expect("classify historical producer"); assert!( classified, - "historical oversized producer must block finality" + "historical oversized producer must block Quiescence" ); let timezone_less_metadata = serde_json::json!({ @@ -726,7 +726,7 @@ fn store_rejects_malformed_reserved_dispatch_metadata() { |row| row.get(0), ) .expect("classify historical timezone-less output"); - assert!(classified, "timezone-less output must block finality"); + assert!(classified, "timezone-less output must block Quiescence"); } #[test] diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog.rs index f2a3b2dd00..94ba20f85e 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog.rs @@ -12,8 +12,8 @@ //! without explicit repair: tasks owned by dead members, stale //! `in_progress` work, and ready ownerless tasks awaiting explicit //! coordinator assignment (issue #272 E1). -//! - **Reconcile the run** when every task is resolved and every worker -//! is terminal. +//! - **Reconcile Team Quiescence** when every formal Task, Inbox delivery, +//! Turn Intent, recovery reservation, and relevant Session is settled. //! //! Failed members are rate-limited by a per-`(run, member)` rewake budget //! (three attempts with 1/5/15-minute backoff) that resets on the next @@ -49,7 +49,7 @@ pub(crate) use reservation::{ }; use std::collections::{BTreeSet, HashMap, HashSet}; -use std::time::Duration; +use std::time::{Duration, Instant}; use chrono::{DateTime, Duration as ChronoDuration, Utc}; use database::db::{get_connection, with_sessions_writer}; @@ -61,9 +61,9 @@ use crate::coordination::agent_inbox::{ }; use crate::coordination::agent_org_plan_approvals::AgentOrgPlanApprovalStore; use crate::coordination::agent_org_runs::{ - recovery_dispatch_recipient_is_available, AgentOrgFinalityBlocker, AgentOrgFinalityDecision, - AgentOrgRunRecord, AgentOrgRunStatus, AgentOrgRunStore, WorkerSessionRuntime, - COORDINATOR_MEMBER_ID, + recovery_dispatch_recipient_is_available, AgentOrgQuiescenceBlocker, + AgentOrgQuiescenceDecision, AgentOrgRunRecord, AgentOrgRunStatus, AgentOrgRunStore, + WorkerSessionRuntime, COORDINATOR_MEMBER_ID, }; use crate::coordination::agent_org_tasks::{self, Task, TaskStatus}; use crate::core::session::SessionStatus; @@ -71,6 +71,8 @@ use crate::tools::impls::orchestration::inbox_wake::AppHandleInboxWakeHook; use crate::tools::impls::orchestration::org_send_message::InboxWakeHook; const WATCHDOG_INTERVAL_SECS: u64 = 60; +const WATCHDOG_MAX_RUNS: usize = 100; +const WATCHDOG_SCAN_BUDGET: Duration = Duration::from_millis(250); const RECOVERY_DELAYS_SECS: [i64; 3] = [60, 5 * 60, 15 * 60]; const PENDING_MATERIALIZATION_GRACE_SECS: i64 = 2 * 60; const MEMBER_REWAKE: &str = "member_rewake"; diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/budget.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/budget.rs index 49ad41c198..b57782a6a4 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/budget.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/budget.rs @@ -21,33 +21,7 @@ pub fn init_schema(conn: &Connection) -> rusqlite::Result<()> { ); CREATE INDEX IF NOT EXISTS idx_agent_org_recovery_attempts_run ON agent_org_recovery_attempts(org_run_id);", - )?; - // Existing databases predate dispatch reservations. Keeping the token in - // the same row lets a failed/coalesced scheduler request refund only its - // own provisional attempt without undoing a newer recovery fingerprint. - ensure_recovery_attempt_column(conn, "reservation_token", "TEXT")?; - Ok(()) -} - -fn ensure_recovery_attempt_column( - conn: &Connection, - column_name: &str, - column_definition: &str, -) -> rusqlite::Result<()> { - let mut stmt = conn.prepare("PRAGMA table_info(agent_org_recovery_attempts)")?; - let columns = stmt.query_map([], |row| row.get::<_, String>(1))?; - for column in columns { - if column? == column_name { - return Ok(()); - } - } - conn.execute( - &format!( - "ALTER TABLE agent_org_recovery_attempts ADD COLUMN {column_name} {column_definition}" - ), - [], - )?; - Ok(()) + ) } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -305,24 +279,3 @@ pub(super) fn member_rewake_fingerprint_from_unread( .map(|unread| format!("unread:{unread}")) .unwrap_or_else(|| format!("status:{}", status.as_str())) } - -/// Drop budget entries whose run is no longer running so the -/// process-global maps cannot grow unbounded over the app lifetime -/// (issue #272 E6). Paused runs also lose their entries; resuming one -/// intentionally grants a fresh set of recovery attempts. -pub(super) fn prune_recovery_budgets() -> Result<(), String> { - with_sessions_writer(|| { - let conn = get_connection().map_err(|err| err.to_string())?; - conn.execute( - "DELETE FROM agent_org_recovery_attempts - WHERE NOT EXISTS ( - SELECT 1 FROM agent_org_runs run - WHERE run.id = agent_org_recovery_attempts.org_run_id - AND run.status = ?1 - )", - params![AgentOrgRunStatus::Running.as_str()], - ) - .map_err(|err| err.to_string())?; - Ok(()) - }) -} diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/inspect.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/inspect.rs index 36b29ad67d..d841e37bee 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/inspect.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/inspect.rs @@ -783,7 +783,8 @@ pub(super) fn inspect_stalled_run_with_connection( return Ok(StallRecoveryPlan::default()); } - let finality_assessment = AgentOrgRunStore::finality_assessment_with_connection(conn, run_id)?; + let quiescence_assessment = + AgentOrgRunStore::quiescence_assessment_with_connection(conn, run_id)?; let unread_counts = AgentInboxStore::unread_counts_by_recipient_with_connection(conn, run_id)?; let unread_fingerprints_by_member = unread_fingerprints_by_member(&unread_counts); let (coordinator_unread, coordinator_unread_wake_member_ids) = @@ -804,8 +805,8 @@ pub(super) fn inspect_stalled_run_with_connection( let coordinator_unread_suppresses_notice = coordinator_unread && !coordinator_unread_is_unavailable; - if finality_assessment.facts.corrupt_task_count > 0 { - let count = finality_assessment.facts.corrupt_task_count; + if quiescence_assessment.facts.corrupt_task_count > 0 { + let count = quiescence_assessment.facts.corrupt_task_count; let mut reasons = vec![format!( "The Agent Org task board has {count} persisted integrity or run-limit violation(s). The watchdog refused to guess task state or declare completion. Use task_list to identify bounded diagnostics. Ordinary task tools intentionally cannot rewrite malformed rows; cancel/delete this run or use a trusted maintenance path to repair the database before continuing." )]; @@ -816,7 +817,7 @@ pub(super) fn inspect_stalled_run_with_connection( &mut repair_facts, ); let has_new_notice = !coordinator_unread_suppresses_notice; - let work_revision = finality_assessment + let work_revision = quiescence_assessment .facts .progress .as_ref() @@ -831,7 +832,7 @@ pub(super) fn inspect_stalled_run_with_connection( .then(|| { recovery_repair_fingerprint(&repair_facts).ok_or_else(|| { format!( - "finality reported {count} corrupt task row(s), but no corrupt identity was found" + "quiescence reported {count} corrupt task row(s), but no corrupt identity was found" ) }) }) @@ -851,7 +852,7 @@ pub(super) fn inspect_stalled_run_with_connection( agent_org_tasks::AgentOrgTaskStore::list_operational_after_validated_with_connection( conn, run_id, )?; - let task_snapshot_work_revision = finality_assessment + let task_snapshot_work_revision = quiescence_assessment .facts .progress .as_ref() @@ -1254,9 +1255,9 @@ pub(super) fn inspect_stalled_run_with_connection( needs_repair.push(ready_unassigned_repair_reason(task)); } - for blocker in &finality_assessment.blockers { + for blocker in &quiescence_assessment.blockers { match blocker { - AgentOrgFinalityBlocker::EmptyTaskBoardRequiresCompletionIntent => { + AgentOrgQuiescenceBlocker::EmptyTaskBoardRequiresCompletionIntent => { repair_facts.push(RecoveryRepairFact::marker( "empty_board_requires_completion_intent", )); @@ -1265,7 +1266,7 @@ pub(super) fn inspect_stalled_run_with_connection( .to_string(), ); } - AgentOrgFinalityBlocker::StaleCompletionIntent { + AgentOrgQuiescenceBlocker::StaleCompletionIntent { requested_work_revision, current_work_revision, } => { @@ -1280,7 +1281,7 @@ pub(super) fn inspect_stalled_run_with_connection( "the previous completion request observed work revision {requested_work_revision:?}, but the board is now revision {current_work_revision}. Re-inspect the current task board and call org_run_complete again only if it is still finished." )); } - AgentOrgFinalityBlocker::CoordinatorHasNotObservedLatestWork { + AgentOrgQuiescenceBlocker::CoordinatorHasNotObservedLatestWork { observed_work_revision, current_work_revision, } if tasks.iter().all(|task| task.status.is_resolved()) => { @@ -1295,36 +1296,39 @@ pub(super) fn inspect_stalled_run_with_connection( "all durable tasks are resolved, but the coordinator has only observed work revision {observed_work_revision:?}; the current revision is {current_work_revision}. Refresh task_list and produce the final user-facing synthesis." )); } - AgentOrgFinalityBlocker::CorruptTaskData { count } => { + AgentOrgQuiescenceBlocker::CorruptTaskData { count } => { repair_facts.extend(corrupt_task_repair_facts(conn, run_id)?); needs_repair.push(format!( "{count} task row(s) contain invalid persisted JSON. Do not declare completion; inspect and repair the task records." )); } - AgentOrgFinalityBlocker::ProgressStateMissing => { + AgentOrgQuiescenceBlocker::ProgressStateMissing => { repair_facts.push(RecoveryRepairFact::marker("missing_run_progress")); needs_repair.push( "the run is missing its durable work-revision record. Do not declare completion until the state is repaired." .to_string(), ); } - AgentOrgFinalityBlocker::RootSessionMissing => { + AgentOrgQuiescenceBlocker::RootSessionMissing => { repair_facts.push(RecoveryRepairFact::marker("missing_coordinator_session")); needs_repair.push( "the run has no materialized coordinator session, so final completion cannot be safely presented." .to_string(), ); } - AgentOrgFinalityBlocker::RunMissing - | AgentOrgFinalityBlocker::RunNotRunning { .. } - | AgentOrgFinalityBlocker::SessionsActive { .. } - | AgentOrgFinalityBlocker::OpenTasks { .. } - | AgentOrgFinalityBlocker::CoordinatorHasNotObservedLatestWork { .. } - | AgentOrgFinalityBlocker::UnreadInbox { .. } - | AgentOrgFinalityBlocker::ActiveInterventions { .. } - | AgentOrgFinalityBlocker::InFlightTurnIntents { .. } - | AgentOrgFinalityBlocker::PendingPlanApprovals { .. } - | AgentOrgFinalityBlocker::TerminalStateInconsistent { .. } => {} + AgentOrgQuiescenceBlocker::RunMissing + | AgentOrgQuiescenceBlocker::RunNotRunning { .. } + | AgentOrgQuiescenceBlocker::SessionsActive { .. } + | AgentOrgQuiescenceBlocker::OpenTasks { .. } + | AgentOrgQuiescenceBlocker::CoordinatorHasNotObservedLatestWork { .. } + | AgentOrgQuiescenceBlocker::UnreadInbox { .. } + | AgentOrgQuiescenceBlocker::ActiveInterventions { .. } + | AgentOrgQuiescenceBlocker::InFlightTurnIntents { .. } + | AgentOrgQuiescenceBlocker::UnknownTurnIntents { .. } + | AgentOrgQuiescenceBlocker::PendingFormalMaterializations { .. } + | AgentOrgQuiescenceBlocker::ActiveRecoveryReservations { .. } + | AgentOrgQuiescenceBlocker::PendingPlanApprovals { .. } + | AgentOrgQuiescenceBlocker::QuietStateInconsistent { .. } => {} } } @@ -1339,8 +1343,8 @@ pub(super) fn inspect_stalled_run_with_connection( .and_then(|_| recovery_repair_fingerprint(&repair_facts)); let terminal_candidate = matches!( - finality_assessment.decision, - AgentOrgFinalityDecision::Complete | AgentOrgFinalityDecision::Abandon + quiescence_assessment.decision, + AgentOrgQuiescenceDecision::Quiescent ); let has_coordinator_repair = coordinator_repair_reason.is_some(); let coordinator_repair_active = !needs_repair.is_empty(); diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/recover.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/recover.rs index 3f6d6db36f..f53d467700 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/recover.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/recover.rs @@ -3,8 +3,7 @@ //! [`super::inspect::inspect_stalled_run`]. use super::budget::{ - budget_disposition_with_connection, prune_recovery_budgets, record_attempt_with_connection, - BudgetDisposition, + budget_disposition_with_connection, record_attempt_with_connection, BudgetDisposition, }; use super::inspect::{ inspect_stalled_run_with_connection, pending_materialization_disposition, @@ -14,31 +13,10 @@ use super::inspect::{ use super::*; pub fn spawn(app_handle: AppHandle) { + if !crate::coordination::agent_org_runs::agent_org_redesign_enabled() { + return; + } tauri::async_runtime::spawn(async move { - match tokio::task::spawn_blocking(|| { - AgentOrgPlanApprovalStore::repair_latest_plan_artifacts() - }) - .await - { - Ok(Ok(report)) => { - if report.repaired > 0 || report.failed > 0 { - tracing::info!( - inspected = report.inspected, - repaired = report.repaired, - failed = report.failed, - "[agent_org_watchdog] reconciled durable plan artifacts at startup" - ); - } - } - Ok(Err(err)) => tracing::warn!( - error = %err, - "[agent_org_watchdog] startup plan artifact reconciliation failed" - ), - Err(err) => tracing::warn!( - error = %err, - "[agent_org_watchdog] startup plan artifact worker failed" - ), - } let mut interval = tokio::time::interval(Duration::from_secs(WATCHDOG_INTERVAL_SECS)); // A slow scan must not be "repaid" with back-to-back burst // ticks afterwards; the next scheduled tick is enough. @@ -60,40 +38,32 @@ pub fn spawn(app_handle: AppHandle) { } fn recover_all_stalled_runs(app_handle: AppHandle) -> Result<(), String> { - let runs = AgentOrgRunStore::list_running_runs(usize::MAX)?; - run_best_effort_cleanup("prune recovery budgets", prune_recovery_budgets); - run_best_effort_cleanup("clear expired member interventions", || { - crate::coordination::agent_member_interventions::AgentMemberInterventionStore::clear_expired_and_legacy() - .map(|_| ()) - }); - run_best_effort_cleanup("cancel stale plan approvals", || { - AgentOrgPlanApprovalStore::cancel_pending_for_terminal_or_missing_runs().map(|_| ()) - }); - recover_listed_runs(app_handle, runs, recover_stalled_run) + let deadline = Instant::now() + WATCHDOG_SCAN_BUDGET; + let runs = AgentOrgRunStore::list_running_runs(WATCHDOG_MAX_RUNS)?; + recover_listed_runs_until(app_handle, runs, deadline, recover_stalled_run) } -/// Auxiliary cleanup is useful but cannot be a global recovery gate. One bad -/// row must not prevent healthy runs from being inspected during this tick. -pub(super) fn run_best_effort_cleanup( - label: &'static str, - cleanup: impl FnOnce() -> Result<(), String>, -) { - if let Err(err) = cleanup() { - tracing::warn!( - cleanup = label, - error = %err, - "[agent_org_watchdog] maintenance failed; continuing run scan" - ); - } +#[cfg(test)] +pub(super) fn recover_listed_runs( + handle: H, + runs: Vec, + recover: impl FnMut(H, &str) -> Result, +) -> Result<(), String> { + let deadline = Instant::now() + WATCHDOG_SCAN_BUDGET; + recover_listed_runs_until(handle, runs, deadline, recover) } -pub(super) fn recover_listed_runs( +fn recover_listed_runs_until( handle: H, runs: Vec, + deadline: Instant, mut recover: impl FnMut(H, &str) -> Result, ) -> Result<(), String> { let mut failed_run_ids = Vec::new(); for run in runs { + if Instant::now() >= deadline { + break; + } if let Err(err) = recover(handle.clone(), &run.id) { tracing::warn!( run_id = %run.id, @@ -136,9 +106,19 @@ fn execute_stall_recovery_plan( // coordinator root session is still open), fall through and deliver // the wakes so pending inbox rows still reach their recipients. if plan.terminal_candidate { - let reconciled = AgentOrgRunStore::reconcile_run_finality(run_id)?; - if reconciled.is_some_and(|status| status != AgentOrgRunStatus::Running) { - return Ok(plan); + let assessment = AgentOrgRunStore::assess_run_quiescence(run_id)?; + if let (Some(generation), Some(work_revision)) = ( + assessment.facts.activation_generation, + assessment + .facts + .progress + .as_ref() + .map(|progress| progress.work_revision), + ) { + if AgentOrgRunStore::try_transition_working_to_idle(run_id, generation, work_revision)? + { + return Ok(plan); + } } } diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/tests.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/tests.rs index 4a777bbf83..b5a2b1f3b2 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/tests.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_watchdog/tests.rs @@ -2,7 +2,7 @@ use super::budget::{ budget_disposition, coordinator_notice_allowed, rewake_budget_exhausted, BudgetDisposition, }; use super::inspect::is_wakeable_status; -use super::recover::{recover_listed_runs, run_best_effort_cleanup}; +use super::recover::recover_listed_runs; use super::*; use crate::coordination::agent_org_runs::{AgentOrgRunEntryMode, AgentOrgRunRecord}; @@ -16,14 +16,18 @@ fn fake_run(id: &str) -> AgentOrgRunRecord { org_snapshot_json: None, entry_mode: AgentOrgRunEntryMode::StandaloneSession, status: AgentOrgRunStatus::Running, + activation_generation: 1, + has_initial_work: true, work_item_id: None, project_slug: None, routine_fire_id: None, summary: None, + failure_json: None, last_error: None, + last_activity_outcome: None, created_at: now.clone(), updated_at: now, - completed_at: None, + idled_at: None, } } @@ -116,8 +120,75 @@ fn one_failed_run_does_not_skip_later_runs() { } #[test] -fn maintenance_failure_is_best_effort() { - run_best_effort_cleanup("injected", || Err("failure".to_string())); +fn watchdog_constants_match_the_single_bounded_design() { + assert_eq!(WATCHDOG_INTERVAL_SECS, 60); + assert_eq!(WATCHDOG_MAX_RUNS, 100); + assert_eq!(WATCHDOG_SCAN_BUDGET, Duration::from_millis(250)); +} + +#[test] +fn shared_scan_deadline_is_checked_at_each_team_boundary() { + let first = fake_run("run-slow"); + let second = fake_run("run-after-budget"); + let mut inspected = Vec::new(); + + recover_listed_runs((), vec![first, second], |(), run_id| { + inspected.push(run_id.to_string()); + if run_id == "run-slow" { + std::thread::sleep(Duration::from_millis(275)); + } + Ok(()) + }) + .expect("budget expiry is a bounded stop, not an error"); + + assert_eq!(inspected, vec!["run-slow"]); +} + +#[test] +fn running_query_is_limited_and_never_visits_quiet_states() { + let _sandbox = test_helpers::test_env::sandbox(); + let conn = get_connection().expect("db"); + crate::coordination::init_agent_org_schemas(&conn).expect("Agent Org schemas"); + let now = Utc::now().to_rfc3339(); + for index in 0..105 { + conn.execute( + "INSERT INTO agent_org_runs ( + id, org_id, coordinator_agent_id, root_session_id, entry_mode, status, + created_at, updated_at + ) VALUES (?1, 'watchdog-org', 'coordinator', ?2, 'standalone_session', + 'running', ?3, ?3)", + params![ + format!("running-{index:03}"), + format!("root-running-{index:03}"), + &now + ], + ) + .expect("seed Working run"); + } + for status in ["starting", "paused", "idle", "failed", "archived"] { + conn.execute( + "INSERT INTO agent_org_runs ( + id, org_id, coordinator_agent_id, root_session_id, entry_mode, status, + created_at, updated_at + ) VALUES (?1, 'watchdog-org', 'coordinator', ?2, 'standalone_session', + ?3, ?4, ?4)", + params![ + format!("quiet-{status}"), + format!("root-quiet-{status}"), + status, + &now + ], + ) + .expect("seed quiet run"); + } + + let runs = AgentOrgRunStore::list_running_runs(WATCHDOG_MAX_RUNS).expect("bounded query"); + assert_eq!(runs.len(), WATCHDOG_MAX_RUNS); + assert!(runs + .iter() + .all(|run| run.status == AgentOrgRunStatus::Running)); + assert_eq!(runs.first().map(|run| run.id.as_str()), Some("running-000")); + assert_eq!(runs.last().map(|run| run.id.as_str()), Some("running-099")); } #[test] diff --git a/src-tauri/crates/agent-core/src/core/definitions/commands.rs b/src-tauri/crates/agent-core/src/core/definitions/commands.rs index ec4b458dc8..0b5ab93091 100644 --- a/src-tauri/crates/agent-core/src/core/definitions/commands.rs +++ b/src-tauri/crates/agent-core/src/core/definitions/commands.rs @@ -128,9 +128,10 @@ pub struct InboxRunSummary { #[tauri::command] pub async fn agent_org_run_list(limit: Option) -> Result, String> { use crate::core::coordination::agent_org_runs::AgentOrgRunStore; + crate::core::coordination::agent_org_runs::require_agent_org_redesign()?; const MAX_LIMIT: usize = 200; let effective_limit = limit.map(|n| n.min(MAX_LIMIT)).unwrap_or(MAX_LIMIT); - // This command backs a read-only Inbox list. Finality reconciliation is a + // This command backs a read-only Inbox list. Quiescence reconciliation is a // lifecycle/watchdog responsibility: doing it here used to turn one UI // refresh into as many as 200 global writer-lock + IMMEDIATE // transactions. Keep the read off the async command executor as well. diff --git a/src-tauri/crates/agent-core/src/core/session/launch/launch_helpers.rs b/src-tauri/crates/agent-core/src/core/session/launch/launch_helpers.rs index 122ec142be..b39871376d 100644 --- a/src-tauri/crates/agent-core/src/core/session/launch/launch_helpers.rs +++ b/src-tauri/crates/agent-core/src/core/session/launch/launch_helpers.rs @@ -7,7 +7,7 @@ use std::collections::HashMap; use core_types::key_source::KeySource; -use crate::coordination::agent_org_runs::AgentOrgRunStore; +use crate::coordination::agent_org_runs::{AgentOrgRunStore, AgentOrgStartingFailure}; use crate::definitions::orgs::{OrgMember, OrgMemberRuntimeConfig}; use crate::session::turn::streaming::{ broadcast_agent_error_structured, classify_streaming_error_message, StreamingError, @@ -30,7 +30,16 @@ pub(super) async fn handle_background_launch_failure( ) { tracing::warn!("{}", message); if let Some(run_id) = agent_org_run_id { - if let Err(mark_err) = AgentOrgRunStore::mark_failed(run_id, message) { + let failure_result = AgentOrgRunStore::load(run_id).and_then(|run| { + let run = run.ok_or_else(|| format!("Agent Org run not found: {run_id}"))?; + AgentOrgRunStore::fail_starting( + run_id, + run.activation_generation, + &AgentOrgStartingFailure::new("starting_convergence_failed", message), + ) + .map(|_| ()) + }); + if let Err(mark_err) = failure_result { tracing::warn!( run_id = %run_id, error = %mark_err, @@ -179,10 +188,6 @@ pub(super) fn member_runtime_account_id( .or_else(|| fallback.clone()) } -pub(super) fn member_runtime_tier(config: Option<&OrgMemberRuntimeConfig>) -> Option { - config.and_then(|cfg| clean_runtime_value(cfg.tier.as_ref())) -} - pub(super) fn member_runtime_key_source( config: Option<&OrgMemberRuntimeConfig>, fallback: &KeySource, diff --git a/src-tauri/crates/agent-core/src/core/session/launch/launch_org.rs b/src-tauri/crates/agent-core/src/core/session/launch/launch_org.rs index 3d621a898d..50c7e4560e 100644 --- a/src-tauri/crates/agent-core/src/core/session/launch/launch_org.rs +++ b/src-tauri/crates/agent-core/src/core/session/launch/launch_org.rs @@ -1,12 +1,17 @@ -//! Org-member materialization for the agent run launch service. +//! Receipt-driven Agent Org Starting convergence. //! -//! Handles spawning background tasks that create member sessions for an -//! Agent Org run, covering both Rust-native and CLI agent members. +//! The stable roster is committed before any Session row is created. Every +//! retry therefore targets the exact same member/session identity instead of +//! minting a replacement after a crash. + +use std::collections::HashMap; use core_types::key_source::KeySource; -use crate::coordination::agent_org_runs::AgentOrgRunStore; -use crate::definitions::orgs::{parse_cli_agent_org_reference, OrgDefinition}; +use crate::coordination::agent_org_runs::{ + AgentOrgMaterializationStatus, AgentOrgRunStore, AgentOrgStartingFailure, COORDINATOR_MEMBER_ID, +}; +use crate::definitions::orgs::{is_cli_agent_org_reference, OrgDefinition, OrgMember}; use crate::session::persistence::{ self as session_persistence, session_type, UnifiedSessionRecord, }; @@ -15,56 +20,46 @@ use crate::state::AgentAppState; use super::launch_helpers::{ flatten_org_members, member_runtime_account_id, member_runtime_key_source, - member_runtime_model, member_runtime_native_harness_type, member_runtime_tier, + member_runtime_model, member_runtime_native_harness_type, }; -#[allow(clippy::too_many_arguments)] -pub(super) fn spawn_agent_org_member_materialization( - org_run_id: String, - org: OrgDefinition, - root_session_id: String, - root_session_name: String, - workspace_path: String, - model: Option, - account_id: Option, - key_source: Option, - agent_exec_mode: Option, - native_harness_type: Option, - work_item_id: Option, - project_slug: Option, -) { - tokio::spawn(async move { - if let Err(err) = materialize_org_member_sessions( - &org_run_id, - &org, - &root_session_id, - &root_session_name, - &workspace_path, - model, - account_id, - key_source, - agent_exec_mode, - native_harness_type, - work_item_id, - project_slug, - ) - .await - { - tracing::warn!( - run_id = %org_run_id, - root_session_id = %root_session_id, - error = %err, - "[session_launch] failed to materialize Agent Org member sessions in background" - ); - if let Err(mark_err) = AgentOrgRunStore::mark_failed(&org_run_id, &err) { - tracing::warn!( - run_id = %org_run_id, - error = %mark_err, - "[session_launch] failed to mark Agent Org run failed after member materialization error" - ); - } +#[derive(Debug)] +pub(super) struct AgentOrgMaterializationError { + failure: AgentOrgStartingFailure, + retryable: bool, +} + +impl AgentOrgMaterializationError { + fn permanent(code: &'static str, message: impl Into) -> Self { + Self { + failure: AgentOrgStartingFailure::new(code, message), + retryable: false, } - }); + } + + fn retryable(message: impl Into) -> Self { + Self { + failure: AgentOrgStartingFailure::new( + "materialization_temporarily_unavailable", + message, + ), + retryable: true, + } + } + + pub(super) fn is_retryable(&self) -> bool { + self.retryable + } + + pub(super) fn failure(&self) -> &AgentOrgStartingFailure { + &self.failure + } +} + +impl std::fmt::Display for AgentOrgMaterializationError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(&self.failure.message) + } } #[allow(clippy::too_many_arguments)] @@ -81,22 +76,7 @@ pub(super) async fn materialize_org_member_sessions( native_harness_type: Option, work_item_id: Option, project_slug: Option, -) -> Result, String> { - let flattened_members = flatten_org_members(&org.children); - if flattened_members.is_empty() { - return Ok(Vec::new()); - } - - let mut rust_members = Vec::new(); - let mut cli_members = Vec::new(); - for member in flattened_members { - if parse_cli_agent_org_reference(&member.agent_id).is_some() { - cli_members.push(member); - } else { - rust_members.push(member); - } - } - +) -> Result, AgentOrgMaterializationError> { let workspace_path = workspace_path.to_string(); let root_session_id = root_session_id.to_string(); let org_name = org.name.clone(); @@ -105,197 +85,165 @@ pub(super) async fn materialize_org_member_sessions( .as_deref() .filter(|value| !value.trim().is_empty()) { - Some(raw) => KeySource::parse(raw).ok_or_else(|| format!("Unknown key_source: {raw:?}"))?, + Some(raw) => KeySource::parse(raw).ok_or_else(|| { + AgentOrgMaterializationError::permanent( + "invalid_member_runtime_config", + format!("Unknown key_source: {raw:?}"), + ) + })?, None => KeySource::default(), }; let native_harness_type = native_harness_type .filter(|value| !value.trim().is_empty()) .map(|raw| { core_types::providers::NativeHarnessType::parse(&raw) - .ok_or_else(|| format!("Unknown native_harness_type: {raw:?}")) + .ok_or_else(|| { + AgentOrgMaterializationError::permanent( + "invalid_member_runtime_config", + format!("Unknown native_harness_type: {raw:?}"), + ) + }) .map(|parsed| parsed.as_str().to_string()) }) .transpose()?; let agent_exec_mode = agent_exec_mode.filter(|mode| !mode.trim().is_empty()); - let org_run_id = org_run_id.to_string(); - let mut created_session_ids = Vec::with_capacity(rust_members.len() + cli_members.len()); - let mut created_rust_session_ids = Vec::new(); - let mut created_cli_session_ids = Vec::new(); - - if !rust_members.is_empty() { - let rust_workspace_path = workspace_path.clone(); - let rust_root_session_id = root_session_id.clone(); - let rust_org_name = org_name.clone(); - let rust_model = model.clone(); - let rust_account_id = account_id.clone(); - let rust_key_source = key_source; - let rust_agent_exec_mode = agent_exec_mode.clone(); - let rust_native_harness_type = native_harness_type.clone(); - let rust_work_item_id = work_item_id.clone(); - let rust_project_slug = project_slug.clone(); - let rust_org_run_id = org_run_id.clone(); - created_rust_session_ids = tokio::task::spawn_blocking(move || { - let now = chrono::Utc::now().to_rfc3339(); - let mut created_session_ids: Vec = Vec::with_capacity(rust_members.len()); - let has_workspace_path = !rust_workspace_path.is_empty(); + let members = flatten_org_members(&org.children) + .into_iter() + .map(|member| (member.id.clone(), member)) + .collect::>(); + let receipts = AgentOrgRunStore::materializations(org_run_id) + .map_err(AgentOrgMaterializationError::retryable)?; + let mut materialized_session_ids = Vec::new(); + for receipt in receipts { + if receipt.member_id == COORDINATOR_MEMBER_ID { + continue; + } + if receipt.status == AgentOrgMaterializationStatus::Succeeded { + materialized_session_ids.push(receipt.session_id); + continue; + } + let member = members.get(&receipt.member_id).ok_or_else(|| { + AgentOrgMaterializationError::permanent( + "materialization_identity_mismatch", + format!( + "materialization receipt references missing canonical member {}", + receipt.member_id + ), + ) + })?; + if member.agent_id != receipt.agent_id { + return Err(AgentOrgMaterializationError::permanent( + "materialization_identity_mismatch", + format!( + "materialization receipt agent mismatch for member {}", + receipt.member_id + ), + )); + } + if is_cli_agent_org_reference(&member.agent_id) { + return Err(AgentOrgMaterializationError::permanent( + "unsupported_member_runtime", + format!( + "CLI Agent Org member {} cannot be materialized on the canonical lifecycle path", + member.id + ), + )); + } - for member in rust_members { - let prefix = crate::definitions::prefix_lookup::session_prefix_for_launch( - Some(&member.agent_id), - has_workspace_path, - ); - let session_id = format!("{}{}", prefix, uuid::Uuid::new_v4()); - let member_config = member.runtime_config.as_ref(); - let member_model = member_runtime_model(member_config, &rust_model); - let member_account_id = member_runtime_account_id(member_config, &rust_account_id); - let member_key_source = member_runtime_key_source(member_config, &rust_key_source) - .map_err(|err| format!("invalid runtime config for member '{}': {}", member.name, err))?; - let member_native_harness_type = - member_runtime_native_harness_type(member_config, &rust_native_harness_type) - .map_err(|err| format!("invalid runtime config for member '{}': {}", member.name, err))?; - let session = UnifiedSessionRecord { - session_id: session_id.clone(), - name: format!("{} · {}", member.name, member.role), - status: crate::session::SessionStatus::Idle.as_str().to_string(), - model: member_model, - account_id: member_account_id, - workspace_path: Some(rust_workspace_path.clone()), - org_id: Some(project_management::projects::types::PERSONAL_ORG_ID.to_string()), - user_input: None, - total_tokens: 0, - created_at: now.clone(), - updated_at: now.clone(), - session_type: session_type::ORG_MEMBER.to_string(), - work_item_id: rust_work_item_id.clone(), - // Same rule as the launch resolver: a work-item-linked - // session is a Project session. Members inherit it so the - // PM tools aren't policy-denied for the team doing the work. - product_mode: rust_work_item_id - .as_ref() - .map(|_| "project".to_string()), - agent_role: Some(member.role.clone()), - project_slug: rust_project_slug.clone(), - agent_definition_id: Some(member.agent_id.clone()), - org_member_id: Some(member.id.clone()), - parent_session_id: Some(rust_root_session_id.clone()), - key_source: member_key_source, - agent_exec_mode: rust_agent_exec_mode.clone(), - native_harness_type: member_native_harness_type, - ..Default::default() - }; - if let Err(err) = session_persistence::upsert_session(&session) { - for created_session_id in &created_session_ids { - if let Err(cleanup_err) = session_persistence::delete_session(created_session_id) - { - tracing::warn!( - session_id = %created_session_id, - error = %cleanup_err, - "[session_launch] failed to clean up materialized Agent Org member session" - ); - } - } - return Err(format!( - "failed to materialize Agent Org member '{}' for run '{}': {}", - member.name, rust_org_run_id, err + let session_id = receipt.session_id.clone(); + let member = member.clone(); + let workspace_path = workspace_path.clone(); + let root_session_id = root_session_id.clone(); + let model = model.clone(); + let account_id = account_id.clone(); + let agent_exec_mode = agent_exec_mode.clone(); + let native_harness_type = native_harness_type.clone(); + let work_item_id = work_item_id.clone(); + let project_slug = project_slug.clone(); + let member_config = member.runtime_config.as_ref(); + let member_model = member_runtime_model(member_config, &model); + let member_account_id = member_runtime_account_id(member_config, &account_id); + let member_key_source = + member_runtime_key_source(member_config, &key_source).map_err(|error| { + AgentOrgMaterializationError::permanent("invalid_member_runtime_config", error) + })?; + let member_native_harness_type = + member_runtime_native_harness_type(member_config, &native_harness_type).map_err( + |error| { + AgentOrgMaterializationError::permanent("invalid_member_runtime_config", error) + }, + )?; + let persisted_session_id = tokio::task::spawn_blocking(move || { + if let Some(existing) = session_persistence::get_session(&session_id) + .map_err(|error| AgentOrgMaterializationError::retryable(error.to_string()))? + { + let identity_matches = existing.agent_definition_id.as_deref() + == Some(member.agent_id.as_str()) + && existing.org_member_id.as_deref() == Some(member.id.as_str()) + && existing.parent_session_id.as_deref() == Some(root_session_id.as_str()); + if !identity_matches { + return Err(AgentOrgMaterializationError::permanent( + "materialization_identity_mismatch", + format!("stable materialization Session identity mismatch: {session_id}"), )); } - created_session_ids.push(session_id); + return Ok(session_id); } - - tracing::info!( - run_id = %rust_org_run_id, - org_name = %rust_org_name, - member_sessions = created_session_ids.len(), - "[session_launch] materialized Rust Agent Org member sessions" - ); - Ok(created_session_ids) - }) - .await - .map_err(|err| err.to_string())??; - created_session_ids.extend(created_rust_session_ids.iter().cloned()); - } - - for member in cli_members { - let cli_agent_type = parse_cli_agent_org_reference(&member.agent_id) - .ok_or_else(|| format!("invalid CLI Agent Org reference: {}", member.agent_id))? - .as_str() - .to_string(); - let member_config = member.runtime_config.as_ref(); - let member_key_source = member_runtime_key_source(member_config, &key_source)?; - let outcome = crate::foundation::session_bridge::launch_cli_agent( - crate::foundation::session_bridge::CliLaunchParams { - name: Some(format!("{} · {}", member.name, member.role)), - cli_agent_type, - model: member_runtime_model(member_config, &model), - tier: member_runtime_tier(member_config), - account_id: member_runtime_account_id(member_config, &account_id), - repo_path: Some(workspace_path.clone()).filter(|path| !path.is_empty()), - branch: None, - worktree_path: None, - worktree_base_ref: None, - hosted_token: None, - isolate: false, - background: true, - key_source: Some(member_key_source.as_ref().to_string()), - additional_directories: None, - parent_session_id: Some(root_session_id.clone()), - org_member_id: Some(member.id.clone()), - org_id: project_management::projects::types::PERSONAL_ORG_ID.to_string(), - project_id: None, - project_name: None, - project_slug: project_slug.clone(), + let now = chrono::Utc::now().to_rfc3339(); + let session = UnifiedSessionRecord { + session_id: session_id.clone(), + name: format!("{} · {}", member.name, member.role), + status: crate::session::SessionStatus::Idle.as_str().to_string(), + model: member_model, + account_id: member_account_id, + workspace_path: Some(workspace_path), + org_id: Some(project_management::projects::types::PERSONAL_ORG_ID.to_string()), + user_input: None, + total_tokens: 0, + created_at: now.clone(), + updated_at: now, + session_type: session_type::ORG_MEMBER.to_string(), work_item_id: work_item_id.clone(), - agent_role: None, product_mode: work_item_id.as_ref().map(|_| "project".to_string()), - durable_run_id: None, - user_input: String::new(), - ide_context: None, - mode: agent_exec_mode.clone(), - images: None, - }, + agent_role: Some(member.role), + project_slug, + agent_definition_id: Some(member.agent_id), + org_member_id: Some(member.id), + parent_session_id: Some(root_session_id), + key_source: member_key_source, + agent_exec_mode, + native_harness_type: member_native_harness_type, + ..Default::default() + }; + session_persistence::upsert_session(&session) + .map_err(|error| AgentOrgMaterializationError::retryable(error.to_string()))?; + Ok::<_, AgentOrgMaterializationError>(session_id) + }) + .await + .map_err(|error| AgentOrgMaterializationError::retryable(error.to_string()))??; + AgentOrgRunStore::mark_materialization_succeeded( + org_run_id, + &receipt.member_id, + receipt.generation, + &persisted_session_id, ) - .await; - match outcome { - Ok(outcome) => { - created_session_ids.push(outcome.session_id.clone()); - created_cli_session_ids.push(outcome.session_id); - } - Err(err) => { - for session_id in &created_rust_session_ids { - if let Err(cleanup_err) = session_persistence::delete_session(session_id) { - tracing::warn!( - session_id = %session_id, - error = %cleanup_err, - "[session_launch] failed to clean up Rust Agent Org member session after CLI materialization failure" - ); - } - } - for session_id in &created_cli_session_ids { - if let Err(cleanup_err) = - crate::foundation::session_bridge::delete_cli_session(session_id) - { - tracing::warn!( - session_id = %session_id, - error = %cleanup_err, - "[session_launch] failed to clean up CLI Agent Org member session" - ); - } - } - return Err(format!( - "failed to materialize CLI Agent Org member '{}' for run '{}': {}", - member.name, org_run_id, err - )); + .map_err(|error| { + if error.contains("identity mismatch") || error.contains("Session identity") { + AgentOrgMaterializationError::permanent("materialization_identity_mismatch", error) + } else { + AgentOrgMaterializationError::retryable(error) } - } + })?; + materialized_session_ids.push(persisted_session_id); } tracing::info!( run_id = %org_run_id, org_name = %org_name, - member_sessions = created_session_ids.len(), - "[session_launch] materialized Agent Org member sessions" + member_sessions = materialized_session_ids.len(), + "[session_launch] converged Agent Org member materialization receipts" ); - Ok(created_session_ids) + Ok(materialized_session_ids) } #[allow(clippy::too_many_arguments)] @@ -313,11 +261,11 @@ pub(super) async fn send_initial_turn( agent_definition_id: Option, sub_agent_ids: Vec, intent_org_run_id: Option, - durable_run_id: Option, + client_message_id: Option, + turn_intent_id: Option, source: crate::foundation::session_bridge::TurnIntentBridgeSource, ) -> Result<(), String> { if sub_agent_ids.is_empty() { - let client_message_id = durable_run_id.clone(); crate::state::commands::session::message::send_message_impl( state, session_id.to_string(), @@ -335,7 +283,7 @@ pub(super) async fn send_initial_turn( false, false, client_message_id, - durable_run_id, + turn_intent_id, None, intent_org_run_id, source, @@ -357,7 +305,6 @@ pub(super) async fn send_initial_turn( .await?; crate::init::init_session(state, launch_spec).await?; - let client_message_id = durable_run_id.clone(); crate::state::commands::session::message::send_message_impl( state, session_id.to_string(), @@ -375,7 +322,7 @@ pub(super) async fn send_initial_turn( false, false, client_message_id, - durable_run_id, + turn_intent_id, None, intent_org_run_id, crate::foundation::session_bridge::TurnIntentBridgeSource::AgentOrg, diff --git a/src-tauri/crates/agent-core/src/core/session/launch/launch_tests.rs b/src-tauri/crates/agent-core/src/core/session/launch/launch_tests.rs index 08813c2dd2..00c608615c 100644 --- a/src-tauri/crates/agent-core/src/core/session/launch/launch_tests.rs +++ b/src-tauri/crates/agent-core/src/core/session/launch/launch_tests.rs @@ -1,7 +1,7 @@ use super::launch_helpers::{ apply_member_launch_overrides_to_snapshot, member_runtime_account_id, member_runtime_key_source, member_runtime_model, member_runtime_native_harness_type, - member_runtime_tier, validate_launch_agent_definitions, + validate_launch_agent_definitions, }; use crate::coordination::agent_org_runs::COORDINATOR_MEMBER_ID; use crate::definitions::builtin::SDE_AGENT_ID; @@ -145,10 +145,6 @@ fn member_runtime_resolution_prefers_member_config_then_falls_back() { member_runtime_account_id(Some(&config), &fallback_account).as_deref(), Some("member-account") ); - assert_eq!( - member_runtime_tier(Some(&config)).as_deref(), - Some("premium") - ); assert_eq!( member_runtime_key_source(Some(&config), &KeySource::OwnKey).expect("key source"), KeySource::HostedKey diff --git a/src-tauri/crates/agent-core/src/core/session/launch/mod.rs b/src-tauri/crates/agent-core/src/core/session/launch/mod.rs index 3f9b5d514d..72d642932b 100644 --- a/src-tauri/crates/agent-core/src/core/session/launch/mod.rs +++ b/src-tauri/crates/agent-core/src/core/session/launch/mod.rs @@ -17,8 +17,8 @@ use std::collections::HashMap; use tauri::Manager; use crate::coordination::agent_org_runs::{ - AgentOrgRunEntryMode, AgentOrgRunStatus, AgentOrgRunStore, CreateAgentOrgRunParams, - COORDINATOR_MEMBER_ID, + AgentOrgRunEntryMode, AgentOrgRunStore, CreateAgentOrgInitialInput, + CreateAgentOrgMaterializationIntent, CreateStartingAgentOrgRunParams, COORDINATOR_MEMBER_ID, }; use crate::definitions::orgs::{AgentOrgsStore, OrgMemberLaunchOverride}; use crate::init::launch_spec::AgentLaunchSpec; @@ -28,12 +28,13 @@ use crate::state::AgentAppState; use project_management::projects::types as project_types; use launch_helpers::{ - apply_member_launch_overrides_to_snapshot, derive_name, handle_background_launch_failure, - provenance_fields, provenance_lock_reason, validate_launch_agent_definitions, + apply_member_launch_overrides_to_snapshot, derive_name, flatten_org_members, + handle_background_launch_failure, provenance_fields, provenance_lock_reason, + validate_launch_agent_definitions, }; use launch_org::{ - cleanup_session_after_org_run_create_failure, send_initial_turn, - spawn_agent_org_member_materialization, + cleanup_session_after_org_run_create_failure, materialize_org_member_sessions, + send_initial_turn, }; use launch_workspace::{ acquire_work_item_execution_lock, prepare_rust_agent_workspace_for_launch, @@ -42,6 +43,36 @@ use launch_workspace::{ pub(crate) const MAX_AUTO_NAME_LEN: usize = 80; +const AGENT_ORG_INITIAL_INPUT_PAYLOAD_VERSION: u8 = 1; + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +struct AgentOrgInitialInputPayload { + version: u8, + images: Option>, + ide_context: Option, + sub_agent_ids: Vec, +} + +fn decode_agent_org_initial_input_payload( + input: &crate::coordination::agent_org_runs::AgentOrgInitialInput, +) -> Result { + let payload: AgentOrgInitialInputPayload = + serde_json::from_str(&input.payload_json).map_err(|error| { + format!( + "invalid Starting initial input payload for {}: {error}", + input.org_run_id + ) + })?; + if payload.version != AGENT_ORG_INITIAL_INPUT_PAYLOAD_VERSION { + return Err(format!( + "unsupported Starting initial input payload version {} for {}", + payload.version, input.org_run_id + )); + } + Ok(payload) +} + #[derive(Debug, Clone)] pub(crate) struct AgentRunLaunchRequest { /// Stable WorkItemRun id used for deterministic Session and turn ids. @@ -405,6 +436,9 @@ pub(crate) async fn launch_rust_agent_run( org_store: Option<&AgentOrgsStore>, request: AgentRunLaunchRequest, ) -> Result { + if matches!(&request.target, AgentRunTarget::AgentOrg { .. }) { + crate::coordination::agent_org_runs::require_agent_org_redesign()?; + } let (workspace_path, branch, isolate, existing_worktree_path, additional_directories) = match &request.workspace { WorkspaceLaunchTarget::LocalWorkspace { @@ -530,6 +564,91 @@ pub(crate) async fn launch_rust_agent_run( .as_ref() .map(|entry| entry.branch.clone()); + let has_initial_content = !request.content.trim().is_empty(); + let org_session_key = agent_org_id.as_ref().map(|_| { + request + .durable_run_id + .clone() + .unwrap_or_else(|| format!("agent-org-{}", uuid::Uuid::new_v4())) + }); + let expected_root_session_id = org_session_key.as_ref().map(|key| { + let prefix = crate::definitions::prefix_lookup::session_prefix_for_launch( + agent_definition_id.as_deref(), + !workspace_path.is_empty(), + ); + format!("{prefix}{key}") + }); + let initial_turn_intent_id = + has_initial_content.then(|| format!("agent-org-initial-turn-{}", uuid::Uuid::new_v4())); + let initial_message_id = + has_initial_content.then(|| format!("agent-org-initial-message-{}", uuid::Uuid::new_v4())); + let initial_input_payload_json = has_initial_content + .then(|| { + serde_json::to_string(&AgentOrgInitialInputPayload { + version: AGENT_ORG_INITIAL_INPUT_PAYLOAD_VERSION, + images: request.images.clone(), + ide_context: request.ide_context.clone(), + sub_agent_ids: request.sub_agent_ids.clone(), + }) + .map_err(|error| format!("serialize Agent Org initial input: {error}")) + }) + .transpose()?; + + // Build the construction envelope before any Team lifecycle row exists. + // The already-persisted coordinator is certified in the same transaction + // as Starting and the remaining stable member identities. + let starting_params = match ( + agent_org_id.as_ref(), + coordinator_agent_id.as_ref(), + effective_org_definition.as_ref(), + expected_root_session_id.as_ref(), + ) { + (Some(org_id), Some(coordinator_id), Some(org_snapshot), Some(root_session_id)) => { + let mut materialization_intents = vec![CreateAgentOrgMaterializationIntent { + member_id: COORDINATOR_MEMBER_ID.to_string(), + agent_id: coordinator_id.clone(), + session_id: root_session_id.clone(), + succeeded: true, + }]; + for member in flatten_org_members(&org_snapshot.children) { + let prefix = crate::definitions::prefix_lookup::session_prefix_for_launch( + Some(&member.agent_id), + !workspace_path.is_empty(), + ); + materialization_intents.push(CreateAgentOrgMaterializationIntent { + member_id: member.id, + agent_id: member.agent_id, + session_id: format!("{prefix}{}", uuid::Uuid::new_v4()), + succeeded: false, + }); + } + Some(CreateStartingAgentOrgRunParams { + org_id: org_id.clone(), + coordinator_agent_id: coordinator_id.clone(), + root_session_id: root_session_id.clone(), + org_snapshot: org_snapshot.clone(), + entry_mode: AgentOrgRunEntryMode::StandaloneSession, + work_item_id: work_item_id.clone(), + project_slug: project_slug.clone(), + routine_fire_id: routine_fire_id.clone(), + materialization_intents, + initial_input: initial_turn_intent_id.as_ref().map(|turn_intent_id| { + CreateAgentOrgInitialInput { + turn_intent_id: turn_intent_id.clone(), + message_id: initial_message_id + .clone() + .expect("initial message id accompanies initial turn"), + content: request.content.clone(), + payload_json: initial_input_payload_json + .clone() + .expect("initial payload accompanies initial turn"), + } + }), + }) + } + _ => None, + }; + let create_result = crate::state::commands::session::create::create_session_impl( None, workspace_path.clone(), @@ -549,7 +668,9 @@ pub(crate) async fn launch_rust_agent_run( request.product_mode.clone(), request.resources.native_harness_type.clone(), request.parent_session_id.clone(), - request.durable_run_id.clone(), + org_session_key + .clone() + .or_else(|| request.durable_run_id.clone()), ) .await?; @@ -558,11 +679,34 @@ pub(crate) async fn launch_rust_agent_run( .and_then(|value| value.as_str()) .ok_or("create_session_impl did not return sessionId")? .to_string(); + if let Some(expected_session_id) = expected_root_session_id.as_deref() { + if session_id != expected_session_id { + return Err(format!( + "coordinator identity mismatch: expected {expected_session_id}, got {session_id}" + )); + } + } let resolved_product_mode = create_result .get("productMode") .and_then(|value| value.as_str()) .map(str::to_string); + if starting_params.is_some() { + persistence::update_org_member_id(&session_id, COORDINATOR_MEMBER_ID) + .map_err(|err| format!("failed to persist coordinator member_id: {err}"))?; + } + + let starting_run = match starting_params { + Some(params) => match AgentOrgRunStore::create_starting(params) { + Ok(run) => Some(run), + Err(error) => { + cleanup_session_after_org_run_create_failure(session_id.clone()).await; + return Err(error); + } + }, + None => None, + }; + if let (Some(project_slug_value), Some(work_item_id_value)) = (project_slug.as_deref(), work_item_id.as_deref()) { @@ -580,61 +724,14 @@ pub(crate) async fn launch_rust_agent_run( } } - let agent_org_run_id = match (agent_org_id.as_ref(), coordinator_agent_id.as_ref()) { - (Some(org_id), Some(coordinator_id)) => { - let org_snapshot = effective_org_definition - .as_ref() - .ok_or("Agent Org launch is missing resolved org definition")? - .clone(); - let run = AgentOrgRunStore::create(CreateAgentOrgRunParams { - org_id: org_id.clone(), - coordinator_agent_id: coordinator_id.clone(), - root_session_id: Some(session_id.clone()), - org_snapshot, - entry_mode: AgentOrgRunEntryMode::StandaloneSession, - status: AgentOrgRunStatus::Running, - work_item_id: work_item_id.clone(), - project_slug: project_slug.clone(), - routine_fire_id, - }); - match run { - Ok(record) => { - persistence::update_org_member_id(&session_id, COORDINATOR_MEMBER_ID) - .map_err(|err| format!("failed to persist coordinator member_id: {err}"))?; - if let Some(org) = effective_org_definition.as_ref() { - spawn_agent_org_member_materialization( - record.id.clone(), - org.clone(), - session_id.clone(), - name.clone(), - workspace_path.clone(), - request.resources.model.clone(), - request.resources.account_id.clone(), - request.resources.key_source.clone(), - request.mode.clone(), - request.resources.native_harness_type.clone(), - work_item_id.clone(), - project_slug.clone(), - ); - } - if apply_member_overrides_for_future { - if let Some(store) = org_store { - store.apply_member_launch_overrides(org_id, &member_overrides)?; - } - } - Some(record.id) - } - Err(err) => { - cleanup_session_after_org_run_create_failure(session_id.clone()).await; - return Err(err); - } - } + let agent_org_run_id = starting_run.as_ref().map(|run| run.id.clone()); + if starting_run.is_some() && apply_member_overrides_for_future { + if let (Some(store), Some(org_id)) = (org_store, agent_org_id.as_ref()) { + store.apply_member_launch_overrides(org_id, &member_overrides)?; } - _ => None, - }; + } let created_at = chrono::Utc::now().to_rfc3339(); - let has_initial_content = !request.content.trim().is_empty(); let native_harness_type_for_send = request .resources .native_harness_type @@ -662,10 +759,20 @@ pub(crate) async fn launch_rust_agent_run( let sub_agent_ids_for_send = request.sub_agent_ids.clone(); let agent_definition_id_for_send = agent_definition_id.clone(); let agent_org_run_id_for_background = agent_org_run_id.clone(); - let durable_run_id_for_background = request.durable_run_id.clone(); let project_slug_for_background = project_slug.clone(); let work_item_id_for_background = work_item_id.clone(); let app_handle_for_background = state.app_handle.clone(); + let request_key_source_for_background = request.resources.key_source.clone(); + let request_native_harness_for_background = request.resources.native_harness_type.clone(); + let org_for_background = effective_org_definition + .clone() + .expect("Agent Org launch has a validated snapshot"); + let starting_generation = starting_run + .as_ref() + .map(|run| run.activation_generation) + .expect("Agent Org launch has a Starting generation"); + let initial_turn_intent_id_for_background = initial_turn_intent_id.clone(); + let initial_message_id_for_background = initial_message_id.clone(); tokio::spawn(async move { let prepared_workspace = match prepare_rust_agent_workspace_for_launch( @@ -699,14 +806,165 @@ pub(crate) async fn launch_rust_agent_run( } }; - if !has_initial_content { - return; - } - let workspace_path_for_send = prepared_workspace .worktree_path .clone() .unwrap_or_else(|| workspace_path_for_background.clone()); + write_agent_session_marker( + &workspace_path_for_send, + &session_id_for_background, + agent_definition_id_for_send.as_deref(), + None, + project_slug_for_background.as_deref(), + Some(project_management::projects::types::PERSONAL_ORG_ID), + ); + + let run_id = agent_org_run_id_for_background + .as_deref() + .expect("Agent Org background launch has a run id"); + if let Err(err) = materialize_org_member_sessions( + run_id, + &org_for_background, + &session_id_for_background, + &name, + &workspace_path_for_send, + model_for_send.clone(), + account_id_for_send.clone(), + request_key_source_for_background.clone(), + mode_for_send.clone(), + request_native_harness_for_background.clone(), + work_item_id_for_background.clone(), + project_slug_for_background.clone(), + ) + .await + { + if err.is_retryable() { + tracing::warn!( + run_id = %run_id, + error = %err, + "[session_launch] retryable Starting materialization deferred" + ); + return; + } + let message = format!( + "[session_launch] member materialization failed for {}: {}", + session_id_for_background, err + ); + handle_background_launch_failure( + &session_id_for_background, + Some(run_id), + project_slug_for_background.as_deref(), + work_item_id_for_background.as_deref(), + app_handle_for_background.as_ref(), + &message, + "[session_launch] failed to mark Starting materialization failure", + "[session_launch] failed to mark coordinator session failed", + ) + .await; + return; + } + + if let Some(turn_intent_id) = initial_turn_intent_id_for_background.as_deref() { + let input = match AgentOrgRunStore::initial_input(run_id) { + Ok(Some(input)) => input, + Ok(None) => { + handle_background_launch_failure( + &session_id_for_background, + Some(run_id), + project_slug_for_background.as_deref(), + work_item_id_for_background.as_deref(), + app_handle_for_background.as_ref(), + "Starting initial input receipt is missing", + "[session_launch] failed to mark missing Starting input", + "[session_launch] failed to mark coordinator session failed", + ) + .await; + return; + } + Err(error) => { + handle_background_launch_failure( + &session_id_for_background, + Some(run_id), + project_slug_for_background.as_deref(), + work_item_id_for_background.as_deref(), + app_handle_for_background.as_ref(), + &error, + "[session_launch] failed to mark Starting input lookup failure", + "[session_launch] failed to mark coordinator session failed", + ) + .await; + return; + } + }; + let session_id_for_persistence = session_id_for_background.clone(); + let input_for_persistence = input.clone(); + let transcript_result = tokio::task::spawn_blocking(move || { + persistence::save_user_msg_with_id( + &input_for_persistence.message_id, + &session_id_for_persistence, + &input_for_persistence.content, + ) + .map(|_| ()) + .map_err(|error| error.to_string()) + }) + .await + .map_err(|error| error.to_string()) + .and_then(|result| result); + let event_result = app_handle_for_background + .as_ref() + .ok_or_else(|| "App handle is unavailable during Starting".to_string()) + .and_then(|handle| { + crate::bus::event_pipeline_bridge::persist_user_message_event( + handle, + &session_id_for_background, + &input.message_id, + &input.content, + None, + images_for_send.as_deref(), + crate::bus::event_pipeline_bridge::PersistedUserMessageSource::User, + turn_intent_id, + ) + }); + if let Err(error) = transcript_result.and(event_result) { + tracing::warn!( + run_id = %run_id, + error = %error, + "[session_launch] retryable Starting input persistence deferred" + ); + return; + } + } + + if let Err(error) = AgentOrgRunStore::finish_starting(run_id, starting_generation) { + if error.starts_with("materialization_identity_mismatch:") + || error.contains("initial input certificate missing") + || error.contains("unexpected initial input certificate") + { + handle_background_launch_failure( + &session_id_for_background, + Some(run_id), + project_slug_for_background.as_deref(), + work_item_id_for_background.as_deref(), + app_handle_for_background.as_ref(), + &format!("Starting convergence failed: {error}"), + "[session_launch] failed to mark Starting convergence failure", + "[session_launch] failed to mark coordinator session failed", + ) + .await; + } else { + tracing::warn!( + run_id = %run_id, + error = %error, + "[session_launch] retryable Starting convergence deferred" + ); + } + return; + } + + if !has_initial_content { + return; + } + // Title generation runs concurrently — it must not delay the // first turn. See `spawn_session_title_generation`. spawn_session_title_generation( @@ -732,7 +990,8 @@ pub(crate) async fn launch_rust_agent_run( agent_definition_id_for_send, sub_agent_ids_for_send, agent_org_run_id_for_background.clone(), - durable_run_id_for_background, + initial_message_id_for_background.clone(), + initial_turn_intent_id_for_background.clone(), crate::foundation::session_bridge::TurnIntentBridgeSource::AgentOrg, ) .await; @@ -753,6 +1012,16 @@ pub(crate) async fn launch_rust_agent_run( "[session_launch] failed to mark session failed after first-message error", ) .await; + } else if let Some(turn_intent_id) = initial_turn_intent_id_for_background.as_deref() { + if let Err(error) = + AgentOrgRunStore::mark_initial_input_dispatched(run_id, turn_intent_id) + { + tracing::warn!( + run_id, + error = %error, + "[session_launch] initial input was accepted but dispatch receipt update failed" + ); + } } }); @@ -883,6 +1152,7 @@ pub(crate) async fn launch_rust_agent_run( agent_definition_id_for_send, sub_agent_ids_for_send, agent_org_run_id_for_send.clone(), + durable_run_id_for_send.clone(), durable_run_id_for_send, crate::foundation::session_bridge::TurnIntentBridgeSource::UserSubmit, ) @@ -945,3 +1215,259 @@ pub(crate) async fn launch_rust_agent_run( product_mode: resolved_product_mode, }) } + +const AGENT_ORG_STARTUP_RECOVERY_LIMIT: usize = 100; + +/// Run the one-shot Starting/initial-input recovery owner after app state and +/// the EventStore bridge are ready. This is intentionally separate from the +/// periodic Working watchdog. +pub fn spawn_agent_org_startup_recovery(state: AgentAppState) { + if !crate::coordination::agent_org_runs::agent_org_redesign_enabled() { + return; + } + tauri::async_runtime::spawn(async move { + if let Err(error) = recover_agent_org_starting_runs(&state).await { + tracing::warn!(error = %error, "[agent-org-startup] Starting recovery failed"); + } + if let Err(error) = recover_agent_org_initial_dispatches(&state).await { + tracing::warn!(error = %error, "[agent-org-startup] initial dispatch recovery failed"); + } + }); +} + +async fn recover_agent_org_starting_runs(state: &AgentAppState) -> Result<(), String> { + let runs = tokio::task::spawn_blocking(|| { + AgentOrgRunStore::list_starting_runs(AGENT_ORG_STARTUP_RECOVERY_LIMIT) + }) + .await + .map_err(|error| error.to_string())??; + + for run in runs { + let Some(root_session_id) = run.root_session_id.as_deref() else { + AgentOrgRunStore::fail_starting( + &run.id, + run.activation_generation, + &crate::coordination::agent_org_runs::AgentOrgStartingFailure::new( + "missing_coordinator_identity", + "Starting run has no coordinator Session identity", + ), + )?; + continue; + }; + let root = persistence::get_session(root_session_id) + .map_err(|error| error.to_string())? + .ok_or_else(|| format!("Starting coordinator Session is missing: {root_session_id}")); + let root = match root { + Ok(root) => root, + Err(message) => { + AgentOrgRunStore::fail_starting( + &run.id, + run.activation_generation, + &crate::coordination::agent_org_runs::AgentOrgStartingFailure::new( + "missing_coordinator_identity", + message, + ), + )?; + continue; + } + }; + let Some(snapshot_raw) = run.org_snapshot_json.as_deref() else { + AgentOrgRunStore::fail_starting( + &run.id, + run.activation_generation, + &crate::coordination::agent_org_runs::AgentOrgStartingFailure::new( + "missing_launch_snapshot", + format!("Starting run {} has no launch snapshot", run.id), + ), + )?; + continue; + }; + let snapshot: crate::definitions::orgs::OrgDefinition = + match serde_json::from_str(snapshot_raw) { + Ok(snapshot) => snapshot, + Err(error) => { + AgentOrgRunStore::fail_starting( + &run.id, + run.activation_generation, + &crate::coordination::agent_org_runs::AgentOrgStartingFailure::new( + "invalid_launch_snapshot", + error.to_string(), + ), + )?; + continue; + } + }; + let workspace_path = root + .worktree_path + .clone() + .or_else(|| root.workspace_path.clone()) + .unwrap_or_default(); + if let Err(error) = materialize_org_member_sessions( + &run.id, + &snapshot, + root_session_id, + &root.name, + &workspace_path, + root.model.clone(), + root.account_id.clone(), + Some(root.key_source.as_ref().to_string()), + root.agent_exec_mode.clone(), + root.native_harness_type.clone(), + root.work_item_id.clone(), + root.project_slug.clone(), + ) + .await + { + if !error.is_retryable() { + AgentOrgRunStore::fail_starting( + &run.id, + run.activation_generation, + error.failure(), + )?; + continue; + } + tracing::warn!(run_id = %run.id, error = %error, "[agent-org-startup] materialization retry deferred"); + continue; + } + + let initial_input = match AgentOrgRunStore::initial_input(&run.id) { + Ok(input) => input, + Err(error) => { + tracing::warn!(run_id = %run.id, error = %error, "[agent-org-startup] initial input lookup retry deferred"); + continue; + } + }; + if let Some(input) = initial_input { + if let Err(error) = persist_starting_initial_input(state, root_session_id, &input).await + { + if error.starts_with("invalid Starting initial input payload") + || error.starts_with("unsupported Starting initial input payload version") + { + AgentOrgRunStore::fail_starting( + &run.id, + run.activation_generation, + &crate::coordination::agent_org_runs::AgentOrgStartingFailure::new( + "invalid_initial_input_payload", + error, + ), + )?; + } else { + tracing::warn!(run_id = %run.id, error = %error, "[agent-org-startup] initial input persistence retry deferred"); + } + continue; + } + } + if let Err(error) = AgentOrgRunStore::finish_starting(&run.id, run.activation_generation) { + if error.starts_with("materialization_identity_mismatch:") + || error.contains("initial input certificate missing") + || error.contains("unexpected initial input certificate") + { + AgentOrgRunStore::fail_starting( + &run.id, + run.activation_generation, + &crate::coordination::agent_org_runs::AgentOrgStartingFailure::new( + "starting_certificate_invalid", + error, + ), + )?; + } else { + tracing::warn!(run_id = %run.id, error = %error, "[agent-org-startup] Starting transition retry deferred"); + } + } + } + Ok(()) +} + +async fn recover_agent_org_initial_dispatches(state: &AgentAppState) -> Result<(), String> { + let inputs = tokio::task::spawn_blocking(|| { + AgentOrgRunStore::recoverable_initial_inputs(AGENT_ORG_STARTUP_RECOVERY_LIMIT) + }) + .await + .map_err(|error| error.to_string())??; + + for input in inputs { + let Some(run) = AgentOrgRunStore::load(&input.org_run_id)? else { + continue; + }; + let Some(root_session_id) = run.root_session_id.as_deref() else { + continue; + }; + let Some(root) = + persistence::get_session(root_session_id).map_err(|error| error.to_string())? + else { + continue; + }; + persistence::update_status(root_session_id, crate::session::SessionStatus::Idle) + .map_err(|error| error.to_string())?; + let workspace_path = root + .worktree_path + .clone() + .or(root.workspace_path.clone()) + .unwrap_or_default(); + let native_harness_type = root + .native_harness_type + .as_deref() + .map(|raw| { + core_types::providers::NativeHarnessType::parse(raw) + .ok_or_else(|| format!("Unknown native_harness_type: {raw:?}")) + }) + .transpose()?; + let payload = decode_agent_org_initial_input_payload(&input)?; + send_initial_turn( + state, + root_session_id, + input.content.clone(), + root.model, + root.account_id, + workspace_path, + native_harness_type, + root.agent_exec_mode, + payload.images, + payload.ide_context, + Some(run.coordinator_agent_id), + payload.sub_agent_ids, + Some(run.id.clone()), + Some(input.message_id.clone()), + Some(input.turn_intent_id.clone()), + crate::foundation::session_bridge::TurnIntentBridgeSource::AgentOrg, + ) + .await?; + AgentOrgRunStore::mark_initial_input_dispatched(&run.id, &input.turn_intent_id)?; + } + Ok(()) +} + +async fn persist_starting_initial_input( + state: &AgentAppState, + session_id: &str, + input: &crate::coordination::agent_org_runs::AgentOrgInitialInput, +) -> Result<(), String> { + let payload = decode_agent_org_initial_input_payload(input)?; + let session_id_owned = session_id.to_string(); + let input_owned = input.clone(); + tokio::task::spawn_blocking(move || { + persistence::save_user_msg_with_id( + &input_owned.message_id, + &session_id_owned, + &input_owned.content, + ) + .map(|_| ()) + .map_err(|error| error.to_string()) + }) + .await + .map_err(|error| error.to_string())??; + let handle = state + .app_handle + .as_ref() + .ok_or_else(|| "App handle is unavailable during Starting recovery".to_string())?; + crate::bus::event_pipeline_bridge::persist_user_message_event( + handle, + session_id, + &input.message_id, + &input.content, + None, + payload.images.as_deref(), + crate::bus::event_pipeline_bridge::PersistedUserMessageSource::User, + &input.turn_intent_id, + ) +} diff --git a/src-tauri/crates/agent-core/src/core/session/scheduler.rs b/src-tauri/crates/agent-core/src/core/session/scheduler.rs index 29e6fed79a..75a4ab51f6 100644 --- a/src-tauri/crates/agent-core/src/core/session/scheduler.rs +++ b/src-tauri/crates/agent-core/src/core/session/scheduler.rs @@ -100,7 +100,7 @@ pub struct ScheduledMessage { /// user-message persistence (resume with empty content). pub turn_intent_id: String, /// Durable Agent Org run that owns this turn, when any. The scheduler - /// uses this only after the intent reaches a terminal state so finality + /// uses this only after the intent reaches a terminal state so Quiescence /// is rechecked after (not before) the current intent stops blocking it. pub org_run_id: Option, /// The user content to process. @@ -500,8 +500,22 @@ impl WorkerTask { if let Some(run_id) = org_run_id { let reconcile_run_id = run_id.clone(); match tokio::task::spawn_blocking(move || { - crate::coordination::agent_org_runs::AgentOrgRunStore::reconcile_run_finality( + let assessment = crate::coordination::agent_org_runs::AgentOrgRunStore::assess_run_quiescence(&reconcile_run_id)?; + let Some(generation) = assessment.facts.activation_generation else { + return Ok(false); + }; + let Some(work_revision) = assessment + .facts + .progress + .as_ref() + .map(|progress| progress.work_revision) + else { + return Ok(false); + }; + crate::coordination::agent_org_runs::AgentOrgRunStore::try_transition_working_to_idle( &reconcile_run_id, + generation, + work_revision, ) }) .await @@ -510,12 +524,12 @@ impl WorkerTask { Ok(Err(error)) => warn!( run_id = %run_id, error = %error, - "[scheduler] post-intent Agent Org finality reconcile failed" + "[scheduler] post-intent Agent Org quiescence reconcile failed" ), Err(error) => warn!( run_id = %run_id, error = %error, - "[scheduler] post-intent Agent Org finality reconcile task failed" + "[scheduler] post-intent Agent Org quiescence reconcile task failed" ), } } @@ -527,16 +541,23 @@ impl WorkerTask { "[scheduler] Message {} failed for session {}: {}", msg.message_id, self.session_id, err ); - // Lifecycle: running → failed. Cancelled turns walk - // here too (the executor returns Err on user stop); a - // future commit can distinguish via the cancel_flag - // probe if we need a separate `cancelled` bucket on - // the round renderer. - crate::foundation::session_bridge::update_turn_intent_status( - &self.session_id, - &turn_intent_id, - crate::foundation::session_bridge::TurnIntentBridgeStatus::Failed, - ); + let assistant_persistence_failed = + should_keep_agent_org_intent_in_flight(org_run_id.as_deref(), err); + if assistant_persistence_failed { + warn!( + session_id = %self.session_id, + turn_intent_id = %turn_intent_id, + "[scheduler] keeping Agent Org turn in-flight because final assistant persistence failed" + ); + } else { + // Lifecycle: running → failed. Cancelled turns walk + // here too (the executor returns Err on user stop). + crate::foundation::session_bridge::update_turn_intent_status( + &self.session_id, + &turn_intent_id, + crate::foundation::session_bridge::TurnIntentBridgeStatus::Failed, + ); + } // Turn-only: an `agent:error` renders as a chat bubble. // Maintenance jobs report failures through their own // channel (e.g. the manual-compact command's reply). @@ -578,11 +599,35 @@ impl WorkerTask { } } +fn should_keep_agent_org_intent_in_flight(org_run_id: Option<&str>, error: &str) -> bool { + org_run_id.is_some() + && error.starts_with( + crate::core::session::turn::event_handler::AGENT_ORG_ASSISTANT_PERSISTENCE_ERROR_PREFIX, + ) +} + #[cfg(test)] mod tests { use super::*; use std::sync::atomic::{AtomicUsize, Ordering}; + #[test] + fn assistant_persistence_failure_keeps_only_agent_org_intent_in_flight() { + let error = format!( + "{} disk full", + crate::core::session::turn::event_handler::AGENT_ORG_ASSISTANT_PERSISTENCE_ERROR_PREFIX + ); + assert!(should_keep_agent_org_intent_in_flight( + Some("run-1"), + &error + )); + assert!(!should_keep_agent_org_intent_in_flight(None, &error)); + assert!(!should_keep_agent_org_intent_in_flight( + Some("run-1"), + "provider failed" + )); + } + #[tokio::test] async fn invalidated_pending_message_is_skipped() { let scheduler = DialogScheduler::new("session-a", 8); diff --git a/src-tauri/crates/agent-core/src/core/session/turn/entry.rs b/src-tauri/crates/agent-core/src/core/session/turn/entry.rs index e265b20972..c82ed01eba 100644 --- a/src-tauri/crates/agent-core/src/core/session/turn/entry.rs +++ b/src-tauri/crates/agent-core/src/core/session/turn/entry.rs @@ -162,6 +162,7 @@ pub async fn process_message( .as_ref() .and_then(|ctx| ctx.repo_path.clone()), agent_org_task_lifecycle: None, + require_durable_assistant_event: false, }; let policy = Arc::clone(&runtime.policy); diff --git a/src-tauri/crates/agent-core/src/core/session/turn/event_handler/mod.rs b/src-tauri/crates/agent-core/src/core/session/turn/event_handler/mod.rs index fc7b33cb98..b7ea16a225 100644 --- a/src-tauri/crates/agent-core/src/core/session/turn/event_handler/mod.rs +++ b/src-tauri/crates/agent-core/src/core/session/turn/event_handler/mod.rs @@ -52,6 +52,9 @@ use core_types::session_event::SessionEvent; use super::super::persistence as unified_persistence; +pub(crate) const AGENT_ORG_ASSISTANT_PERSISTENCE_ERROR_PREFIX: &str = + "agent_org_assistant_persistence_failed:"; + fn tool_result_is_error(result: &str) -> bool { if result.starts_with("Error") { return true; @@ -123,6 +126,10 @@ pub struct EventHandlerConfig { /// Agent Org worker identity used by the bounded task-lifecycle stop gate. /// Coordinators and non-org sessions leave this unset. pub agent_org_task_lifecycle: Option, + + /// Agent Org work-capable turns may not become terminal until their + /// assistant EventStore rows are durably committed. + pub require_durable_assistant_event: bool, } /// Durable identity needed to verify that an Agent Org worker did not end a @@ -170,6 +177,7 @@ pub struct UnifiedEventHandler { /// A second miss is reported durably to the coordinator by `MemberIdle` /// rather than looping the provider. agent_org_lifecycle_correction_emitted: AtomicBool, + assistant_persistence_error: Mutex>, } /// Accumulated state for one streaming `create_plan` call. @@ -242,6 +250,7 @@ impl UnifiedEventHandler { plan_draft_streams: Mutex::new(std::collections::HashMap::new()), last_context_tokens: std::sync::atomic::AtomicI64::new(0), agent_org_lifecycle_correction_emitted: AtomicBool::new(false), + assistant_persistence_error: Mutex::new(None), } } @@ -279,7 +288,7 @@ impl UnifiedEventHandler { sessions.insert(session_id.to_string()); } self.track_retractable_segment(session_id, &event.id); - self.push_to_store(session_id, event.clone()); + self.push_to_store_durable_assistant(session_id, event.clone()); broadcast_event( "agent:streaming_complete", serde_json::json!({ @@ -310,6 +319,38 @@ impl UnifiedEventHandler { self.agent_called.load(Ordering::Relaxed) } + pub fn take_assistant_persistence_error(&self) -> Option { + self.assistant_persistence_error + .lock() + .ok() + .and_then(|mut error| error.take()) + } + + fn record_assistant_persistence_error(&self, error: String) { + if let Ok(mut slot) = self.assistant_persistence_error.lock() { + if slot.is_none() { + *slot = Some(error); + } + } + } + + fn push_to_store_durable_assistant(&self, session_id: &str, event: SessionEvent) { + if self.is_cancelled() || !self.is_current_turn_generation() { + return; + } + if self.config.require_durable_assistant_event { + if let Err(error) = event_pipeline_bridge::persist_events( + "agent-org-assistant-final", + session_id, + std::slice::from_ref(&event), + 5, + ) { + self.record_assistant_persistence_error(error); + } + } + self.push_to_store(session_id, event); + } + /// Push a SessionEvent into the session's EventStore so frontend /// subscribers receive it via `es:changed`. Silently no-op when the /// handler was constructed without an app handle (tests / non-Tauri @@ -743,6 +784,11 @@ impl TurnEventHandler for UnifiedEventHandler { "[unified_handler] Failed to persist assistant iteration: {}", err ); + if self.config.require_durable_assistant_event { + self.record_assistant_persistence_error(format!( + "assistant transcript persistence failed: {err}" + )); + } } let has_active_message_stream = self @@ -761,7 +807,7 @@ impl TurnEventHandler for UnifiedEventHandler { ) { let mut event = event_factory::build_assistant_message_event(session_id, text); attach_turn_id(&mut event, self.config.turn_id.as_deref()); - self.push_to_store(session_id, event); + self.push_to_store_durable_assistant(session_id, event); } } @@ -877,7 +923,7 @@ impl TurnEventHandler for UnifiedEventHandler { // steering queue and is about to be presented to the model. // Never leave its durable intent queued merely because the // transcript write failed: that would block Agent Org - // finality forever. Failed is terminal and truthfully records + // Quiescence forever. Failed is terminal and truthfully records // that durable persistence did not complete. crate::foundation::session_bridge::update_turn_intent_status( session_id, @@ -1140,6 +1186,39 @@ mod tests { assert!(should_push_assistant_event(false, false, true)); } + #[test] + fn agent_org_assistant_persistence_failure_is_retained_for_turn_owner() { + let handler = UnifiedEventHandler::new(EventHandlerConfig { + require_durable_assistant_event: true, + ..Default::default() + }); + let event = super::event_factory::build_assistant_message_event( + "agent-org-session", + "durable final answer", + ); + + handler.push_to_store_durable_assistant("agent-org-session", event); + + let error = handler + .take_assistant_persistence_error() + .expect("unregistered durable EventStore bridge must fail closed"); + assert!(error.contains("event pipeline persistence is not registered")); + assert!(handler.take_assistant_persistence_error().is_none()); + } + + #[test] + fn generic_assistant_event_does_not_require_synchronous_eventstore_commit() { + let handler = UnifiedEventHandler::new(EventHandlerConfig::default()); + let event = super::event_factory::build_assistant_message_event( + "generic-session", + "ordinary answer", + ); + + handler.push_to_store_durable_assistant("generic-session", event); + + assert!(handler.take_assistant_persistence_error().is_none()); + } + #[test] fn retractable_segments_drained_once_on_retry() { let handler = UnifiedEventHandler::new(EventHandlerConfig::default()); diff --git a/src-tauri/crates/agent-core/src/core/session/turn/processor/execute.rs b/src-tauri/crates/agent-core/src/core/session/turn/processor/execute.rs index 2a854a3fde..744d0c2d52 100644 --- a/src-tauri/crates/agent-core/src/core/session/turn/processor/execute.rs +++ b/src-tauri/crates/agent-core/src/core/session/turn/processor/execute.rs @@ -101,6 +101,8 @@ impl UnifiedMessageProcessor { let mut event_handler_config = self.event_handler_config.clone(); event_handler_config.turn_id = Some(turn_id.to_string()); + event_handler_config.require_durable_assistant_event = + self.runtime.agent_org_context.is_some(); event_handler_config.agent_org_task_lifecycle = self .runtime .agent_org_context diff --git a/src-tauri/crates/agent-core/src/core/session/turn/processor/inbox_drain/guard.rs b/src-tauri/crates/agent-core/src/core/session/turn/processor/inbox_drain/guard.rs index 1ea018c45a..feb04ce8ff 100644 --- a/src-tauri/crates/agent-core/src/core/session/turn/processor/inbox_drain/guard.rs +++ b/src-tauri/crates/agent-core/src/core/session/turn/processor/inbox_drain/guard.rs @@ -74,7 +74,7 @@ impl DrainGuard { /// Exact source rows this turn will acknowledge only after successful /// provider execution. Threaded into tool-call context so prospective - /// finality can project this turn's guaranteed commit without treating + /// Quiescence can project this turn's guaranteed commit without treating /// unrelated unread mail as consumed. pub fn pending_ids(&self) -> &[i64] { &self.pending_ids diff --git a/src-tauri/crates/agent-core/src/core/session/turn/processor/mod.rs b/src-tauri/crates/agent-core/src/core/session/turn/processor/mod.rs index f1f07f30d3..e051fef593 100644 --- a/src-tauri/crates/agent-core/src/core/session/turn/processor/mod.rs +++ b/src-tauri/crates/agent-core/src/core/session/turn/processor/mod.rs @@ -546,13 +546,39 @@ impl UnifiedMessageProcessor { // ("text content is empty") on the very next request. let should_save_user_msg = !(context.is_resume && content.is_empty()); if should_save_user_msg { - let message_id = tokio::task::block_in_place(|| { - unified_persistence::save_user_msg(session_id, content, context.images.as_deref()) - }) + let initial_input = tokio::task::block_in_place(|| { + crate::coordination::agent_org_runs::AgentOrgRunStore::initial_input_for_turn( + &context.turn_intent_id, + ) + })?; + let message_id = if let Some(input) = initial_input.as_ref() { + if input.content != content { + return Err(format!( + "Starting input content mismatch for turn {}", + context.turn_intent_id + )); + } + tokio::task::block_in_place(|| { + unified_persistence::save_user_msg_with_id( + &input.message_id, + session_id, + content, + ) + }) + .map(|(message_id, _inserted)| message_id) + } else { + tokio::task::block_in_place(|| { + unified_persistence::save_user_msg( + session_id, + content, + context.images.as_deref(), + ) + }) + } .map_err(|err| format!("Failed to save user message: {}", err))?; if let Some(handle) = self.app_handle.as_ref() { - if let Err(err) = tokio::task::block_in_place(|| { + let event_result = tokio::task::block_in_place(|| { crate::bus::event_pipeline_bridge::persist_user_message_event( handle, session_id, @@ -563,7 +589,13 @@ impl UnifiedMessageProcessor { crate::bus::event_pipeline_bridge::PersistedUserMessageSource::User, context.turn_intent_id.as_str(), ) - }) { + }); + if let Err(err) = event_result { + if initial_input.is_some() { + return Err(format!( + "Failed to persist Starting user-message event: {err}" + )); + } tracing::warn!( session_id, error = %err, @@ -926,6 +958,12 @@ impl UnifiedMessageProcessor { // Flush any pending streaming content before completing the turn. handler.flush_streaming(session_id); + if let Some(error) = handler.take_assistant_persistence_error() { + return Err(format!( + "{} {error}", + super::event_handler::AGENT_ORG_ASSISTANT_PERSISTENCE_ERROR_PREFIX + )); + } // Update nag-reminder counter based on whether manage_todo was called // during this turn. Reset to 0 on any todo call; increment otherwise. diff --git a/src-tauri/crates/agent-core/src/core/tools/call_context.rs b/src-tauri/crates/agent-core/src/core/tools/call_context.rs index c6ee85adb9..f08cab7938 100644 --- a/src-tauri/crates/agent-core/src/core/tools/call_context.rs +++ b/src-tauri/crates/agent-core/src/core/tools/call_context.rs @@ -29,7 +29,7 @@ //! //! - `turn_intent_id` and `projected_inbox_ids`: identify the exact durable //! turn and Inbox batch whose effects become committed only after this turn -//! succeeds. Agent Org finality uses them to build a prospective +//! succeeds. Agent Org Quiescence uses them to build a prospective //! completion certificate without guessing or subtracting unrelated work. //! //! ## Defaults diff --git a/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/inbox_repair.rs b/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/inbox_repair.rs index c3aada258b..70612e31db 100644 --- a/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/inbox_repair.rs +++ b/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/inbox_repair.rs @@ -179,7 +179,7 @@ async fn resolve( "outcome": resolution.resolution_kind.as_str(), "org_run_id": run_id, "delivery_resolution": resolution, - "guidance": "The original Inbox row remains durable and unread as audit evidence, but no longer blocks delivery/finality. Re-inspect task_list and the replacement work before requesting completion." + "guidance": "The original Inbox row remains durable and unread as audit evidence, but no longer blocks delivery/Quiescence. Re-inspect task_list and the replacement work before requesting completion." })) .map_err(|err| { ToolError::ExecutionFailed(format!( @@ -377,10 +377,10 @@ mod tests { let fixture = fixture(); let conn = get_connection().expect("test sqlite connection"); conn.execute( - "UPDATE agent_org_runs SET status='completed' WHERE id=?1", + "UPDATE agent_org_runs SET status='archived' WHERE id=?1", params![&fixture.run_id], ) - .expect("complete run"); + .expect("archive run"); let error = OrgInboxRepairTool::new(fixture.coordinator) .execute_text( json!({ diff --git a/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/run_complete.rs b/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/run_complete.rs index 9015290f8c..3755b396cd 100644 --- a/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/run_complete.rs +++ b/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/run_complete.rs @@ -21,7 +21,7 @@ pub struct OrgRunCompleteParams { /// Coordinator-only explicit completion intent. /// /// This does not force a terminal state. It records a durable request at the -/// current work revision; the canonical finality reconciler still waits for a +/// current work revision; the canonical Quiescence reconciler still waits for a /// successful coordinator turn, resolved tasks, drained inbox, settled /// approvals/interventions, and no in-flight turns. pub struct OrgRunCompleteTool { @@ -41,7 +41,7 @@ impl Tool for OrgRunCompleteTool { } fn description(&self) -> &str { - "Request safe completion of the current Agent Org run. Coordinator-only. Records a durable summary at the current work revision; it never bypasses open tasks or finality checks. Use it when task_list says an empty task board requires explicit completion intent." + "Request safe completion of the current Agent Org run. Coordinator-only. Records a durable summary at the current work revision; it never bypasses open tasks or Quiescence checks. Use it when task_list says an empty task board requires explicit completion intent." } fn category(&self) -> &str { @@ -80,7 +80,7 @@ impl Tool for OrgRunCompleteTool { "outcome": "recorded", "org_run_id": run_id, "work_revision": progress.work_revision, - "guidance": "Completion was requested durably. Finish this coordinator turn normally; the canonical finality reconciler will close the run only after every remaining delivery and lifecycle blocker has settled." + "guidance": "Completion was requested durably. Finish this coordinator turn normally; the canonical Quiescence reconciler will move the Team to Idle only after every remaining delivery and lifecycle blocker has settled." }), AgentOrgCompletionRequestOutcome::OpenTasks { unresolved_task_ids, diff --git a/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/send_message/persistence.rs b/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/send_message/persistence.rs index a2c7a2d8fa..517003592b 100644 --- a/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/send_message/persistence.rs +++ b/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/send_message/persistence.rs @@ -130,7 +130,7 @@ pub(super) fn persist_ordinary_message_if_running( "reason": "run_not_running", "org_run_id": run_id, "run_status": run_status, - "guidance": "The Agent Org run is paused or terminal, so this message was not persisted. Resume a paused run before sending new work; terminal runs cannot be reopened.", + "guidance": "The Agent Org Team is not Running, so this formal peer message was not persisted. Starting, Paused, Idle, Failed, and Archived Teams do not accept this mutation in PR1.", })) .map_err(|err| ToolError::ExecutionFailed(err.to_string()))?; tx.commit() diff --git a/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/send_message/tests.rs b/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/send_message/tests.rs index 9f6cb4d14e..2579d7723a 100644 --- a/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/send_message/tests.rs +++ b/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/send_message/tests.rs @@ -375,14 +375,14 @@ async fn plain_message_to_worker_without_task_returns_guidance_and_does_not_wake } #[tokio::test(flavor = "multi_thread", worker_threads = 1)] -async fn ordinary_message_does_not_create_unread_work_after_run_is_terminal() { +async fn ordinary_message_does_not_create_unread_work_after_run_is_archived() { let _sandbox = init_inbox_schema(); let conn = database::db::get_connection().expect("test sqlite connection"); conn.execute( - "UPDATE agent_org_runs SET status='completed' WHERE id='run-1'", + "UPDATE agent_org_runs SET status='archived' WHERE id='run-1'", [], ) - .expect("complete run"); + .expect("archive run"); let wake = Arc::new(RecordingWakeHook::default()); let tool = OrgSendMessageTool::with_hooks( context(), diff --git a/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/task_list_get.rs b/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/task_list_get.rs index 5c3bc6b4d7..9563b33c2c 100644 --- a/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/task_list_get.rs +++ b/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/task_list_get.rs @@ -9,7 +9,7 @@ use serde_json::{json, Value}; use crate::coordination::agent_org_payload_limits::validate_task_identifier; use crate::coordination::agent_org_runs::{ - guaranteed_current_turn_effects_with_connection, AgentOrgFinalityDecision, AgentOrgRunStore, + guaranteed_current_turn_effects_with_connection, AgentOrgQuiescenceDecision, AgentOrgRunStore, }; use crate::coordination::agent_org_tasks::AgentOrgTaskStore; use crate::tools::names as tool_names; @@ -137,7 +137,7 @@ impl Tool for TaskListTool { .map_err(ToolError::InvalidParams)?; } - // Task summaries and run finality facts must describe the same + // Task summaries and Team Quiescence facts must describe the same // database moment. Use one deferred read transaction and project only // bounded columns; routine task_list calls never deserialize full // descriptions, raw metadata, or output content for the entire board. @@ -158,7 +158,7 @@ impl Tool for TaskListTool { let tx = conn .transaction_with_behavior(TransactionBehavior::Deferred) .map_err(|err| err.to_string())?; - let completion = AgentOrgRunStore::finality_assessment_with_connection(&tx, &run_id)?; + let completion = AgentOrgRunStore::quiescence_assessment_with_connection(&tx, &run_id)?; let guaranteed_turn_effects = guaranteed_current_turn_effects_with_connection( &tx, &run_id, @@ -203,7 +203,7 @@ impl Tool for TaskListTool { completion.after_successful_coordinator_turn_with_effects(guaranteed_turn_effects); let completion_ready = matches!( completion_after_turn.decision, - AgentOrgFinalityDecision::Complete + AgentOrgQuiescenceDecision::Quiescent ); let body = json!({ "tasks": page.tasks.iter().map(compact_task_summary_to_json).collect::>(), @@ -236,8 +236,8 @@ impl Tool for TaskListTool { "unread_inbox_count": completion.facts.unread_inbox_count, "pending_plan_approval_count": completion.facts.pending_plan_approval_count, "completion_ready": completion_ready, - "finality_decision": completion.decision, - "current_finality_blockers": &completion.blockers, + "quiescence_decision": completion.decision, + "current_quiescence_blockers": &completion.blockers, "completion_blockers": &completion_after_turn.blockers, }, "org_run_id": self.ctx.org_context.run_id, diff --git a/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/task_tests.rs b/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/task_tests.rs index 63b1510775..281ab235f2 100644 --- a/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/task_tests.rs +++ b/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent_org/task_tests.rs @@ -226,13 +226,7 @@ fn task_tools_sandbox() -> test_env::SandboxGuard { crate::foundation::persistence::session_snapshots::ensure_tables_with(&conn) .expect("agent sessions schema"); crate::session::persistence::init(&conn).expect("unified session schema"); - crate::coordination::agent_inbox::init_schema(&conn).expect("agent inbox schema"); - crate::coordination::agent_org_runs::init_schema(&conn).expect("agent org runs schema"); - crate::coordination::agent_member_interventions::init_schema(&conn) - .expect("member intervention schema"); - crate::coordination::agent_org_tasks::init_schema(&conn).expect("agent team tasks schema"); - crate::coordination::agent_org_plan_approvals::init_schema(&conn) - .expect("agent org plan approval schema"); + crate::coordination::init_agent_org_schemas(&conn).expect("canonical Agent Org schemas"); conn.execute_batch( "CREATE TABLE IF NOT EXISTS code_sessions ( session_id TEXT PRIMARY KEY, @@ -2836,7 +2830,7 @@ async fn task_list_defaults_to_fifty_compact_rows() { assert!(value["page"]["next_cursor"].is_string()); } -fn seed_task_list_current_turn_finality_fixture(materialize_inbox: bool) -> i64 { +fn seed_task_list_current_turn_quiescence_fixture(materialize_inbox: bool) -> i64 { let now = chrono::Utc::now().to_rfc3339(); let conn = database::db::get_connection().expect("test sqlite connection"); conn.execute( @@ -2906,10 +2900,10 @@ fn seed_task_list_current_turn_finality_fixture(materialize_inbox: bool) -> i64 inbox.id } -fn has_finality_blocker(value: &Value, field: &str, kind: &str, count: i64) -> bool { +fn has_quiescence_blocker(value: &Value, field: &str, kind: &str, count: i64) -> bool { value["run_summary"][field] .as_array() - .expect("finality blocker array") + .expect("Quiescence blocker array") .iter() .any(|blocker| blocker["kind"] == kind && blocker["count"] == count) } @@ -2917,7 +2911,7 @@ fn has_finality_blocker(value: &Value, field: &str, kind: &str, count: i64) -> b #[tokio::test] async fn task_list_projects_exact_current_root_turn_and_materialized_inbox() { let _sandbox = task_tools_sandbox(); - let inbox_id = seed_task_list_current_turn_finality_fixture(true); + let inbox_id = seed_task_list_current_turn_quiescence_fixture(true); let call_ctx = crate::tools::call_context::CallContext::for_turn( "task-list-current-turn", "root-tools-1", @@ -2937,15 +2931,15 @@ async fn task_list_projects_exact_current_root_turn_and_materialized_inbox() { "the raw snapshot still includes the currently running coordinator intent" ); assert_eq!(value["run_summary"]["unread_inbox_count"], 1); - assert!(has_finality_blocker( + assert!(has_quiescence_blocker( &value, - "current_finality_blockers", + "current_quiescence_blockers", "in_flight_turn_intents", 1, )); - assert!(has_finality_blocker( + assert!(has_quiescence_blocker( &value, - "current_finality_blockers", + "current_quiescence_blockers", "unread_inbox", 1, )); @@ -2956,7 +2950,7 @@ async fn task_list_projects_exact_current_root_turn_and_materialized_inbox() { #[tokio::test] async fn task_list_current_turn_projection_keeps_unrelated_durable_blockers() { let _sandbox = task_tools_sandbox(); - let projected_inbox_id = seed_task_list_current_turn_finality_fixture(true); + let projected_inbox_id = seed_task_list_current_turn_quiescence_fixture(true); let now = chrono::Utc::now().to_rfc3339(); let conn = database::db::get_connection().expect("test sqlite connection"); conn.execute( @@ -2996,13 +2990,13 @@ async fn task_list_current_turn_projection_keeps_unrelated_durable_blockers() { assert_eq!(value["run_summary"]["pending_worker_turn_intent_count"], 2); assert_eq!(value["run_summary"]["unread_inbox_count"], 2); assert_eq!(value["run_summary"]["completion_ready"], false); - assert!(has_finality_blocker( + assert!(has_quiescence_blocker( &value, "completion_blockers", "in_flight_turn_intents", 1, )); - assert!(has_finality_blocker( + assert!(has_quiescence_blocker( &value, "completion_blockers", "unread_inbox", @@ -3013,7 +3007,7 @@ async fn task_list_current_turn_projection_keeps_unrelated_durable_blockers() { #[tokio::test] async fn task_list_current_turn_projection_fails_closed_for_wrong_identity_or_receipt() { let _sandbox = task_tools_sandbox(); - let inbox_id = seed_task_list_current_turn_finality_fixture(true); + let inbox_id = seed_task_list_current_turn_quiescence_fixture(true); let list = TaskListTool::new(ctx(COORDINATOR_MEMBER_ID)); let wrong_session_ctx = crate::tools::call_context::CallContext::for_turn( @@ -3030,13 +3024,13 @@ async fn task_list_current_turn_projection_fails_closed_for_wrong_identity_or_re ) .expect("decode wrong-session result"); assert_eq!(wrong_session["run_summary"]["completion_ready"], false); - assert!(has_finality_blocker( + assert!(has_quiescence_blocker( &wrong_session, "completion_blockers", "in_flight_turn_intents", 1, )); - assert!(has_finality_blocker( + assert!(has_quiescence_blocker( &wrong_session, "completion_blockers", "unread_inbox", @@ -3057,13 +3051,13 @@ async fn task_list_current_turn_projection_fails_closed_for_wrong_identity_or_re ) .expect("decode wrong-intent result"); assert_eq!(wrong_intent["run_summary"]["completion_ready"], false); - assert!(has_finality_blocker( + assert!(has_quiescence_blocker( &wrong_intent, "completion_blockers", "in_flight_turn_intents", 1, )); - assert!(has_finality_blocker( + assert!(has_quiescence_blocker( &wrong_intent, "completion_blockers", "unread_inbox", @@ -3091,13 +3085,13 @@ async fn task_list_current_turn_projection_fails_closed_for_wrong_identity_or_re ) .expect("decode missing-receipt result"); assert_eq!(missing_receipt["run_summary"]["completion_ready"], false); - assert!(has_finality_blocker( + assert!(has_quiescence_blocker( &missing_receipt, "completion_blockers", "unread_inbox", 1, )); - assert!(!has_finality_blocker( + assert!(!has_quiescence_blocker( &missing_receipt, "completion_blockers", "in_flight_turn_intents", diff --git a/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/member_idle.rs b/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/member_idle.rs index d68fb54676..5393dfd7c6 100644 --- a/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/member_idle.rs +++ b/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/member_idle.rs @@ -28,8 +28,8 @@ use crate::tools::impls::orchestration::org_send_message::{InboxWakeHook, NoopIn /// Production hook: persist a `MemberIdle` envelope into the inbox, then wake the coordinator. /// -/// The hook contract is synchronous because finality must observe this durable -/// notification before it can complete the Run. When called from Tokio's +/// The hook contract is synchronous because Quiescence must observe this durable +/// notification before it can move the Team to Idle. When called from Tokio's /// multi-thread runtime we therefore use an explicit `block_in_place` section: /// executor capacity is handed to another worker while ordering is preserved. pub struct InboxStoreMemberIdleHook { @@ -310,7 +310,7 @@ mod tests { let _sandbox = test_env::sandbox(); let conn = database::db::get_connection().expect("test connection"); agent_inbox::init_schema(&conn).expect("agent inbox schema"); - seed_run(&conn, "run-terminal", "completed"); + seed_run(&conn, "run-terminal", "archived"); let wake_hook = Arc::new(RecordingWakeHook::default()); let hook = InboxStoreMemberIdleHook::new(wake_hook.clone()); diff --git a/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/subagent_handler/persistence.rs b/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/subagent_handler/persistence.rs index 3eb7fad7e0..9c0b15f89f 100644 --- a/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/subagent_handler/persistence.rs +++ b/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/subagent_handler/persistence.rs @@ -72,7 +72,15 @@ impl UnifiedSubagentHandler { .collect(); let count = persistable.len(); - event_pipeline_bridge::persist_events("subagent-child-persist", &sid, &persistable, 5); + if let Err(error) = + event_pipeline_bridge::persist_events("subagent-child-persist", &sid, &persistable, 5) + { + tracing::warn!( + session_id = %sid, + error = %error, + "failed to persist subagent child events" + ); + } tracing::info!( "[subagent:{}] Persisted {} child events for session {}", self.config.subagent_type, diff --git a/src-tauri/crates/agent-core/src/foundation/bus/event_pipeline_bridge.rs b/src-tauri/crates/agent-core/src/foundation/bus/event_pipeline_bridge.rs index 18e3b202ef..fe81f34134 100644 --- a/src-tauri/crates/agent-core/src/foundation/bus/event_pipeline_bridge.rs +++ b/src-tauri/crates/agent-core/src/foundation/bus/event_pipeline_bridge.rs @@ -120,8 +120,12 @@ pub type FinalizePlanRevisionEventsFn = /// Persist a batch of `SessionEvent`s synchronously with retry. The wire /// side converts each to its on-disk `CachedEvent` representation. `label` /// is used in retry log lines. -pub type PersistEventsFn = - fn(label: &'static str, session_id: &str, events: &[SessionEvent], max_retries: u32); +pub type PersistEventsFn = fn( + label: &'static str, + session_id: &str, + events: &[SessionEvent], + max_retries: u32, +) -> Result<(), String>; /// Fire-and-forget variant: spawns `persist_events` onto a blocking thread. pub type PersistEventsAsyncFn = @@ -444,16 +448,13 @@ pub fn persist_events( session_id: &str, events: &[SessionEvent], max_retries: u32, -) { +) -> Result<(), String> { if let Some(f) = PERSIST_EVENTS.get() { - f(label, session_id, events, max_retries); - } else { - tracing::warn!( - "[event-pipeline-bridge] persist_events ({}) called before register for {}", - label, - session_id - ); + return f(label, session_id, events, max_retries); } + Err(format!( + "event pipeline persistence is not registered ({label}, session {session_id})" + )) } pub fn persist_events_async( diff --git a/src-tauri/crates/agent-core/src/foundation/session_bridge.rs b/src-tauri/crates/agent-core/src/foundation/session_bridge.rs index 116293e068..b4f49ff71b 100644 --- a/src-tauri/crates/agent-core/src/foundation/session_bridge.rs +++ b/src-tauri/crates/agent-core/src/foundation/session_bridge.rs @@ -444,7 +444,7 @@ impl TurnIntentBridgeStatus { } /// Canonical persisted wire values for turn intents that may still execute. -/// Agent Org finality queries bind these values instead of independently +/// Agent Org Quiescence queries bind these values instead of independently /// hard-coding a second lifecycle definition. pub const IN_FLIGHT_TURN_INTENT_STATUSES: [&str; 3] = [ TurnIntentBridgeStatus::Optimistic.as_str(), @@ -499,6 +499,16 @@ pub type UpsertTurnIntentFn = fn( status: TurnIntentBridgeStatus, ); +pub type UpsertTurnIntentWithConnectionFn = fn( + connection: &rusqlite::Connection, + session_id: &str, + turn_intent_id: &str, + client_message_id: Option<&str>, + org_run_id: Option<&str>, + source: TurnIntentBridgeSource, + status: TurnIntentBridgeStatus, +) -> Result<(), String>; + pub type UpdateTurnIntentStatusFn = fn(session_id: &str, turn_intent_id: &str, new_status: TurnIntentBridgeStatus); @@ -508,6 +518,8 @@ pub type GetTurnIntentStatusFn = pub type MarkPendingTurnIntentsStaleFn = fn(session_id: &str); static UPSERT_TURN_INTENT: OnceLock = OnceLock::new(); +static UPSERT_TURN_INTENT_WITH_CONNECTION: OnceLock = + OnceLock::new(); static UPDATE_TURN_INTENT_STATUS: OnceLock = OnceLock::new(); static GET_TURN_INTENT_STATUS: OnceLock = OnceLock::new(); static MARK_PENDING_TURN_INTENTS_STALE: OnceLock = OnceLock::new(); @@ -516,6 +528,12 @@ pub fn register_upsert_turn_intent(implementation: UpsertTurnIntentFn) { let _ = UPSERT_TURN_INTENT.set(implementation); } +pub fn register_upsert_turn_intent_with_connection( + implementation: UpsertTurnIntentWithConnectionFn, +) { + let _ = UPSERT_TURN_INTENT_WITH_CONNECTION.set(implementation); +} + pub fn register_update_turn_intent_status(implementation: UpdateTurnIntentStatusFn) { let _ = UPDATE_TURN_INTENT_STATUS.set(implementation); } @@ -553,6 +571,34 @@ pub fn upsert_turn_intent( } } +/// Connection-scoped form for lifecycle owners that must accept an intent in +/// the same SQLite transaction as an adjacent Agent Org state transition. +pub fn upsert_turn_intent_with_connection( + connection: &rusqlite::Connection, + session_id: &str, + turn_intent_id: &str, + client_message_id: Option<&str>, + org_run_id: Option<&str>, + source: TurnIntentBridgeSource, + status: TurnIntentBridgeStatus, +) -> Result<(), String> { + if turn_intent_id.is_empty() { + return Err("turn_intent_id must not be empty".to_string()); + } + let implementation = UPSERT_TURN_INTENT_WITH_CONNECTION + .get() + .ok_or_else(|| "turn-intent persistence bridge is not registered".to_string())?; + implementation( + connection, + session_id, + turn_intent_id, + client_message_id, + org_run_id, + source, + status, + ) +} + /// Patch the status of an existing lifecycle row. Illegal transitions are /// silently rejected by the implementation — callers do not need to handle /// the error case. diff --git a/src-tauri/crates/agent-core/src/lifecycle.rs b/src-tauri/crates/agent-core/src/lifecycle.rs index 1893c7da64..a9430a96c4 100644 --- a/src-tauri/crates/agent-core/src/lifecycle.rs +++ b/src-tauri/crates/agent-core/src/lifecycle.rs @@ -11,9 +11,7 @@ use tauri::Emitter; use crate::bus::{broadcast_event, event_pipeline_bridge}; use crate::coordination::agent_inbox::MemberIdleReason; -use crate::coordination::agent_org_runs::{ - AgentOrgRunContext, AgentOrgRunStatus, AgentOrgRunStore, -}; +use crate::coordination::agent_org_runs::{AgentOrgRunContext, AgentOrgRunStore}; use crate::coordination::agent_org_tasks::{ self, AgentOrgTaskStore, Task, TASK_METADATA_REQUIRED_ROLE, }; @@ -476,13 +474,27 @@ pub fn finalize_agent_org_member_turn( // after every member/coordinator boundary so an all-completed, fully // quiescent run closes without requiring the user to pause/resume it. // The store re-checks tasks, inbox, interventions and queued turn - // intents in one IMMEDIATE transaction before committing finality. + // intents in one IMMEDIATE transaction before committing quiescence. if let Some(run_id) = reconcile_run_id { - match AgentOrgRunStore::reconcile_run_finality(&run_id) { - Ok(Some(AgentOrgRunStatus::Completed)) => { - tracing::info!(run_id = %run_id, "[lifecycle] completed quiescent Agent Org run"); + let transition = AgentOrgRunStore::assess_run_quiescence(&run_id).and_then(|assessment| { + let Some(generation) = assessment.facts.activation_generation else { + return Ok(false); + }; + let Some(work_revision) = assessment + .facts + .progress + .as_ref() + .map(|progress| progress.work_revision) + else { + return Ok(false); + }; + AgentOrgRunStore::try_transition_working_to_idle(&run_id, generation, work_revision) + }); + match transition { + Ok(true) => { + tracing::info!(run_id = %run_id, "[lifecycle] idled quiescent Agent Org run"); } - Ok(_) => {} + Ok(false) => {} Err(err) => { tracing::warn!(run_id = %run_id, error = %err, "[lifecycle] failed to reconcile Agent Org run after turn finalization"); } @@ -576,7 +588,7 @@ pub async fn finalize_session( if is_agent_org_member_session { // Member finalization performs several synchronous SQLite operations // under the shared writer lock (task requeue, recovery-budget cleanup, - // MemberIdle persistence, and run finality reconciliation). Keep the + // MemberIdle persistence, and Team quiescence reconciliation). Keep the // complete blocking phase off the Tokio worker that is finalizing the // provider turn; moving only the first query still leaves the later // writes able to stall unrelated async sessions. diff --git a/src-tauri/crates/agent-core/src/state/commands/session/identity.rs b/src-tauri/crates/agent-core/src/state/commands/session/identity.rs index 0db3a75639..844b5db7d0 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/identity.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/identity.rs @@ -32,6 +32,14 @@ pub(super) struct SessionIdentity { pub(super) account_id: Option, pub(super) native_harness_type: Option, pub(super) workspace_root: PathBuf, + /// A loaded Agent Org runtime already knows its authoritative Run id. + /// Keeping that hint here lets the send boundary enforce Team lifecycle + /// before it creates a Provider without querying ordinary SDE sessions. + pub(super) agent_org_run_id_hint: Option, + /// Unloaded Agent Org root/member sessions carry a canonical member id in + /// persistence. Only those sessions need the bounded parent-walk lookup; + /// ordinary SDE sessions stay on the existing zero-Agent-Org-query path. + pub(super) has_persisted_agent_org_identity: bool, } /// Caller-supplied overrides. Fields that are `None` are resolved from @@ -104,6 +112,10 @@ pub(super) async fn resolve_session_identity( let native_harness_after_l2 = overrides .native_harness_type .or_else(|| cached_runtime.as_ref().and_then(|r| r.native_harness_type)); + let agent_org_run_id_hint = cached_runtime + .as_ref() + .and_then(|runtime| runtime.agent_org_context.as_ref()) + .map(|context| context.run_id.clone()); // ── Layer 3: DB (lazy — only when at least one field still needs it) ─ let needs_db = model_after_l2.is_none() @@ -163,6 +175,10 @@ pub(super) async fn resolve_session_identity( }) .transpose()?; let native_harness_type = native_harness_after_l2.or(native_harness_from_db); + let has_persisted_agent_org_identity = db_record + .as_ref() + .and_then(|record| record.org_member_id.as_deref()) + .is_some(); // ── Workspace Root ─────────────────────────────────────────────────── // @@ -211,6 +227,8 @@ pub(super) async fn resolve_session_identity( account_id, workspace_root, native_harness_type, + agent_org_run_id_hint, + has_persisted_agent_org_identity, }) } diff --git a/src-tauri/crates/agent-core/src/state/commands/session/message/org_wake.rs b/src-tauri/crates/agent-core/src/state/commands/session/message/org_wake.rs index 093539ef57..fbeeff52e1 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/message/org_wake.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/message/org_wake.rs @@ -93,11 +93,10 @@ pub(super) fn promote_agent_org_wake_session_to_running( .map_err(|error| error.to_string()) } -/// Promote a direct Rust Agent Org turn unless deletion has established the -/// run's terminal `cancelled` fence. Direct user turns intentionally retain -/// their existing behavior for completed/failed historical runs; this guard -/// only closes the race where a message was queued while hierarchy deletion -/// was stopping the run. +/// Promote a direct Rust Agent Org turn only while its Team is still Running. +/// Submit preflight is only a snapshot: a queued turn must re-check the +/// durable lifecycle fence immediately before execution so Starting, Idle, +/// Paused, Failed, or Archived can never start a Provider turn. pub(super) fn promote_agent_org_direct_session_to_running( conn: &rusqlite::Connection, run_id: &str, @@ -113,7 +112,9 @@ pub(super) fn promote_agent_org_direct_session_to_running( ) .optional() .map_err(|error| error.to_string())?; - if run_status.as_deref() == Some("cancelled") || run_status.is_none() { + if run_status.as_deref() + != Some(crate::coordination::agent_org_runs::AgentOrgRunStatus::Running.as_str()) + { return Ok(0); } diff --git a/src-tauri/crates/agent-core/src/state/commands/session/message/send.rs b/src-tauri/crates/agent-core/src/state/commands/session/message/send.rs index 183b8153fc..949fa37e97 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/message/send.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/message/send.rs @@ -27,6 +27,77 @@ use super::org_wake::{ resolve_agent_org_wake_mode, }; +pub(super) fn ensure_agent_org_turn_is_runnable( + run_id: &str, + status: crate::coordination::agent_org_runs::AgentOrgRunStatus, +) -> Result<(), String> { + use crate::coordination::agent_org_runs::AgentOrgRunStatus; + + match status { + AgentOrgRunStatus::Running => Ok(()), + AgentOrgRunStatus::Starting => Err(format!( + "team_not_ready: Agent Org run {run_id} is still materializing" + )), + AgentOrgRunStatus::Paused => Err(format!( + "team_paused: Agent Org run {run_id} cannot start a turn in this lifecycle slice" + )), + AgentOrgRunStatus::Idle => Err(format!( + "team_idle: Agent Org run {run_id} has no formal activation for a new turn" + )), + AgentOrgRunStatus::Failed => Err(format!( + "team_unavailable: Agent Org run {run_id} failed during materialization" + )), + AgentOrgRunStatus::Archived => Err(format!( + "team_archived: Agent Org run {run_id} is read-only" + )), + } +} + +async fn preflight_agent_org_turn_before_runtime( + session_id: &str, + explicit_run_id: Option<&str>, + run_id_hint: Option<&str>, + has_persisted_agent_org_identity: bool, +) -> Result, String> { + if let (Some(explicit), Some(hint)) = (explicit_run_id, run_id_hint) { + if explicit != hint { + return Err(format!( + "Agent Org turn intent run mismatch for session {session_id}: explicit run {explicit}, runtime run {hint}" + )); + } + } + + let run_id = match explicit_run_id.or(run_id_hint) { + Some(run_id) => Some(run_id.to_string()), + None if has_persisted_agent_org_identity => { + let session_id = session_id.to_string(); + tokio::task::spawn_blocking(move || { + crate::coordination::agent_org_runs::AgentOrgRunStore::run_id_for_session_with_parent_walk( + &session_id, + ) + }) + .await + .map_err(|error| format!("Agent Org run lookup worker failed: {error}"))?? + } + None => None, + }; + + let Some(run_id) = run_id else { + return Ok(None); + }; + + crate::coordination::agent_org_runs::require_agent_org_redesign()?; + let status_run_id = run_id.clone(); + let status = tokio::task::spawn_blocking(move || { + crate::coordination::agent_org_runs::AgentOrgRunStore::get_run_status(&status_run_id) + }) + .await + .map_err(|error| format!("Agent Org status worker failed: {error}"))?? + .ok_or_else(|| format!("team_unavailable: Agent Org run {run_id} does not exist"))?; + ensure_agent_org_turn_is_runnable(&run_id, status)?; + Ok(Some(run_id)) +} + pub(super) fn should_divert_to_mid_turn_steering( source: TurnIntentBridgeSource, is_resume: bool, @@ -108,6 +179,23 @@ pub(crate) async fn send_message_impl( // ── 1. Resolve session identity (unified — single code path) ───────── let identity = resolve_session_identity(state, &session_id, overrides).await?; + let explicit_org_run_id = match (org_wake_run_id.as_deref(), intent_org_run_id.as_deref()) { + (Some(wake_run_id), Some(intent_run_id)) if wake_run_id != intent_run_id => { + return Err(format!( + "Agent Org wake/intent run mismatch for session {session_id}: wake run {wake_run_id}, intent run {intent_run_id}" + )); + } + (Some(run_id), _) | (None, Some(run_id)) => Some(run_id), + (None, None) => None, + }; + let preflight_org_run_id = preflight_agent_org_turn_before_runtime( + &session_id, + explicit_org_run_id, + identity.agent_org_run_id_hint.as_deref(), + identity.has_persisted_agent_org_identity, + ) + .await?; + // Goal loop: a real user submission becomes (or replaces) the // session's standing goal and resets the continuation counter. // `Queue`-sourced messages (goal continuations, queued flushes) and @@ -157,7 +245,7 @@ pub(crate) async fn send_message_impl( } (Some(_), _) => intent_org_run_id, (None, Some(_)) => runtime_org_run_id, - (None, None) => None, + (None, None) => preflight_org_run_id, }; // Wingman resume: reopen the bottom bar. On fresh start the frontend @@ -477,9 +565,9 @@ pub(crate) async fn send_message_impl( // Queued and coalesced messages are not running sessions. Promote // the DB state only when the scheduler actually begins execution. - // Agent Org wakes require a running run. Direct Agent Org turns - // retain their historical-run behavior but refuse the terminal - // `cancelled` fence established by hierarchy deletion. + // Both Agent Org wakes and direct turns require a Running Team. + // This execute-time check closes the race after submit preflight: + // a queued turn becomes a no-op if lifecycle advances first. let status_sid = sid.clone(); let status_wake_run_id = org_wake_run_id.clone(); let status_intent_run_id = intent_org_run_id.clone(); diff --git a/src-tauri/crates/agent-core/src/state/commands/session/message/tests.rs b/src-tauri/crates/agent-core/src/state/commands/session/message/tests.rs index b5a4d0c019..7e90f77657 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/message/tests.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/message/tests.rs @@ -10,7 +10,10 @@ use super::org_wake::{ promote_agent_org_direct_session_to_running, promote_agent_org_wake_session_to_running, resolve_agent_org_wake_mode, }; -use super::send::{should_divert_to_mid_turn_steering, terminal_intent_status_override}; +use super::send::{ + ensure_agent_org_turn_is_runnable, should_divert_to_mid_turn_steering, + terminal_intent_status_override, +}; use crate::coordination::agent_inbox::{ AgentInboxStore, AgentMessage, InsertInboxParams, RequestId, }; @@ -287,28 +290,71 @@ fn queued_agent_org_wake_rechecks_run_member_and_intervention_at_turn_start() { } #[test] -fn direct_agent_org_turn_refuses_cancelled_delete_fence() { +fn direct_agent_org_turn_only_promotes_while_run_is_running() { let fixture = setup_wake_mode_fixture("build", TaskStatus::Pending); let conn = database::db::get_connection().expect("test db"); + for status in [ + AgentOrgRunStatus::Starting, + AgentOrgRunStatus::Paused, + AgentOrgRunStatus::Idle, + AgentOrgRunStatus::Failed, + AgentOrgRunStatus::Archived, + ] { + conn.execute( + "UPDATE agent_org_runs SET status=?1 WHERE id=?2", + rusqlite::params![status.as_str(), &fixture.run_id], + ) + .expect("set non-runnable run status"); + assert_eq!( + promote_agent_org_direct_session_to_running( + &conn, + &fixture.run_id, + &fixture.session_id, + ) + .expect("non-running run claim is a no-op"), + 0, + "{status:?} must not promote the member Session" + ); + let session_status = conn + .query_row( + "SELECT status FROM agent_sessions WHERE session_id=?1", + [&fixture.session_id], + |row| row.get::<_, String>(0), + ) + .expect("load member status"); + assert_eq!(session_status, "idle"); + } + conn.execute( - "UPDATE agent_org_runs SET status='cancelled' WHERE id=?1", - [&fixture.run_id], + "UPDATE agent_org_runs SET status=?1 WHERE id=?2", + rusqlite::params![AgentOrgRunStatus::Running.as_str(), &fixture.run_id], ) - .expect("establish delete fence"); - + .expect("restore running run"); assert_eq!( promote_agent_org_direct_session_to_running(&conn, &fixture.run_id, &fixture.session_id) - .expect("cancelled run claim is a no-op"), - 0 + .expect("running run promotes the member Session"), + 1 ); - let status = conn - .query_row( - "SELECT status FROM agent_sessions WHERE session_id=?1", - [&fixture.session_id], - |row| row.get::<_, String>(0), - ) - .expect("load member status"); - assert_eq!(status, "idle"); +} + +#[test] +fn provider_preflight_exhaustively_rejects_every_non_running_team_status() { + assert!(ensure_agent_org_turn_is_runnable("run", AgentOrgRunStatus::Running).is_ok()); + + for (status, code) in [ + (AgentOrgRunStatus::Starting, "team_not_ready"), + (AgentOrgRunStatus::Paused, "team_paused"), + (AgentOrgRunStatus::Idle, "team_idle"), + (AgentOrgRunStatus::Failed, "team_unavailable"), + (AgentOrgRunStatus::Archived, "team_archived"), + ] { + let error = ensure_agent_org_turn_is_runnable("run", status) + .expect_err("non-running Team cannot initialize a turn"); + assert!( + error.starts_with(code), + "{status:?} should return {code}, got {error}" + ); + } } #[test] diff --git a/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/group_chat.rs b/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/group_chat.rs index 7fb67a4ce7..9c03bb1d2d 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/group_chat.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/group_chat.rs @@ -12,7 +12,9 @@ use serde::Serialize; use crate::coordination::agent_inbox::{ AgentInboxRecord, AgentInboxStore, AgentMessage, InsertInboxParams, USER_SENDER_ID, }; -use crate::coordination::agent_org_runs::{AgentOrgRunContext, COORDINATOR_MEMBER_ID}; +use crate::coordination::agent_org_runs::{ + AgentOrgRunContext, AgentOrgRunStatus, COORDINATOR_MEMBER_ID, +}; use crate::state::AgentAppState; use super::context::session_org_read_context; @@ -75,6 +77,7 @@ pub async fn agent_org_group_chat_history_page_impl( before_id: Option, limit: Option, ) -> Result { + crate::coordination::agent_org_runs::require_agent_org_redesign()?; if before_id.is_some_and(|id| id <= 0) { return Err("before_id must be a positive Inbox row id".to_string()); } @@ -307,6 +310,7 @@ async fn agent_org_send_group_chat_message_impl_with_display( content: String, display_text: Option, ) -> Result { + crate::coordination::agent_org_runs::require_agent_org_redesign()?; let content = content.trim(); if content.is_empty() { return Err("Agent Org group chat message content is required".to_string()); @@ -418,17 +422,26 @@ pub(super) fn persist_group_chat_message( ) .optional() .map_err(|err| err.to_string())?; - match run_status.as_deref() { - Some("running" | "paused") => {} - Some(status) => { + let Some(run_status) = run_status else { + return Err(format!("Agent Org run {} no longer exists", context.run_id)); + }; + let run_status = AgentOrgRunStatus::parse(&run_status).ok_or_else(|| { + format!( + "Agent Org run {} has an unrecognized status", + context.run_id + ) + })?; + match run_status { + AgentOrgRunStatus::Running | AgentOrgRunStatus::Paused => {} + AgentOrgRunStatus::Starting + | AgentOrgRunStatus::Idle + | AgentOrgRunStatus::Failed + | AgentOrgRunStatus::Archived => { return Err(format!( - "Agent Org run {} is {status}; terminal runs do not accept new group messages", - context.run_id + "Agent Org run {} is {}; this status does not accept new group messages", + context.run_id, run_status )); } - None => { - return Err(format!("Agent Org run {} no longer exists", context.run_id)); - } } let row = AgentInboxStore::insert_in_tx( diff --git a/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/intervention.rs b/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/intervention.rs index 4335760be9..1f5576ec41 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/intervention.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/intervention.rs @@ -43,6 +43,7 @@ pub async fn agent_org_session_enter_intervention( state: tauri::State<'_, AgentAppState>, session_id: String, ) -> Result { + crate::coordination::agent_org_runs::require_agent_org_redesign()?; let Some(read_context) = session_org_read_context(&state, &session_id).await? else { return Ok(false); }; @@ -76,6 +77,7 @@ pub async fn agent_org_session_intervention_state( state: tauri::State<'_, AgentAppState>, session_id: String, ) -> Result { + crate::coordination::agent_org_runs::require_agent_org_redesign()?; let Some(read_context) = session_org_read_context(&state, &session_id).await? else { return Ok(AgentOrgSessionInterventionState { intervention: None }); }; @@ -146,6 +148,7 @@ pub async fn agent_org_session_return_to_work_impl( state: &AgentAppState, session_id: String, ) -> Result { + crate::coordination::agent_org_runs::require_agent_org_redesign()?; let Some(read_context) = session_org_read_context(state, &session_id).await? else { return Ok(false); }; @@ -189,6 +192,7 @@ pub async fn agent_org_send_user_message_to_member_impl( member_id: String, content: String, ) -> Result { + crate::coordination::agent_org_runs::require_agent_org_redesign()?; let member_id = member_id.trim(); if member_id.is_empty() { return Err("Agent Org member id is required".to_string()); diff --git a/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/lifecycle.rs b/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/lifecycle.rs index 8c567f979c..711106f764 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/lifecycle.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/lifecycle.rs @@ -24,13 +24,14 @@ use super::context::session_org_read_context; /// Pause the Agent Org run that the given session belongs to. Transitions /// `running → paused`; already non-running runs return `Ok(false)` (idempotent). -/// The run remains queryable while paused — polling and member switching are -/// unaffected. The coordinator and members stop receiving dispatch until resumed. +/// The run remains available to explicit reads while paused, but PR1's +/// fallback poller deliberately observes only Starting and Running Teams. #[tauri::command] pub async fn agent_org_pause_run( state: tauri::State<'_, AgentAppState>, session_id: String, ) -> Result { + crate::coordination::agent_org_runs::require_agent_org_redesign()?; let Some(read_context) = session_org_read_context(&state, &session_id).await? else { return Ok(false); }; @@ -61,6 +62,7 @@ pub async fn agent_org_resume_run( state: tauri::State<'_, AgentAppState>, session_id: String, ) -> Result { + crate::coordination::agent_org_runs::require_agent_org_redesign()?; let Some(read_context) = session_org_read_context(&state, &session_id).await? else { return Ok(false); }; diff --git a/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/plan_approval.rs b/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/plan_approval.rs index 195c0e4bdb..adf4e2cee0 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/plan_approval.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/plan_approval.rs @@ -3,7 +3,7 @@ //! When a run's plan-approval policy routes a plan revision to the user, these //! commands fetch the revision detail and record the user's decision (approve, //! approve-with-edits, or request-changes), then wake the affected members and -//! reconcile run finality off the durable transaction. +//! reconcile Team quiescence off the durable transaction. use crate::coordination::agent_inbox::USER_SENDER_ID; use crate::coordination::agent_org_plan_approvals::{ @@ -31,6 +31,7 @@ pub async fn agent_org_plan_approval_detail( approval_id: String, plan_revision_id: String, ) -> Result { + crate::coordination::agent_org_runs::require_agent_org_redesign()?; let Some(read_context) = session_org_read_context(&state, &session_id).await? else { return Err(format!( "Session {session_id} is not part of an Agent Org run" @@ -71,6 +72,7 @@ pub async fn agent_org_plan_approval_respond( edited_content: Option, feedback: Option, ) -> Result { + crate::coordination::agent_org_runs::require_agent_org_redesign()?; let Some(read_context) = session_org_read_context(&state, &session_id).await? else { return Err(format!( "Session {session_id} is not part of an Agent Org run" @@ -169,7 +171,23 @@ pub async fn agent_org_plan_approval_respond( let reconcile_run_id = run_id.clone(); tokio::spawn(async move { match tokio::task::spawn_blocking(move || { - AgentOrgRunStore::reconcile_run_finality(&reconcile_run_id) + let assessment = AgentOrgRunStore::assess_run_quiescence(&reconcile_run_id)?; + let Some(generation) = assessment.facts.activation_generation else { + return Ok(false); + }; + let Some(work_revision) = assessment + .facts + .progress + .as_ref() + .map(|progress| progress.work_revision) + else { + return Ok(false); + }; + AgentOrgRunStore::try_transition_working_to_idle( + &reconcile_run_id, + generation, + work_revision, + ) }) .await { diff --git a/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/run_view.rs b/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/run_view.rs index 01d743df6a..a409fcd3ad 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/run_view.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/run_view.rs @@ -141,6 +141,7 @@ pub struct AgentOrgRunTaskOverview { #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] #[serde(rename_all = "snake_case")] pub enum AgentOrgRunPhase { + Starting, Coordinating, Dispatching, MembersWorking, @@ -148,10 +149,9 @@ pub enum AgentOrgRunPhase { AwaitingPlanApproval, Finalizing, Paused, - Completed, + Idle, Failed, - Cancelled, - Abandoned, + Archived, } /// The Run View is a live operational snapshot, not an inbox-history API. @@ -172,6 +172,7 @@ pub async fn agent_org_session_run_view_impl( state: &AgentAppState, session_id: &str, ) -> Result, String> { + crate::coordination::agent_org_runs::require_agent_org_redesign()?; let Some(read_context) = session_org_read_context(state, session_id).await? else { return Ok(None); }; @@ -193,7 +194,7 @@ pub async fn agent_org_session_run_view_impl( Ok(Some(view)) } -fn build_agent_org_run_view( +pub(super) fn build_agent_org_run_view( context: &AgentOrgRunContext, current_member_id: String, ) -> Result { @@ -201,8 +202,8 @@ fn build_agent_org_run_view( let tx = conn .transaction_with_behavior(rusqlite::TransactionBehavior::Deferred) .map_err(|err| err.to_string())?; - let finality = AgentOrgRunStore::finality_assessment_with_connection(&tx, &context.run_id)?; - let run_status_value = finality + let quiescence = AgentOrgRunStore::quiescence_assessment_with_connection(&tx, &context.run_id)?; + let run_status_value = quiescence .facts .run_status .ok_or_else(|| format!("Agent Org run {} no longer exists", context.run_id))?; @@ -217,11 +218,11 @@ fn build_agent_org_run_view( RUN_VIEW_TASK_LIMIT, )?; let task_overview = AgentOrgRunTaskOverview { - total: finality.facts.task_count, - pending: finality.facts.pending_task_count, - in_progress: finality.facts.in_progress_task_count, - completed: finality.facts.completed_task_count, - corrupt: finality.facts.corrupt_task_count, + total: quiescence.facts.task_count, + pending: quiescence.facts.pending_task_count, + in_progress: quiescence.facts.in_progress_task_count, + completed: quiescence.facts.completed_task_count, + corrupt: quiescence.facts.corrupt_task_count, visible: task_page.tasks.len(), truncated: task_page.has_more, }; @@ -307,7 +308,7 @@ fn build_agent_org_run_view( run_status_value, &members, &task_overview, - finality.facts.unread_inbox_count, + quiescence.facts.unread_inbox_count, &pending_plan_approvals, ); @@ -322,7 +323,7 @@ fn build_agent_org_run_view( tasks, task_overview, inbox, - unread_inbox_count: finality.facts.unread_inbox_count, + unread_inbox_count: quiescence.facts.unread_inbox_count, pending_plan_approvals, }) } @@ -335,11 +336,11 @@ pub(super) fn project_run_phase( pending_plan_approvals: &[AgentOrgPlanApprovalSummary], ) -> AgentOrgRunPhase { match run_status { + AgentOrgRunStatus::Starting => AgentOrgRunPhase::Starting, AgentOrgRunStatus::Paused => AgentOrgRunPhase::Paused, - AgentOrgRunStatus::Completed => AgentOrgRunPhase::Completed, + AgentOrgRunStatus::Idle => AgentOrgRunPhase::Idle, AgentOrgRunStatus::Failed => AgentOrgRunPhase::Failed, - AgentOrgRunStatus::Cancelled => AgentOrgRunPhase::Cancelled, - AgentOrgRunStatus::Abandoned => AgentOrgRunPhase::Abandoned, + AgentOrgRunStatus::Archived => AgentOrgRunPhase::Archived, AgentOrgRunStatus::Running => { let all_tasks_completed = task_overview.total > 0 && task_overview.pending == 0 diff --git a/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/tests.rs b/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/tests.rs index b5bf8d3a47..8e160cd8f0 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/tests.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/tests.rs @@ -68,7 +68,7 @@ fn prepare_command_run(status: &str) -> AgentOrgRunContext { id, org_id, coordinator_agent_id, root_session_id, org_snapshot_json, entry_mode, status, work_item_id, project_slug, routine_fire_id, summary, last_error, - created_at, updated_at, completed_at + created_at, updated_at, idled_at ) VALUES (?1, ?2, ?3, ?4, NULL, 'standalone_session', ?5, NULL, NULL, NULL, NULL, NULL, ?6, ?6, NULL)", params![ @@ -167,7 +167,7 @@ fn task_for_resume(owner: Option<&str>, status: TaskStatus) -> Task { } #[test] -fn run_phase_projects_all_completed_running_board_as_finalizing() { +fn run_phase_projects_completed_work_as_finalizing_then_idle() { let overview = AgentOrgRunTaskOverview { total: 1, pending: 0, @@ -183,7 +183,7 @@ fn run_phase_projects_all_completed_running_board_as_finalizing() { ); assert_eq!( project_run_phase( - AgentOrgRunStatus::Completed, + AgentOrgRunStatus::Idle, &[], &AgentOrgRunTaskOverview { total: 0, @@ -197,10 +197,88 @@ fn run_phase_projects_all_completed_running_board_as_finalizing() { 0, &[], ), - AgentOrgRunPhase::Completed + AgentOrgRunPhase::Idle ); } +#[test] +fn run_view_is_a_pure_read_and_does_not_advance_updated_at() { + let _sandbox = test_helpers::test_env::sandbox(); + let conn = get_connection().expect("db connection"); + crate::foundation::persistence::test_schema::ensure_agent_sessions_schema(&conn); + crate::foundation::persistence::session_snapshots::ensure_tables_with(&conn) + .expect("session snapshot schema"); + crate::session::persistence::init(&conn).expect("session schema"); + crate::coordination::init_agent_org_schemas(&conn).expect("Agent Org schemas"); + conn.execute_batch( + "CREATE TABLE IF NOT EXISTS code_sessions ( + session_id TEXT PRIMARY KEY, + cli_agent_type TEXT NOT NULL, + status TEXT NOT NULL, + parent_session_id TEXT, + org_member_id TEXT, + updated_at TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS session_turn_intents ( + session_id TEXT NOT NULL, + turn_intent_id TEXT NOT NULL, + client_message_id TEXT, + org_run_id TEXT, + source TEXT NOT NULL, + status TEXT NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (session_id, turn_intent_id) + );", + ) + .expect("runtime support schemas"); + drop(conn); + + let context = prepare_command_run("running"); + crate::session::persistence::upsert_session( + &crate::session::persistence::UnifiedSessionRecord { + session_id: "root-shared-agent".to_string(), + name: "Coordinator".to_string(), + status: crate::session::SessionStatus::Idle.as_str().to_string(), + session_type: "agent".to_string(), + agent_definition_id: Some("builtin:sde".to_string()), + org_member_id: Some(COORDINATOR_MEMBER_ID.to_string()), + created_at: "2026-05-28T00:00:00Z".to_string(), + updated_at: "2026-05-28T00:00:00Z".to_string(), + ..Default::default() + }, + ) + .expect("persist coordinator Session"); + let observer = get_connection().expect("observer connection"); + let before_data_version: i64 = observer + .query_row("PRAGMA data_version", [], |row| row.get(0)) + .expect("read data version"); + let before_updated_at: String = observer + .query_row( + "SELECT updated_at FROM agent_org_runs WHERE id=?1", + [&context.run_id], + |row| row.get(0), + ) + .expect("read run timestamp"); + + let view = build_agent_org_run_view(&context, COORDINATOR_MEMBER_ID.to_string()) + .expect("build pure Run View"); + + let after_data_version: i64 = observer + .query_row("PRAGMA data_version", [], |row| row.get(0)) + .expect("read data version after Run View"); + let after_updated_at: String = observer + .query_row( + "SELECT updated_at FROM agent_org_runs WHERE id=?1", + [&context.run_id], + |row| row.get(0), + ) + .expect("read run timestamp after Run View"); + assert_eq!(view.run_status, "running"); + assert_eq!(after_data_version, before_data_version); + assert_eq!(after_updated_at, before_updated_at); +} + #[test] fn task_runtime_projects_execution_mode_on_the_wire() { let task = AgentOrgTaskRuntime { @@ -325,9 +403,9 @@ fn resume_wake_requires_unread_inbox() { } #[test] -fn terminal_group_message_writes_neither_inbox_nor_intervention_clear() { +fn archived_group_message_writes_neither_inbox_nor_intervention_clear() { let _sandbox = test_helpers::test_env::sandbox(); - let context = prepare_command_run("completed"); + let context = prepare_command_run("archived"); AgentMemberInterventionStore::enter(EnterMemberInterventionParams { org_run_id: context.run_id.clone(), member_id: "member-planner".to_string(), @@ -342,12 +420,12 @@ fn terminal_group_message_writes_neither_inbox_nor_intervention_clear() { &context, "builtin:sde", "member-planner", - "This must not enter a terminal run", + "This must not enter an Archived run", None, ) - .expect_err("terminal run rejects group message"); + .expect_err("Archived run rejects group message"); - assert!(error.contains("terminal runs do not accept")); + assert!(error.contains("this status does not accept")); assert_eq!(inbox_count_for_member(&context, "member-planner"), 0); assert!( AgentMemberInterventionStore::active_for_member(&context.run_id, "member-planner") @@ -460,10 +538,10 @@ fn group_chat_history_pages_all_rows_and_preserves_long_display_text_after_reloa let conn = get_connection().expect("db connection"); conn.execute( - "UPDATE agent_org_runs SET status='completed' WHERE id=?1", + "UPDATE agent_org_runs SET status='archived' WHERE id=?1", params![&context.run_id], ) - .expect("terminalize run"); + .expect("archive run"); assert_eq!( load_group_chat_history_page(&context, None, 100) .expect("terminal history stays readable") diff --git a/src-tauri/crates/agent-core/src/state/commands/session/persistence.rs b/src-tauri/crates/agent-core/src/state/commands/session/persistence.rs index 89964e14bb..dcd9fac74c 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/persistence.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/persistence.rs @@ -2,7 +2,6 @@ use std::collections::HashSet; use std::sync::Arc; -use std::time::Duration; use crate::coordination::agent_org_runs::AgentOrgRunStore; use crate::interaction::plan_approval::persistence::PlanApprovalStore; @@ -52,8 +51,6 @@ pub async fn agent_list_all_sessions() -> Result, String> } const MAX_AGENT_ORG_DELETE_SESSIONS: usize = 1_024; -const AGENT_ORG_DELETE_STOP_TIMEOUT: Duration = Duration::from_secs(10); -const AGENT_ORG_DELETE_STOP_POLL_INTERVAL: Duration = Duration::from_millis(50); #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "camelCase")] @@ -103,46 +100,14 @@ pub async fn agent_delete_session( }); }; - let (plan, quiesced_runtime_session_ids) = if matches!( - plan.run_status, - crate::coordination::agent_org_runs::AgentOrgRunStatus::Running - | crate::coordination::agent_org_runs::AgentOrgRunStatus::Paused - | crate::coordination::agent_org_runs::AgentOrgRunStatus::Cancelled - ) { - let fenced_plan = - tokio::task::spawn_blocking(move || establish_agent_org_delete_fence(&plan)) - .await - .map_err(|err| format!("Agent Org deletion fence worker failed: {err}"))??; - let quiesced_runtime_session_ids = if fenced_plan.run_status - == crate::coordination::agent_org_runs::AgentOrgRunStatus::Cancelled - { - stop_agent_org_runtime_sessions(&state, &fenced_plan).await? - } else { - ensure_agent_org_runtime_sessions_idle(&state, &fenced_plan).await?; - HashSet::new() - }; - let root_session_id = fenced_plan.root_session_id.clone(); - let current_plan = tokio::task::spawn_blocking(move || { - let conn = get_connection().map_err(|err| err.to_string())?; - load_agent_org_session_delete_plan(&conn, &root_session_id)?.ok_or_else(|| { - format!( - "Refusing to delete Agent Org root {root_session_id}: ownership disappeared while stopping" - ) - }) - }) - .await - .map_err(|err| format!("Agent Org post-stop planning worker failed: {err}"))??; - if !agent_org_delete_topology_matches(&fenced_plan, ¤t_plan) { - return Err(format!( - "Refusing to delete Agent Org run {}: session hierarchy changed while stopping", - fenced_plan.run_id - )); - } - (current_plan, quiesced_runtime_session_ids) - } else { - ensure_agent_org_runtime_sessions_idle(&state, &plan).await?; - (plan, HashSet::new()) - }; + if plan.run_status != crate::coordination::agent_org_runs::AgentOrgRunStatus::Archived { + return Err(format!( + "Refusing to delete Agent Org run {}: Archive is required before Delete", + plan.run_id + )); + } + ensure_agent_org_runtime_sessions_idle(&state, &plan).await?; + let quiesced_runtime_session_ids = HashSet::new(); validate_agent_org_delete_ready(&plan, &quiesced_runtime_session_ids)?; ensure_agent_org_runtime_sessions_idle(&state, &plan).await?; @@ -375,89 +340,11 @@ fn load_agent_org_session_delete_plan( })) } -fn agent_org_delete_topology_matches( - expected: &AgentOrgSessionDeletePlan, - current: &AgentOrgSessionDeletePlan, -) -> bool { - expected.run_id == current.run_id - && expected.root_session_id == current.root_session_id - && expected.sessions.len() == current.sessions.len() - && expected - .sessions - .iter() - .zip(¤t.sessions) - .all(|(left, right)| { - left.session_id == right.session_id - && left.parent_session_id == right.parent_session_id - && left.depth == right.depth - }) -} - -fn establish_agent_org_delete_fence( - expected_plan: &AgentOrgSessionDeletePlan, -) -> Result { - let (current_plan, changed) = with_sessions_writer(|| { - let mut conn = get_connection().map_err(|err| err.to_string())?; - let tx = conn - .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate) - .map_err(|err| err.to_string())?; - let mut current_plan = - load_agent_org_session_delete_plan(&tx, &expected_plan.root_session_id)?.ok_or_else( - || { - format!( - "Refusing to delete Agent Org run {}: root ownership changed before stopping", - expected_plan.run_id - ) - }, - )?; - if !agent_org_delete_topology_matches(expected_plan, ¤t_plan) { - return Err(format!( - "Refusing to delete Agent Org run {}: session hierarchy changed before stopping", - expected_plan.run_id - )); - } - - let changed = match current_plan.run_status { - crate::coordination::agent_org_runs::AgentOrgRunStatus::Running - | crate::coordination::agent_org_runs::AgentOrgRunStatus::Paused => { - let changed = - AgentOrgRunStore::cancel_for_delete_with_connection(&tx, ¤t_plan.run_id)?; - if !changed { - return Err(format!( - "Refusing to delete Agent Org run {}: run status changed before cancellation", - current_plan.run_id - )); - } - current_plan.run_status = - crate::coordination::agent_org_runs::AgentOrgRunStatus::Cancelled; - true - } - crate::coordination::agent_org_runs::AgentOrgRunStatus::Cancelled => false, - status if status.is_terminal() => false, - status => { - return Err(format!( - "Refusing to delete Agent Org run {}: unsupported run status {}", - current_plan.run_id, - status.as_str() - )); - } - }; - tx.commit().map_err(|err| err.to_string())?; - Ok::<_, String>((current_plan, changed)) - })?; - if changed { - crate::coordination::agent_org_run_events::notify_agent_org_run_changed( - ¤t_plan.run_id, - ); - } - Ok(current_plan) -} - fn validate_agent_org_delete_ready( plan: &AgentOrgSessionDeletePlan, - quiesced_runtime_session_ids: &HashSet, + _quiesced_runtime_session_ids: &HashSet, ) -> Result<(), String> { - if !plan.run_status.is_terminal() { + if plan.run_status != crate::coordination::agent_org_runs::AgentOrgRunStatus::Archived { return Err(format!( "Refusing to delete Agent Org run {}: run status is {}", plan.run_id, @@ -466,13 +353,7 @@ fn validate_agent_org_delete_ready( } for node in &plan.sessions { - let allowed = node.status == SessionStatus::Idle - || node.status.is_terminal() - || (plan.run_status - == crate::coordination::agent_org_runs::AgentOrgRunStatus::Cancelled - && (matches!(node.status, SessionStatus::Pending | SessionStatus::Paused) - || (node.status.is_in_flight() - && quiesced_runtime_session_ids.contains(&node.session_id)))); + let allowed = node.status == SessionStatus::Idle || node.status.is_terminal(); if !allowed { return Err(format!( "Refusing to delete Agent Org run {}: session {} status is {}", @@ -518,47 +399,6 @@ async fn agent_org_runtime_blockers( blockers } -async fn stop_agent_org_runtime_sessions( - state: &AgentAppState, - plan: &AgentOrgSessionDeletePlan, -) -> Result, String> { - stop_agent_org_runtime_sessions_with_timeout(state, plan, AGENT_ORG_DELETE_STOP_TIMEOUT).await -} - -async fn stop_agent_org_runtime_sessions_with_timeout( - state: &AgentAppState, - plan: &AgentOrgSessionDeletePlan, - timeout: Duration, -) -> Result, String> { - let runtime_sessions = agent_org_runtime_sessions(state, plan).await; - let runtime_session_ids = runtime_sessions - .iter() - .map(|(session_id, _)| session_id.clone()) - .collect::>(); - - for (_, session) in &runtime_sessions { - session - .cancel_active_turn(CancelReason::AgentOrgDelete) - .await; - } - - let deadline = tokio::time::Instant::now() + timeout; - loop { - let blockers = agent_org_runtime_blockers(&runtime_sessions).await; - if blockers.is_empty() { - return Ok(runtime_session_ids); - } - if tokio::time::Instant::now() >= deadline { - return Err(format!( - "Timed out stopping Agent Org run {} before deletion: {}", - plan.run_id, - blockers.join(", ") - )); - } - tokio::time::sleep(AGENT_ORG_DELETE_STOP_POLL_INTERVAL).await; - } -} - async fn ensure_agent_org_runtime_sessions_idle( state: &AgentAppState, plan: &AgentOrgSessionDeletePlan, @@ -1102,7 +942,7 @@ mod tests { } fn seed_run(run_id: &str, root_session_id: &str) { - seed_run_with_status(run_id, root_session_id, "completed"); + seed_run_with_status(run_id, root_session_id, "archived"); } fn seed_session_owned_rows(session_id: &str) { @@ -1273,7 +1113,7 @@ mod tests { } #[test] - fn session_hierarchy_delete_fences_active_run_and_requires_quiesced_sessions() { + fn session_hierarchy_delete_requires_archived_without_mutating_active_run() { let _sandbox = test_helpers::test_env::sandbox(); ensure_test_schemas(); let root = "hierarchy-active-root"; @@ -1287,11 +1127,9 @@ mod tests { .expect("load running hierarchy") .expect("root owns run"); drop(conn); - let fenced = establish_agent_org_delete_fence(&plan).expect("cancel run for deletion"); - assert_eq!( - fenced.run_status, - crate::coordination::agent_org_runs::AgentOrgRunStatus::Cancelled - ); + let error = validate_agent_org_delete_ready(&plan, &HashSet::new()) + .expect_err("Delete must fail closed before the Archive transition exists"); + assert!(error.contains("run status is running")); assert_eq!( get_connection() .expect("sandbox DB") @@ -1300,18 +1138,9 @@ mod tests { [], |row| row.get::<_, String>(0) ) - .expect("load fenced status"), - "cancelled" + .expect("load unchanged status"), + "running" ); - - let error = validate_agent_org_delete_ready(&fenced, &HashSet::new()) - .expect_err("unobserved running worker must fail closed"); - assert!(error.contains(worker)); - assert!(error.contains("running")); - - let quiesced = HashSet::from([worker.to_string()]); - validate_agent_org_delete_ready(&fenced, &quiesced) - .expect("a stopped live runtime may retain a stale running row"); assert!(row_exists("agent_sessions", "session_id", root)); assert!(row_exists("agent_sessions", "session_id", worker)); assert!(row_exists("agent_org_runs", "id", "hierarchy-active-run")); @@ -1600,185 +1429,4 @@ mod tests { "hierarchy-trigger-change-run" )); } - - #[tokio::test] - async fn session_hierarchy_delete_stops_active_runtime_and_discards_pending_work() { - let _sandbox = test_helpers::test_env::sandbox(); - ensure_test_schemas(); - let root = "hierarchy-runtime-root"; - let state = AgentAppState::new(); - let root_runtime = std::sync::Arc::new(crate::state::AgentSession::new( - root.to_string(), - crate::definitions::AgentDefinition::default(), - )); - let turn_started = std::sync::Arc::new(tokio::sync::Notify::new()); - let turn_started_for_job = std::sync::Arc::clone(&turn_started); - let runtime_for_job = std::sync::Arc::clone(&root_runtime); - root_runtime - .scheduler - .enqueue(crate::session::ScheduledMessage { - kind: crate::session::ScheduledKind::Turn, - message_id: "hierarchy-runtime-processing".to_string(), - generation: 0, - client_message_id: None, - turn_intent_id: "hierarchy-runtime-processing-intent".to_string(), - org_run_id: Some("hierarchy-runtime-run".to_string()), - content: String::new(), - execute: Box::new(move || { - let runtime = std::sync::Arc::clone(&runtime_for_job); - let started = std::sync::Arc::clone(&turn_started_for_job); - Box::pin(async move { - runtime.begin_turn("still running".to_string()).await; - started.notify_one(); - while !runtime - .cancel_flag - .load(std::sync::atomic::Ordering::SeqCst) - { - tokio::task::yield_now().await; - } - runtime - .end_turn( - crate::session::DialogTurnState::Cancelled, - crate::session::TurnStats::default(), - ) - .await; - Err("cancelled for hierarchy deletion".to_string()) - }) - }), - }) - .await - .expect("enqueue processing work"); - tokio::time::timeout(std::time::Duration::from_secs(1), turn_started.notified()) - .await - .expect("turn starts processing"); - let pending_executed = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); - let pending_executed_for_job = std::sync::Arc::clone(&pending_executed); - root_runtime - .scheduler - .enqueue(crate::session::ScheduledMessage { - kind: crate::session::ScheduledKind::Turn, - message_id: "hierarchy-runtime-pending".to_string(), - generation: 0, - client_message_id: None, - turn_intent_id: "hierarchy-runtime-pending-intent".to_string(), - org_run_id: Some("hierarchy-runtime-run".to_string()), - content: String::new(), - execute: Box::new(move || { - let executed = std::sync::Arc::clone(&pending_executed_for_job); - Box::pin(async move { - executed.store(true, std::sync::atomic::Ordering::SeqCst); - Ok(String::new()) - }) - }), - }) - .await - .expect("enqueue pending work"); - state - .sessions - .lock() - .await - .insert(root.to_string(), std::sync::Arc::clone(&root_runtime)); - let plan = AgentOrgSessionDeletePlan { - run_id: "hierarchy-runtime-run".to_string(), - root_session_id: root.to_string(), - run_status: crate::coordination::agent_org_runs::AgentOrgRunStatus::Cancelled, - sessions: vec![AgentOrgSessionDeleteNode { - session_id: root.to_string(), - parent_session_id: None, - status: SessionStatus::Running, - depth: 0, - }], - }; - - let quiesced = stop_agent_org_runtime_sessions_with_timeout( - &state, - &plan, - std::time::Duration::from_secs(1), - ) - .await - .expect("active Rust runtime stops"); - assert_eq!(quiesced, HashSet::from([root.to_string()])); - assert_eq!(root_runtime.scheduler.pending_count(), 0); - assert!(!root_runtime.scheduler.is_processing()); - assert!(root_runtime.active_turn.lock().await.is_none()); - assert!(!pending_executed.load(std::sync::atomic::Ordering::SeqCst)); - validate_agent_org_delete_ready(&plan, &quiesced) - .expect("quiesced active status is safe behind cancelled fence"); - } - - #[tokio::test] - async fn session_hierarchy_delete_times_out_without_removing_runtime() { - let _sandbox = test_helpers::test_env::sandbox(); - ensure_test_schemas(); - let root = "hierarchy-runtime-timeout-root"; - let state = AgentAppState::new(); - let runtime = std::sync::Arc::new(crate::state::AgentSession::new( - root.to_string(), - crate::definitions::AgentDefinition::default(), - )); - let release = std::sync::Arc::new(tokio::sync::Notify::new()); - let release_for_job = std::sync::Arc::clone(&release); - runtime - .scheduler - .enqueue(crate::session::ScheduledMessage { - kind: crate::session::ScheduledKind::Maintenance, - message_id: "hierarchy-runtime-timeout".to_string(), - generation: 0, - client_message_id: None, - turn_intent_id: "hierarchy-runtime-timeout-intent".to_string(), - org_run_id: Some("hierarchy-runtime-timeout-run".to_string()), - content: String::new(), - execute: Box::new(move || { - let release = std::sync::Arc::clone(&release_for_job); - Box::pin(async move { - release.notified().await; - Ok(String::new()) - }) - }), - }) - .await - .expect("enqueue non-cooperative maintenance"); - tokio::time::timeout(std::time::Duration::from_secs(1), async { - while !runtime.scheduler.is_processing() { - tokio::task::yield_now().await; - } - }) - .await - .expect("maintenance starts"); - state - .sessions - .lock() - .await - .insert(root.to_string(), std::sync::Arc::clone(&runtime)); - let plan = AgentOrgSessionDeletePlan { - run_id: "hierarchy-runtime-timeout-run".to_string(), - root_session_id: root.to_string(), - run_status: crate::coordination::agent_org_runs::AgentOrgRunStatus::Cancelled, - sessions: vec![AgentOrgSessionDeleteNode { - session_id: root.to_string(), - parent_session_id: None, - status: SessionStatus::Running, - depth: 0, - }], - }; - - let error = stop_agent_org_runtime_sessions_with_timeout( - &state, - &plan, - std::time::Duration::from_millis(50), - ) - .await - .expect_err("non-cooperative work must time out"); - assert!(error.contains("Timed out stopping")); - assert!(error.contains(root)); - assert!(state.get_session(root).await.is_some()); - release.notify_one(); - tokio::time::timeout(std::time::Duration::from_secs(1), async { - while runtime.scheduler.is_processing() { - tokio::task::yield_now().await; - } - }) - .await - .expect("maintenance finishes after the timeout assertion"); - } } diff --git a/src-tauri/crates/agent-core/src/state/unified.rs b/src-tauri/crates/agent-core/src/state/unified.rs index 14ea6f05c9..413bd51af1 100644 --- a/src-tauri/crates/agent-core/src/state/unified.rs +++ b/src-tauri/crates/agent-core/src/state/unified.rs @@ -165,7 +165,7 @@ impl AgentAppState { } // Interventions cannot survive a process restart: their in-memory - // sessions were abandoned above. Clear them before finality checks so + // sessions were abandoned above. Clear them before Quiescence checks so // a fully-resolved run is not needlessly paused by an expired control // lease from the previous process. match crate::coordination::agent_member_interventions::AgentMemberInterventionStore::clear_all_active_on_startup() { @@ -180,38 +180,6 @@ impl AgentAppState { ), } - // Runs whose tasks were already resolved may have been kept open only - // by an orphaned queued intent. Close them through the normal atomic - // finality path before pausing genuinely unfinished work. - match crate::coordination::agent_org_runs::AgentOrgRunStore::reconcile_resolved_running_runs_on_startup() { - Ok(0) => {} - Ok(n) => info!( - "[agent-state] Completed {} fully-resolved Agent Org run(s) during startup recovery", - n - ), - Err(err) => warn!( - "[agent-state] Failed to reconcile resolved Agent Org runs on startup: {}", - err - ), - } - - // Transition any Agent Org runs that were `running` when the previous - // process exited to `paused`. Their member sessions are now `abandoned` - // (see above), so `reconcile_run_finality` would auto-terminate the run - // if it remained `running`. By moving to `paused` instead, the run stays - // visible (non-terminal) and can be resumed from the UI. - match crate::coordination::agent_org_runs::AgentOrgRunStore::mark_all_running_as_paused_on_startup() { - Ok(0) => {} - Ok(n) => info!( - "[agent-state] Paused {} Agent Org run(s) interrupted by app exit", - n - ), - Err(err) => warn!( - "[agent-state] Failed to pause interrupted Agent Org runs on startup: {}", - err - ), - } - let bus = Arc::new(Mutex::new(AgentMessageBus::new())); let sessions: Arc>>> = Arc::new(Mutex::new(HashMap::new())); diff --git a/src-tauri/crates/session-persistence/src/agent_core_bridge.rs b/src-tauri/crates/session-persistence/src/agent_core_bridge.rs index 3c296f918d..b92c63e21c 100644 --- a/src-tauri/crates/session-persistence/src/agent_core_bridge.rs +++ b/src-tauri/crates/session-persistence/src/agent_core_bridge.rs @@ -154,6 +154,28 @@ fn upsert_turn_intent_adapter( } } +fn upsert_turn_intent_with_connection_adapter( + connection: &rusqlite::Connection, + session_id: &str, + turn_intent_id: &str, + client_message_id: Option<&str>, + org_run_id: Option<&str>, + source: session_bridge::TurnIntentBridgeSource, + status: session_bridge::TurnIntentBridgeStatus, +) -> Result<(), String> { + turn_intents::upsert_initial_on( + connection, + session_id, + turn_intent_id, + client_message_id, + org_run_id, + map_bridge_source(source), + map_bridge_status(status), + ) + .map(|_| ()) + .map_err(|error| error.to_string()) +} + fn update_turn_intent_status_adapter( session_id: &str, turn_intent_id: &str, @@ -209,6 +231,9 @@ pub fn register() { session_bridge::register_record_token_usage(record_token_usage_adapter); session_bridge::register_record_usage_telemetry_batch(record_usage_telemetry_batch_adapter); session_bridge::register_upsert_turn_intent(upsert_turn_intent_adapter); + session_bridge::register_upsert_turn_intent_with_connection( + upsert_turn_intent_with_connection_adapter, + ); session_bridge::register_update_turn_intent_status(update_turn_intent_status_adapter); session_bridge::register_get_turn_intent_status(get_turn_intent_status_adapter); session_bridge::register_mark_pending_turn_intents_stale( diff --git a/src-tauri/crates/session-persistence/src/turn_intents.rs b/src-tauri/crates/session-persistence/src/turn_intents.rs index 1f8aeeb6f2..b4e1bf3eb0 100644 --- a/src-tauri/crates/session-persistence/src/turn_intents.rs +++ b/src-tauri/crates/session-persistence/src/turn_intents.rs @@ -421,13 +421,13 @@ pub fn mark_pending_stale(session_id: &str) -> Result { Ok(affected) } -/// Close every in-flight intent left by a previous process. +/// Close ordinary SDE in-flight intents left by a previous process. /// /// The scheduler queue is memory-only, so after a process restart no /// `optimistic` or `queued` row can still execute; they are stale. A `running` /// row means the process died during execution and is recorded as failed. -/// This is intentionally connection-scoped so the app can call it from the -/// database initialization hook without recursively opening the database. +/// Agent Org intents have their own durable Starting recovery and must be +/// reconciled only after the Agent Org schema is available. pub fn reconcile_in_flight_after_restart(conn: &Connection) -> Result { let now = Utc::now().to_rfc3339(); let affected = conn.execute( @@ -437,7 +437,37 @@ pub fn reconcile_in_flight_after_restart(conn: &Connection) -> Result Result { + let now = Utc::now().to_rfc3339(); + let affected = conn.execute( + "UPDATE session_turn_intents + SET status = 'stale', updated_at = ?1 + WHERE org_run_id IS NOT NULL + AND status IN ('optimistic', 'queued') + AND NOT ( + status = 'queued' + AND EXISTS ( + SELECT 1 FROM agent_org_initial_inputs initial + WHERE initial.org_run_id=session_turn_intents.org_run_id + AND initial.turn_intent_id=session_turn_intents.turn_intent_id + AND initial.status IN ('queued', 'dispatched') + ) + )", [now], )?; Ok(affected) @@ -822,7 +852,7 @@ mod tests { } #[test] - fn restart_reconciliation_closes_every_in_flight_intent() { + fn restart_reconciliation_closes_every_generic_in_flight_intent() { with_temp_orgii_home(|| { let session = "test-session-restart-intents"; for (intent, status) in [ @@ -870,4 +900,84 @@ mod tests { assert_eq!(by_id["completed-d"], TurnIntentStatus::Completed); }); } + + #[test] + fn agent_org_restart_preserves_running_and_replayable_initial_input_only() { + with_temp_orgii_home(|| { + let session = "test-agent-org-restart"; + let run_id = "agent-org-restart-run"; + let conn = get_connection().expect("open sessions DB"); + agent_core::coordination::init_agent_org_schemas(&conn) + .expect("init Agent Org schemas"); + let now = Utc::now().to_rfc3339(); + conn.execute( + "INSERT INTO agent_org_runs ( + id, org_id, coordinator_agent_id, root_session_id, + entry_mode, status, has_initial_work, created_at, updated_at + ) VALUES (?1, 'restart-org', 'coordinator', ?2, + 'standalone_session', 'running', 1, ?3, ?3)", + params![run_id, session, &now], + ) + .expect("seed run"); + + for (intent, status) in [ + ("optimistic-noninitial", TurnIntentStatus::Optimistic), + ("queued-noninitial", TurnIntentStatus::Queued), + ("running-final-not-committed", TurnIntentStatus::Running), + ("queued-canonical-initial", TurnIntentStatus::Queued), + ] { + upsert_initial( + session, + intent, + Some(&format!("message-{intent}")), + Some(run_id), + TurnIntentSource::AgentOrg, + status, + ) + .expect("seed Agent Org intent"); + } + conn.execute( + "INSERT INTO agent_org_initial_inputs ( + org_run_id, turn_intent_id, message_id, content, + payload_json, status, created_at, updated_at + ) VALUES (?1, 'queued-canonical-initial', 'initial-message', + 'initial input', ?2, 'queued', ?3, ?3)", + params![ + run_id, + serde_json::json!({ + "version": 1, + "images": null, + "ideContext": null, + "subAgentIds": [], + }) + .to_string(), + &now, + ], + ) + .expect("seed canonical initial input receipt"); + + assert_eq!(reconcile_in_flight_after_restart(&conn).unwrap(), 0); + assert_eq!( + reconcile_agent_org_in_flight_after_restart(&conn).unwrap(), + 2 + ); + let rows = list_for_session(session).expect("load reconciled intents"); + let by_id = rows + .into_iter() + .map(|row| (row.turn_intent_id, row.status)) + .collect::>(); + assert_eq!(by_id["optimistic-noninitial"], TurnIntentStatus::Stale); + assert_eq!(by_id["queued-noninitial"], TurnIntentStatus::Stale); + assert_eq!( + by_id["running-final-not-committed"], + TurnIntentStatus::Running, + "unknown post-crash side effects and an uncommitted final answer must block Idle" + ); + assert_eq!( + by_id["queued-canonical-initial"], + TurnIntentStatus::Queued, + "only the stable initial input receipt is safe to replay with the same ids" + ); + }); + } } diff --git a/src-tauri/src/agent_sessions/cli/session_runner/finalize.rs b/src-tauri/src/agent_sessions/cli/session_runner/finalize.rs index fcd8d170ed..45cd1da6f3 100644 --- a/src-tauri/src/agent_sessions/cli/session_runner/finalize.rs +++ b/src-tauri/src/agent_sessions/cli/session_runner/finalize.rs @@ -267,8 +267,8 @@ pub(super) async fn finalize_session_run( // CLI member sessions inside an Agent Org run must land on `Idle` after each // successful turn so they remain available for the next coordinator dispatch. - // `Completed` is terminal (is_terminal() == true) and would cause - // `reconcile_run_finality` to prematurely end the run. + // `Completed` is terminal (is_terminal() == true) and would make the + // member unavailable to the canonical Team Quiescence projection. let is_org_member = session.org_member_id.is_some(); let final_status = if raw_final_status == SessionStatus::Completed && is_org_member { SessionStatus::Idle diff --git a/src-tauri/src/agent_sessions/cli/types.rs b/src-tauri/src/agent_sessions/cli/types.rs index 7c20829188..62bf82e199 100644 --- a/src-tauri/src/agent_sessions/cli/types.rs +++ b/src-tauri/src/agent_sessions/cli/types.rs @@ -20,7 +20,7 @@ pub enum SessionStatus { Running, /// Session is idle — waiting for the next dispatch (non-terminal). /// Used for Agent Org member sessions after each successful turn so - /// `reconcile_run_finality` does not prematurely end the run. + /// Team Quiescence can distinguish a reusable member from active work. Idle, Completed, Failed, diff --git a/src-tauri/src/agent_sessions/event_pipeline/agent_core_bridge.rs b/src-tauri/src/agent_sessions/event_pipeline/agent_core_bridge.rs index 9bb7c1d078..e63275b6db 100644 --- a/src-tauri/src/agent_sessions/event_pipeline/agent_core_bridge.rs +++ b/src-tauri/src/agent_sessions/event_pipeline/agent_core_bridge.rs @@ -398,9 +398,9 @@ fn persist_events_adapter( session_id: &str, events: &[SessionEvent], max_retries: u32, -) { +) -> Result<(), String> { let cached: Vec<_> = events.iter().map(session_event_to_cached_event).collect(); - let _ = save_events_retry(label, session_id, &cached, max_retries); + save_events_retry(label, session_id, &cached, max_retries).map_err(|error| error.to_string()) } fn persist_events_async_adapter( diff --git a/src-tauri/src/api/agent/test/agent_org.rs b/src-tauri/src/api/agent/test/agent_org.rs index 8820ad53f3..bab0589a0e 100644 --- a/src-tauri/src/api/agent/test/agent_org.rs +++ b/src-tauri/src/api/agent/test/agent_org.rs @@ -2991,7 +2991,7 @@ pub async fn test_agent_org_tasks_list( /// /// Seeds a minimal Agent Org run with a CLI member session at a specified /// status in `code_sessions`. Used by deterministic E2E scenarios that -/// verify `reconcile_run_finality` does not prematurely end a run when a +/// verify Team Quiescence does not prematurely idle a run when a /// CLI member session is `idle` (non-terminal, between turns). /// /// Body: @@ -3206,11 +3206,13 @@ pub async fn test_agent_org_pause_run( /// member task disposition as production startup. /// 5. `clear_all_active_on_startup` — clears interventions whose in-memory /// sessions no longer exist. -/// 6. `reconcile_resolved_running_runs_on_startup` — completes runs whose -/// tasks were already fully resolved. -/// 7. `mark_all_running_as_paused_on_startup` — transitions -/// every `running` org run to `paused` so `reconcile_run_finality` cannot -/// auto-terminate the run when it sees all sessions abandoned. +/// 6. `reconcile_agent_org_in_flight_after_restart` — preserves Agent Org +/// Running intents as explicit quiescence blockers and only retains a +/// replayable queued canonical initial input. +/// +/// Run lifecycle state is deliberately not changed here. In particular, +/// startup never maps `running` to `paused` and never infers terminality from +/// abandoned Session rows. /// /// Caller-path probe: drives the same sequence that `AgentAppState:: /// with_browser` calls, so this endpoint stays in sync if any of those @@ -3228,6 +3230,11 @@ pub async fn test_agent_org_simulate_app_restart() -> Json { let intents_reconciled = session_persistence::turn_intents::reconcile_in_flight_after_restart(&conn) .map_err(|err| format!("reconcile_in_flight_after_restart failed: {err}"))?; + let agent_org_intents_reconciled = + session_persistence::turn_intents::reconcile_agent_org_in_flight_after_restart(&conn) + .map_err(|err| { + format!("reconcile_agent_org_in_flight_after_restart failed: {err}") + })?; let terminal_sessions_reconciled = reconcile_sessions_with_terminal_turn_markers() .map_err(|err| { format!("reconcile_sessions_with_terminal_turn_markers failed: {err}") @@ -3238,18 +3245,17 @@ pub async fn test_agent_org_simulate_app_restart() -> Json { .map_err(|err| format!("requeue_abandoned_member_tasks_on_startup failed: {err}"))?; let interventions_cleared = AgentMemberInterventionStore::clear_all_active_on_startup() .map_err(|err| format!("clear_all_active_on_startup failed: {err}"))?; - let runs_completed = AgentOrgRunStore::reconcile_resolved_running_runs_on_startup() - .map_err(|err| format!("reconcile_resolved_running_runs_on_startup failed: {err}"))?; - let runs_paused = AgentOrgRunStore::mark_all_running_as_paused_on_startup() - .map_err(|err| format!("mark_all_running_as_paused_on_startup failed: {err}"))?; Ok::(serde_json::json!({ "ok": true, "intents_reconciled": intents_reconciled, + "agent_org_intents_reconciled": agent_org_intents_reconciled, "terminal_sessions_reconciled": terminal_sessions_reconciled, "sessions_abandoned": sessions_abandoned, "tasks_requeued": tasks_requeued, - "runs_completed": runs_completed, - "runs_paused": runs_paused, + // Kept for old E2E clients; canonical PR1 startup performs neither + // transition, so both counters are intentionally always zero. + "runs_completed": 0, + "runs_paused": 0, "interventions_cleared": interventions_cleared, })) }) diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 7c7795501e..fd6b6f7f84 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -740,8 +740,40 @@ pub fn run() { ); tracing::info!("[MemberIdle] Member idle hook installed"); - agent_core::coordination::agent_org_watchdog::spawn(app.handle().clone()); - tracing::info!("[AgentOrgWatchdog] Agent Org watchdog started"); + if agent_core::coordination::agent_org_runs::agent_org_redesign_enabled() { + agent_core::coordination::agent_org_watchdog::spawn(app.handle().clone()); + tracing::info!("[AgentOrgWatchdog] Agent Org watchdog started"); + } else { + tracing::info!("[AgentOrgWatchdog] Agent Org redesign is disabled"); + } + + // Plan artifacts have their own one-shot startup owner. Keep this + // repair independent from the bounded Working-only watchdog so a + // global filesystem scan can never consume its 250 ms Team scan + // budget or run every 60 seconds. + tauri::async_runtime::spawn(async move { + match tokio::task::spawn_blocking(|| { + agent_core::coordination::agent_org_plan_approvals::AgentOrgPlanApprovalStore::repair_latest_plan_artifacts() + }) + .await + { + Ok(Ok(report)) if report.repaired > 0 || report.failed > 0 => tracing::info!( + inspected = report.inspected, + repaired = report.repaired, + failed = report.failed, + "[AgentOrgPlanArtifacts] one-shot startup reconciliation finished" + ), + Ok(Ok(_)) => {} + Ok(Err(error)) => tracing::warn!( + error = %error, + "[AgentOrgPlanArtifacts] startup reconciliation failed" + ), + Err(error) => tracing::warn!( + error = %error, + "[AgentOrgPlanArtifacts] startup worker failed" + ), + } + }); // Install the production `SubagentCompletionWakeHook` so a // background subagent that finishes while its parent is idle @@ -757,9 +789,15 @@ pub fn run() { tracing::info!("[SubagentWake] Subagent completion wake hook installed"); let housekeeper_compaction_state = unified_state.clone(); + let agent_org_startup_state = unified_state.clone(); app.manage(unified_state); tracing::info!("[UnifiedAgent] Unified agent state initialized"); + agent_core::core::session::launch::spawn_agent_org_startup_recovery( + agent_org_startup_state, + ); + tracing::info!("[AgentOrgStartup] one-shot lifecycle recovery scheduled"); + agent_core::session::housekeeper_compaction::spawn( housekeeper_compaction_state, ); diff --git a/src-tauri/src/setup/hooks.rs b/src-tauri/src/setup/hooks.rs index 97f88ddf09..a64424558a 100644 --- a/src-tauri/src/setup/hooks.rs +++ b/src-tauri/src/setup/hooks.rs @@ -65,6 +65,17 @@ pub(crate) fn register_database_schemas() { } agent_core::coordination::init_agent_org_schemas(conn)?; + match session_persistence::turn_intents::reconcile_agent_org_in_flight_after_restart(conn) { + Ok(0) => {} + Ok(count) => tracing::info!( + "[startup] Reconciled {} Agent Org turn intent(s) from the previous process", + count + ), + Err(err) => tracing::warn!( + "[startup] Failed to reconcile Agent Org turn intents: {}", + err + ), + } // Pending plan-approval snapshots (one row per session with a Build // button still awaiting the user). Persists the pending action so the diff --git a/src/api/tauri/agent/orgTasks.ts b/src/api/tauri/agent/orgTasks.ts index a2185b3227..b0139999d1 100644 --- a/src/api/tauri/agent/orgTasks.ts +++ b/src/api/tauri/agent/orgTasks.ts @@ -79,18 +79,19 @@ export interface AgentOrgRunMemberView { } export const AGENT_ORG_RUN_STATUS = { + STARTING: "starting", RUNNING: "running", PAUSED: "paused", - COMPLETED: "completed", + IDLE: "idle", FAILED: "failed", - CANCELLED: "cancelled", - ABANDONED: "abandoned", + ARCHIVED: "archived", } as const; export type AgentOrgRunStatus = (typeof AGENT_ORG_RUN_STATUS)[keyof typeof AGENT_ORG_RUN_STATUS]; export const AGENT_ORG_RUN_PHASE = { + STARTING: "starting", COORDINATING: "coordinating", DISPATCHING: "dispatching", MEMBERS_WORKING: "members_working", @@ -98,10 +99,9 @@ export const AGENT_ORG_RUN_PHASE = { AWAITING_PLAN_APPROVAL: "awaiting_plan_approval", FINALIZING: "finalizing", PAUSED: "paused", - COMPLETED: "completed", + IDLE: "idle", FAILED: "failed", - CANCELLED: "cancelled", - ABANDONED: "abandoned", + ARCHIVED: "archived", } as const; export type AgentOrgRunPhase = diff --git a/src/api/tauri/agent/session.ts b/src/api/tauri/agent/session.ts index 5cc9606b39..9f3c2cbc57 100644 --- a/src/api/tauri/agent/session.ts +++ b/src/api/tauri/agent/session.ts @@ -7,6 +7,7 @@ import { invoke } from "@tauri-apps/api/core"; import { rpc } from "@src/api/tauri/rpc"; import type { CliAgentType, NativeHarnessType } from "@src/api/types/keys"; +import { requireAgentOrgRedesign } from "@src/config/agentOrgRedesign"; import type { OrgMemberLaunchOverride } from "@src/modules/MainApp/AgentOrgs/types"; import type { WorkspaceSnapshot } from "@src/services/context/workspaceSnapshot"; import type { SessionStatus } from "@src/types/session/session"; @@ -520,6 +521,7 @@ export interface SessionLaunchResult { export async function sessionLaunch( params: SessionLaunchParams ): Promise { + if (params.agentOrgId) requireAgentOrgRedesign(); return rpc.agentSession.sessionLaunch({ params, }) as Promise; diff --git a/src/config/agentOrgRedesign.ts b/src/config/agentOrgRedesign.ts new file mode 100644 index 0000000000..bb0a7cb01f --- /dev/null +++ b/src/config/agentOrgRedesign.ts @@ -0,0 +1,15 @@ +/** + * One internal rollout gate for the long-lived Agent Org redesign stack. + * It is intentionally absent from Team settings and model/tool context. + */ +export const AGENT_ORG_REDESIGN_ENABLED = + process.env.NODE_ENV === "test" || + process.env.ORGII_AGENT_ORG_REDESIGN === "1"; + +export function requireAgentOrgRedesign(): void { + if (!AGENT_ORG_REDESIGN_ENABLED) { + throw new Error( + "agent_org_redesign_disabled: the long-lived Agent Team lifecycle is not enabled" + ); + } +} diff --git a/src/engines/ChatPanel/InputArea/components/agentOrgRunViewStore.test.ts b/src/engines/ChatPanel/InputArea/components/agentOrgRunViewStore.test.ts index b859de01db..f0829e06ca 100644 --- a/src/engines/ChatPanel/InputArea/components/agentOrgRunViewStore.test.ts +++ b/src/engines/ChatPanel/InputArea/components/agentOrgRunViewStore.test.ts @@ -33,7 +33,7 @@ async function flushPromises(): Promise { } function runView( - runStatus: "running" | "paused" | "completed", + runStatus: "starting" | "running" | "paused" | "idle" | "failed" | "archived", interventionResumeAfter?: string ) { return { @@ -106,6 +106,18 @@ function runView( }; } +function runViewForRoot( + runStatus: "starting" | "running" | "paused" | "idle" | "failed" | "archived", + runId: string, + rootSessionId: string +) { + const view = runView(runStatus); + view.context.runId = runId; + view.context.rootSessionId = rootSessionId; + view.members[0].sessionRuntime.sessionId = rootSessionId; + return view; +} + function deferred() { let resolve!: (value: T) => void; const promise = new Promise((resolvePromise) => { @@ -118,10 +130,11 @@ afterEach(() => { agentOrgRunViewStoreTestApi.reset(); vi.useRealTimers(); vi.clearAllMocks(); + vi.unstubAllGlobals(); }); describe("Agent Org run-view store", () => { - it("shares one fallback per run, coalesces pushes, and stops terminal runs", async () => { + it("shares one fallback per run, coalesces pushes, and stops immediately on Idle", async () => { vi.useFakeTimers(); let stateChangeHandler: ((sessionId: string) => void) | undefined; let backendChangeHandler: @@ -151,7 +164,7 @@ describe("Agent Org run-view store", () => { ); mocks.getAgentOrgSessionRunView .mockResolvedValueOnce(runView("running")) - .mockResolvedValueOnce(runView("completed")); + .mockResolvedValueOnce(runView("idle")); const rootSubscriber = vi.fn(); const secondRootSubscriber = vi.fn(); @@ -213,6 +226,104 @@ describe("Agent Org run-view store", () => { unsubscribe(); }); + it.each(["paused", "idle", "failed", "archived"] as const)( + "does not retain a fallback interval when the initial Team is %s", + async (status) => { + vi.useFakeTimers(); + mocks.subscribeAgentOrgStateChanges.mockReturnValue( + mocks.unsubscribeStateChanges + ); + mocks.getAgentOrgSessionRunView.mockResolvedValue(runView(status)); + + const unsubscribe = subscribeAgentOrgRunView("session-root", vi.fn()); + await flushPromises(); + + expect(agentOrgRunViewStoreTestApi.hasPollingTimer()).toBe(false); + await vi.advanceTimersByTimeAsync(AGENT_ORG_RUN_VIEW_FALLBACK_MS * 5); + expect(mocks.getAgentOrgSessionRunView).toHaveBeenCalledTimes(1); + unsubscribe(); + } + ); + + it("destroys the shared interval when the last pollable Team becomes Idle", async () => { + vi.useFakeTimers(); + mocks.subscribeAgentOrgStateChanges.mockReturnValue( + mocks.unsubscribeStateChanges + ); + mocks.getAgentOrgSessionRunView.mockImplementation((sessionId: string) => { + if (sessionId === "root-a") { + return Promise.resolve(runViewForRoot("running", "run-a", "root-a")); + } + return Promise.resolve(runViewForRoot("running", "run-b", "root-b")); + }); + + const unsubscribeA = subscribeAgentOrgRunView("root-a", vi.fn()); + const unsubscribeB = subscribeAgentOrgRunView("root-b", vi.fn()); + await flushPromises(); + expect(agentOrgRunViewStoreTestApi.hasPollingTimer()).toBe(true); + + mocks.getAgentOrgSessionRunView.mockImplementation((sessionId: string) => + Promise.resolve( + sessionId === "root-a" + ? runViewForRoot("idle", "run-a", "root-a") + : runViewForRoot("running", "run-b", "root-b") + ) + ); + await agentOrgRunViewStoreTestApi.refresh("root-a"); + expect(agentOrgRunViewStoreTestApi.hasPollingTimer()).toBe(true); + + mocks.getAgentOrgSessionRunView.mockImplementation((sessionId: string) => + Promise.resolve( + sessionId === "root-b" + ? runViewForRoot("idle", "run-b", "root-b") + : runViewForRoot("idle", "run-a", "root-a") + ) + ); + await agentOrgRunViewStoreTestApi.refresh("root-b"); + expect(agentOrgRunViewStoreTestApi.hasPollingTimer()).toBe(false); + + unsubscribeA(); + unsubscribeB(); + }); + + it("clears polling while hidden and performs one bounded refresh when visible", async () => { + vi.useFakeTimers(); + let hidden = false; + let visibilityChange: (() => void) | undefined; + vi.stubGlobal("document", { + get hidden() { + return hidden; + }, + addEventListener: vi.fn((event: string, handler: () => void) => { + if (event === "visibilitychange") visibilityChange = handler; + }), + removeEventListener: vi.fn(), + }); + mocks.subscribeAgentOrgStateChanges.mockReturnValue( + mocks.unsubscribeStateChanges + ); + mocks.getAgentOrgSessionRunView.mockResolvedValue(runView("running")); + + const unsubscribe = subscribeAgentOrgRunView("session-root", vi.fn()); + await flushPromises(); + expect(agentOrgRunViewStoreTestApi.hasPollingTimer()).toBe(true); + expect(mocks.getAgentOrgSessionRunView).toHaveBeenCalledTimes(1); + + hidden = true; + visibilityChange?.(); + expect(agentOrgRunViewStoreTestApi.hasPollingTimer()).toBe(false); + await vi.advanceTimersByTimeAsync(AGENT_ORG_RUN_VIEW_FALLBACK_MS * 5); + expect(mocks.getAgentOrgSessionRunView).toHaveBeenCalledTimes(1); + + hidden = false; + visibilityChange?.(); + await flushPromises(); + expect(mocks.getAgentOrgSessionRunView).toHaveBeenCalledTimes(2); + expect(agentOrgRunViewStoreTestApi.hasPollingTimer()).toBe(true); + + unsubscribe(); + }); + it("refreshes a retained view immediately when the session is reopened", async () => { vi.useFakeTimers(); mocks.subscribeAgentOrgStateChanges.mockReturnValue( @@ -263,13 +374,13 @@ describe("Agent Org run-view store", () => { // discovery hangs, the second is released after the bounded join timeout; // request ordering must still reject the first request's late result. await vi.advanceTimersByTimeAsync(AGENT_ORG_BOOTSTRAP_JOIN_TIMEOUT_MS); - workerRequest.resolve(runView("completed")); + workerRequest.resolve(runView("idle")); await flushPromises(); rootRequest.resolve(runView("running")); await flushPromises(); expect(getAgentOrgRunViewSnapshot("session-root").view?.runStatus).toBe( - "completed" + "idle" ); unsubscribeRoot(); unsubscribeWorker(); diff --git a/src/engines/ChatPanel/InputArea/components/agentOrgRunViewStore.ts b/src/engines/ChatPanel/InputArea/components/agentOrgRunViewStore.ts index 38252a6e86..b5a58f6f10 100644 --- a/src/engines/ChatPanel/InputArea/components/agentOrgRunViewStore.ts +++ b/src/engines/ChatPanel/InputArea/components/agentOrgRunViewStore.ts @@ -12,11 +12,9 @@ export const AGENT_ORG_RUN_VIEW_CACHE_RETENTION_MS = 30_000; export const AGENT_ORG_BOOTSTRAP_JOIN_TIMEOUT_MS = 1_000; const MAX_NON_ORG_DISCOVERY_ATTEMPTS = 1; -const TERMINAL_RUN_STATUSES: ReadonlySet = new Set([ - "completed", - "failed", - "cancelled", - "abandoned", +export const POLLABLE_RUN_STATUSES: ReadonlySet = new Set([ + "starting", + "running", ]); export interface AgentOrgRunViewSnapshot { @@ -94,8 +92,8 @@ function viewForSession( return { ...view, currentMemberId }; } -function isTerminal(view: AgentOrgRunView | null): boolean { - return view !== null && TERMINAL_RUN_STATUSES.has(view.runStatus); +function isPollable(view: AgentOrgRunView | null): boolean { + return view !== null && POLLABLE_RUN_STATUSES.has(view.runStatus); } function findEntryCoveringSession(sessionId: string): RunViewEntry | undefined { @@ -138,6 +136,7 @@ function evictEntry(entry: RunViewEntry): void { if (!isCurrentEntry(entry) || entry.subscribers.size > 0) return; entry.retired = true; entriesBySessionId.delete(entry.sessionId); + reconcilePollingTimer(); if (bootstrapOwner === entry) bootstrapOwner = null; const sessionKey = `session:${entry.sessionId}`; @@ -207,6 +206,7 @@ function publishEntry( entry.snapshot = { view, error }; entry.serializedView = serializedView; for (const subscriber of entry.subscribers) subscriber(); + reconcilePollingTimer(); return true; } @@ -438,7 +438,10 @@ function pollActiveRuns(): void { const representatives = new Map(); for (const entry of entriesBySessionId.values()) { - if (entry.subscribers.size === 0 || isTerminal(entry.snapshot.view)) + if ( + entry.subscribers.size === 0 || + (entry.snapshot.view !== null && !isPollable(entry.snapshot.view)) + ) continue; if ( entry.snapshot.view === null && @@ -454,13 +457,42 @@ function pollActiveRuns(): void { } function handleVisibilityChange(): void { - if (isDocumentVisible()) pollActiveRuns(); + if (!isDocumentVisible()) { + reconcilePollingTimer(); + return; + } + pollActiveRuns(); + reconcilePollingTimer(); } -function startScheduler(): void { +function hasPollableSubscriber(): boolean { + for (const entry of entriesBySessionId.values()) { + if (entry.subscribers.size === 0) continue; + if (entry.snapshot.view !== null) { + if (isPollable(entry.snapshot.view)) return true; + continue; + } + if (entry.discoveryAttempts < MAX_NON_ORG_DISCOVERY_ATTEMPTS) return true; + } + return false; +} + +function reconcilePollingTimer(): void { + const shouldPoll = + activeSubscriberCount > 0 && isDocumentVisible() && hasPollableSubscriber(); + if (!shouldPoll) { + if (pollingTimer) { + clearInterval(pollingTimer); + pollingTimer = undefined; + } + return; + } if (!pollingTimer) { pollingTimer = setInterval(pollActiveRuns, AGENT_ORG_RUN_VIEW_FALLBACK_MS); } +} + +function startScheduler(): void { if (!unsubscribeStateChanges) { unsubscribeStateChanges = subscribeAgentOrgStateChanges((sessionId) => { const entry = @@ -514,6 +546,7 @@ function startScheduler(): void { scheduledRuns.add(view.context.runId); scheduleInterventionExpiryRefresh(view); } + reconcilePollingTimer(); } function stopScheduler(): void { @@ -552,6 +585,7 @@ export function subscribeAgentOrgRunView( entry.subscribers.add(subscription); activeSubscriberCount += 1; if (activeSubscriberCount === 1) startScheduler(); + else reconcilePollingTimer(); if (entry.snapshot.view === null && entry.discoveryAttempts === 0) { void refreshAgentOrgRunViewInternal(sessionId, false); } else if (isReturningSubscriber && entry.snapshot.view !== null) { @@ -565,6 +599,7 @@ export function subscribeAgentOrgRunView( if (!entry.subscribers.delete(subscription)) return; activeSubscriberCount -= 1; if (activeSubscriberCount === 0) stopScheduler(); + else reconcilePollingTimer(); scheduleEntryEviction(entry); }; } @@ -592,6 +627,9 @@ export const agentOrgRunViewStoreTestApi = { )?.sessionId ?? null ); }, + hasPollingTimer(): boolean { + return pollingTimer !== undefined; + }, reset(): void { stopScheduler(); for (const entry of entriesBySessionId.values()) { diff --git a/src/engines/ChatPanel/InputArea/components/useAgentOrgRunView.ts b/src/engines/ChatPanel/InputArea/components/useAgentOrgRunView.ts index a1582aefaa..696f8e7149 100644 --- a/src/engines/ChatPanel/InputArea/components/useAgentOrgRunView.ts +++ b/src/engines/ChatPanel/InputArea/components/useAgentOrgRunView.ts @@ -1,5 +1,6 @@ import { useCallback, useMemo, useSyncExternalStore } from "react"; +import { AGENT_ORG_REDESIGN_ENABLED } from "@src/config/agentOrgRedesign"; import { getAgentOrgRunViewSnapshot, refreshAgentOrgRunView, @@ -21,6 +22,7 @@ const EMPTY_RESULT = { view: null, error: null } as const; */ export function useAgentOrgRunView(sessionId: string | null) { const canFetchRunView = + AGENT_ORG_REDESIGN_ENABLED && !!sessionId && !isCliSession(sessionId) && !isImportedHistorySession(sessionId); diff --git a/src/i18n/locales/en/sessions.json b/src/i18n/locales/en/sessions.json index 93f0f63114..aff5b7dfe3 100644 --- a/src/i18n/locales/en/sessions.json +++ b/src/i18n/locales/en/sessions.json @@ -2518,6 +2518,7 @@ "submitFailed": "The plan response could not be saved. Please try again." }, "phase": { + "starting": "Starting", "coordinating": "Coordinating", "dispatching": "Dispatching", "members_working": "Members working", @@ -2525,10 +2526,9 @@ "awaiting_plan_approval": "Awaiting plan approval", "finalizing": "Finalizing", "paused": "Paused", - "completed": "Completed", + "idle": "Idle", "failed": "Failed", - "cancelled": "Cancelled", - "abandoned": "Abandoned" + "archived": "Archived" } }, "agentOrgInbox": { diff --git a/src/i18n/locales/zh/sessions.json b/src/i18n/locales/zh/sessions.json index 8bd60e4c98..7d8dac511f 100644 --- a/src/i18n/locales/zh/sessions.json +++ b/src/i18n/locales/zh/sessions.json @@ -2492,6 +2492,7 @@ "submitFailed": "计划审批结果未能保存，请重试。" }, "phase": { + "starting": "正在建立团队", "coordinating": "正在协调", "dispatching": "正在派发", "members_working": "成员工作中", @@ -2499,10 +2500,9 @@ "awaiting_plan_approval": "等待计划审批", "finalizing": "正在收尾", "paused": "已暂停", - "completed": "已完成", + "idle": "待命", "failed": "失败", - "cancelled": "已取消", - "abandoned": "已放弃" + "archived": "已归档" } }, "agentOrgInbox": { diff --git a/webpack.config.js b/webpack.config.js index 0ad0e91df9..b43d3895e5 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -564,6 +564,9 @@ module.exports = (env, argv) => { "process.env.ORGII_DEEP_LINK_SCHEME": JSON.stringify( process.env.ORGII_DEEP_LINK_SCHEME ?? "orgii" ), + "process.env.ORGII_AGENT_ORG_REDESIGN": JSON.stringify( + process.env.ORGII_AGENT_ORG_REDESIGN ?? "0" + ), "process.env.E2E_BASE_URL": JSON.stringify( process.env.E2E_BASE_URL ?? `http://127.0.0.1:${process.env.ORGII_IDE_SERVER_PORT ?? "13847"}`