From 29f8261f92f6e144bc7cf1d0dc2878285d550c5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Carter=20=E7=A5=81=E6=98=8E=E6=80=9D?= Date: Thu, 6 Aug 2026 02:28:55 +0000 Subject: [PATCH 1/2] feat: add bounded autoresearch and context telemetry --- docs/autoresearch.md | 65 ++++ src/agent/loop_runner.rs | 1 + src/agent/rotary_bridge.rs | 56 +++- src/autoresearch.rs | 622 +++++++++++++++++++++++++++++++++++++ src/cost.rs | 62 ++++ src/lib.rs | 1 + src/main.rs | 158 ++++++++++ 7 files changed, 963 insertions(+), 2 deletions(-) create mode 100644 docs/autoresearch.md create mode 100644 src/autoresearch.rs diff --git a/docs/autoresearch.md b/docs/autoresearch.md new file mode 100644 index 0000000..1e77956 --- /dev/null +++ b/docs/autoresearch.md @@ -0,0 +1,65 @@ +# Apollo autoresearch + +`apollo autoresearch` runs bounded experiments against a numeric local metric. +It measures a baseline, asks the agent for one hypothesis at a time, validates +the candidate, keeps only improvements, and records every decision in a TOML +ledger. + +Create `.apollo/autoresearch.toml` in the workspace: + +```toml +objective = "Reduce warm startup latency" +metric_command = "cargo bench --bench startup -- --output-format json | jq -r .median_ms" +direction = "minimize" +validation_command = "cargo test --workspace --all-features" +validation_retries = 2 +command_timeout_secs = 300 +samples = 3 +max_iterations = 10 +max_duration_secs = 1800 +ledger_path = ".apollo/autoresearch-ledger.toml" +``` + +Run it with: + +```bash +apollo autoresearch --workspace . +apollo autoresearch --workspace . --resume +``` + +The metric command must print at least one finite numeric value. Commands in +the specification are trusted local code and run through `sh -c`; do not use a +specification copied from an untrusted source. The workspace must be clean at +startup. Rejected iterations are restored to their checkpoint and untracked +files created by that iteration are removed, so run autoresearch in a dedicated +worktree when experimenting with valuable uncommitted files. + +If `ledger_path` is inside the workspace, it must be Git-ignored; an external +ledger path is also supported. This keeps the durable ledger from becoming an +unrelated dirty change after an accepted iteration. + +Accepted iterations are committed locally as `autoresearch: iteration N`. +Pushing is intentionally not automatic. + +The autoresearch runner exposes only the runtime and filesystem tool groups to +the experiment agent. Network, messaging, memory, MCP, dynamic tools, host +plugins, and workspace skills are not ambient capabilities for this workflow. + +Validation and metric processes are bounded by `command_timeout_secs`, and a +validation command may be retried with `validation_retries`. The whole run is +bounded by `max_iterations` and `max_duration_secs`. + +Apollo also records estimated system, history, and tool-definition context for +each provider request in the existing cost tracker. The estimates use four +characters per token and are useful for comparing harness configurations, not +for billing. + +## Design notes + +The loop follows two useful ideas from adjacent agent systems: bounded +autonomous runs with explicit quality gates and budgets (as in +[Prime Agent](https://github.com/PrimeIntellect-ai/prime-agent)), and +capability-oriented access instead of ambient tools (as in +[Cloudflare OS](https://github.com/cloudflare/cloudflare-os)). Apollo keeps the +implementation local and Git-backed: there is no hosted worker or remote +control plane in this workflow. diff --git a/src/agent/loop_runner.rs b/src/agent/loop_runner.rs index 8c02272..0233df5 100644 --- a/src/agent/loop_runner.rs +++ b/src/agent/loop_runner.rs @@ -757,6 +757,7 @@ impl AgentRunner { workspace: self.workspace.clone(), max_tool_iterations: self.agent_config.max_rounds, auto_compact_after: self.agent_config.auto_compact_after, + cost_tracker: Some(Arc::clone(&self.cost_tracker)), // Both engines must run the same hooks and emit the same events. hook_ctx: crate::agent::rotary_bridge::ToolHookContext::new( self.hooks.read().unwrap().clone(), diff --git a/src/agent/rotary_bridge.rs b/src/agent/rotary_bridge.rs index 2665738..39f4687 100644 --- a/src/agent/rotary_bridge.rs +++ b/src/agent/rotary_bridge.rs @@ -23,6 +23,7 @@ use rx4::provider::{ use crate::agent::hooks::{run_post_hooks, run_pre_hooks, HookDecision, ToolHook}; use crate::agent::stream::{emit, AgentStreamEvent, AgentStreamTx}; +use crate::cost::{ContextSnapshot, CostTracker, TokenUsage}; use crate::plugin::{HookManager, LifecycleEvent, PluginRegistry}; use crate::providers::{ChatMessage, ChatRequest, Provider as UnthinkclawProvider}; use crate::tools::{Tool as UnthinkclawTool, ToolResult as UnthinkclawToolResult, ToolSpec}; @@ -214,16 +215,21 @@ pub struct RotaryProviderAdapter { inner: Arc, id: String, name: String, + cost_tracker: Option>, } impl RotaryProviderAdapter { - pub fn new(provider: Arc) -> Self { + pub fn new( + provider: Arc, + cost_tracker: Option>, + ) -> Self { let id = provider.name().to_string(); let name = format!("apollo-{}", provider.name()); Self { inner: provider, id, name, + cost_tracker, } } } @@ -300,12 +306,53 @@ impl Rx4Provider for RotaryProviderAdapter { max_tokens: Some(8192), }; + if let Some(tracker) = &self.cost_tracker { + let system_chars = system + .as_ref() + .map(|value| value.chars().count()) + .unwrap_or(0); + let history_chars = messages + .iter() + .map(|message| message.content.chars().count()) + .sum::(); + let tool_chars = tools + .iter() + .map(|tool| { + serde_json::to_string(tool) + .unwrap_or_default() + .chars() + .count() + }) + .sum::(); + tracker + .record_context(ContextSnapshot { + system_chars, + history_chars, + tool_chars, + estimated_input_tokens: (system_chars + history_chars + tool_chars).div_ceil(4), + }) + .await; + } + let response = self .inner .chat(&request) .await .map_err(|e| Rx4ProviderError::Api(e.to_string()))?; + if let (Some(tracker), Some(usage)) = (&self.cost_tracker, response.usage.as_ref()) { + let _ = tracker + .record( + model, + TokenUsage { + input_tokens: usage.input_tokens as usize, + output_tokens: usage.output_tokens as usize, + total_tokens: usage.input_tokens as usize + usage.output_tokens as usize, + }, + ) + .await; + } + // Build a stream that emits the response as events let text = response.text.unwrap_or_default(); let tool_calls = response.tool_calls; @@ -402,6 +449,8 @@ pub struct RotaryBridgeConfig { /// rx4 auto-compaction threshold. `0` leaves compaction off; a non-zero /// value is forwarded to `Agent::auto_compact_after`. pub auto_compact_after: usize, + /// Optional tracker used for provider usage and context-shape telemetry. + pub cost_tracker: Option>, /// Pre/post tool hooks, so rx4 enforces the same permissions as the /// legacy loop. pub hook_ctx: ToolHookContext, @@ -428,7 +477,10 @@ pub struct RotaryAgentBridge { impl RotaryAgentBridge { /// Build a new bridge from the given configuration. pub fn new(config: RotaryBridgeConfig) -> Self { - let rx4_provider = Arc::new(RotaryProviderAdapter::new(config.provider)); + let rx4_provider = Arc::new(RotaryProviderAdapter::new( + config.provider, + config.cost_tracker, + )); let mut agent = rx4::Agent::new(); agent.set_model(&config.model); diff --git a/src/autoresearch.rs b/src/autoresearch.rs new file mode 100644 index 0000000..a44697c --- /dev/null +++ b/src/autoresearch.rs @@ -0,0 +1,622 @@ +//! Metric-driven autonomous experimentation. +//! +//! Autoresearch is deliberately separate from [`crate::autonomous`]. The +//! autonomous TODO loop completes one task and validates it; this loop keeps a +//! measurable objective, accepts only improvements, and resumes from a small +//! durable ledger. + +use std::cmp::Ordering; +use std::future::Future; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; + +use anyhow::{bail, Context}; +use serde::{Deserialize, Serialize}; + +use crate::agent::NullChannel; +use crate::channels::IncomingMessage; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(default)] +pub struct AutoresearchConfig { + /// Human-readable objective supplied to the agent. + pub objective: String, + /// Trusted local command that prints one numeric metric value. + pub metric_command: String, + /// `minimize` or `maximize`. + pub direction: String, + /// Optional command that must pass before the metric is considered. + pub validation_command: String, + /// Number of attempts for the validation command. + pub validation_retries: usize, + /// Timeout for each validation and metric process. + pub command_timeout_secs: u64, + /// Number of metric samples per measurement. + pub samples: usize, + /// Minimum accepted improvement as a percentage. Defaults to strict. + pub min_improvement_percent: f64, + /// Maximum number of iterations for one invocation. + pub max_iterations: usize, + /// Wall-clock budget for one invocation. Zero disables the budget. + pub max_duration_secs: u64, + /// Durable ledger path, relative to the workspace unless absolute. + pub ledger_path: String, + /// Model override. Empty uses the runner's default. + pub model: String, +} + +impl Default for AutoresearchConfig { + fn default() -> Self { + Self { + objective: String::new(), + metric_command: String::new(), + direction: "minimize".to_string(), + validation_command: String::new(), + validation_retries: 1, + command_timeout_secs: 300, + samples: 3, + min_improvement_percent: 0.0, + max_iterations: 10, + max_duration_secs: 1800, + ledger_path: ".apollo/autoresearch-ledger.toml".to_string(), + model: String::new(), + } + } +} + +impl AutoresearchConfig { + fn validate(&self) -> anyhow::Result<()> { + if self.objective.trim().is_empty() { + bail!("autoresearch objective must not be empty"); + } + if self.metric_command.trim().is_empty() { + bail!("autoresearch metric_command must not be empty"); + } + if !matches!( + self.direction.trim().to_ascii_lowercase().as_str(), + "minimize" | "maximize" + ) { + bail!("autoresearch direction must be 'minimize' or 'maximize'"); + } + if self.samples == 0 { + bail!("autoresearch samples must be at least 1"); + } + if self.max_iterations == 0 { + bail!("autoresearch max_iterations must be at least 1"); + } + if self.validation_retries == 0 { + bail!("autoresearch validation_retries must be at least 1"); + } + if self.command_timeout_secs == 0 { + bail!("autoresearch command_timeout_secs must be at least 1"); + } + if !self.min_improvement_percent.is_finite() || self.min_improvement_percent < 0.0 { + bail!("autoresearch min_improvement_percent must be a finite non-negative number"); + } + Ok(()) + } + + pub fn load(path: &Path) -> anyhow::Result { + let content = std::fs::read_to_string(path) + .with_context(|| format!("reading autoresearch spec {}", path.display()))?; + let config: Self = toml::from_str(&content) + .with_context(|| format!("parsing autoresearch spec {}", path.display()))?; + config.validate()?; + Ok(config) + } + + fn ledger_path(&self, workspace: &Path) -> PathBuf { + let path = PathBuf::from(&self.ledger_path); + if path.is_absolute() { + path + } else { + workspace.join(path) + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum ExperimentDecision { + Baseline, + Accepted, + Rejected, + Failed, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ExperimentRecord { + pub iteration: usize, + pub hypothesis: String, + pub commit: Option, + pub metric: Option, + pub baseline: f64, + pub delta_percent: Option, + pub decision: ExperimentDecision, + pub reason: String, + pub timestamp: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AutoresearchLedger { + pub objective: String, + pub direction: String, + pub best_metric: f64, + pub best_commit: String, + pub records: Vec, +} + +impl AutoresearchLedger { + fn new(config: &AutoresearchConfig, baseline: f64, commit: String) -> Self { + Self { + objective: config.objective.clone(), + direction: config.direction.trim().to_ascii_lowercase(), + best_metric: baseline, + best_commit: commit, + records: vec![ExperimentRecord { + iteration: 0, + hypothesis: "Initial measurement".to_string(), + commit: None, + metric: Some(baseline), + baseline, + delta_percent: Some(0.0), + decision: ExperimentDecision::Baseline, + reason: "baseline".to_string(), + timestamp: chrono::Utc::now().to_rfc3339(), + }], + } + } +} + +/// Controller for one bounded autoresearch run. +pub struct AutoresearchLoop { + config: AutoresearchConfig, + workspace: PathBuf, +} + +impl AutoresearchLoop { + pub fn new(config: AutoresearchConfig, workspace: PathBuf) -> Self { + Self { config, workspace } + } + + pub async fn run( + &self, + agent: std::sync::Arc, + resume: bool, + ) -> anyhow::Result { + self.config.validate()?; + let started = Instant::now(); + let ledger_path = self.config.ledger_path(&self.workspace); + ensure_ledger_path_safe(&self.workspace, &ledger_path).await?; + ensure_clean_workspace(&self.workspace).await?; + + let mut ledger = if resume { + load_ledger(&ledger_path).await? + } else { + let commit = git_rev(&self.workspace).await?; + let baseline = run_with_budget( + started, + self.config.max_duration_secs, + measure_metric(&self.config, &self.workspace), + ) + .await?; + let ledger = AutoresearchLedger::new(&self.config, baseline, commit); + save_ledger(&ledger_path, &ledger).await?; + ledger + }; + + if ledger.objective != self.config.objective + || ledger.direction != self.config.direction.trim().to_ascii_lowercase() + { + bail!("autoresearch ledger does not match the current spec; use a new ledger path"); + } + + let start = ledger + .records + .iter() + .map(|record| record.iteration) + .max() + .unwrap_or(0) + .saturating_add(1); + + for iteration in start..start.saturating_add(self.config.max_iterations) { + if self.config.max_duration_secs > 0 + && started.elapsed() >= Duration::from_secs(self.config.max_duration_secs) + { + tracing::info!("autoresearch wall-clock budget exhausted"); + break; + } + let checkpoint = git_rev(&self.workspace).await?; + let previous_best = ledger.best_metric; + let prompt = format!( + "You are running one bounded autoresearch iteration.\n\n\ + Objective: {objective}\n\ + Metric command (must print one numeric value): {metric}\n\ + Direction: {direction}\n\ + Current best metric: {best}\n\n\ + Form exactly one concrete hypothesis, implement only that experiment,\ + and leave the workspace in the candidate state. Do not commit, reset,\ + edit the autoresearch ledger, or claim success without making a change.\n\n\ + Hypothesis: ", + objective = self.config.objective, + metric = self.config.metric_command, + direction = self.config.direction, + best = ledger.best_metric, + ); + + let null_channel = NullChannel::new("autoresearch"); + let message = IncomingMessage { + id: uuid::Uuid::new_v4().to_string(), + sender_id: "autoresearch".to_string(), + sender_name: Some("Autoresearch".to_string()), + chat_id: "autoresearch".to_string(), + text: prompt, + is_group: false, + reply_to: None, + timestamp: chrono::Utc::now(), + }; + let turn = async { + if self.config.model.trim().is_empty() { + agent.handle_message(&message, &null_channel).await + } else { + agent + .handle_message_with_model( + &message, + &null_channel, + Some(self.config.model.trim()), + ) + .await + } + }; + let result = run_with_budget(started, self.config.max_duration_secs, turn).await; + + let hypothesis = result + .as_ref() + .map(|response| first_line(response).unwrap_or_else(|| "agent experiment".into())) + .unwrap_or_else(|error| format!("agent error: {error}")); + + let (decision, metric, reason) = match result { + Err(error) => ( + ExperimentDecision::Failed, + None, + format!("agent error: {error}"), + ), + Ok(_) => { + let evaluation = + run_with_budget(started, self.config.max_duration_secs, async { + if !run_validation(&self.config, &self.workspace).await? { + return Ok(( + ExperimentDecision::Rejected, + None, + "validation failed".to_string(), + )); + } + match measure_metric(&self.config, &self.workspace).await { + Ok(value) if is_better(&self.config, value, previous_best) => Ok(( + ExperimentDecision::Accepted, + Some(value), + "metric improved".to_string(), + )), + Ok(value) => Ok(( + ExperimentDecision::Rejected, + Some(value), + "metric did not improve".to_string(), + )), + Err(error) => { + Ok((ExperimentDecision::Failed, None, error.to_string())) + } + } + }) + .await; + evaluation.unwrap_or_else(|error| { + ( + ExperimentDecision::Failed, + None, + format!("evaluation error: {error}"), + ) + }) + } + }; + + let delta_percent = metric.map(|value| percent_delta(previous_best, value)); + let accepted = decision == ExperimentDecision::Accepted; + let commit = if accepted { + Some(commit_experiment(&self.workspace, iteration).await?) + } else { + restore_checkpoint(&self.workspace, &checkpoint).await?; + None + }; + + if let Some(value) = metric.filter(|_| accepted) { + ledger.best_metric = value; + ledger.best_commit = commit.clone().unwrap_or(checkpoint); + } + ledger.records.push(ExperimentRecord { + iteration, + hypothesis, + commit, + metric, + baseline: previous_best, + delta_percent, + decision, + reason, + timestamp: chrono::Utc::now().to_rfc3339(), + }); + save_ledger(&ledger_path, &ledger).await?; + tracing::info!( + iteration, + best_metric = ledger.best_metric, + "autoresearch iteration complete" + ); + } + + Ok(ledger) + } +} + +async fn load_ledger(path: &Path) -> anyhow::Result { + let content = tokio::fs::read_to_string(path) + .await + .with_context(|| format!("reading autoresearch ledger {}", path.display()))?; + Ok(toml::from_str(&content)?) +} + +async fn run_with_budget( + started: Instant, + max_duration_secs: u64, + future: F, +) -> anyhow::Result +where + F: Future>, +{ + if max_duration_secs == 0 { + return future.await; + } + let remaining = Duration::from_secs(max_duration_secs).saturating_sub(started.elapsed()); + tokio::time::timeout(remaining, future) + .await + .with_context(|| "autoresearch wall-clock budget exhausted")? +} + +async fn save_ledger(path: &Path, ledger: &AutoresearchLedger) -> anyhow::Result<()> { + if let Some(parent) = path.parent() { + tokio::fs::create_dir_all(parent).await?; + } + let content = toml::to_string_pretty(ledger)?; + let temporary = path.with_extension("tmp"); + tokio::fs::write(&temporary, content).await?; + tokio::fs::rename(&temporary, path).await?; + Ok(()) +} + +async fn ensure_clean_workspace(workspace: &Path) -> anyhow::Result<()> { + let output = git_command(workspace, &["status", "--porcelain"]).await?; + if !output.trim().is_empty() { + bail!("autoresearch requires a clean workspace; commit or stash existing changes first"); + } + Ok(()) +} + +async fn ensure_ledger_path_safe(workspace: &Path, ledger_path: &Path) -> anyhow::Result<()> { + let Ok(relative) = ledger_path.strip_prefix(workspace) else { + return Ok(()); + }; + let relative = relative.to_string_lossy(); + let status = tokio::process::Command::new("git") + .args(["check-ignore", "--quiet", "--", relative.as_ref()]) + .current_dir(workspace) + .status() + .await + .with_context(|| format!("checking whether ledger path is ignored: {relative}"))?; + if !status.success() { + bail!( + "autoresearch ledger path {} is inside the workspace but is not git-ignored; choose an ignored path or store the ledger outside the workspace", + ledger_path.display() + ); + } + Ok(()) +} + +async fn git_rev(workspace: &Path) -> anyhow::Result { + Ok(git_command(workspace, &["rev-parse", "HEAD"]) + .await? + .trim() + .to_string()) +} + +async fn commit_experiment(workspace: &Path, iteration: usize) -> anyhow::Result { + git_command(workspace, &["add", "-A"]).await?; + let status = git_command(workspace, &["status", "--porcelain"]).await?; + if status.trim().is_empty() { + bail!("experiment iteration {iteration} made no changes"); + } + git_command( + workspace, + &[ + "commit", + "-m", + &format!("autoresearch: iteration {iteration}"), + ], + ) + .await?; + git_rev(workspace).await +} + +async fn restore_checkpoint(workspace: &Path, checkpoint: &str) -> anyhow::Result<()> { + // The clean-workspace precondition makes these scoped resets recoverable: + // only changes made by the rejected iteration can exist at this point. + git_command(workspace, &["reset", "--hard", checkpoint]).await?; + git_command(workspace, &["clean", "-fd"]).await?; + Ok(()) +} + +async fn git_command(workspace: &Path, args: &[&str]) -> anyhow::Result { + let output = tokio::process::Command::new("git") + .args(args) + .current_dir(workspace) + .output() + .await + .with_context(|| format!("running git {}", args.join(" ")))?; + if !output.status.success() { + bail!( + "git {} failed: {}", + args.join(" "), + crate::text::truncate_chars(&String::from_utf8_lossy(&output.stderr), 1000) + ); + } + Ok(String::from_utf8_lossy(&output.stdout).into_owned()) +} + +async fn run_validation(config: &AutoresearchConfig, workspace: &Path) -> anyhow::Result { + if config.validation_command.trim().is_empty() { + return Ok(true); + } + for attempt in 1..=config.validation_retries { + let output = match run_shell( + &config.validation_command, + workspace, + config.command_timeout_secs, + ) + .await + { + Ok(output) => output, + Err(error) => { + tracing::warn!(attempt, "autoresearch validation could not run: {error}"); + continue; + } + }; + if output.status.success() { + return Ok(true); + } + tracing::warn!( + attempt, + "autoresearch validation failed: {}", + crate::text::truncate_chars(&String::from_utf8_lossy(&output.stderr), 1000) + ); + } + Ok(false) +} + +async fn measure_metric(config: &AutoresearchConfig, workspace: &Path) -> anyhow::Result { + let mut values = Vec::with_capacity(config.samples); + for _ in 0..config.samples { + let output = run_shell( + &config.metric_command, + workspace, + config.command_timeout_secs, + ) + .await?; + if !output.status.success() { + bail!( + "metric command failed: {}", + crate::text::truncate_chars(&String::from_utf8_lossy(&output.stderr), 1000) + ); + } + values.push(parse_metric(&String::from_utf8_lossy(&output.stdout))?); + } + values.sort_by(|a, b| a.partial_cmp(b).unwrap_or(Ordering::Equal)); + Ok(values[values.len() / 2]) +} + +async fn run_shell( + command: &str, + workspace: &Path, + timeout_secs: u64, +) -> anyhow::Result { + let child = tokio::process::Command::new("sh") + .arg("-c") + .arg(command) + .current_dir(workspace) + .kill_on_drop(true) + .spawn() + .with_context(|| format!("starting autoresearch command: {command}"))?; + tokio::time::timeout(Duration::from_secs(timeout_secs), child.wait_with_output()) + .await + .with_context(|| format!("command timed out after {timeout_secs}s"))? + .with_context(|| format!("waiting for autoresearch command: {command}")) +} + +fn parse_metric(output: &str) -> anyhow::Result { + output + .split_whitespace() + .find_map(|token| { + token + .trim_matches(|c: char| !c.is_ascii_digit() && c != '.' && c != '-') + .parse::() + .ok() + }) + .filter(|value| value.is_finite()) + .ok_or_else(|| anyhow::anyhow!("metric command must print a numeric value")) +} + +fn is_better(config: &AutoresearchConfig, candidate: f64, current: f64) -> bool { + let improvement = current.abs() * config.min_improvement_percent / 100.0; + match config.direction.trim().to_ascii_lowercase().as_str() { + "maximize" => candidate > current + improvement, + _ => candidate < current - improvement, + } +} + +fn percent_delta(previous: f64, candidate: f64) -> f64 { + if previous == 0.0 { + 0.0 + } else { + ((candidate - previous) / previous.abs()) * 100.0 + } +} + +fn first_line(text: &str) -> Option { + text.lines() + .map(str::trim) + .find(|line| !line.is_empty()) + .map(str::to_string) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_numeric_metric_from_command_output() { + assert_eq!(parse_metric("median_ms=19.125\n").unwrap(), 19.125); + } + + #[test] + fn median_sampling_is_sorted() { + let mut values = [9.0, 1.0, 4.0]; + values.sort_by(|a, b| a.partial_cmp(b).unwrap_or(Ordering::Equal)); + assert_eq!(values[values.len() / 2], 4.0); + } + + #[test] + fn direction_and_tolerance_are_respected() { + let config = AutoresearchConfig { + direction: "minimize".into(), + min_improvement_percent: 1.0, + ..AutoresearchConfig::default() + }; + assert!(is_better(&config, 98.9, 100.0)); + assert!(!is_better(&config, 100.5, 100.0)); + let strict = AutoresearchConfig::default(); + assert!(!is_better(&strict, 100.0, 100.0)); + } + + #[tokio::test] + async fn command_timeout_is_enforced() { + let error = run_shell("sleep 5", Path::new("."), 1) + .await + .expect_err("long-running metric should time out"); + assert!(error.to_string().contains("timed out")); + } + + #[tokio::test] + async fn validation_timeout_is_a_rejected_attempt() { + let config = AutoresearchConfig { + validation_command: "sleep 5".into(), + validation_retries: 1, + command_timeout_secs: 1, + ..AutoresearchConfig::default() + }; + assert!(!run_validation(&config, Path::new(".")).await.unwrap()); + } +} diff --git a/src/cost.rs b/src/cost.rs index f74d7e9..5a4cf7f 100644 --- a/src/cost.rs +++ b/src/cost.rs @@ -40,6 +40,26 @@ pub struct CostRecord { pub timestamp: chrono::DateTime, } +/// Estimated input shape for one provider request. These counts are based on +/// characters because providers do not expose tokenizers uniformly; they are +/// intended for comparing Apollo configurations, not billing. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct ContextSnapshot { + pub system_chars: usize, + pub history_chars: usize, + pub tool_chars: usize, + pub estimated_input_tokens: usize, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct ContextSummary { + pub request_count: usize, + pub system_chars: usize, + pub history_chars: usize, + pub tool_chars: usize, + pub estimated_input_tokens: usize, +} + /// Claude API rate limit status (from response headers) #[derive(Debug, Clone, Serialize, Deserialize)] pub struct RateLimitStatus { @@ -55,6 +75,7 @@ pub struct RateLimitStatus { /// Cost tracker (in-memory + persistent accounting hooks) pub struct CostTracker { costs: Arc>>, + contexts: Arc>>, models: Arc>>, rate_limit_status: Arc>>, } @@ -96,6 +117,7 @@ impl CostTracker { Self { costs: Arc::new(RwLock::new(Vec::new())), + contexts: Arc::new(RwLock::new(Vec::new())), models: Arc::new(RwLock::new(models)), rate_limit_status: Arc::new(RwLock::new(None)), } @@ -129,6 +151,26 @@ impl CostTracker { Ok(()) } + /// Record the estimated shape of a provider request for harness telemetry. + pub async fn record_context(&self, snapshot: ContextSnapshot) { + self.contexts.write().await.push(snapshot); + } + + /// Aggregate prompt-shape telemetry since this tracker was created. + pub async fn context_summary(&self) -> ContextSummary { + let contexts = self.contexts.read().await; + ContextSummary { + request_count: contexts.len(), + system_chars: contexts.iter().map(|item| item.system_chars).sum(), + history_chars: contexts.iter().map(|item| item.history_chars).sum(), + tool_chars: contexts.iter().map(|item| item.tool_chars).sum(), + estimated_input_tokens: contexts + .iter() + .map(|item| item.estimated_input_tokens) + .sum(), + } + } + /// Get cost summary pub async fn summary(&self) -> CostSummary { let costs = self.costs.read().await; @@ -143,11 +185,13 @@ impl CostTracker { *by_model.entry(cost.model.clone()).or_insert(0.0) += cost.cost_usd; } + let context = self.context_summary().await; CostSummary { total_cost, total_tokens, by_model, call_count: costs.len(), + context, } } @@ -210,6 +254,7 @@ pub struct CostSummary { pub total_tokens: usize, pub by_model: std::collections::HashMap, pub call_count: usize, + pub context: ContextSummary, } #[cfg(test)] @@ -313,4 +358,21 @@ mod tests { Some("2023-11-20T12:00:00Z".to_string()) ); } + + #[tokio::test] + async fn context_telemetry_aggregates_request_shape() { + let tracker = CostTracker::new(); + tracker + .record_context(ContextSnapshot { + system_chars: 40, + history_chars: 80, + tool_chars: 20, + estimated_input_tokens: 35, + }) + .await; + let summary = tracker.context_summary().await; + assert_eq!(summary.request_count, 1); + assert_eq!(summary.estimated_input_tokens, 35); + assert_eq!(summary.system_chars, 40); + } } diff --git a/src/lib.rs b/src/lib.rs index 914af54..77402c3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,6 +16,7 @@ pub mod agent; pub mod agent_http; pub mod autonomous; +pub mod autoresearch; pub mod bootstrap; pub mod channel_check; pub mod channels; diff --git a/src/main.rs b/src/main.rs index 6721549..92b73c2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -9,6 +9,7 @@ use clap::{Parser, Subcommand}; use apollo::agent::hooks::PermissionHook; use apollo::agent::{agent_mode_from_permission_profile, AgentRunner}; use apollo::autonomous::{AutonomousConfig, AutonomousLoop}; +use apollo::autoresearch::{AutoresearchConfig, AutoresearchLoop}; use apollo::bootstrap::{ build_base_tools, build_embedding_provider, build_memory_backend, build_provider, load_config, require_config_file, @@ -284,6 +285,29 @@ enum Commands { resume: bool, }, + /// Run bounded metric-driven experiments and keep only improvements + Autoresearch { + /// Configuration file path + #[arg(short, long, default_value = "apollo.json")] + config: String, + + /// Workspace directory + #[arg(short, long)] + workspace: Option, + + /// Autoresearch specification (TOML) + #[arg(long, default_value = ".apollo/autoresearch.toml")] + spec: PathBuf, + + /// Continue from the persisted ledger + #[arg(long, default_value_t = false)] + resume: bool, + + /// Override the spec's iteration limit + #[arg(long)] + iterations: Option, + }, + /// Swarm commands (multi-agent coordination) Swarm { #[command(subcommand)] @@ -537,6 +561,105 @@ enum SwarmAction { Status, } +/// Build the same unattended runner used by autonomous and autoresearch modes. +async fn build_automation_agent( + config_path: &str, + workspace: &Path, + restricted: bool, +) -> anyhow::Result<(Arc, Config)> { + let mut cfg = apollo::bootstrap::load_config_workspace(config_path, Some(workspace)); + if restricted { + // Autoresearch only needs local code execution and filesystem edits. + // Keep network, messaging, memory, MCP, and plugin capabilities out of + // the model's ambient tool catalog; this mirrors capability-oriented + // agent designs where access is introduced deliberately. + cfg.toolsets.enabled = vec!["runtime".into(), "fs".into()]; + cfg.toolsets.disabled = vec![ + "web".into(), + "browser".into(), + "memory".into(), + "sessions".into(), + "messaging".into(), + "advanced".into(), + "desktop".into(), + "media".into(), + "skills".into(), + ]; + } + let provider = build_provider(&cfg); + let policy = Arc::new(ExecutionPolicy::from_config(&cfg.policy)); + let memory = build_memory_backend(workspace, &cfg).await?; + let embedding_provider = build_embedding_provider(&cfg)?; + let system_prompt = prompt::build_system_prompt(workspace).await; + let discovered_skills = if restricted { + Vec::new() + } else { + skills::discover_skills_for_workspace(Some(workspace)) + }; + + #[cfg(feature = "zkr-memory")] + let zkr_store = apollo::bootstrap::build_zkr_store(workspace, &cfg) + .ok() + .flatten(); + #[cfg(feature = "zkr-memory")] + let mut tools = build_base_tools( + workspace, + Arc::clone(&policy), + memory.clone(), + embedding_provider, + Arc::clone(&provider), + &cfg, + zkr_store.clone(), + ); + #[cfg(not(feature = "zkr-memory"))] + let mut tools = build_base_tools( + workspace, + Arc::clone(&policy), + memory.clone(), + embedding_provider, + Arc::clone(&provider), + &cfg, + ); + + if !restricted { + for tool in apollo::tools::dynamic::DynamicTool::load_all(Arc::clone(&policy)) { + tools.push(Arc::new(tool)); + } + } + + let mut runner = AgentRunner::new(provider, tools, memory, &system_prompt, cfg.model.clone()) + .with_config(cfg.agent.clone()) + .with_mode(agent_mode_from_permission_profile( + &cfg.agent.permission_profile, + )) + .with_workspace(workspace.to_path_buf()) + .with_memory_ideas(cfg.memory.clone()) + .with_group_chat(cfg.group_chat.clone()) + .with_skills(discovered_skills) + .await; + #[cfg(feature = "zkr-memory")] + { + runner = runner.with_zkr(zkr_store, cfg.zkr.clone()); + } + + if !restricted { + let mut host_reg = apollo::plugin::PluginRegistry::new(); + host_reg.ingest_host_plugins_trusting( + workspace, + &cfg.plugin_layer.host_plugin_roots, + &cfg.plugin_layer.trusted_host_plugins, + ); + runner = runner.with_plugin_registry(host_reg).await; + } + + let runner = Arc::new(runner); + runner.add_hook(Arc::new(PermissionHook::new( + cfg.agent.permissions.deny.clone(), + cfg.agent.permissions.allow.clone(), + ))); + Ok((runner, cfg)) +} + #[tokio::main] async fn main() -> anyhow::Result<()> { // Load .env if present — allows running without manually exporting env vars @@ -1254,6 +1377,40 @@ async fn main() -> anyhow::Result<()> { } } + Commands::Autoresearch { + config, + workspace, + spec, + resume, + iterations, + } => { + let workspace = workspace.unwrap_or_else(|| load_config(&config).workspace.clone()); + let spec_path = if spec.is_absolute() { + spec + } else { + workspace.join(spec) + }; + let mut autoresearch_config = AutoresearchConfig::load(&spec_path)?; + if let Some(iterations) = iterations { + autoresearch_config.max_iterations = iterations; + } + let (runner, _cfg) = build_automation_agent(&config, &workspace, true).await?; + println!( + "apollo v{} — autoresearch (objective: {})", + env!("CARGO_PKG_VERSION"), + autoresearch_config.objective + ); + println!(" Workspace: {}", workspace.display()); + let ledger = AutoresearchLoop::new(autoresearch_config, workspace) + .run(runner, resume) + .await?; + println!( + " Best metric: {} ({} records)", + ledger.best_metric, + ledger.records.len() + ); + } + Commands::Autonomous { config, workspace, @@ -2093,6 +2250,7 @@ fn config_path_for_cli(cli: &Cli) -> Option { | Some(Commands::SelfUpdate { config, .. }) | Some(Commands::Mcp { config, .. }) | Some(Commands::Autonomous { config, .. }) + | Some(Commands::Autoresearch { config, .. }) | Some(Commands::Serve { config, .. }) | Some(Commands::Tui { config, .. }) => Some(config.clone()), Some(_) => None, From 4ce86947ad3979aa4474c338bdc6ea26f8b2d3be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Carter=20=E7=A5=81=E6=98=8E=E6=80=9D?= Date: Thu, 6 Aug 2026 15:59:05 +0000 Subject: [PATCH 2/2] fix: harden autoresearch resume and isolation --- docs/autoresearch.md | 21 +- src/agent/loop_runner.rs | 67 +++-- src/autoresearch.rs | 565 ++++++++++++++++++++++++++++++++++++--- src/cost.rs | 36 ++- src/main.rs | 61 +++-- 5 files changed, 659 insertions(+), 91 deletions(-) diff --git a/docs/autoresearch.md b/docs/autoresearch.md index 1e77956..f3e6c1d 100644 --- a/docs/autoresearch.md +++ b/docs/autoresearch.md @@ -30,9 +30,11 @@ apollo autoresearch --workspace . --resume The metric command must print at least one finite numeric value. Commands in the specification are trusted local code and run through `sh -c`; do not use a specification copied from an untrusted source. The workspace must be clean at -startup. Rejected iterations are restored to their checkpoint and untracked -files created by that iteration are removed, so run autoresearch in a dedicated -worktree when experimenting with valuable uncommitted files. +startup. Rejected iterations are restored to their checkpoint; ignored +configuration/state files are restored to their pre-iteration contents, and +untracked files created by that iteration are removed. Build output under +`target/` is treated as disposable process state. Run autoresearch in a +dedicated worktree when experimenting with valuable local files. If `ledger_path` is inside the workspace, it must be Git-ignored; an external ledger path is also supported. This keeps the durable ledger from becoming an @@ -41,16 +43,23 @@ unrelated dirty change after an accepted iteration. Accepted iterations are committed locally as `autoresearch: iteration N`. Pushing is intentionally not automatic. +The ledger records the branch, the accepted commit, a fingerprint of the +metric/validation definition, and a unique run chat id. `--resume` refuses to +continue if the branch, HEAD, or experiment definition no longer matches the +ledger. + The autoresearch runner exposes only the runtime and filesystem tool groups to the experiment agent. Network, messaging, memory, MCP, dynamic tools, host -plugins, and workspace skills are not ambient capabilities for this workflow. +plugins, and workspace skills are not ambient capabilities for this workflow; +history, personal-context injection, ZKR recall/capture, and reflection are +also disabled. Validation and metric processes are bounded by `command_timeout_secs`, and a validation command may be retried with `validation_retries`. The whole run is bounded by `max_iterations` and `max_duration_secs`. -Apollo also records estimated system, history, and tool-definition context for -each provider request in the existing cost tracker. The estimates use four +Apollo also records estimated system, history, and tool-definition context in +aggregate counters in the existing cost tracker. The estimates use four characters per token and are useful for comparing harness configurations, not for billing. diff --git a/src/agent/loop_runner.rs b/src/agent/loop_runner.rs index 0233df5..d13cf54 100644 --- a/src/agent/loop_runner.rs +++ b/src/agent/loop_runner.rs @@ -50,6 +50,8 @@ pub struct AgentRunner { plugin_registry: Arc>, /// Current trajectory being recorded (per chat) trajectories: Arc>>, + /// Whether this runner may read or persist conversational memory. + memory_enabled: bool, memory_ideas: crate::config::MemoryIdeasConfig, group_chat: crate::config::GroupChatConfig, #[cfg(feature = "zkr-memory")] @@ -89,6 +91,7 @@ impl AgentRunner { hook_manager: Arc::new(HookManager::new()), plugin_registry: Arc::new(RwLock::new(PluginRegistry::new())), trajectories: Arc::new(RwLock::new(HashMap::new())), + memory_enabled: true, memory_ideas: crate::config::MemoryIdeasConfig::default(), group_chat: crate::config::GroupChatConfig::default(), #[cfg(feature = "zkr-memory")] @@ -180,6 +183,15 @@ impl AgentRunner { self } + /// Enable or disable conversational memory for this runner. + /// + /// Restricted automation uses this to keep prior conversations and + /// experiment output out of the model context and persistent stores. + pub fn with_memory_enabled(mut self, enabled: bool) -> Self { + self.memory_enabled = enabled; + self + } + pub fn with_group_chat(mut self, cfg: crate::config::GroupChatConfig) -> Self { self.group_chat = cfg; self @@ -589,7 +601,7 @@ impl AgentRunner { let base_prompt = self.system_prompt.read().await.clone(); #[cfg(feature = "zkr-memory")] - let system_prompt = if self.zkr_config.self_improve { + let system_prompt = if self.memory_enabled && self.zkr_config.self_improve { if let Some(store) = &self.zkr { match store.augment_prompt(&effective_text, &base_prompt).await { Ok(augmented) => augmented, @@ -644,7 +656,7 @@ impl AgentRunner { } } } - if msg.is_group { + if self.memory_enabled && msg.is_group { if let Some(group_memory) = self.load_group_memory(&msg.chat_id).await? { if !group_memory.trim().is_empty() { messages.push(ChatMessage::system(crate::context::group_memory_prompt( @@ -655,23 +667,25 @@ impl AgentRunner { } } - let history = crate::memory::context_inject::merged_history( - &self.memory, - &msg.chat_id, - self.memory_ideas.principal_id.as_deref(), - self.agent_config.max_history_messages, - ) - .await?; - for (role, content) in history { - match role.as_str() { - "user" => messages.push(ChatMessage::user(&content)), - "assistant" => messages.push(ChatMessage::assistant(&content)), - _ => {} + if self.memory_enabled { + let history = crate::memory::context_inject::merged_history( + &self.memory, + &msg.chat_id, + self.memory_ideas.principal_id.as_deref(), + self.agent_config.max_history_messages, + ) + .await?; + for (role, content) in history { + match role.as_str() { + "user" => messages.push(ChatMessage::user(&content)), + "assistant" => messages.push(ChatMessage::assistant(&content)), + _ => {} + } } } let mut user_turn = effective_text.clone(); - if self.memory_ideas.inject_context { + if self.memory_enabled && self.memory_ideas.inject_context { let blocks = crate::memory::context_inject::personal_context_blocks( &self.memory, crate::memory::context_inject::InjectConfig { @@ -687,7 +701,7 @@ impl AgentRunner { } } #[cfg(feature = "zkr-memory")] - if self.zkr_config.inject_recall { + if self.memory_enabled && self.zkr_config.inject_recall { if let Some(store) = &self.zkr { match store .context(&effective_text, self.zkr_config.recall_limit) @@ -811,7 +825,9 @@ impl AgentRunner { text: &str, delivery: &Delivery<'_>, ) -> anyhow::Result { - self.persist_conversation(msg, text).await?; + if self.memory_enabled { + self.persist_conversation(msg, text).await?; + } // Mark trajectory as successful, record final response { @@ -830,11 +846,16 @@ impl AgentRunner { text.to_string(), )) .await; - if let Some(ws) = &self.session_note_workspace { - let preview: String = text.chars().take(200).collect(); - if !preview.is_empty() { - let _ = - crate::memory::session_note::append_session_note(ws, &msg.chat_id, &preview); + if self.memory_enabled { + if let Some(ws) = &self.session_note_workspace { + let preview: String = text.chars().take(200).collect(); + if !preview.is_empty() { + let _ = crate::memory::session_note::append_session_note( + ws, + &msg.chat_id, + &preview, + ); + } } } @@ -852,7 +873,7 @@ impl AgentRunner { let delivered = delivery.deliver(&msg.chat_id, text).await?; #[cfg(feature = "zkr-memory")] - if self.zkr_config.self_improve { + if self.memory_enabled && self.zkr_config.self_improve { if let Some(store) = &self.zkr { let _ = store .record_reflection(&msg.text, "agent turn", text, "completed") diff --git a/src/autoresearch.rs b/src/autoresearch.rs index a44697c..06798b1 100644 --- a/src/autoresearch.rs +++ b/src/autoresearch.rs @@ -6,12 +6,14 @@ //! durable ledger. use std::cmp::Ordering; +use std::collections::HashSet; use std::future::Future; use std::path::{Path, PathBuf}; use std::time::{Duration, Instant}; use anyhow::{bail, Context}; use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; use crate::agent::NullChannel; use crate::channels::IncomingMessage; @@ -143,16 +145,28 @@ pub struct AutoresearchLedger { pub direction: String, pub best_metric: f64, pub best_commit: String, + /// Branch on which this experiment started and accepted commits land. + #[serde(default)] + pub branch: String, + /// Stable chat id so a fresh run cannot inherit another run's history. + #[serde(default)] + pub chat_id: String, + /// Hash of the metric/validation definition used to produce this ledger. + #[serde(default)] + pub spec_fingerprint: String, pub records: Vec, } impl AutoresearchLedger { - fn new(config: &AutoresearchConfig, baseline: f64, commit: String) -> Self { + fn new(config: &AutoresearchConfig, baseline: f64, commit: String, branch: String) -> Self { Self { objective: config.objective.clone(), direction: config.direction.trim().to_ascii_lowercase(), best_metric: baseline, best_commit: commit, + branch, + chat_id: format!("autoresearch-{}", uuid::Uuid::new_v4()), + spec_fingerprint: spec_fingerprint(config), records: vec![ExperimentRecord { iteration: 0, hypothesis: "Initial measurement".to_string(), @@ -189,28 +203,68 @@ impl AutoresearchLoop { let ledger_path = self.config.ledger_path(&self.workspace); ensure_ledger_path_safe(&self.workspace, &ledger_path).await?; ensure_clean_workspace(&self.workspace).await?; + let branch = git_branch(&self.workspace).await?; + if branch.is_empty() { + bail!("autoresearch requires a named branch; detached HEAD is not supported"); + } let mut ledger = if resume { - load_ledger(&ledger_path).await? + let ledger = load_ledger(&ledger_path).await?; + if ledger.objective != self.config.objective + || ledger.direction != self.config.direction.trim().to_ascii_lowercase() + { + bail!("autoresearch ledger does not match the current spec; use a new ledger path"); + } + if ledger.spec_fingerprint != spec_fingerprint(&self.config) { + bail!( + "autoresearch ledger uses a different metric definition; use a new ledger path" + ); + } + if ledger.branch.is_empty() || ledger.branch != branch { + bail!( + "autoresearch ledger belongs to branch `{}`, current branch is `{}`", + if ledger.branch.is_empty() { + "" + } else { + &ledger.branch + }, + branch + ); + } + let head = git_rev(&self.workspace).await?; + if ledger.best_commit.is_empty() || ledger.best_commit != head { + bail!( + "autoresearch ledger best commit {} does not match workspace HEAD {}; restore the recorded commit or use a new ledger path", + if ledger.best_commit.is_empty() { "" } else { &ledger.best_commit }, + head + ); + } + if ledger.chat_id.is_empty() { + bail!("autoresearch ledger has no run identity; use a new ledger path"); + } + ledger } else { let commit = git_rev(&self.workspace).await?; + if !run_with_budget( + started, + self.config.max_duration_secs, + run_validation(&self.config, &self.workspace), + ) + .await? + { + bail!("autoresearch baseline validation failed"); + } let baseline = run_with_budget( started, self.config.max_duration_secs, measure_metric(&self.config, &self.workspace), ) .await?; - let ledger = AutoresearchLedger::new(&self.config, baseline, commit); + let ledger = AutoresearchLedger::new(&self.config, baseline, commit, branch.clone()); save_ledger(&ledger_path, &ledger).await?; ledger }; - if ledger.objective != self.config.objective - || ledger.direction != self.config.direction.trim().to_ascii_lowercase() - { - bail!("autoresearch ledger does not match the current spec; use a new ledger path"); - } - let start = ledger .records .iter() @@ -227,6 +281,17 @@ impl AutoresearchLoop { break; } let checkpoint = git_rev(&self.workspace).await?; + if checkpoint != ledger.best_commit { + bail!( + "autoresearch workspace HEAD {} does not match ledger best commit {}", + checkpoint, + ledger.best_commit + ); + } + if git_branch(&self.workspace).await? != ledger.branch { + bail!("autoresearch branch changed while the run was in progress"); + } + let ignored_state = capture_ignored_state(&self.workspace).await?; let previous_best = ledger.best_metric; let prompt = format!( "You are running one bounded autoresearch iteration.\n\n\ @@ -249,7 +314,7 @@ impl AutoresearchLoop { id: uuid::Uuid::new_v4().to_string(), sender_id: "autoresearch".to_string(), sender_name: Some("Autoresearch".to_string()), - chat_id: "autoresearch".to_string(), + chat_id: ledger.chat_id.clone(), text: prompt, is_group: false, reply_to: None, @@ -270,6 +335,12 @@ impl AutoresearchLoop { }; let result = run_with_budget(started, self.config.max_duration_secs, turn).await; + // The agent may edit files, but it must not move the experiment's + // branch or checkpoint. Verify before running trusted commands so + // a rejection can never reset an unrelated branch. + ensure_experiment_state(&self.workspace, &ledger.branch, &checkpoint).await?; + restore_ignored_state(&self.workspace, &ignored_state).await?; + let hypothesis = result .as_ref() .map(|response| first_line(response).unwrap_or_else(|| "agent experiment".into())) @@ -318,12 +389,44 @@ impl AutoresearchLoop { } }; + // Validation and metric commands are trusted shell, but they are + // still not allowed to move the branch or checkpoint. Their + // ignored-file side effects are also discarded before deciding. + ensure_experiment_state(&self.workspace, &ledger.branch, &checkpoint).await?; + restore_ignored_state(&self.workspace, &ignored_state).await?; + let delta_percent = metric.map(|value| percent_delta(previous_best, value)); let accepted = decision == ExperimentDecision::Accepted; let commit = if accepted { - Some(commit_experiment(&self.workspace, iteration).await?) + match run_with_budget( + started, + self.config.max_duration_secs, + commit_experiment( + &self.workspace, + iteration, + started, + self.config.max_duration_secs, + ), + ) + .await + { + Ok(commit) => Some(commit), + Err(error) => { + // A timed-out hook may have left git add's index + // changes behind. Reset only if the branch and HEAD + // are still the checkpoint we verified above. + let current_branch = git_branch(&self.workspace).await?; + let current_head = git_rev(&self.workspace).await?; + if current_branch == ledger.branch && current_head == checkpoint { + restore_checkpoint(&self.workspace, &checkpoint).await?; + restore_ignored_state(&self.workspace, &ignored_state).await?; + } + bail!("autoresearch acceptance commit failed: {error}"); + } + } } else { restore_checkpoint(&self.workspace, &checkpoint).await?; + restore_ignored_state(&self.workspace, &ignored_state).await?; None }; @@ -361,6 +464,197 @@ async fn load_ledger(path: &Path) -> anyhow::Result { Ok(toml::from_str(&content)?) } +#[derive(Serialize)] +struct ExperimentDefinition<'a> { + objective: &'a str, + metric_command: &'a str, + direction: &'a str, + validation_command: &'a str, + validation_retries: usize, + command_timeout_secs: u64, + samples: usize, + min_improvement_percent: f64, + max_iterations: usize, + max_duration_secs: u64, + model: &'a str, +} + +fn spec_fingerprint(config: &AutoresearchConfig) -> String { + let direction = config.direction.trim().to_ascii_lowercase(); + let definition = ExperimentDefinition { + objective: &config.objective, + metric_command: &config.metric_command, + direction: &direction, + validation_command: &config.validation_command, + validation_retries: config.validation_retries, + command_timeout_secs: config.command_timeout_secs, + samples: config.samples, + min_improvement_percent: config.min_improvement_percent, + max_iterations: config.max_iterations, + max_duration_secs: config.max_duration_secs, + model: &config.model, + }; + let encoded = serde_json::to_vec(&definition).expect("experiment definition is serializable"); + format!("{:x}", Sha256::digest(encoded)) +} + +#[derive(Debug)] +struct IgnoredFile { + relative: PathBuf, + contents: Option>, + symlink_target: Option, + #[cfg(unix)] + mode: u32, +} + +#[derive(Debug)] +struct IgnoredWorkspaceState { + files: Vec, +} + +async fn ignored_paths(workspace: &Path) -> anyhow::Result> { + let output = git_command( + workspace, + &[ + "ls-files", + "--others", + "--ignored", + "--exclude-standard", + "-z", + ], + ) + .await?; + output + .split('\0') + .filter(|path| !path.is_empty()) + // Build output can contain millions of files and is deliberately + // treated as disposable process state rather than experiment input. + // The source/configuration files that can affect an experiment are + // still snapshotted and restored below. + .filter(|path| { + !matches!( + Path::new(path).components().next(), + Some(std::path::Component::Normal(component)) if component == "target" + ) + }) + .map(|path| { + let relative = PathBuf::from(path); + validate_workspace_relative_path(&relative)?; + Ok(relative) + }) + .collect() +} + +fn validate_workspace_relative_path(path: &Path) -> anyhow::Result<()> { + if path.is_absolute() + || path + .components() + .any(|component| matches!(component, std::path::Component::ParentDir)) + { + bail!( + "git returned an unsafe workspace-relative path: {}", + path.display() + ); + } + Ok(()) +} + +async fn capture_ignored_state(workspace: &Path) -> anyhow::Result { + let mut files = Vec::new(); + for relative in ignored_paths(workspace).await? { + let path = workspace.join(&relative); + let metadata = tokio::fs::symlink_metadata(&path) + .await + .with_context(|| format!("reading ignored path metadata: {}", path.display()))?; + let file_type = metadata.file_type(); + if file_type.is_symlink() { + files.push(IgnoredFile { + relative, + contents: None, + symlink_target: Some(tokio::fs::read_link(&path).await?), + #[cfg(unix)] + mode: 0, + }); + } else if file_type.is_file() { + files.push(IgnoredFile { + relative, + contents: Some(tokio::fs::read(&path).await?), + symlink_target: None, + #[cfg(unix)] + mode: { + use std::os::unix::fs::PermissionsExt; + metadata.permissions().mode() + }, + }); + } + } + Ok(IgnoredWorkspaceState { files }) +} + +async fn remove_workspace_path(path: &Path) -> anyhow::Result<()> { + let Ok(metadata) = tokio::fs::symlink_metadata(path).await else { + return Ok(()); + }; + if metadata.file_type().is_dir() { + tokio::fs::remove_dir_all(path).await?; + } else { + tokio::fs::remove_file(path).await?; + } + Ok(()) +} + +async fn restore_ignored_state( + workspace: &Path, + state: &IgnoredWorkspaceState, +) -> anyhow::Result<()> { + // Restore tracked/untracked files separately; git clean intentionally does + // not touch ignored files, which is exactly where autoresearch configs, + // generated artifacts, and local credentials commonly live. + git_command(workspace, &["clean", "-fd"]).await?; + + let baseline: HashSet<&Path> = state + .files + .iter() + .map(|file| file.relative.as_path()) + .collect(); + for relative in ignored_paths(workspace).await? { + if !baseline.contains(relative.as_path()) { + remove_workspace_path(&workspace.join(relative)).await?; + } + } + + for file in &state.files { + let path = workspace.join(&file.relative); + if let Some(target) = &file.symlink_target { + remove_workspace_path(&path).await?; + if let Some(parent) = path.parent() { + tokio::fs::create_dir_all(parent).await?; + } + #[cfg(unix)] + std::os::unix::fs::symlink(target, &path)?; + #[cfg(windows)] + std::os::windows::fs::symlink_file(target, &path)?; + } else if let Some(contents) = &file.contents { + if let Some(parent) = path.parent() { + tokio::fs::create_dir_all(parent).await?; + } + if let Ok(metadata) = tokio::fs::symlink_metadata(&path).await { + if !metadata.file_type().is_file() { + remove_workspace_path(&path).await?; + } + } + tokio::fs::write(&path, contents).await?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + tokio::fs::set_permissions(&path, std::fs::Permissions::from_mode(file.mode)) + .await?; + } + } + } + Ok(()) +} + async fn run_with_budget( started: Instant, max_duration_secs: u64, @@ -424,22 +718,69 @@ async fn git_rev(workspace: &Path) -> anyhow::Result { .to_string()) } -async fn commit_experiment(workspace: &Path, iteration: usize) -> anyhow::Result { - git_command(workspace, &["add", "-A"]).await?; - let status = git_command(workspace, &["status", "--porcelain"]).await?; +async fn git_branch(workspace: &Path) -> anyhow::Result { + Ok(git_command(workspace, &["branch", "--show-current"]) + .await? + .trim() + .to_string()) +} + +async fn ensure_experiment_state( + workspace: &Path, + expected_branch: &str, + expected_head: &str, +) -> anyhow::Result<()> { + let branch = git_branch(workspace).await?; + if branch != expected_branch { + bail!( + "autoresearch agent moved from branch `{expected_branch}` to `{branch}`; refusing to reset" + ); + } + let head = git_rev(workspace).await?; + if head != expected_head { + bail!( + "autoresearch agent moved HEAD from `{expected_head}` to `{head}`; refusing to reset" + ); + } + Ok(()) +} + +async fn commit_experiment( + workspace: &Path, + iteration: usize, + started: Instant, + max_duration_secs: u64, +) -> anyhow::Result { + git_command_with_budget(workspace, &["add", "-A"], started, max_duration_secs).await?; + let status = git_command_with_budget( + workspace, + &["status", "--porcelain"], + started, + max_duration_secs, + ) + .await?; if status.trim().is_empty() { bail!("experiment iteration {iteration} made no changes"); } - git_command( + git_command_with_budget( workspace, &[ "commit", "-m", &format!("autoresearch: iteration {iteration}"), ], + started, + max_duration_secs, ) .await?; - git_rev(workspace).await + git_command_with_budget( + workspace, + &["rev-parse", "HEAD"], + started, + max_duration_secs, + ) + .await + .map(|commit| commit.trim().to_string()) } async fn restore_checkpoint(workspace: &Path, checkpoint: &str) -> anyhow::Result<()> { @@ -451,12 +792,35 @@ async fn restore_checkpoint(workspace: &Path, checkpoint: &str) -> anyhow::Resul } async fn git_command(workspace: &Path, args: &[&str]) -> anyhow::Result { - let output = tokio::process::Command::new("git") - .args(args) - .current_dir(workspace) - .output() - .await - .with_context(|| format!("running git {}", args.join(" ")))?; + git_command_with_timeout(workspace, args, None).await +} + +async fn git_command_with_budget( + workspace: &Path, + args: &[&str], + started: Instant, + max_duration_secs: u64, +) -> anyhow::Result { + let timeout = if max_duration_secs == 0 { + None + } else { + let remaining = Duration::from_secs(max_duration_secs).saturating_sub(started.elapsed()); + if remaining.is_zero() { + bail!("autoresearch wall-clock budget exhausted"); + } + Some(remaining) + }; + git_command_with_timeout(workspace, args, timeout).await +} + +async fn git_command_with_timeout( + workspace: &Path, + args: &[&str], + timeout: Option, +) -> anyhow::Result { + let mut command = tokio::process::Command::new("git"); + command.args(args).current_dir(workspace); + let output = run_process(&mut command, timeout, &format!("git {}", args.join(" "))).await?; if !output.status.success() { bail!( "git {} failed: {}", @@ -523,17 +887,64 @@ async fn run_shell( workspace: &Path, timeout_secs: u64, ) -> anyhow::Result { - let child = tokio::process::Command::new("sh") - .arg("-c") - .arg(command) - .current_dir(workspace) + let mut process = tokio::process::Command::new("sh"); + process.arg("-c").arg(command).current_dir(workspace); + run_process( + &mut process, + Some(Duration::from_secs(timeout_secs)), + "autoresearch shell command", + ) + .await +} + +fn configure_process_group(command: &mut tokio::process::Command) { + #[cfg(unix)] + { + // A separate process group lets timeout cleanup terminate descendants + // spawned by `sh -c`, such as cargo/compiler children. + command.process_group(0); + } +} + +fn terminate_process_group(pid: Option) { + #[cfg(unix)] + if let Some(pid) = pid { + // SAFETY: pid is the process-group leader created by this command. + unsafe { + libc::killpg(pid as libc::pid_t, libc::SIGKILL); + } + } +} + +async fn run_process( + command: &mut tokio::process::Command, + timeout: Option, + label: &str, +) -> anyhow::Result { + configure_process_group(command); + command + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()); + let child = command .kill_on_drop(true) .spawn() - .with_context(|| format!("starting autoresearch command: {command}"))?; - tokio::time::timeout(Duration::from_secs(timeout_secs), child.wait_with_output()) - .await - .with_context(|| format!("command timed out after {timeout_secs}s"))? - .with_context(|| format!("waiting for autoresearch command: {command}")) + .with_context(|| format!("starting {label}"))?; + let pid = child.id(); + let output = if let Some(timeout) = timeout { + match tokio::time::timeout(timeout, child.wait_with_output()).await { + Ok(result) => result.with_context(|| format!("waiting for {label}"))?, + Err(_) => { + terminate_process_group(pid); + bail!("{label} timed out"); + } + } + } else { + child + .wait_with_output() + .await + .with_context(|| format!("waiting for {label}"))? + }; + Ok(output) } fn parse_metric(output: &str) -> anyhow::Result { @@ -601,6 +1012,32 @@ mod tests { assert!(!is_better(&strict, 100.0, 100.0)); } + #[test] + fn experiment_definition_fingerprint_changes_when_metric_changes() { + let config = AutoresearchConfig { + objective: "startup".into(), + metric_command: "./measure-a".into(), + ..AutoresearchConfig::default() + }; + let mut changed = config.clone(); + changed.metric_command = "./measure-b".into(); + assert_ne!(spec_fingerprint(&config), spec_fingerprint(&changed)); + } + + #[test] + fn new_ledger_is_bound_to_branch_and_has_a_unique_run_id() { + let config = AutoresearchConfig { + objective: "startup".into(), + metric_command: "./measure".into(), + ..AutoresearchConfig::default() + }; + let first = AutoresearchLedger::new(&config, 10.0, "abc".into(), "feature/x".into()); + let second = AutoresearchLedger::new(&config, 10.0, "abc".into(), "feature/x".into()); + assert_eq!(first.branch, "feature/x"); + assert_eq!(first.spec_fingerprint, spec_fingerprint(&config)); + assert_ne!(first.chat_id, second.chat_id); + } + #[tokio::test] async fn command_timeout_is_enforced() { let error = run_shell("sleep 5", Path::new("."), 1) @@ -609,6 +1046,72 @@ mod tests { assert!(error.to_string().contains("timed out")); } + #[cfg(unix)] + #[tokio::test] + async fn command_timeout_terminates_shell_descendants() { + let directory = tempfile::tempdir().unwrap(); + let marker = directory.path().join("descendant-finished"); + let command = format!( + "sleep 2; touch {}", + shlex::try_quote(&marker.to_string_lossy()).unwrap() + ); + run_shell(&command, directory.path(), 1) + .await + .expect_err("command should time out"); + tokio::time::sleep(Duration::from_secs(2)).await; + assert!(!marker.exists(), "timed-out descendant survived"); + } + + #[tokio::test] + async fn ignored_state_restores_existing_files_and_removes_new_files() { + let directory = tempfile::tempdir().unwrap(); + git_command(directory.path(), &["init", "-q"]) + .await + .unwrap(); + tokio::fs::write(directory.path().join(".gitignore"), ".env\nnew-*\n") + .await + .unwrap(); + git_command(directory.path(), &["add", ".gitignore"]) + .await + .unwrap(); + git_command( + directory.path(), + &[ + "-c", + "user.name=Autoresearch Test", + "-c", + "user.email=autoresearch@example.invalid", + "commit", + "-qm", + "initial", + ], + ) + .await + .unwrap(); + tokio::fs::write(directory.path().join(".env"), "before\n") + .await + .unwrap(); + + let state = capture_ignored_state(directory.path()).await.unwrap(); + tokio::fs::write(directory.path().join(".env"), "candidate\n") + .await + .unwrap(); + tokio::fs::write(directory.path().join("new-output"), "candidate\n") + .await + .unwrap(); + restore_ignored_state(directory.path(), &state) + .await + .unwrap(); + + assert_eq!( + tokio::fs::read_to_string(directory.path().join(".env")) + .await + .unwrap(), + "before\n" + ); + assert!(!directory.path().join("new-output").exists()); + } + #[tokio::test] async fn validation_timeout_is_a_rejected_attempt() { let config = AutoresearchConfig { diff --git a/src/cost.rs b/src/cost.rs index 5a4cf7f..a62bfd3 100644 --- a/src/cost.rs +++ b/src/cost.rs @@ -75,7 +75,7 @@ pub struct RateLimitStatus { /// Cost tracker (in-memory + persistent accounting hooks) pub struct CostTracker { costs: Arc>>, - contexts: Arc>>, + contexts: Arc>, models: Arc>>, rate_limit_status: Arc>>, } @@ -117,7 +117,7 @@ impl CostTracker { Self { costs: Arc::new(RwLock::new(Vec::new())), - contexts: Arc::new(RwLock::new(Vec::new())), + contexts: Arc::new(RwLock::new(ContextSummary::default())), models: Arc::new(RwLock::new(models)), rate_limit_status: Arc::new(RwLock::new(None)), } @@ -153,22 +153,17 @@ impl CostTracker { /// Record the estimated shape of a provider request for harness telemetry. pub async fn record_context(&self, snapshot: ContextSnapshot) { - self.contexts.write().await.push(snapshot); + let mut summary = self.contexts.write().await; + summary.request_count += 1; + summary.system_chars += snapshot.system_chars; + summary.history_chars += snapshot.history_chars; + summary.tool_chars += snapshot.tool_chars; + summary.estimated_input_tokens += snapshot.estimated_input_tokens; } /// Aggregate prompt-shape telemetry since this tracker was created. pub async fn context_summary(&self) -> ContextSummary { - let contexts = self.contexts.read().await; - ContextSummary { - request_count: contexts.len(), - system_chars: contexts.iter().map(|item| item.system_chars).sum(), - history_chars: contexts.iter().map(|item| item.history_chars).sum(), - tool_chars: contexts.iter().map(|item| item.tool_chars).sum(), - estimated_input_tokens: contexts - .iter() - .map(|item| item.estimated_input_tokens) - .sum(), - } + self.contexts.read().await.clone() } /// Get cost summary @@ -374,5 +369,18 @@ mod tests { assert_eq!(summary.request_count, 1); assert_eq!(summary.estimated_input_tokens, 35); assert_eq!(summary.system_chars, 40); + + tracker + .record_context(ContextSnapshot { + system_chars: 2, + history_chars: 3, + tool_chars: 4, + estimated_input_tokens: 5, + }) + .await; + let summary = tracker.context_summary().await; + assert_eq!(summary.request_count, 2); + assert_eq!(summary.system_chars, 42); + assert_eq!(summary.estimated_input_tokens, 40); } } diff --git a/src/main.rs b/src/main.rs index 92b73c2..b8721f8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -562,6 +562,31 @@ enum SwarmAction { } /// Build the same unattended runner used by autonomous and autoresearch modes. +fn configure_restricted_automation(cfg: &mut Config) { + // Autoresearch only needs local code execution and filesystem edits. Keep + // all conversational memory disabled as well: tool filtering alone does + // not prevent history, personal-context, or ZKR recall from being added to + // the model prompt by AgentRunner. + cfg.toolsets.enabled = vec!["runtime".into(), "fs".into()]; + cfg.toolsets.disabled = vec![ + "web".into(), + "browser".into(), + "memory".into(), + "sessions".into(), + "messaging".into(), + "advanced".into(), + "desktop".into(), + "media".into(), + "skills".into(), + ]; + cfg.memory.inject_context = false; + cfg.memory.principal_id = None; + cfg.zkr.enabled = false; + cfg.zkr.auto_capture = false; + cfg.zkr.inject_recall = false; + cfg.zkr.self_improve = false; +} + async fn build_automation_agent( config_path: &str, workspace: &Path, @@ -569,22 +594,7 @@ async fn build_automation_agent( ) -> anyhow::Result<(Arc, Config)> { let mut cfg = apollo::bootstrap::load_config_workspace(config_path, Some(workspace)); if restricted { - // Autoresearch only needs local code execution and filesystem edits. - // Keep network, messaging, memory, MCP, and plugin capabilities out of - // the model's ambient tool catalog; this mirrors capability-oriented - // agent designs where access is introduced deliberately. - cfg.toolsets.enabled = vec!["runtime".into(), "fs".into()]; - cfg.toolsets.disabled = vec![ - "web".into(), - "browser".into(), - "memory".into(), - "sessions".into(), - "messaging".into(), - "advanced".into(), - "desktop".into(), - "media".into(), - "skills".into(), - ]; + configure_restricted_automation(&mut cfg); } let provider = build_provider(&cfg); let policy = Arc::new(ExecutionPolicy::from_config(&cfg.policy)); @@ -633,6 +643,7 @@ async fn build_automation_agent( &cfg.agent.permission_profile, )) .with_workspace(workspace.to_path_buf()) + .with_memory_enabled(!restricted) .with_memory_ideas(cfg.memory.clone()) .with_group_chat(cfg.group_chat.clone()) .with_skills(discovered_skills) @@ -2279,7 +2290,23 @@ fn init_tracing(cfg: &apollo::config::ObservabilityConfig) -> anyhow::Result<()> #[cfg(test)] mod autostart_tests { - use super::{launchd_plist, systemd_unit, validate_autostart_config_path}; + use super::{ + configure_restricted_automation, launchd_plist, systemd_unit, + validate_autostart_config_path, + }; + use apollo::config::Config; + + #[test] + fn restricted_automation_disables_ambient_memory() { + let mut config = Config::default(); + configure_restricted_automation(&mut config); + assert!(!config.memory.inject_context); + assert!(config.memory.principal_id.is_none()); + assert!(!config.zkr.enabled); + assert!(!config.zkr.inject_recall); + assert!(!config.zkr.self_improve); + assert!(!config.zkr.auto_capture); + } /// A path with a space is ordinary on macOS, so it must be accepted and /// escaped rather than refused.