diff --git a/SPEC.md b/SPEC.md index adb8bc59a8..65a88fcaa9 100644 --- a/SPEC.md +++ b/SPEC.md @@ -752,6 +752,9 @@ Retry entry creation: - Cancel any existing retry timer for the same issue. - Store `attempt`, `identifier`, `error`, `due_at_ms`, and new timer handle. +- Keep the issue claimed while retrying so a queued handoff cannot be dispatched twice. +- If a claim lease exists, update it to `retrying` with retry due/backoff metadata and the last + seen worker/workspace details. Backoff formula: @@ -776,6 +779,23 @@ Note: - Retry handling mainly operates on active candidates and releases claims when the issue is absent, rather than performing terminal cleanup itself. +Claim lease behavior: + +- When an issue is claimed for a worker, create a visible tracker comment marker headed + `## Symphony Claim Lease`. +- Persist these lease fields in runtime state and tracker-visible marker text: + `worker_id`, `worker_host`, `workspace_path`, `attempt`, `last_seen_at`, and + `lease_expires_at`. +- The default lease TTL is derived from polling cadence: at least 60 seconds and at least three poll + intervals. +- Active workers refresh the lease during poll reconciliation and when runtime/Codex activity is + observed. +- Retry and blocked transitions update the lease marker with `retrying` or `blocked` state and + relevant error/backoff details. +- Expired leases are recovered only when no live running or blocked worker exists for the issue. + Recovery logs the expiration, records an `expired` claim entry for observability, and requeues the + issue for retry handoff. + ### 8.5 Active Run Reconciliation Reconciliation runs every tick and has two parts. @@ -1394,7 +1414,9 @@ Minimum endpoints: "generated_at": "2026-02-24T20:15:30Z", "counts": { "running": 2, - "retrying": 1 + "retrying": 1, + "blocked": 0, + "expired": 0 }, "running": [ { @@ -1423,6 +1445,19 @@ Minimum endpoints: "error": "no available orchestrator slots" } ], + "claim_leases": [ + { + "issue_id": "abc123", + "issue_identifier": "MT-649", + "state": "active", + "worker_id": "local:#PID<0.123.0>", + "workspace_path": "/tmp/symphony_workspaces/MT-649", + "attempt": 1, + "last_seen_at": "2026-02-24T20:14:59Z", + "lease_expires_at": "2026-02-24T20:16:30Z" + } + ], + "expired": [], "codex_totals": { "input_tokens": 5000, "output_tokens": 2400, diff --git a/elixir/README.md b/elixir/README.md index b35bae2358..e2c68a0fed 100644 --- a/elixir/README.md +++ b/elixir/README.md @@ -31,6 +31,12 @@ issue claimed and exposes it as blocked in the runtime state, JSON API, and dash entries are in memory only; restarting the orchestrator clears that blocked map, so any still-active Linear issue can become a dispatch candidate again after restart. +Claimed issues also get a Symphony claim lease marker through the tracker comment API. The lease +records the last-seen worker id, workspace path, attempt number, last heartbeat time, and expiry. +Active workers refresh the lease during poll and Codex activity; retry and blocked transitions +update the same lease state. If a non-live claim lease expires, Symphony logs the recovery and +requeues the issue without starting a duplicate worker for a still-running claim. + ## How to use it 1. Make sure your codebase is set up to work well with agents: see @@ -162,6 +168,7 @@ The observability UI now runs on a minimal Phoenix stack: - LiveView for the dashboard at `/` - JSON API for operational debugging under `/api/v1/*` +- Active, retrying, blocked, and expired claim lease visibility - Bandit as the HTTP server - Phoenix dependency static assets for the LiveView client bootstrap diff --git a/elixir/lib/symphony_elixir/orchestrator.ex b/elixir/lib/symphony_elixir/orchestrator.ex index 0445776945..21a1a76bf0 100644 --- a/elixir/lib/symphony_elixir/orchestrator.ex +++ b/elixir/lib/symphony_elixir/orchestrator.ex @@ -12,6 +12,9 @@ defmodule SymphonyElixir.Orchestrator do @continuation_retry_delay_ms 1_000 @failure_retry_base_ms 10_000 + @minimum_claim_lease_ttl_ms 60_000 + @claim_lease_ttl_poll_multiplier 3 + @claim_lease_marker_interval_ms 60_000 # Slightly above the dashboard render interval so "checking now…" can render. @poll_transition_render_delay_ms 20 @empty_codex_totals %{ @@ -26,6 +29,8 @@ defmodule SymphonyElixir.Orchestrator do Runtime state for the orchestrator polling loop. """ + @type t :: %__MODULE__{} + defstruct [ :poll_interval_ms, :max_concurrent_agents, @@ -38,6 +43,8 @@ defmodule SymphonyElixir.Orchestrator do claimed: MapSet.new(), blocked: %{}, retry_attempts: %{}, + claim_leases: %{}, + expired_claims: %{}, codex_totals: nil, codex_rate_limits: nil ] @@ -151,8 +158,13 @@ defmodule SymphonyElixir.Orchestrator do |> maybe_put_runtime_value(:worker_host, runtime_info[:worker_host]) |> maybe_put_runtime_value(:workspace_path, runtime_info[:workspace_path]) + state = + state + |> Map.put(:running, Map.put(running, issue_id, updated_running_entry)) + |> refresh_claim_lease_from_running(issue_id, updated_running_entry) + notify_dashboard() - {:noreply, %{state | running: Map.put(running, issue_id, updated_running_entry)}} + {:noreply, state} end end @@ -171,9 +183,11 @@ defmodule SymphonyElixir.Orchestrator do state |> apply_codex_token_delta(token_delta) |> apply_codex_rate_limits(update) + |> Map.put(:running, Map.put(running, issue_id, updated_running_entry)) + |> refresh_claim_lease_from_running(issue_id, updated_running_entry) notify_dashboard() - {:noreply, %{state | running: Map.put(running, issue_id, updated_running_entry)}} + {:noreply, state} end end @@ -209,7 +223,8 @@ defmodule SymphonyElixir.Orchestrator do identifier: running_entry.identifier, delay_type: :continuation, worker_host: Map.get(running_entry, :worker_host), - workspace_path: Map.get(running_entry, :workspace_path) + workspace_path: Map.get(running_entry, :workspace_path), + worker_id: lease_worker_id(running_entry) }) end end @@ -239,7 +254,8 @@ defmodule SymphonyElixir.Orchestrator do identifier: running_entry.identifier, error: "agent exited: #{inspect(reason)}", worker_host: Map.get(running_entry, :worker_host), - workspace_path: Map.get(running_entry, :workspace_path) + workspace_path: Map.get(running_entry, :workspace_path), + worker_id: lease_worker_id(running_entry) }) end @@ -248,11 +264,17 @@ defmodule SymphonyElixir.Orchestrator do state |> reconcile_running_issues() |> reconcile_blocked_issues() + |> refresh_running_claim_leases() with :ok <- Config.validate!(), - {:ok, issues} <- Tracker.fetch_candidate_issues(), - true <- available_slots(state) > 0 do - choose_issues(issues, state) + {:ok, issues} <- Tracker.fetch_candidate_issues() do + state = recover_expired_claim_leases(state, issues) + + if available_slots(state) > 0 do + choose_issues(issues, state) + else + state + end else {:error, :missing_linear_api_token} -> Logger.error("Linear API token missing in WORKFLOW.md") @@ -291,9 +313,6 @@ defmodule SymphonyElixir.Orchestrator do {:error, reason} -> Logger.error("Failed to fetch from Linear: #{inspect(reason)}") state - - false -> - state end end @@ -382,6 +401,26 @@ defmodule SymphonyElixir.Orchestrator do select_worker_host(state, preferred_worker_host) end + @doc false + @spec start_claim_lease_for_test(State.t(), Issue.t(), map(), integer() | nil) :: State.t() + def start_claim_lease_for_test(%State{} = state, %Issue{} = issue, running_entry, attempt \\ nil) + when is_map(running_entry) do + start_claim_lease(state, issue, running_entry, attempt) + end + + @doc false + @spec refresh_claim_lease_from_running_for_test(State.t(), String.t(), map()) :: State.t() + def refresh_claim_lease_from_running_for_test(%State{} = state, issue_id, running_entry) + when is_binary(issue_id) and is_map(running_entry) do + refresh_claim_lease_from_running(state, issue_id, running_entry) + end + + @doc false + @spec recover_expired_claim_leases_for_test(State.t(), [Issue.t()]) :: State.t() + def recover_expired_claim_leases_for_test(%State{} = state, issues) when is_list(issues) do + recover_expired_claim_leases(state, issues) + end + defp reconcile_running_issue_states([], state, _active_states, _terminal_states), do: state defp reconcile_running_issue_states([issue | rest], state, active_states, terminal_states) do @@ -526,6 +565,401 @@ defmodule SymphonyElixir.Orchestrator do end end + defp refresh_running_claim_leases(%State{} = state) do + Enum.reduce(state.running, state, fn {issue_id, running_entry}, state_acc -> + refresh_claim_lease_from_running(state_acc, issue_id, running_entry) + end) + end + + defp start_claim_lease(%State{} = state, %Issue{} = issue, running_entry, attempt) + when is_map(running_entry) do + times = claim_lease_times() + + lease = + state.claim_leases + |> Map.get(issue.id, %{}) + |> Map.merge(%{ + issue_id: issue.id, + identifier: issue.identifier || issue.id, + state: :active, + worker_id: lease_worker_id(running_entry), + worker_host: Map.get(running_entry, :worker_host), + workspace_path: Map.get(running_entry, :workspace_path), + attempt: claim_attempt_number(attempt), + last_seen_at: times.now, + lease_started_at: times.now, + lease_expires_at: times.expires_at, + lease_expires_at_ms: times.expires_at_ms, + heartbeat_count: 1, + retry_due_at: nil, + retry_due_at_ms: nil, + retry_backoff_ms: nil, + error: nil + }) + + put_claim_lease(state, issue.id, lease, times.now_ms, force: true) + end + + defp refresh_claim_lease_from_running(%State{} = state, issue_id, running_entry) + when is_binary(issue_id) and is_map(running_entry) do + case Map.get(state.claim_leases, issue_id) do + nil -> + state + + lease -> + times = claim_lease_times() + + refreshed_lease = + lease + |> Map.merge(%{ + state: :active, + worker_id: lease_worker_id(running_entry), + worker_host: Map.get(running_entry, :worker_host), + workspace_path: Map.get(running_entry, :workspace_path), + attempt: Map.get(lease, :attempt, 1), + last_seen_at: times.now, + lease_expires_at: times.expires_at, + lease_expires_at_ms: times.expires_at_ms, + heartbeat_count: Map.get(lease, :heartbeat_count, 0) + 1, + retry_due_at: nil, + retry_due_at_ms: nil, + retry_backoff_ms: nil, + error: nil + }) + + put_claim_lease(state, issue_id, refreshed_lease, times.now_ms) + end + end + + defp mark_retry_claim_lease(%State{} = state, issue_id, retry_entry) when is_binary(issue_id) do + case Map.get(state.claim_leases, issue_id) do + nil -> + state + + lease -> + times = claim_lease_times() + retry_due_at_ms = Map.get(retry_entry, :due_at_ms) + retry_expires_at_ms = retry_lease_expires_at_ms(times, retry_due_at_ms) + + retry_lease = + lease + |> Map.merge(%{ + state: :retrying, + identifier: retry_entry_value(retry_entry, lease, :identifier, issue_id), + worker_id: retry_entry_value(retry_entry, lease, :worker_id), + worker_host: Map.get(retry_entry, :worker_host), + workspace_path: Map.get(retry_entry, :workspace_path), + attempt: retry_entry_value(retry_entry, lease, :attempt, 1), + lease_expires_at: DateTime.add(times.now, retry_expires_at_ms - times.now_ms, :millisecond), + lease_expires_at_ms: retry_expires_at_ms, + retry_due_at: retry_due_at(times, retry_due_at_ms), + retry_due_at_ms: retry_due_at_ms, + retry_backoff_ms: retry_backoff_ms(times, retry_due_at_ms), + error: Map.get(retry_entry, :error) + }) + + put_claim_lease(state, issue_id, retry_lease, times.now_ms, force: true) + end + end + + defp retry_entry_value(retry_entry, lease, key, fallback \\ nil) do + Map.get(retry_entry, key) || Map.get(lease, key) || fallback + end + + defp retry_lease_expires_at_ms(times, retry_due_at_ms) do + max(times.expires_at_ms, (retry_due_at_ms || times.now_ms) + claim_lease_ttl_ms()) + end + + defp retry_due_at(_times, nil), do: nil + + defp retry_due_at(times, retry_due_at_ms) when is_integer(retry_due_at_ms) do + DateTime.add(times.now, retry_due_at_ms - times.now_ms, :millisecond) + end + + defp retry_backoff_ms(_times, nil), do: nil + + defp retry_backoff_ms(times, retry_due_at_ms) when is_integer(retry_due_at_ms) do + max(0, retry_due_at_ms - times.now_ms) + end + + defp mark_blocked_claim_lease(%State{} = state, issue_id, blocked_entry) when is_binary(issue_id) do + case Map.get(state.claim_leases, issue_id) do + nil -> + state + + lease -> + times = claim_lease_times() + + blocked_lease = + lease + |> Map.merge(%{ + state: :blocked, + identifier: Map.get(blocked_entry, :identifier) || Map.get(lease, :identifier) || issue_id, + worker_id: Map.get(lease, :worker_id), + worker_host: Map.get(blocked_entry, :worker_host), + workspace_path: Map.get(blocked_entry, :workspace_path), + session_id: Map.get(blocked_entry, :session_id), + lease_expires_at: times.expires_at, + lease_expires_at_ms: times.expires_at_ms, + error: Map.get(blocked_entry, :error) + }) + + put_claim_lease(state, issue_id, blocked_lease, times.now_ms, force: true) + end + end + + defp recover_expired_claim_leases(%State{} = state, issues) when is_list(issues) do + now_ms = System.monotonic_time(:millisecond) + issues_by_id = Map.new(issues, &{&1.id, &1}) + + state.claim_leases + |> Enum.filter(fn {issue_id, lease} -> + claim_lease_expired?(lease, now_ms) and not live_claim?(state, issue_id) + end) + |> Enum.reduce(state, fn {issue_id, lease}, state_acc -> + recover_expired_claim_lease(state_acc, issue_id, lease, Map.get(issues_by_id, issue_id), now_ms) + end) + end + + defp recover_expired_claim_leases(state, _issues), do: state + + defp recover_expired_claim_lease(%State{} = state, issue_id, lease, %Issue{} = issue, _now_ms) do + if retry_candidate_issue?(issue, terminal_state_set()) do + expired_at = DateTime.utc_now() + error = "claim lease expired at #{iso8601(expired_at)}; requeueing" + attempt = expired_retry_attempt(state, issue_id, lease) + + Logger.warning("Claim lease expired; requeueing issue_id=#{issue_id} issue_identifier=#{issue.identifier} attempt=#{attempt}") + + state + |> put_expired_claim(issue_id, lease, expired_at, error) + |> schedule_issue_retry(issue_id, attempt, %{ + identifier: issue.identifier, + error: error, + worker_host: Map.get(lease, :worker_host), + workspace_path: Map.get(lease, :workspace_path), + worker_id: Map.get(lease, :worker_id), + delay_ms: 0 + }) + else + Logger.warning("Claim lease expired for non-candidate issue_id=#{issue_id}; releasing claim") + release_issue_claim(state, issue_id) + end + end + + defp recover_expired_claim_lease(%State{} = state, issue_id, _lease, _issue, _now_ms) do + Logger.warning("Claim lease expired for invisible issue_id=#{issue_id}; releasing claim") + release_issue_claim(state, issue_id) + end + + defp put_expired_claim(%State{} = state, issue_id, lease, expired_at, error) do + expired_claim = + lease + |> Map.merge(%{ + state: :expired, + expired_at: expired_at, + requeued_at: DateTime.utc_now(), + error: error + }) + + %{state | expired_claims: Map.put(state.expired_claims, issue_id, expired_claim)} + end + + defp put_claim_lease(%State{} = state, issue_id, lease, now_ms, opts \\ []) do + previous = Map.get(state.claim_leases, issue_id) + lease = maybe_publish_claim_lease_marker(issue_id, previous, lease, now_ms, opts) + + %{state | claim_leases: Map.put(state.claim_leases, issue_id, lease)} + end + + defp maybe_publish_claim_lease_marker(issue_id, previous, lease, now_ms, opts) do + if claim_lease_marker_due?(previous, lease, now_ms, opts) do + body = claim_lease_marker_body(lease) + + case safe_create_tracker_comment(issue_id, body) do + :ok -> Map.put(lease, :last_marker_at_ms, now_ms) + {:error, _reason} -> inherit_last_marker_at(lease, previous) + end + else + inherit_last_marker_at(lease, previous) + end + end + + defp claim_lease_marker_due?(nil, _lease, _now_ms, _opts), do: true + + defp claim_lease_marker_due?(previous, lease, now_ms, opts) do + Keyword.get(opts, :force, false) or + claim_lease_material_change?(previous, lease) or + claim_lease_marker_interval_due?(previous, now_ms) + end + + defp claim_lease_material_change?(previous, lease) do + fields = [:state, :worker_id, :worker_host, :workspace_path, :attempt, :retry_due_at_ms, :error] + + Enum.any?(fields, fn field -> + Map.get(previous, field) != Map.get(lease, field) + end) + end + + defp claim_lease_marker_interval_due?(previous, now_ms) do + case Map.get(previous, :last_marker_at_ms) do + marker_at_ms when is_integer(marker_at_ms) -> now_ms - marker_at_ms >= @claim_lease_marker_interval_ms + _ -> true + end + end + + defp inherit_last_marker_at(lease, nil), do: lease + + defp inherit_last_marker_at(lease, previous) do + Map.put(lease, :last_marker_at_ms, Map.get(previous, :last_marker_at_ms)) + end + + defp safe_create_tracker_comment(issue_id, body) do + Tracker.create_comment(issue_id, body) + rescue + error -> + Logger.warning("Failed to write claim lease marker for issue_id=#{issue_id}: #{Exception.message(error)}") + {:error, error} + catch + kind, reason -> + Logger.warning("Failed to write claim lease marker for issue_id=#{issue_id}: #{inspect({kind, reason})}") + {:error, reason} + end + + defp claim_lease_marker_body(lease) do + ["## Symphony Claim Lease", "" | claim_lease_marker_lines(lease)] + |> Enum.join("\n") + end + + defp claim_lease_marker_lines(lease) do + [ + {:state, Map.get(lease, :state), "n/a"}, + {:worker_id, Map.get(lease, :worker_id), "n/a"}, + {:worker_host, Map.get(lease, :worker_host), "local"}, + {:workspace_path, Map.get(lease, :workspace_path), "pending"}, + {:attempt, Map.get(lease, :attempt), 1}, + {:last_seen_at, iso8601(Map.get(lease, :last_seen_at)), "n/a"}, + {:lease_expires_at, iso8601(Map.get(lease, :lease_expires_at)), "n/a"}, + {:retry_due_at, iso8601(Map.get(lease, :retry_due_at)), "n/a"}, + {:retry_backoff_ms, Map.get(lease, :retry_backoff_ms), "n/a"}, + {:error, Map.get(lease, :error), "n/a"} + ] + |> Enum.map(fn {key, value, fallback} -> "- #{key}: #{value || fallback}" end) + end + + defp claim_lease_expired?(lease, now_ms) when is_map(lease) and is_integer(now_ms) do + case Map.get(lease, :lease_expires_at_ms) do + expires_at_ms when is_integer(expires_at_ms) -> expires_at_ms <= now_ms + _ -> false + end + end + + defp live_claim?(%State{} = state, issue_id) do + Map.has_key?(state.running, issue_id) or Map.has_key?(state.blocked, issue_id) + end + + defp expired_retry_attempt(%State{} = state, issue_id, lease) do + lease_attempt = Map.get(lease, :attempt, 1) + retry_attempt = state.retry_attempts |> Map.get(issue_id, %{}) |> Map.get(:attempt, 0) + + max(lease_attempt, retry_attempt) + 1 + end + + defp claim_lease_times do + now = DateTime.utc_now() + now_ms = System.monotonic_time(:millisecond) + ttl_ms = claim_lease_ttl_ms() + + %{ + now: now, + now_ms: now_ms, + expires_at: DateTime.add(now, ttl_ms, :millisecond), + expires_at_ms: now_ms + ttl_ms + } + end + + defp claim_lease_ttl_ms do + Config.settings!().polling.interval_ms + |> Kernel.*(@claim_lease_ttl_poll_multiplier) + |> max(@minimum_claim_lease_ttl_ms) + end + + defp claim_attempt_number(attempt) when is_integer(attempt) and attempt > 0, do: attempt + defp claim_attempt_number(_attempt), do: 1 + + defp lease_worker_id(running_entry) when is_map(running_entry) do + case Map.get(running_entry, :worker_id) do + worker_id when is_binary(worker_id) and worker_id != "" -> + worker_id + + _ -> + host = Map.get(running_entry, :worker_host) || "local" + pid = Map.get(running_entry, :pid) + "#{host}:#{if(is_pid(pid), do: inspect(pid), else: "unknown")}" + end + end + + defp restore_claim_lease(state, _issue_id, nil), do: state + + defp restore_claim_lease(%State{} = state, issue_id, lease) when is_binary(issue_id) and is_map(lease) do + %{state | claim_leases: Map.put(state.claim_leases, issue_id, lease)} + end + + defp claim_lease_snapshot_entry(lease, now_ms) do + %{ + issue_id: Map.get(lease, :issue_id), + identifier: Map.get(lease, :identifier), + state: lease_state_string(Map.get(lease, :state)), + worker_id: Map.get(lease, :worker_id), + worker_host: Map.get(lease, :worker_host), + workspace_path: Map.get(lease, :workspace_path), + attempt: Map.get(lease, :attempt), + last_seen_at: Map.get(lease, :last_seen_at), + lease_expires_at: Map.get(lease, :lease_expires_at), + lease_expires_in_ms: lease_expires_in_ms(lease, now_ms), + retry_due_at: Map.get(lease, :retry_due_at), + retry_due_in_ms: retry_due_in_ms(lease, now_ms), + retry_backoff_ms: Map.get(lease, :retry_backoff_ms), + error: Map.get(lease, :error) + } + end + + defp expired_claim_snapshot_entry(lease, now_ms) do + lease + |> claim_lease_snapshot_entry(now_ms) + |> Map.merge(%{ + state: "expired", + expired_at: Map.get(lease, :expired_at), + requeued_at: Map.get(lease, :requeued_at) + }) + end + + defp lease_expires_in_ms(lease, now_ms) do + case Map.get(lease, :lease_expires_at_ms) do + expires_at_ms when is_integer(expires_at_ms) -> expires_at_ms - now_ms + _ -> nil + end + end + + defp retry_due_in_ms(lease, now_ms) do + case Map.get(lease, :retry_due_at_ms) do + due_at_ms when is_integer(due_at_ms) -> max(0, due_at_ms - now_ms) + _ -> nil + end + end + + defp lease_state_string(state) when is_atom(state), do: Atom.to_string(state) + defp lease_state_string(state) when is_binary(state), do: state + defp lease_state_string(_state), do: nil + + defp iso8601(%DateTime{} = datetime) do + datetime + |> DateTime.truncate(:second) + |> DateTime.to_iso8601() + end + + defp iso8601(_datetime), do: nil + defp terminate_running_issue(%State{} = state, issue_id, cleanup_workspace) do case Map.get(state.running, issue_id) do nil -> @@ -546,7 +980,9 @@ defmodule SymphonyElixir.Orchestrator do | running: Map.delete(state.running, issue_id), claimed: MapSet.delete(state.claimed, issue_id), blocked: Map.delete(state.blocked, issue_id), - retry_attempts: Map.delete(state.retry_attempts, issue_id) + retry_attempts: Map.delete(state.retry_attempts, issue_id), + claim_leases: Map.delete(state.claim_leases, issue_id), + expired_claims: Map.delete(state.expired_claims, issue_id) } _ -> @@ -600,12 +1036,17 @@ defmodule SymphonyElixir.Orchestrator do Logger.warning("Issue stalled: issue_id=#{issue_id} issue_identifier=#{identifier} session_id=#{session_id} elapsed_ms=#{elapsed_ms}; restarting with backoff") next_attempt = next_retry_attempt_from_running(running_entry) + previous_lease = Map.get(state.claim_leases, issue_id) state |> terminate_running_issue(issue_id, false) + |> restore_claim_lease(issue_id, previous_lease) |> schedule_issue_retry(issue_id, next_attempt, %{ identifier: identifier, - error: "stalled for #{elapsed_ms}ms without codex activity" + error: "stalled for #{elapsed_ms}ms without codex activity", + worker_host: Map.get(running_entry, :worker_host), + workspace_path: Map.get(running_entry, :workspace_path), + worker_id: lease_worker_id(running_entry) }) end else @@ -739,13 +1180,15 @@ defmodule SymphonyElixir.Orchestrator do last_codex_timestamp: Map.get(running_entry, :last_codex_timestamp) } - %{ + state = %{ state | running: Map.delete(state.running, issue_id), retry_attempts: Map.delete(state.retry_attempts, issue_id), claimed: MapSet.put(state.claimed, issue_id), blocked: Map.put(state.blocked, issue_id, blocked_entry) } + + mark_blocked_claim_lease(state, issue_id, blocked_entry) end defp choose_issues(issues, state) do @@ -932,37 +1375,41 @@ defmodule SymphonyElixir.Orchestrator do Logger.info("Dispatching issue to agent: #{issue_context(issue)} pid=#{inspect(pid)} attempt=#{inspect(attempt)} worker_host=#{worker_host || "local"}") - running = - Map.put(state.running, issue.id, %{ - pid: pid, - ref: ref, - identifier: issue.identifier, - issue: issue, - worker_host: worker_host, - workspace_path: nil, - session_id: nil, - last_codex_message: nil, - last_codex_timestamp: nil, - last_codex_event: nil, - codex_app_server_pid: nil, - codex_input_tokens: 0, - codex_output_tokens: 0, - codex_total_tokens: 0, - codex_last_reported_input_tokens: 0, - codex_last_reported_output_tokens: 0, - codex_last_reported_total_tokens: 0, - turn_count: 0, - retry_attempt: normalize_retry_attempt(attempt), - started_at: DateTime.utc_now() - }) + running_entry = %{ + pid: pid, + ref: ref, + identifier: issue.identifier, + issue: issue, + worker_host: worker_host, + workspace_path: nil, + session_id: nil, + last_codex_message: nil, + last_codex_timestamp: nil, + last_codex_event: nil, + codex_app_server_pid: nil, + codex_input_tokens: 0, + codex_output_tokens: 0, + codex_total_tokens: 0, + codex_last_reported_input_tokens: 0, + codex_last_reported_output_tokens: 0, + codex_last_reported_total_tokens: 0, + turn_count: 0, + retry_attempt: normalize_retry_attempt(attempt), + started_at: DateTime.utc_now() + } - %{ + running = Map.put(state.running, issue.id, running_entry) + + state = %{ state | running: running, claimed: MapSet.put(state.claimed, issue.id), - retry_attempts: Map.delete(state.retry_attempts, issue.id) + retry_attempts: Map.delete(state.retry_attempts, issue.id), + expired_claims: Map.delete(state.expired_claims, issue.id) } + start_claim_lease(state, issue, running_entry, attempt) + {:error, reason} -> Logger.error("Unable to spawn agent for #{issue_context(issue)}: #{inspect(reason)}") next_attempt = if is_integer(attempt), do: attempt + 1, else: nil @@ -1015,6 +1462,7 @@ defmodule SymphonyElixir.Orchestrator do error = pick_retry_error(previous_retry, metadata) worker_host = pick_retry_worker_host(previous_retry, metadata) workspace_path = pick_retry_workspace_path(previous_retry, metadata) + worker_id = pick_retry_worker_id(previous_retry, metadata) if is_reference(old_timer) do Process.cancel_timer(old_timer) @@ -1026,20 +1474,25 @@ defmodule SymphonyElixir.Orchestrator do Logger.warning("Retrying issue_id=#{issue_id} issue_identifier=#{identifier} in #{delay_ms}ms (attempt #{next_attempt})#{error_suffix}") - %{ + retry_entry = %{ + attempt: next_attempt, + timer_ref: timer_ref, + retry_token: retry_token, + due_at_ms: due_at_ms, + identifier: identifier, + error: error, + worker_host: worker_host, + workspace_path: workspace_path, + worker_id: worker_id + } + + state = %{ state - | retry_attempts: - Map.put(state.retry_attempts, issue_id, %{ - attempt: next_attempt, - timer_ref: timer_ref, - retry_token: retry_token, - due_at_ms: due_at_ms, - identifier: identifier, - error: error, - worker_host: worker_host, - workspace_path: workspace_path - }) + | claimed: MapSet.put(state.claimed, issue_id), + retry_attempts: Map.put(state.retry_attempts, issue_id, retry_entry) } + + mark_retry_claim_lease(state, issue_id, retry_entry) end defp pop_retry_attempt_state(%State{} = state, issue_id, retry_token) when is_reference(retry_token) do @@ -1049,7 +1502,8 @@ defmodule SymphonyElixir.Orchestrator do identifier: Map.get(retry_entry, :identifier), error: Map.get(retry_entry, :error), worker_host: Map.get(retry_entry, :worker_host), - workspace_path: Map.get(retry_entry, :workspace_path) + workspace_path: Map.get(retry_entry, :workspace_path), + worker_id: Map.get(retry_entry, :worker_id) } {:ok, attempt, metadata, %{state | retry_attempts: Map.delete(state.retry_attempts, issue_id)}} @@ -1165,15 +1619,22 @@ defmodule SymphonyElixir.Orchestrator do state | claimed: MapSet.delete(state.claimed, issue_id), blocked: Map.delete(state.blocked, issue_id), - retry_attempts: Map.delete(state.retry_attempts, issue_id) + retry_attempts: Map.delete(state.retry_attempts, issue_id), + claim_leases: Map.delete(state.claim_leases, issue_id), + expired_claims: Map.delete(state.expired_claims, issue_id) } end defp retry_delay(attempt, metadata) when is_integer(attempt) and attempt > 0 and is_map(metadata) do - if metadata[:delay_type] == :continuation and attempt == 1 do - @continuation_retry_delay_ms - else - failure_retry_delay(attempt) + cond do + is_integer(metadata[:delay_ms]) and metadata[:delay_ms] >= 0 -> + metadata[:delay_ms] + + metadata[:delay_type] == :continuation and attempt == 1 -> + @continuation_retry_delay_ms + + true -> + failure_retry_delay(attempt) end end @@ -1208,6 +1669,10 @@ defmodule SymphonyElixir.Orchestrator do metadata[:workspace_path] || Map.get(previous_retry, :workspace_path) end + defp pick_retry_worker_id(previous_retry, metadata) do + metadata[:worker_id] || Map.get(previous_retry, :worker_id) + end + defp maybe_put_runtime_value(running_entry, _key, nil), do: running_entry defp maybe_put_runtime_value(running_entry, key, value) when is_map(running_entry) do @@ -1402,11 +1867,21 @@ defmodule SymphonyElixir.Orchestrator do } end) + claim_leases = + state.claim_leases + |> Enum.map(fn {_issue_id, lease} -> claim_lease_snapshot_entry(lease, now_ms) end) + + expired = + state.expired_claims + |> Enum.map(fn {_issue_id, lease} -> expired_claim_snapshot_entry(lease, now_ms) end) + {:reply, %{ running: running, retrying: retrying, blocked: blocked, + claim_leases: claim_leases, + expired: expired, codex_totals: state.codex_totals, rate_limits: Map.get(state, :codex_rate_limits), polling: %{ diff --git a/elixir/lib/symphony_elixir_web/live/dashboard_live.ex b/elixir/lib/symphony_elixir_web/live/dashboard_live.ex index 43fc3ef642..ccaa1f6e8c 100644 --- a/elixir/lib/symphony_elixir_web/live/dashboard_live.ex +++ b/elixir/lib/symphony_elixir_web/live/dashboard_live.ex @@ -97,6 +97,12 @@ defmodule SymphonyElixirWeb.DashboardLive do

Issues paused for operator input or approval.

+
+

Expired

+

<%= @payload.counts.expired %>

+

Claim leases recovered and handed back to retry.

+
+

Total tokens

<%= format_int(@payload.codex_totals.total_tokens) %>

@@ -123,6 +129,112 @@ defmodule SymphonyElixirWeb.DashboardLive do
<%= pretty_value(@payload.rate_limits) %>
+
+
+
+

Claim leases

+

Durable worker claim heartbeat, retry, and lease expiry state.

+
+
+ + <%= if @payload.claim_leases == [] do %> +

No active claim leases.

+ <% else %> +
+ + + + + + + + + + + + + + + + + + + + + + + + + +
IssueLease stateAttemptWorkerWorkspaceLast seenExpiresRetry/backoff
+
+ <%= entry.issue_identifier %> + JSON details +
+
+ + <%= entry.state || "claimed" %> + + <%= entry.attempt || "n/a" %> +
+ <%= entry.worker_id || "n/a" %> + <%= entry.worker_host || "local" %> +
+
<%= entry.workspace_path || "pending" %><%= entry.last_seen_at || "n/a" %><%= entry.lease_expires_at || "n/a" %> +
+ <%= entry.retry_due_at || "n/a" %> + + backoff=<%= entry.retry_backoff_ms || "n/a" %> error=<%= entry.error || "n/a" %> + +
+
+
+ <% end %> +
+ +
+
+
+

Expired leases

+

Recovered stale claims that were requeued without duplicating live workers.

+
+
+ + <%= if @payload.expired == [] do %> +

No expired leases recovered.

+ <% else %> +
+ + + + + + + + + + + + + + + + + + + + + +
IssueAttemptWorkerExpired atRequeued atError
+
+ <%= entry.issue_identifier %> + JSON details +
+
<%= entry.attempt || "n/a" %><%= entry.worker_id || "n/a" %><%= entry.expired_at || "n/a" %><%= entry.requeued_at || "n/a" %><%= entry.error || "n/a" %>
+
+ <% end %> +
+
@@ -395,7 +507,7 @@ defmodule SymphonyElixirWeb.DashboardLive do cond do String.contains?(normalized, ["progress", "running", "active"]) -> "#{base} state-badge-active" - String.contains?(normalized, ["blocked", "error", "failed"]) -> "#{base} state-badge-danger" + String.contains?(normalized, ["blocked", "error", "failed", "expired"]) -> "#{base} state-badge-danger" String.contains?(normalized, ["todo", "queued", "pending", "retry"]) -> "#{base} state-badge-warning" true -> base end diff --git a/elixir/lib/symphony_elixir_web/presenter.ex b/elixir/lib/symphony_elixir_web/presenter.ex index 3595cfeb33..133815e18f 100644 --- a/elixir/lib/symphony_elixir_web/presenter.ex +++ b/elixir/lib/symphony_elixir_web/presenter.ex @@ -16,11 +16,14 @@ defmodule SymphonyElixirWeb.Presenter do counts: %{ running: length(snapshot.running), retrying: length(snapshot.retrying), - blocked: length(Map.get(snapshot, :blocked, [])) + blocked: length(Map.get(snapshot, :blocked, [])), + expired: length(Map.get(snapshot, :expired, [])) }, running: Enum.map(snapshot.running, &running_entry_payload/1), retrying: Enum.map(snapshot.retrying, &retry_entry_payload/1), blocked: Enum.map(Map.get(snapshot, :blocked, []), &blocked_entry_payload/1), + claim_leases: Enum.map(Map.get(snapshot, :claim_leases, []), &claim_lease_payload/1), + expired: Enum.map(Map.get(snapshot, :expired, []), &expired_claim_payload/1), codex_totals: snapshot.codex_totals, rate_limits: snapshot.rate_limits } @@ -40,11 +43,13 @@ defmodule SymphonyElixirWeb.Presenter do running = Enum.find(snapshot.running, &(&1.identifier == issue_identifier)) retry = Enum.find(snapshot.retrying, &(&1.identifier == issue_identifier)) blocked = Enum.find(Map.get(snapshot, :blocked, []), &(&1.identifier == issue_identifier)) + claim_lease = Enum.find(Map.get(snapshot, :claim_leases, []), &(&1.identifier == issue_identifier)) + expired = Enum.find(Map.get(snapshot, :expired, []), &(&1.identifier == issue_identifier)) - if is_nil(running) and is_nil(retry) and is_nil(blocked) do + if is_nil(running) and is_nil(retry) and is_nil(blocked) and is_nil(claim_lease) and is_nil(expired) do {:error, :issue_not_found} else - {:ok, issue_payload_body(issue_identifier, running, retry, blocked)} + {:ok, issue_payload_body(issue_identifier, running, retry, blocked, claim_lease, expired)} end _ -> @@ -63,14 +68,14 @@ defmodule SymphonyElixirWeb.Presenter do end end - defp issue_payload_body(issue_identifier, running, retry, blocked) do + defp issue_payload_body(issue_identifier, running, retry, blocked, claim_lease, expired) do %{ issue_identifier: issue_identifier, - issue_id: issue_id_from_entries(running, retry, blocked), - status: issue_status(running, retry, blocked), + issue_id: issue_id_from_entries(running, retry, blocked, claim_lease, expired), + status: issue_status(running, retry, blocked, expired, claim_lease), workspace: %{ - path: workspace_path(issue_identifier, running, retry, blocked), - host: workspace_host(running, retry, blocked) + path: workspace_path(issue_identifier, running, retry, blocked, claim_lease, expired), + host: workspace_host(running, retry, blocked, claim_lease, expired) }, attempts: %{ restart_count: restart_count(retry), @@ -84,20 +89,24 @@ defmodule SymphonyElixirWeb.Presenter do }, recent_events: recent_events_payload(running || blocked), last_error: (blocked && blocked.error) || (retry && retry.error), - tracked: %{} + tracked: tracked_payload(claim_lease, expired) } end - defp issue_id_from_entries(running, retry, blocked), - do: (running && running.issue_id) || (retry && retry.issue_id) || (blocked && blocked.issue_id) + defp issue_id_from_entries(running, retry, blocked, claim_lease, expired) do + first_entry_value([running, retry, blocked, claim_lease, expired], :issue_id) + end defp restart_count(retry), do: max(retry_attempt(retry) - 1, 0) defp retry_attempt(nil), do: 0 defp retry_attempt(retry), do: retry.attempt || 0 - defp issue_status(running, _retry, _blocked) when not is_nil(running), do: "running" - defp issue_status(nil, retry, _blocked) when not is_nil(retry), do: "retrying" - defp issue_status(nil, nil, _blocked), do: "blocked" + defp issue_status(running, _retry, _blocked, _expired, _claim_lease) when not is_nil(running), do: "running" + defp issue_status(nil, retry, _blocked, _expired, _claim_lease) when not is_nil(retry), do: "retrying" + defp issue_status(nil, nil, blocked, _expired, _claim_lease) when not is_nil(blocked), do: "blocked" + defp issue_status(nil, nil, nil, expired, _claim_lease) when not is_nil(expired), do: "expired" + defp issue_status(nil, nil, nil, nil, claim_lease) when not is_nil(claim_lease), do: claim_lease.state || "claimed" + defp issue_status(nil, nil, nil, nil, nil), do: "unknown" defp running_entry_payload(entry) do %{ @@ -148,6 +157,35 @@ defmodule SymphonyElixirWeb.Presenter do } end + defp claim_lease_payload(entry) do + %{ + issue_id: entry.issue_id, + issue_identifier: entry.identifier, + state: entry.state, + worker_id: entry.worker_id, + worker_host: Map.get(entry, :worker_host), + workspace_path: Map.get(entry, :workspace_path), + attempt: entry.attempt, + last_seen_at: iso8601(entry.last_seen_at), + lease_expires_at: iso8601(entry.lease_expires_at), + lease_expires_in_ms: entry.lease_expires_in_ms, + retry_due_at: iso8601(entry.retry_due_at), + retry_due_in_ms: entry.retry_due_in_ms, + retry_backoff_ms: entry.retry_backoff_ms, + error: entry.error + } + end + + defp expired_claim_payload(entry) do + entry + |> claim_lease_payload() + |> Map.merge(%{ + state: "expired", + expired_at: iso8601(entry.expired_at), + requeued_at: iso8601(entry.requeued_at) + }) + end + defp running_issue_payload(running) do %{ worker_host: Map.get(running, :worker_host), @@ -191,17 +229,31 @@ defmodule SymphonyElixirWeb.Presenter do } end - defp workspace_path(issue_identifier, running, retry, blocked) do - (running && Map.get(running, :workspace_path)) || - (retry && Map.get(retry, :workspace_path)) || - (blocked && Map.get(blocked, :workspace_path)) || + defp tracked_payload(nil, nil), do: %{} + + defp tracked_payload(claim_lease, expired) do + %{} + |> maybe_put(:claim_lease, claim_lease && claim_lease_payload(claim_lease)) + |> maybe_put(:expired_claim, expired && expired_claim_payload(expired)) + end + + defp maybe_put(map, _key, nil), do: map + defp maybe_put(map, key, value), do: Map.put(map, key, value) + + defp workspace_path(issue_identifier, running, retry, blocked, claim_lease, expired) do + first_entry_value([running, retry, blocked, claim_lease, expired], :workspace_path) || Path.join(Config.settings!().workspace.root, issue_identifier) end - defp workspace_host(running, retry, blocked) do - (running && Map.get(running, :worker_host)) || - (retry && Map.get(retry, :worker_host)) || - (blocked && Map.get(blocked, :worker_host)) + defp workspace_host(running, retry, blocked, claim_lease, expired) do + first_entry_value([running, retry, blocked, claim_lease, expired], :worker_host) + end + + defp first_entry_value(entries, key) when is_list(entries) do + Enum.find_value(entries, fn + entry when is_map(entry) -> Map.get(entry, key) + _ -> nil + end) end defp recent_events_payload(nil), do: [] diff --git a/elixir/test/symphony_elixir/extensions_test.exs b/elixir/test/symphony_elixir/extensions_test.exs index 789eb3d06f..fc9a63be4a 100644 --- a/elixir/test/symphony_elixir/extensions_test.exs +++ b/elixir/test/symphony_elixir/extensions_test.exs @@ -342,7 +342,7 @@ defmodule SymphonyElixir.ExtensionsTest do assert state_payload == %{ "generated_at" => state_payload["generated_at"], - "counts" => %{"running" => 1, "retrying" => 1, "blocked" => 1}, + "counts" => %{"running" => 1, "retrying" => 1, "blocked" => 1, "expired" => 1}, "running" => [ %{ "issue_id" => "issue-http", @@ -385,6 +385,76 @@ defmodule SymphonyElixir.ExtensionsTest do "last_event_at" => state_payload["blocked"] |> List.first() |> Map.fetch!("last_event_at") } ], + "claim_leases" => [ + %{ + "issue_id" => "issue-http", + "issue_identifier" => "MT-HTTP", + "state" => "active", + "worker_id" => "local:#PID<0.1.0>", + "worker_host" => nil, + "workspace_path" => "/workspaces/MT-HTTP", + "attempt" => 1, + "last_seen_at" => state_payload["claim_leases"] |> Enum.at(0) |> Map.fetch!("last_seen_at"), + "lease_expires_at" => state_payload["claim_leases"] |> Enum.at(0) |> Map.fetch!("lease_expires_at"), + "lease_expires_in_ms" => 90_000, + "retry_due_at" => nil, + "retry_due_in_ms" => nil, + "retry_backoff_ms" => nil, + "error" => nil + }, + %{ + "issue_id" => "issue-retry", + "issue_identifier" => "MT-RETRY", + "state" => "retrying", + "worker_id" => "local:#PID<0.2.0>", + "worker_host" => nil, + "workspace_path" => "/workspaces/MT-RETRY", + "attempt" => 2, + "last_seen_at" => state_payload["claim_leases"] |> Enum.at(1) |> Map.fetch!("last_seen_at"), + "lease_expires_at" => state_payload["claim_leases"] |> Enum.at(1) |> Map.fetch!("lease_expires_at"), + "lease_expires_in_ms" => 120_000, + "retry_due_at" => state_payload["claim_leases"] |> Enum.at(1) |> Map.fetch!("retry_due_at"), + "retry_due_in_ms" => 2_000, + "retry_backoff_ms" => 2_000, + "error" => "boom" + }, + %{ + "issue_id" => "issue-blocked", + "issue_identifier" => "MT-BLOCKED", + "state" => "blocked", + "worker_id" => "dm-dev2:#PID<0.3.0>", + "worker_host" => "dm-dev2", + "workspace_path" => "/workspaces/MT-BLOCKED", + "attempt" => 1, + "last_seen_at" => state_payload["claim_leases"] |> Enum.at(2) |> Map.fetch!("last_seen_at"), + "lease_expires_at" => state_payload["claim_leases"] |> Enum.at(2) |> Map.fetch!("lease_expires_at"), + "lease_expires_in_ms" => 90_000, + "retry_due_at" => nil, + "retry_due_in_ms" => nil, + "retry_backoff_ms" => nil, + "error" => "codex turn requires operator input" + } + ], + "expired" => [ + %{ + "issue_id" => "issue-expired", + "issue_identifier" => "MT-EXPIRED", + "state" => "expired", + "worker_id" => "local:#PID<0.4.0>", + "worker_host" => nil, + "workspace_path" => "/workspaces/MT-EXPIRED", + "attempt" => 3, + "last_seen_at" => state_payload["expired"] |> List.first() |> Map.fetch!("last_seen_at"), + "lease_expires_at" => state_payload["expired"] |> List.first() |> Map.fetch!("lease_expires_at"), + "lease_expires_in_ms" => -1_000, + "retry_due_at" => nil, + "retry_due_in_ms" => nil, + "retry_backoff_ms" => nil, + "error" => "claim lease expired at 2026-05-29T17:00:00Z; requeueing", + "expired_at" => state_payload["expired"] |> List.first() |> Map.fetch!("expired_at"), + "requeued_at" => state_payload["expired"] |> List.first() |> Map.fetch!("requeued_at") + } + ], "codex_totals" => %{ "input_tokens" => 4, "output_tokens" => 8, @@ -402,7 +472,7 @@ defmodule SymphonyElixir.ExtensionsTest do "issue_id" => "issue-http", "status" => "running", "workspace" => %{ - "path" => Path.join(Config.settings!().workspace.root, "MT-HTTP"), + "path" => "/workspaces/MT-HTTP", "host" => nil }, "attempts" => %{"restart_count" => 0, "current_retry_attempt" => 0}, @@ -423,7 +493,24 @@ defmodule SymphonyElixir.ExtensionsTest do "logs" => %{"codex_session_logs" => []}, "recent_events" => [], "last_error" => nil, - "tracked" => %{} + "tracked" => %{ + "claim_lease" => %{ + "issue_id" => "issue-http", + "issue_identifier" => "MT-HTTP", + "state" => "active", + "worker_id" => "local:#PID<0.1.0>", + "worker_host" => nil, + "workspace_path" => "/workspaces/MT-HTTP", + "attempt" => 1, + "last_seen_at" => issue_payload["tracked"]["claim_lease"]["last_seen_at"], + "lease_expires_at" => issue_payload["tracked"]["claim_lease"]["lease_expires_at"], + "lease_expires_in_ms" => 90_000, + "retry_due_at" => nil, + "retry_due_in_ms" => nil, + "retry_backoff_ms" => nil, + "error" => nil + } + } } conn = get(build_conn(), "/api/v1/MT-RETRY") @@ -443,6 +530,15 @@ defmodule SymphonyElixir.ExtensionsTest do } } = json_response(conn, 200) + conn = get(build_conn(), "/api/v1/MT-EXPIRED") + + assert %{ + "status" => "expired", + "tracked" => %{ + "expired_claim" => %{"state" => "expired", "error" => "claim lease expired" <> _} + } + } = json_response(conn, 200) + conn = get(build_conn(), "/api/v1/MT-MISSING") assert json_response(conn, 404) == %{ @@ -571,6 +667,9 @@ defmodule SymphonyElixir.ExtensionsTest do assert html =~ "MT-HTTP" assert html =~ "MT-RETRY" assert html =~ "MT-BLOCKED" + assert html =~ "MT-EXPIRED" + assert html =~ "Claim leases" + assert html =~ "Expired leases" assert html =~ "rendered" assert html =~ "turn blocked: waiting for user input" assert html =~ "Runtime" @@ -671,7 +770,7 @@ defmodule SymphonyElixir.ExtensionsTest do response = Req.get!("http://127.0.0.1:#{port}/api/v1/state") assert response.status == 200 - assert response.body["counts"] == %{"running" => 1, "retrying" => 1, "blocked" => 1} + assert response.body["counts"] == %{"running" => 1, "retrying" => 1, "blocked" => 1, "expired" => 1} dashboard_css = Req.get!("http://127.0.0.1:#{port}/dashboard.css") assert dashboard_css.status == 200 @@ -714,6 +813,8 @@ defmodule SymphonyElixir.ExtensionsTest do end defp static_snapshot do + now = DateTime.utc_now() + %{ running: [ %{ @@ -729,7 +830,7 @@ defmodule SymphonyElixir.ExtensionsTest do codex_input_tokens: 4, codex_output_tokens: 8, codex_total_tokens: 12, - started_at: DateTime.utc_now() + started_at: now } ], retrying: [ @@ -750,14 +851,84 @@ defmodule SymphonyElixir.ExtensionsTest do worker_host: "dm-dev2", workspace_path: "/workspaces/MT-BLOCKED", session_id: "thread-blocked", - blocked_at: DateTime.utc_now(), + blocked_at: now, last_codex_event: :turn_input_required, last_codex_message: %{ event: :turn_input_required, message: %{"method" => "turn/input_required"}, - timestamp: DateTime.utc_now() + timestamp: now }, - last_codex_timestamp: DateTime.utc_now() + last_codex_timestamp: now + } + ], + claim_leases: [ + %{ + issue_id: "issue-http", + identifier: "MT-HTTP", + state: "active", + worker_id: "local:#PID<0.1.0>", + worker_host: nil, + workspace_path: "/workspaces/MT-HTTP", + attempt: 1, + last_seen_at: now, + lease_expires_at: DateTime.add(now, 90_000, :millisecond), + lease_expires_in_ms: 90_000, + retry_due_at: nil, + retry_due_in_ms: nil, + retry_backoff_ms: nil, + error: nil + }, + %{ + issue_id: "issue-retry", + identifier: "MT-RETRY", + state: "retrying", + worker_id: "local:#PID<0.2.0>", + worker_host: nil, + workspace_path: "/workspaces/MT-RETRY", + attempt: 2, + last_seen_at: now, + lease_expires_at: DateTime.add(now, 120_000, :millisecond), + lease_expires_in_ms: 120_000, + retry_due_at: DateTime.add(now, 2_000, :millisecond), + retry_due_in_ms: 2_000, + retry_backoff_ms: 2_000, + error: "boom" + }, + %{ + issue_id: "issue-blocked", + identifier: "MT-BLOCKED", + state: "blocked", + worker_id: "dm-dev2:#PID<0.3.0>", + worker_host: "dm-dev2", + workspace_path: "/workspaces/MT-BLOCKED", + attempt: 1, + last_seen_at: now, + lease_expires_at: DateTime.add(now, 90_000, :millisecond), + lease_expires_in_ms: 90_000, + retry_due_at: nil, + retry_due_in_ms: nil, + retry_backoff_ms: nil, + error: "codex turn requires operator input" + } + ], + expired: [ + %{ + issue_id: "issue-expired", + identifier: "MT-EXPIRED", + state: "expired", + worker_id: "local:#PID<0.4.0>", + worker_host: nil, + workspace_path: "/workspaces/MT-EXPIRED", + attempt: 3, + last_seen_at: DateTime.add(now, -90_000, :millisecond), + lease_expires_at: DateTime.add(now, -1_000, :millisecond), + lease_expires_in_ms: -1_000, + retry_due_at: nil, + retry_due_in_ms: nil, + retry_backoff_ms: nil, + error: "claim lease expired at 2026-05-29T17:00:00Z; requeueing", + expired_at: DateTime.add(now, -1_000, :millisecond), + requeued_at: now } ], codex_totals: %{input_tokens: 4, output_tokens: 8, total_tokens: 12, seconds_running: 42.5}, diff --git a/elixir/test/symphony_elixir/orchestrator_status_test.exs b/elixir/test/symphony_elixir/orchestrator_status_test.exs index 35a2977ede..b5037acdbb 100644 --- a/elixir/test/symphony_elixir/orchestrator_status_test.exs +++ b/elixir/test/symphony_elixir/orchestrator_status_test.exs @@ -1134,6 +1134,201 @@ defmodule SymphonyElixir.OrchestratorStatusTest do } = state.blocked[issue_id] end + test "claim lease marker is persisted when an issue is claimed" do + write_workflow_file!(Workflow.workflow_file_path(), tracker_kind: "memory") + Application.put_env(:symphony_elixir, :memory_tracker_recipient, self()) + + issue = %Issue{ + id: "issue-lease", + identifier: "MT-LEASE", + title: "Lease marker", + state: "In Progress" + } + + running_entry = %{ + pid: self(), + ref: make_ref(), + identifier: issue.identifier, + issue: issue, + worker_host: nil, + workspace_path: "/tmp/symphony_workspaces/MT-LEASE", + started_at: DateTime.utc_now() + } + + state = + %Orchestrator.State{poll_interval_ms: 30_000, max_concurrent_agents: 1} + |> Orchestrator.start_claim_lease_for_test(issue, running_entry) + + assert %{ + state: :active, + worker_id: "local:" <> _, + workspace_path: "/tmp/symphony_workspaces/MT-LEASE", + attempt: 1, + lease_expires_at: %DateTime{} + } = state.claim_leases[issue.id] + + assert_receive {:memory_tracker_comment, "issue-lease", body} + assert body =~ "## Symphony Claim Lease" + assert body =~ "- state: active" + assert body =~ "- worker_id: local:" + assert body =~ "- workspace_path: /tmp/symphony_workspaces/MT-LEASE" + assert body =~ "- attempt: 1" + assert body =~ "- lease_expires_at:" + end + + test "claim lease heartbeat refresh updates last seen and lease expiry" do + write_workflow_file!(Workflow.workflow_file_path(), tracker_kind: "memory") + Application.put_env(:symphony_elixir, :memory_tracker_recipient, self()) + + issue = %Issue{id: "issue-heartbeat", identifier: "MT-HB", title: "Heartbeat", state: "In Progress"} + + running_entry = %{ + pid: self(), + ref: make_ref(), + identifier: issue.identifier, + issue: issue, + worker_host: "worker-a", + workspace_path: "/workspaces/MT-HB", + started_at: DateTime.utc_now() + } + + state = + %Orchestrator.State{poll_interval_ms: 30_000, max_concurrent_agents: 1} + |> Orchestrator.start_claim_lease_for_test(issue, running_entry) + + assert_receive {:memory_tracker_comment, "issue-heartbeat", _body} + + stale_seen_at = DateTime.add(DateTime.utc_now(), -120, :second) + stale_marker_at_ms = System.monotonic_time(:millisecond) - 90_000 + stale_expires_at_ms = System.monotonic_time(:millisecond) + 1_000 + old_lease = state.claim_leases[issue.id] + + state = + put_in(state.claim_leases[issue.id], %{ + old_lease + | last_seen_at: stale_seen_at, + lease_expires_at_ms: stale_expires_at_ms, + last_marker_at_ms: stale_marker_at_ms + }) + + refreshed_state = Orchestrator.refresh_claim_lease_from_running_for_test(state, issue.id, running_entry) + refreshed_lease = refreshed_state.claim_leases[issue.id] + + assert DateTime.compare(refreshed_lease.last_seen_at, stale_seen_at) == :gt + assert refreshed_lease.lease_expires_at_ms > stale_expires_at_ms + assert refreshed_lease.heartbeat_count == old_lease.heartbeat_count + 1 + + assert_receive {:memory_tracker_comment, "issue-heartbeat", body} + assert body =~ "- state: active" + assert body =~ "- worker_host: worker-a" + end + + test "expired claim leases are requeued and logged" do + write_workflow_file!(Workflow.workflow_file_path(), tracker_kind: "memory") + Application.put_env(:symphony_elixir, :memory_tracker_recipient, self()) + + issue = %Issue{ + id: "issue-expired-lease", + identifier: "MT-EXPIRED-LEASE", + title: "Expired lease", + state: "In Progress" + } + + expired_lease = %{ + issue_id: issue.id, + identifier: issue.identifier, + state: :active, + worker_id: "local:#PID<0.1.0>", + worker_host: nil, + workspace_path: "/tmp/symphony_workspaces/MT-EXPIRED-LEASE", + attempt: 1, + last_seen_at: DateTime.add(DateTime.utc_now(), -120, :second), + lease_expires_at: DateTime.add(DateTime.utc_now(), -1, :second), + lease_expires_at_ms: System.monotonic_time(:millisecond) - 1, + last_marker_at_ms: System.monotonic_time(:millisecond) - 120_000 + } + + state = %Orchestrator.State{ + poll_interval_ms: 30_000, + max_concurrent_agents: 1, + claimed: MapSet.new([issue.id]), + claim_leases: %{issue.id => expired_lease} + } + + log = + capture_log(fn -> + recovered_state = Orchestrator.recover_expired_claim_leases_for_test(state, [issue]) + send(self(), {:recovered_state, recovered_state}) + end) + + assert_receive {:recovered_state, recovered_state} + assert log =~ "Claim lease expired; requeueing" + + assert %{attempt: 2, error: "claim lease expired" <> _} = recovered_state.retry_attempts[issue.id] + assert %{state: :expired, error: "claim lease expired" <> _} = recovered_state.expired_claims[issue.id] + assert %{state: :retrying, attempt: 2, retry_backoff_ms: 0} = recovered_state.claim_leases[issue.id] + assert MapSet.member?(recovered_state.claimed, issue.id) + + assert_receive {:memory_tracker_comment, "issue-expired-lease", body} + assert body =~ "- state: retrying" + assert body =~ "- retry_backoff_ms: 0" + end + + test "expired claim lease recovery does not duplicate live workers" do + issue = %Issue{ + id: "issue-live-expired", + identifier: "MT-LIVE-EXPIRED", + title: "Live expired lease", + state: "In Progress" + } + + worker_pid = + spawn(fn -> + receive do + :done -> :ok + end + end) + + on_exit(fn -> + if Process.alive?(worker_pid), do: Process.exit(worker_pid, :normal) + end) + + expired_lease = %{ + issue_id: issue.id, + identifier: issue.identifier, + state: :active, + worker_id: "local:#PID<0.2.0>", + worker_host: nil, + workspace_path: "/tmp/symphony_workspaces/MT-LIVE-EXPIRED", + attempt: 1, + last_seen_at: DateTime.add(DateTime.utc_now(), -120, :second), + lease_expires_at: DateTime.add(DateTime.utc_now(), -1, :second), + lease_expires_at_ms: System.monotonic_time(:millisecond) - 1 + } + + state = %Orchestrator.State{ + poll_interval_ms: 30_000, + max_concurrent_agents: 1, + claimed: MapSet.new([issue.id]), + running: %{ + issue.id => %{ + pid: worker_pid, + ref: make_ref(), + identifier: issue.identifier, + issue: issue, + started_at: DateTime.utc_now() + } + }, + claim_leases: %{issue.id => expired_lease} + } + + recovered_state = Orchestrator.recover_expired_claim_leases_for_test(state, [issue]) + + assert recovered_state.retry_attempts == %{} + assert recovered_state.expired_claims == %{} + assert recovered_state.running[issue.id].pid == worker_pid + end + test "status dashboard renders offline marker to terminal" do rendered = ExUnit.CaptureIO.capture_io(fn ->