From 991d66d5ecc528f9ed6cc089dfd5cdc3f5988d4a Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 13 Jun 2026 14:40:44 -0700 Subject: [PATCH] feat: add provider-backed workspace deletion --- CHANGELOG.md | 3 +- README.md | 10 +- cmd/crabbox-ssh-gateway/main.go | 52 ++++ cmd/crabbox-ssh-gateway/main_test.go | 98 ++++++++ cmd/crabfleet/main.go | 34 ++- cmd/crabfleet/main_test.go | 87 +++++++ docs/api.md | 11 +- docs/architecture.md | 2 +- docs/github-actions-sessions.md | 26 +- docs/index.md | 3 +- docs/quickstart.md | 2 + src/app/fleet.jsx | 31 ++- src/app/main.jsx | 57 ++++- src/app/utils.js | 11 +- src/index.ts | 352 ++++++++++++++++++++++----- src/runtime-adapter.ts | 7 + tests/app-utils.test.ts | 8 + tests/html-dialogs.test.ts | 20 ++ tests/runtime-adapter.test.ts | 166 ++++++++++++- 19 files changed, 880 insertions(+), 100 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8adac92..caf5663 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,8 @@ - Bound Crabbox terminal output with negotiated acknowledgements and legacy-client compatibility. - Enable the OpenClaw deployment's versioned Crabbox runtime adapter with a stable tenant namespace. - Add comprehensive documentation for durable GitHub Actions sessions, including registration, runner and viewer relay, work-state heartbeats, Codex steering, resumption, completion, cancellation, authentication, archives, and troubleshooting. -- Keep GitHub Actions sessions out of legacy workspace-stop reconciliation and give them a dedicated cancel lifecycle. +- Name versioned provider-backed workspace lifecycle actions Delete across Fleet, the Go CLI, and SSH while retaining explicit Stop wording for legacy sessions and `stop` as a CLI compatibility alias; keep the provider stop wire action internal; and fail closed without adopting or deleting a pre-existing adapter workspace on an explicit ID conflict. +- Keep GitHub Actions sessions out of legacy workspace-stop reconciliation and let operators end their Crabfleet terminal session without claiming to cancel the underlying workflow run. - Add durable steerable GitHub Actions sessions with service registration, scoped runner URLs, work-state heartbeats, Fleet metadata, and a SessionControlDO PTY relay. - Add a tenant-namespaced versioned runtime lifecycle adapter with replayable idempotent create, monotonic workspace identities, CAS reconciliation, durable terminal finalization, confirmed provider release before failure/stop, presence-aware capability/expiry tracking, authenticated transient VNC redirects, and deployment-neutral configuration. - Reconcile runtime lifecycles and every adapter's terminal archives on cron and direct access, preserve partial capability-object defaults while honoring authoritative lists and explicit terminal withdrawal, make PTY availability server-authoritative, preserve opaque signed terminal and desktop URLs byte-for-byte, retain adapter failure evidence through confirmed release and exact session-version archive finalization, preflight adapter credentials before session allocation, bind every external lifecycle to its immutable registered control plane, generation-fence managed and standalone Sandbox credential ownership across crashes and late requests, repair incomplete equal-count archives, run teardown only after an exact cleanup CAS, use unique concurrent archive attempts, and transactionally remove D1 archive pointers before best-effort R2 object cleanup. diff --git a/README.md b/README.md index 0e44683..5ca40aa 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ Crabfleet gives OpenClaw maintainers a fleet dashboard where every Codex crabbox - **Ghostty WebAssembly** for the fullscreen attach grid and run log replay. - **Cloudflare Sandbox containers** for standalone interactive Codex CLI workspaces with live PTY attach. - **Runtime adapter descriptors** for Container and Crabbox selection, capability display, interactive lifecycle handoff, and guarded takeover. -- **Versioned lifecycle adapter** for idempotent external workspace creation, bounded status reconciliation, provider-backed stop, terminal attachment, and authenticated transient desktop connections. +- **Versioned lifecycle adapter** for idempotent external workspace creation, bounded status reconciliation, provider-backed deletion, terminal attachment, and authenticated transient desktop connections. - **Provision endpoint** at `/api/provision/interactive` that can use the built-in Sandbox backend or retain a legacy create-only adapter or ClawFleet integration, with durable ownership and a bearer-authenticated standalone PTY route. - **SessionControlDO relay** for one outbound GitHub Actions runner and multiple authenticated Ghostty viewers per action session. - **R2 session archives** for crabbox event NDJSON, transcripts, and summaries. @@ -300,8 +300,9 @@ go run ./cmd/crabbox-ssh-gateway ``` Unknown public keys get a short GitHub OAuth link through `ssh link@host`. Linked keys can -run `whoami`, `list`, `new`, and `attach SESSION_ID`; `new` creates an interactive Codex -session and attaches. +run `whoami`, `list`, `new`, `attach SESSION_ID`, and `delete SESSION_ID`; `new` creates an +interactive Codex session and attaches. Delete confirms runtime release for versioned lifecycle +adapters; legacy create-only and ClawFleet sessions stop locally and may need provider cleanup. Production should expose the gateway at `crabd.sh` as a DNS-only `A` record. Use `ssh link@crabd.sh` once to connect a GitHub-backed SSH key, then run @@ -319,9 +320,12 @@ go run ./cmd/crabfleet login go run ./cmd/crabfleet list go run ./cmd/crabfleet new --repo openclaw/crabfleet "start on the release checklist" go run ./cmd/crabfleet attach +go run ./cmd/crabfleet delete go run ./cmd/crabfleet vnc --open ``` +`crabfleet stop ` remains a compatibility alias for `delete`. + ### CLI Release Tagged releases publish `crabfleet` with GoReleaser and dispatch the OpenClaw Homebrew tap updater: diff --git a/cmd/crabbox-ssh-gateway/main.go b/cmd/crabbox-ssh-gateway/main.go index d73c833..569f2f4 100644 --- a/cmd/crabbox-ssh-gateway/main.go +++ b/cmd/crabbox-ssh-gateway/main.go @@ -55,6 +55,7 @@ type interactiveSession struct { Repo string `json:"repo"` Branch string `json:"branch"` Runtime string `json:"runtime"` + Adapter string `json:"adapter"` Status string `json:"status"` Owner string `json:"owner"` CreatedBy string `json:"createdBy"` @@ -69,6 +70,28 @@ type interactiveSession struct { LogArchive logArchive `json:"logArchive"` } +func legacyProviderCleanupMayBeRequired(session interactiveSession) bool { + if session.Adapter == "runtime-v1" || session.Runtime == "github_actions" { + return false + } + switch session.Status { + case "stopping", "stopped", "expired": + return true + default: + return false + } +} + +func lifecycleStopNote(session interactiveSession) string { + if session.Runtime == "github_actions" && session.Status == "stopped" { + return "GitHub Actions workflow run was not canceled and may continue on GitHub" + } + if legacyProviderCleanupMayBeRequired(session) { + return "provider deletion was not confirmed; legacy runtimes may require separate cleanup" + } + return "" +} + type sessionCapabilities struct { Terminal bool `json:"terminal"` } @@ -399,6 +422,21 @@ func runCommand(ctx context.Context, out io.ReadWriter, perms *ssh.Permissions, } fmt.Fprintf(out, "session %s not found\n", terminalSafe(args[1])) return 1 + case "delete", "stop": + if len(args) != 2 { + fmt.Fprintln(out, "usage: delete SESSION_ID") + return 2 + } + session, err := client.action(ctx, auth.fingerprint, args[1], "stop") + if err != nil { + fmt.Fprintf(out, "error: %v\n", err) + return 1 + } + fmt.Fprintf(out, "session: %s\nstatus: %s\n", terminalSafe(session.ID), terminalSafe(session.Status)) + if note := lifecycleStopNote(session); note != "" { + fmt.Fprintf(out, "note: %s\n", note) + } + return 0 case "logs": if len(args) < 2 { fmt.Fprintln(out, "usage: logs SESSION_ID") @@ -527,6 +565,7 @@ func printHelp(out io.Writer, user user) { fmt.Fprintln(out, " --runtime overrides the deployment default") fmt.Fprintln(out, " attach SESSION_ID") fmt.Fprintln(out, " vnc SESSION_ID") + fmt.Fprintln(out, " delete SESSION_ID") fmt.Fprintln(out, " logs SESSION_ID") fmt.Fprintln(out, " transcript SESSION_ID") fmt.Fprintln(out, " message SESSION_ID [--no-enter] TEXT") @@ -874,6 +913,19 @@ func (c *apiClient) createSession(ctx context.Context, fingerprint string, reque return response.Session, err } +func (c *apiClient) action(ctx context.Context, fingerprint string, id string, action string) (interactiveSession, error) { + var response sessionResponse + err := c.do( + ctx, + http.MethodPost, + "/api/ssh/interactive-sessions/"+url.PathEscape(id)+"/actions", + fingerprint, + map[string]string{"action": action}, + &response, + ) + return response.Session, err +} + func (c *apiClient) logs(ctx context.Context, fingerprint string, id string) (sessionLogResponse, error) { var response sessionLogResponse err := c.do(ctx, http.MethodGet, "/api/ssh/interactive-sessions/"+url.PathEscape(id)+"/logs", fingerprint, nil, &response) diff --git a/cmd/crabbox-ssh-gateway/main_test.go b/cmd/crabbox-ssh-gateway/main_test.go index ac269ad..064a7b2 100644 --- a/cmd/crabbox-ssh-gateway/main_test.go +++ b/cmd/crabbox-ssh-gateway/main_test.go @@ -2,9 +2,15 @@ package main import ( "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" "reflect" "strings" "testing" + + "golang.org/x/crypto/ssh" ) func TestSplitCommandKeepsQuotedValues(t *testing.T) { @@ -127,6 +133,98 @@ func TestCreateAutoAttachRequiresReadyResolvablePTY(t *testing.T) { } } +func TestDeleteCommandAndStopAliasUseWorkspaceStopAction(t *testing.T) { + var action string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != "/api/ssh/interactive-sessions/IS-7/actions" { + t.Errorf("request = %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusBadRequest) + return + } + if got := r.Header.Get("X-Crabfleet-SSH-Fingerprint"); got != "SHA256:test" { + t.Errorf("fingerprint = %q", got) + w.WriteHeader(http.StatusBadRequest) + return + } + var body map[string]string + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Error(err) + w.WriteHeader(http.StatusBadRequest) + return + } + action = body["action"] + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"session":{"id":"IS-7","status":"stopping"}}`)) + })) + defer server.Close() + + client := &apiClient{baseURL: server.URL, token: "gateway-token", client: server.Client()} + permissions := &ssh.Permissions{Extensions: map[string]string{ + "authorized": "true", + "fingerprint": "SHA256:test", + "login": "operator", + "role": "owner", + }} + for _, command := range []string{"delete IS-7", "stop IS-7"} { + action = "" + var output bytes.Buffer + if exit := runCommand(context.Background(), &output, permissions, client, command, sessionPTY{}); exit != 0 { + t.Fatalf("command=%q exit=%d output=%q", command, exit, output.String()) + } + if action != "stop" { + t.Fatalf("command=%q action=%q, want stop", command, action) + } + if !strings.Contains(output.String(), "provider deletion was not confirmed") { + t.Fatalf("command=%q missing legacy cleanup warning: %q", command, output.String()) + } + if got := output.String(); !strings.Contains(got, "session: IS-7\nstatus: stopping\n") { + t.Fatalf("command=%q output=%q", command, got) + } + } + for _, command := range []string{"delete", "delete IS-7 extra", "stop", "stop IS-7 extra"} { + action = "" + var output bytes.Buffer + if exit := runCommand(context.Background(), &output, permissions, client, command, sessionPTY{}); exit != 2 { + t.Fatalf("command=%q exit=%d output=%q", command, exit, output.String()) + } + if action != "" { + t.Fatalf("command=%q unexpectedly submitted action=%q", command, action) + } + if got := output.String(); got != "usage: delete SESSION_ID\n" { + t.Fatalf("command=%q output=%q", command, got) + } + } +} + +func TestHelpNamesDeleteAsCanonicalCommand(t *testing.T) { + var output bytes.Buffer + printHelp(&output, user{Login: "operator", Role: "owner"}) + if got := output.String(); !strings.Contains(got, "delete SESSION_ID") || strings.Contains(got, "stop SESSION_ID") { + t.Fatalf("help = %q", got) + } +} + +func TestLegacyProviderCleanupWarningRequiresConfirmedLegacyStop(t *testing.T) { + if !legacyProviderCleanupMayBeRequired(interactiveSession{Status: "stopped"}) { + t.Fatal("confirmed legacy stop should retain the cleanup warning") + } + for _, session := range []interactiveSession{ + {Status: "failed"}, + {Status: "stopped", Adapter: "runtime-v1"}, + {Status: "stopped", Runtime: "github_actions"}, + } { + if legacyProviderCleanupMayBeRequired(session) { + t.Fatalf("session %#v must not recommend provider cleanup", session) + } + } + if got := lifecycleStopNote(interactiveSession{Status: "stopped", Runtime: "github_actions"}); !strings.Contains(got, "not canceled") { + t.Fatalf("GitHub Actions note = %q", got) + } + if got := lifecycleStopNote(interactiveSession{Status: "failed"}); got != "" { + t.Fatalf("failed unowned workspace note = %q", got) + } +} + func TestPrintListShowsOwnersAndSessionTree(t *testing.T) { var out bytes.Buffer printList(&out, stateResponse{ diff --git a/cmd/crabfleet/main.go b/cmd/crabfleet/main.go index 7a2e3bb..389a2fa 100644 --- a/cmd/crabfleet/main.go +++ b/cmd/crabfleet/main.go @@ -43,7 +43,7 @@ type cli struct { New newCmd `cmd:"" help:"Create a repo-ready crabbox and attach."` Attach attachCmd `cmd:"" help:"Attach to a crabbox terminal."` Status statusCmd `cmd:"" help:"Show one crabbox lifecycle state."` - Stop stopCmd `cmd:"" help:"Stop a crabbox workspace."` + Delete deleteCmd `cmd:"" aliases:"stop" help:"End a crabbox session through its configured lifecycle."` Doctor doctorCmd `cmd:"" help:"Check API, auth, and linked lifecycle access."` Checkpoints checkpointsCmd `cmd:"" help:"List sandbox checkpoints."` Checkpoint checkpointCmd `cmd:"" help:"Create a sandbox checkpoint."` @@ -82,7 +82,7 @@ type statusCmd struct { ID string `arg:"" help:"Crabbox session id."` } -type stopCmd struct { +type deleteCmd struct { ID string `arg:"" help:"Crabbox session id."` } @@ -157,6 +157,7 @@ type interactiveSession struct { Repo string `json:"repo"` Branch string `json:"branch"` Runtime string `json:"runtime"` + Adapter string `json:"adapter"` Status string `json:"status"` Owner string `json:"owner"` CreatedBy string `json:"createdBy"` @@ -171,6 +172,28 @@ type interactiveSession struct { LogArchive logArchive `json:"logArchive"` } +func legacyProviderCleanupMayBeRequired(session interactiveSession) bool { + if session.Adapter == "runtime-v1" || session.Runtime == "github_actions" { + return false + } + switch session.Status { + case "stopping", "stopped", "expired": + return true + default: + return false + } +} + +func lifecycleStopNote(session interactiveSession) string { + if session.Runtime == "github_actions" && session.Status == "stopped" { + return "GitHub Actions workflow run was not canceled and may continue on GitHub" + } + if legacyProviderCleanupMayBeRequired(session) { + return "provider deletion was not confirmed; legacy runtimes may require separate cleanup" + } + return "" +} + type sessionCapabilities struct { Terminal bool `json:"terminal"` } @@ -436,18 +459,21 @@ func (cmd statusCmd) Run(app *cli, api *apiClient) error { return nil } -func (cmd stopCmd) Run(app *cli, api *apiClient) error { +func (cmd deleteCmd) Run(app *cli, api *apiClient) error { session, err := api.action(context.Background(), cmd.ID, "stop") if err != nil { if app.NoInput || app.JSON { return err } - return runSSH(app, "stop", cmd.ID) + return runSSH(app, "delete", cmd.ID) } if app.JSON { return json.NewEncoder(os.Stdout).Encode(session) } fmt.Fprintf(os.Stdout, "session: %s\nstatus: %s\n", session.ID, session.Status) + if note := lifecycleStopNote(session); note != "" { + fmt.Fprintf(os.Stdout, "note: %s\n", note) + } return nil } diff --git a/cmd/crabfleet/main_test.go b/cmd/crabfleet/main_test.go index a27fc64..e9b7ba2 100644 --- a/cmd/crabfleet/main_test.go +++ b/cmd/crabfleet/main_test.go @@ -3,6 +3,8 @@ package main import ( "bytes" "encoding/json" + "net/http" + "net/http/httptest" "strings" "testing" @@ -97,6 +99,91 @@ func TestNewRuntimeOverrideIsOptional(t *testing.T) { } } +func TestDeleteCommandUsesProviderStopAction(t *testing.T) { + var action string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != "/api/ssh/interactive-sessions/IS-7/actions" { + t.Errorf("request = %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusBadRequest) + return + } + if got := r.Header.Get("X-Crabfleet-SSH-Fingerprint"); got != "SHA256:test" { + t.Errorf("fingerprint = %q", got) + w.WriteHeader(http.StatusBadRequest) + return + } + var body map[string]string + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Error(err) + w.WriteHeader(http.StatusBadRequest) + return + } + action = body["action"] + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"session":{"id":"IS-7","status":"stopping"}}`)) + })) + defer server.Close() + + app := &cli{ + API: server.URL, + Token: "gateway-token", + Fingerprint: "SHA256:test", + NoInput: true, + } + if err := (deleteCmd{ID: "IS-7"}).Run(app, app.apiClient()); err != nil { + t.Fatal(err) + } + if action != "stop" { + t.Fatalf("action = %q, want stop", action) + } +} + +func TestCLIUsesDeleteCanonicalNameWithStopAlias(t *testing.T) { + var app cli + parser, err := kong.New(&app, kong.Name("crabfleet"), kong.Vars{"version": version}) + if err != nil { + t.Fatal(err) + } + if _, err := parser.Parse([]string{"delete", "IS-7"}); err != nil { + t.Fatal(err) + } + if app.Delete.ID != "IS-7" { + t.Fatalf("delete id = %q", app.Delete.ID) + } + var legacy cli + legacyParser, err := kong.New(&legacy, kong.Name("crabfleet"), kong.Vars{"version": version}) + if err != nil { + t.Fatal(err) + } + if _, err := legacyParser.Parse([]string{"stop", "IS-8"}); err != nil { + t.Fatal(err) + } + if legacy.Delete.ID != "IS-8" { + t.Fatalf("stop alias id = %q", legacy.Delete.ID) + } +} + +func TestLegacyProviderCleanupWarningRequiresConfirmedLegacyStop(t *testing.T) { + if !legacyProviderCleanupMayBeRequired(interactiveSession{Status: "stopped"}) { + t.Fatal("confirmed legacy stop should retain the cleanup warning") + } + for _, session := range []interactiveSession{ + {Status: "failed"}, + {Status: "stopped", Adapter: "runtime-v1"}, + {Status: "stopped", Runtime: "github_actions"}, + } { + if legacyProviderCleanupMayBeRequired(session) { + t.Fatalf("session %#v must not recommend provider cleanup", session) + } + } + if got := lifecycleStopNote(interactiveSession{Status: "stopped", Runtime: "github_actions"}); !strings.Contains(got, "not canceled") { + t.Fatalf("GitHub Actions note = %q", got) + } + if got := lifecycleStopNote(interactiveSession{Status: "failed"}); got != "" { + t.Fatalf("failed unowned workspace note = %q", got) + } +} + func TestAttachableRequiresReadySessionWithAttachURL(t *testing.T) { if !attachable(interactiveSession{Status: "ready", AttachURL: "/api/interactive-sessions/IS-1/pty"}) { t.Fatal("ready session with sandbox attach URL should be attachable") diff --git a/docs/api.md b/docs/api.md index 94bb479..e5568f9 100644 --- a/docs/api.md +++ b/docs/api.md @@ -271,18 +271,19 @@ For a successful direct built-in Sandbox provision, `attachUrl` is an absolute ` Crabfleet authenticates every adapter request with `Authorization: Bearer CRABBOX_RUNTIME_ADAPTER_TOKEN`. Adapter URLs must use HTTPS, except literal loopback HTTP for same-host deployments. The original base URL may include a nested path, whose semantics are preserved, but any raw `?` or `#` delimiter is rejected even when its query or fragment is empty. Authenticated adapter requests reject redirects so the bearer token cannot cross origins. The canonical control-plane base URL is persisted with the lifecycle registration before create. Replay, inspect, desktop connection, and delete fail closed unless the current configured URL exactly matches that registered identity, so a configuration change cannot redirect an existing workspace ID or turn a 404 from another origin into release proof. -- `POST /v1/workspaces`: idempotent create. Crabfleet persists the deterministic adapter identity, TTL, idle timeout, requested capabilities, and exact serialized create payload before the request, then sends the same namespaced DNS-safe lowercase `id` and `Idempotency-Key`, plus repo, branch, runtime, opaque profile, command, prompt, ownership/lineage, and lifecycle settings. A definitive non-2xx response is read once, sanitized, and durably recorded as the failure reason before provider release begins. After an ambiguous result, a bounded reconciliation pass retries only that immutable payload and key when inspect returns 404; later edits to session metadata do not alter it. +- `POST /v1/workspaces`: idempotent create. Crabfleet persists the deterministic adapter identity, TTL, idle timeout, requested capabilities, and exact serialized create payload before the request, then sends the same namespaced DNS-safe lowercase `id` and `Idempotency-Key`, plus repo, branch, runtime, opaque profile, command, prompt, ownership/lineage, and lifecycle settings. A definitive non-2xx response to the initial request is read once, sanitized, and durably recorded as the failure reason before provider release begins. After an ambiguous result, a bounded reconciliation pass retries only that immutable payload and key before any inspect; later edits to session metadata do not alter it. Replay-time authentication, routing, validation, or other non-success responses cannot prove the original request failed and therefore keep create ambiguity pending. +- An adapter that finds the requested ID already bound to a different immutable request returns `409` with `error.code = "workspace_id_conflict"`. Crabfleet marks only its local session failed and atomically drops that adapter identity when the exact pending create attempt still owns the lifecycle revision and reconciliation claim; a stale conflict response is ignored. It never adopts, inspects, or deletes the pre-existing workspace. Other `409` responses remain ambiguous and retryable. - `GET /v1/workspaces/:id`: inspect current status, capabilities, terminal URL, expiry, and provider resource identity. Status-only responses preserve previously stored capabilities and expiry; explicit `null` clears those fields. Active external sessions are reconciled in bounded batches; state responses wait only for a short foreground budget while remaining work continues in the Worker background. - `DELETE /v1/workspaces/:id`: stop/release. Crabfleet enters `stopping` before calling the adapter and marks the session stopped only after `204`, `404`, or a valid exact-ID terminal response confirms release; malformed successful bodies remain `stopping`. Plain-text and malformed-JSON responses are read once and sanitized before their evidence is retained. An explicit stop whose ownership claim loses returns success only when the exact workspace is already stopping or terminal; otherwise it returns a lifecycle conflict. -- `POST /v1/workspaces/:id/connections/desktop`: mint a current transient desktop URL. `expiresAt` is optional; when present it must be in the future and no more than 15 minutes away. Accepted HTTPS URLs are treated as opaque signed connection material and redirected byte-for-byte without URL normalization. After minting, Crabfleet re-reads the exact current session status, control grant, capabilities, and registered adapter identity before redirecting; a concurrent stop, revocation, capability withdrawal, or lifecycle replacement discards the URL and denies access. +- `POST /v1/workspaces/:id/connections/desktop`: mint a current transient desktop URL. The request has no body. `expiresAt` is optional; when present it must be in the future and no more than 15 minutes away. Accepted HTTPS URLs are treated as opaque signed connection material and redirected byte-for-byte without URL normalization. After minting, Crabfleet re-reads the exact current session status, control grant, capabilities, and registered adapter identity before redirecting; a concurrent stop, revocation, capability withdrawal, or lifecycle replacement discards the URL and denies access. -`CRABBOX_RUNTIME_ADAPTER_NAMESPACE` is required and must remain stable for the deployment. It prevents workspace and idempotency collisions when an adapter serves more than one Crabfleet tenant. The adapter workspace `id` is an immutable lifecycle route key and remains separate from an opaque `providerResourceId`; the provider identity is never interpreted as a legacy lease or sandbox ID. Create, inspect, and stop responses must echo the byte-exact requested DNS-safe `id`; whitespace normalization is not accepted. Responses use `status`, `id`, optional `providerResourceId`, `attachUrl`, `capabilities`, `expiresAt`, and `message`. Only a literal `null` clears a previously stored expiry; a malformed non-null timestamp invalidates the response. A terminal URL implies terminal capability only when the response omits a terminal capability; an explicit `terminal: false` wins. Supported status values include `provisioning`, `ready`, `stopping`, `stopped`, `expired`, and `failed`. Create-only legacy adapters cannot return `stopping`, because they do not own a later reconciliation lifecycle. +`CRABBOX_RUNTIME_ADAPTER_NAMESPACE` is required and must remain stable for the deployment. It prevents workspace and idempotency collisions when an adapter serves more than one Crabfleet tenant. The adapter workspace `id` is an immutable lifecycle route key and remains separate from an opaque `providerResourceId`; the provider identity is never interpreted as a legacy lease or sandbox ID. Create, inspect, and stop responses must echo the byte-exact requested DNS-safe `id`; whitespace normalization is not accepted. Responses use `status`, `id`, optional `providerResourceId`, `attachUrl`, `capabilities`, `expiresAt`, and `message`. Only a literal `null` clears a previously stored expiry; a malformed non-null timestamp invalidates the response. A terminal URL implies terminal capability only when the response omits a terminal capability; an explicit `terminal: false` wins. Supported status values include `provisioning`, `ready`, `stopping`, `stopped`, `expired`, and `failed`. Create-only legacy adapters cannot return `stopping`, because they do not own a later reconciliation lifecycle. Every session-bound provider DELETE is gated on the persisted create ambiguity marker being clear. Every create, inspect, delete, and desktop response body is consumed through one 64 KiB bounded stream reader before JSON or text parsing. Declared or chunked oversized bodies are cancelled and fail safely: ambiguous create remains reconcilable, delete remains pending, inspect retries later, and desktop access is denied. Adapter messages are untrusted display text. Crabfleet removes raw and slash-escaped HTTP/WebSocket connection URLs directly from arbitrary message/detail text; Bearer and Basic credentials; authorization, cookie, and API-key headers; and sensitive assignments such as quoted JSON, colon fields, `token`, `ticket`, `access_token`, passwords, signatures, and secrets before replacing opaque provider identifiers and storing text in `lastEvent`, events, terminal failure evidence, or archives. For non-successful or malformed responses, opaque `providerResourceId`, `provider_resource_id`, `leaseId`, and `lease_id` values are collected from body, workspace, and error envelopes before redaction. Sanitizing credential structure first prevents an identifier such as `token` from hiding `token=secret`. -An adapter-reported `failed` workspace is not locally terminal until Crabfleet calls DELETE and confirms release. Crabfleet durably clears create ambiguity and records the requested failed terminal state and original failure reason before awaiting DELETE, so reconciliation cannot replay the create or replace useful failure evidence with a generic release message. Asynchronous or uncertain release remains `stopping`; reconciliation records `failed` only after the workspace is gone. A stop racing an ambiguous create also remains `stopping`: every reconciliation pass uses a dedicated replay path fenced to the exact `stopping` row, pending marker, registered control plane, immutable payload, settings, and session version, then issues DELETE before recording the requested terminal state. Generic provisioning cannot restage that row. After confirmed release, Crabfleet re-reads and compare-and-swaps the current ambiguity marker and terminal intent: a cleared marker terminalizes immediately, while a still-pending create remains `stopping`. A valid exact-ID create response, including `provisioning`, or a definitive create rejection resolves that ambiguity. +An adapter-reported `failed` workspace is not locally terminal until Crabfleet calls DELETE and confirms release. Crabfleet durably clears create ambiguity and records the requested failed terminal state and original failure reason before awaiting DELETE, so reconciliation cannot replay the create or replace useful failure evidence with a generic release message. Asynchronous or uncertain release remains `stopping`; reconciliation records `failed` only after the workspace is gone. A stop racing an ambiguous create also remains `stopping`: every reconciliation pass uses a dedicated replay path fenced to the exact `stopping` row, pending marker, registered control plane, immutable payload, settings, and session version, then issues DELETE before recording the requested terminal state. Generic provisioning cannot restage that row. After confirmed release, Crabfleet re-reads and compare-and-swaps the current ambiguity marker and terminal intent: a cleared marker terminalizes immediately, while a still-pending create remains `stopping`. Only a valid exact-ID successful replay, including `provisioning`, proves ownership and clears ambiguity. An explicit `workspace_id_conflict` proves non-ownership and atomically detaches the local lifecycle without DELETE; every other replay response leaves the marker pending. ### GET /api/terminal/ws @@ -454,7 +455,7 @@ Actions: - `revoke_control`: owner/maintainer, revoke active delegated control. - `enable_multiplayer`: session creator, prefix submitted terminal prompts with the actor. - `disable_multiplayer`: session creator, stop prefixing submitted terminal prompts with the actor. -- `stop`: owner/maintainer, stop the provider workspace first, then mark stopped; asynchronous releases remain `stopping` until reconciliation confirms completion. +- `stop`: owner/maintainer, internal wire action behind user-facing Delete, Stop, or End. Versioned adapters release the provider workspace before marking stopped, and asynchronous releases remain `stopping` until reconciliation confirms completion. Legacy create-only and ClawFleet sessions stop only in Crabfleet because those integrations expose no release lifecycle. For GitHub Actions, End disconnects and finalizes only the Crabfleet terminal session; it does not call GitHub's workflow-cancellation API, so the workflow run may continue. Response: diff --git a/docs/architecture.md b/docs/architecture.md index f991c06..9412291 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -67,7 +67,7 @@ Interactive sessions can use the versioned external lifecycle adapter configured - `DELETE /v1/workspaces/:id` releases the provider workspace before Crabfleet marks the session stopped. - `POST /v1/workspaces/:id/connections/desktop` mints a current desktop connection after Crabfleet authorization. -Active external sessions are reconciled with compare-and-swap updates so a stale inspect cannot overwrite a concurrent stop. The reconciliation claim and completion both fence the original session revision, and changed state commits a completion-time revision strictly newer than the snapshot; slow provider I/O therefore cannot regress `updated_at` or overwrite a concurrent same-status edit. Credential-cleanup completion and confirmed runtime release use the same ownership CAS and `MAX(updated_at + 1, now)` rule, so terminal archive versions cannot move backward. A durable create-ambiguity marker prevents a stop from becoming terminal while an idempotent request outcome is unknown; a parsed response clears the marker, while ambiguous outcomes replay the exact serialized original request and issue DELETE on every stopping pass until release is confirmed. Replay during `stopping` has its own exact-row path fenced by the pending marker, registered control plane, immutable payload/settings, requested terminal state, and session version; it does not pass through generic provision staging. Definitive create failures read the provider response once, redact it through the shared sanitizer, durably enter `stopping`, clear create ambiguity, and record failed terminal intent plus the actionable failure reason before the DELETE request. All adapter bodies use one 64 KiB bounded stream reader before parsing, including chunked responses, so oversized provider output cannot consume unbounded Worker memory or starve reconciliation. Redaction removes credential and URL structure before substituting opaque provider identifiers, preventing identifier text from masking a secret-bearing field. Redacted provider messages from successful, pending, and failed DELETE responses are persisted while release is pending and retained as events in the final archive. After confirmed release, Crabfleet re-reads and compare-and-swaps the current marker and terminal intent, preventing a pre-DELETE snapshot from either losing failed intent or leaving a resolved create stuck in `stopping`; the original reason remains in the terminal event, API state, and archive. Every terminal event insert and finalization-marker update share one D1 batch, including the finalizer's idempotent synthetic event; summary, sharing, multiplayer, and control mutations include their row update in that same batch. Legacy local stop records its request event, stopped event, terminal state, and finalization marker in one exact-owner D1 batch; cron and targeted reconciliation recover pre-existing or interrupted `stopping` rows. The winning terminal transition forces the final archive, R2 transcript, and summary; equal-count archive replacement is monotonic by mutable session version, while concurrent writers use unique object keys and delete only keys proven not to be the committed archive. Status-only inspection preserves omitted capability, expiry, and terminal fields, while explicit clears remain explicit, including explicit terminal-capability withdrawal despite a terminal URL. Raw terminal URLs, attachability, and UI terminal/SSH affordances are redacted from every outward session shape while terminal capability is false. Known signed connection URLs are also removed from JSON and common slash-escaped provider-message representations before persistence or display. Adapter failures enter provider release first and become locally failed only after release confirmation. State reads with an `ExecutionContext` have a short reconciliation budget and hand at most one three-workspace wave to the Worker background, keeping worst-case adapter timeouts inside the platform lifetime; callers without a context await completion. D1 stores the immutable adapter workspace ID and optional opaque provider resource ID separately, plus the immutable payload and TTL/idle/capability snapshot, create-ambiguity marker, current capabilities, expiry, desired terminal status, last-reconcile, and reconcile-error state. Provider resource IDs never enter legacy lease parsing. Authenticated adapter calls reject redirects. Versioned-adapter terminal and VNC bearer URLs are validated without normalization and accepted values retain their exact signed bytes. Terminal URLs remain server-side and API clients receive only the authenticated Worker PTY route; VNC URLs are never persisted, and browser, CLI, and SSH views receive an absolute canonical Crabfleet browser route which authenticates control before redirecting to the transient adapter URL. After desktop mint, Crabfleet re-reads the exact session status, control grant, capability, and adapter identity before releasing the transient redirect, so concurrent revocation discards the URL. Legacy adapters keep their existing absolute URL contract. +Active external sessions are reconciled with compare-and-swap updates so a stale inspect cannot overwrite a concurrent stop. The reconciliation claim and completion both fence the original session revision, and changed state commits a completion-time revision strictly newer than the snapshot; slow provider I/O therefore cannot regress `updated_at` or overwrite a concurrent same-status edit. Credential-cleanup completion and confirmed runtime release use the same ownership CAS and `MAX(updated_at + 1, now)` rule, so terminal archive versions cannot move backward. A durable create-ambiguity marker prevents a stop from becoming terminal while an idempotent request outcome is unknown; ambiguous outcomes replay the exact serialized original request, and every session-bound DELETE remains blocked until that replay resolves create ownership. Replay during `stopping` has its own exact-row path fenced by the pending marker, registered control plane, immutable payload/settings, requested terminal state, and session version; it does not pass through generic provision staging. Definitive failures of the initial create read the provider response once, redact it through the shared sanitizer, durably enter `stopping`, clear create ambiguity, and record failed terminal intent plus the actionable failure reason before the DELETE request. Once any create result is ambiguous, only an exact-ID successful replay proves ownership and clears the marker; an explicit workspace-ID conflict proves non-ownership and detaches locally without DELETE, while authentication, routing, validation, and all other replay failures remain pending. All adapter bodies use one 64 KiB bounded stream reader before parsing, including chunked responses, so oversized provider output cannot consume unbounded Worker memory or starve reconciliation. Redaction removes credential and URL structure before substituting opaque provider identifiers, preventing identifier text from masking a secret-bearing field. Redacted provider messages from successful, pending, and failed DELETE responses are persisted while release is pending and retained as events in the final archive. After confirmed release, Crabfleet re-reads and compare-and-swaps the current marker and terminal intent, preventing a pre-DELETE snapshot from either losing failed intent or leaving a resolved create stuck in `stopping`; the original reason remains in the terminal event, API state, and archive. Every terminal event insert and finalization-marker update share one D1 batch, including the finalizer's idempotent synthetic event; summary, sharing, multiplayer, and control mutations include their row update in that same batch. Legacy local stop records its request event, stopped event, terminal state, and finalization marker in one exact-owner D1 batch; cron and targeted reconciliation recover pre-existing or interrupted `stopping` rows. The winning terminal transition forces the final archive, R2 transcript, and summary; equal-count archive replacement is monotonic by mutable session version, while concurrent writers use unique object keys and delete only keys proven not to be the committed archive. Status-only inspection preserves omitted capability, expiry, and terminal fields, while explicit clears remain explicit, including explicit terminal-capability withdrawal despite a terminal URL. Raw terminal URLs, attachability, and UI terminal/SSH affordances are redacted from every outward session shape while terminal capability is false. Known signed connection URLs are also removed from JSON and common slash-escaped provider-message representations before persistence or display. Adapter failures enter provider release first and become locally failed only after release confirmation. State reads with an `ExecutionContext` have a short reconciliation budget and hand at most one three-workspace wave to the Worker background, keeping worst-case adapter timeouts inside the platform lifetime; callers without a context await completion. D1 stores the immutable adapter workspace ID and optional opaque provider resource ID separately, plus the immutable payload and TTL/idle/capability snapshot, create-ambiguity marker, current capabilities, expiry, desired terminal status, last-reconcile, and reconcile-error state. Provider resource IDs never enter legacy lease parsing. Authenticated adapter calls reject redirects. Versioned-adapter terminal and VNC bearer URLs are validated without normalization and accepted values retain their exact signed bytes. Terminal URLs remain server-side and API clients receive only the authenticated Worker PTY route; VNC URLs are never persisted, and browser, CLI, and SSH views receive an absolute canonical Crabfleet browser route which authenticates control before redirecting to the transient adapter URL. After desktop mint, Crabfleet re-reads the exact session status, control grant, capability, and adapter identity before releasing the transient redirect, so concurrent revocation discards the URL. Legacy adapters keep their existing absolute URL contract. The versioned adapter is reachable only through the durable interactive-session lifecycle, never the stateless provision hook. Existing lifecycle operations resolve through the persisted canonical control-plane identity and fail closed if the live deployment binding differs or disappears. An explicit stop that loses its initial compare-and-swap succeeds only after rereading the exact workspace in `stopping` or terminal state; a concurrent active mutation instead returns conflict. Terminal attach and recurring socket grants require the adapter's current `terminal` capability; withdrawing it closes active sockets. Direct adapter `attachUrl` values stay server-side and receive the configured adapter bearer only when their origin matches both the persisted and currently configured control plane, so reusable shell credentials never enter arbitrary origins, URLs, or the browser; only Crabfleet-owned bridge and runner endpoints receive `cols` and `rows` query parameters. A PTY transport failure leaves the lifecycle workspace detached and retryable rather than terminalizing it without provider release. diff --git a/docs/github-actions-sessions.md b/docs/github-actions-sessions.md index bdae9da..ee6fb73 100644 --- a/docs/github-actions-sessions.md +++ b/docs/github-actions-sessions.md @@ -203,7 +203,7 @@ ClawSweeper include: - `plan_complete` - `gates_passed` - `action_failed` -- `stopped from Crabfleet` +- `workflow_canceled` CrabFleet records the state transition as a session event and exposes the latest state in Fleet, Sessions, API, CLI, and logs. @@ -346,28 +346,34 @@ For ClawSweeper: - `completed / done / gates_passed` means repair and all configured deterministic gates passed. - `blocked / action_failed` means required workflow gates did not complete. -- `canceled / stopped from Crabfleet` means an authorized operator canceled the - session. +- A user-ended Crabfleet terminal session does not claim a terminal workflow + state. The GitHub run remains authoritative and may continue. CrabFleet completion is status evidence, not GitHub mutation authority. The ClawSweeper result ledger and target repository state describe what was actually changed. -## Cancellation +## Ending the Crabfleet terminal session -GitHub Actions sessions use a dedicated cancel lifecycle. +GitHub Actions sessions use a dedicated terminal-session end lifecycle. This +does not call GitHub's workflow-cancellation API. -An authorized stop: +An authorized End action: -1. Atomically appends the cancellation event and updates the session. +1. Atomically appends the terminal-session event and updates the session. 2. Sets `status = stopped`. -3. Sets `workState = canceled`. -4. Sets `workPhase = canceled`. -5. Records `completionReason = stopped from Crabfleet`. +3. Clears Crabfleet's synthetic work state instead of claiming the workflow was + canceled. +4. Sets `workPhase = session_ended`. +5. Records that the Crabfleet terminal ended without canceling the workflow. 6. Clears the agent token, attach URL, and control state. 7. Disconnects the current runner. 8. Archives and finalizes terminal logs. +The browser, CLI, and SSH surfaces warn that the GitHub Actions workflow run +may continue. Cancel the run in GitHub when provider-side cancellation is +required. + `github_actions` sessions are excluded from the legacy workspace-stop reconciler. They do not have a provider workspace lease for that reconciler to release. diff --git a/docs/index.md b/docs/index.md index bd719b1..d91f2e1 100644 --- a/docs/index.md +++ b/docs/index.md @@ -23,6 +23,7 @@ ssh crabd.sh attach # Or use the Go CLI. crabfleet new --repo openclaw/crabfleet "fix the failing check" +crabfleet delete crabfleet vnc ``` @@ -31,7 +32,7 @@ The web app at [crabfleet.openclaw.ai/app](https://crabfleet.openclaw.ai/app/) e ## What Crabfleet Does - **SSH-first onboarding.** Connect through `ssh link@crabd.sh`, complete GitHub sign-in, then use linked-key auth. -- **Crabbox control.** Create, attach, share, open WebVNC, and clean up interactive Codex sessions backed by Ghostty WASM tiles. +- **Crabbox control.** Create, attach, share, open WebVNC, delete provider-backed runtime workspaces, stop legacy sessions locally, and clean up retained Codex session history. - **Fleet visibility.** The app groups all org Codex instances by person so OpenClaw can supervise live work. - **Repo-gated cards.** Prompt cards and GitHub issue/PR previews stay scoped to enabled OpenClaw repos. - **Runtime policy.** Crabfleet records runtime selection, capabilities, heartbeat, stall state, and operator intent. diff --git a/docs/quickstart.md b/docs/quickstart.md index e40d975..e9e70d6 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -82,6 +82,8 @@ crabfleet new --repo openclaw/crabfleet "fix the failing check" The CLI omits `runtime` unless `--runtime` is passed, so the deployment chooses via `CRABFLEET_DEFAULT_RUNTIME` (`container` when unset). Set a deployment to `crabbox` when its configured adapter provides terminal and WebVNC capabilities. +End a session with `crabfleet delete `. Versioned lifecycle adapters confirm runtime release; legacy create-only and ClawFleet sessions stop only in Crabfleet and may require separate provider cleanup. Crabfleet retains the final status and logs until you clean up the dead session record. + ## 6. Create a Card Click New card. diff --git a/src/app/fleet.jsx b/src/app/fleet.jsx index 8816895..5aae0dc 100644 --- a/src/app/fleet.jsx +++ b/src/app/fleet.jsx @@ -1,5 +1,6 @@ import { CopyCommand } from "./components.jsx"; import { + canDeleteInteractiveWorkspace, canMaintain, elapsed, humanStatus, @@ -121,6 +122,8 @@ export function FleetPage(props) { key={session.id} session={{ ...session, fleet: fleetSessionsById.get(session.id) }} openSessionGrid={props.openSessionGrid} + deleteInteractiveSession={props.deleteInteractiveSession} + canManage={session.canManage || canMaintain(props.state.user)} sshHost={sshHost} /> ))} @@ -263,7 +266,7 @@ function ConnectionDeck({ signedIn, userLabel, beginLogin, sshHost, preferredRep ); } -function FleetBox({ session, openSessionGrid, sshHost }) { +function FleetBox({ session, openSessionGrid, deleteInteractiveSession, canManage, sshHost }) { const capabilities = runCapabilities(session); const attachable = isFleetSessionAttachable(session); const archiveCount = session.logArchive?.eventCount || session.logs?.length || 0; @@ -271,6 +274,23 @@ function FleetBox({ session, openSessionGrid, sshHost }) { const status = interactiveSessionStatus(session); const seen = session.lastSeenAt || session.updatedAt || session.createdAt; const desktopEligible = !["stopping", "stopped", "expired", "failed"].includes(session.status); + const ending = session.status === "stopping"; + const deletesWorkspace = canDeleteInteractiveWorkspace(session); + const endsWorkflowSession = session.runtime === "github_actions"; + const endLabel = ending + ? deletesWorkspace + ? "Deleting…" + : endsWorkflowSession + ? "Ending…" + : "Stopping…" + : deletesWorkspace + ? "Delete" + : endsWorkflowSession + ? "End" + : "Stop"; + const actionable = + !String(session.id).startsWith("LOCAL-") && + !["stopping", "stopped", "expired", "failed", "unavailable"].includes(session.status); return (
@@ -333,6 +353,15 @@ function FleetBox({ session, openSessionGrid, sshHost }) { Logs ) : null} + {canManage && actionable ? ( + + ) : canManage && ending ? ( + + ) : null}
); diff --git a/src/app/main.jsx b/src/app/main.jsx index 6c920f1..053b604 100644 --- a/src/app/main.jsx +++ b/src/app/main.jsx @@ -4,6 +4,7 @@ import { api } from "./api.js"; import { CopyCommand, Icon } from "./components.jsx"; import { FleetPage } from "./fleet.jsx"; import { + canDeleteInteractiveWorkspace, canMaintain, canOwn, elapsed, @@ -654,16 +655,34 @@ function App() { return result; } - function closeInteractiveSession(id) { + function deleteInteractiveSession(id) { const session = findInteractiveSession(id); const label = session ? `${session.repo} (${session.id})` : id; + const deletesWorkspace = canDeleteInteractiveWorkspace(session); + const endsWorkflowSession = session?.runtime === "github_actions"; openActionDialog({ kind: "danger", - eyebrow: "Live session", - title: "End Codex session?", - description: "This stops the terminal. Its final status and logs stay visible in Crabfleet.", + eyebrow: deletesWorkspace + ? "Live workspace" + : endsWorkflowSession + ? "Live workflow terminal" + : "Live session", + title: deletesWorkspace + ? "Delete Crabbox workspace?" + : endsWorkflowSession + ? "End GitHub Actions terminal session?" + : "Stop Crabbox session?", + description: deletesWorkspace + ? "This releases the runtime workspace and cannot be undone. Its final status and logs stay visible in Crabfleet." + : endsWorkflowSession + ? "This ends the Crabfleet terminal session and disconnects it. It does not cancel the GitHub Actions workflow run, which may continue on GitHub. Final Crabfleet logs stay visible." + : "This stops Crabfleet access and marks the session stopped. This legacy backend does not expose provider deletion, so its runtime may require separate cleanup.", subject: label, - confirmLabel: "End session", + confirmLabel: deletesWorkspace + ? "Delete workspace" + : endsWorkflowSession + ? "End session" + : "Stop session", action: () => interactiveSessionAction(id, "stop"), }); } @@ -944,7 +963,7 @@ function App() { cardAction, attachCard, interactiveSessionAction, - closeInteractiveSession, + deleteInteractiveSession, cleanupInteractiveSession, cleanupDeadInteractiveSessions, shareInteractiveSession, @@ -2061,6 +2080,22 @@ function InteractiveSessionActions(props) { const session = props.session; if (String(session.id).startsWith("LOCAL-")) return null; const stopped = isDeadInteractiveSession(session); + const ending = session.status === "stopping"; + const deletesWorkspace = canDeleteInteractiveWorkspace(session); + const endsWorkflowSession = session.runtime === "github_actions"; + const endLabel = stopped + ? "Clean up" + : ending + ? deletesWorkspace + ? "Deleting…" + : endsWorkflowSession + ? "Ending…" + : "Stopping…" + : deletesWorkspace + ? "Delete" + : endsWorkflowSession + ? "End" + : "Stop"; const canManage = session.canManage || canMaintain(props.state.user); const canChangeMultiplayer = Boolean(session.canChangeMultiplayer); const shareAction = session.shareMode === "link_read" ? "disable_share" : "share_link"; @@ -2097,13 +2132,14 @@ function InteractiveSessionActions(props) { {canManage ? ( ) : null} @@ -2155,13 +2191,14 @@ function InteractiveSessionActions(props) { {canManage ? ( ) : null} diff --git a/src/app/utils.js b/src/app/utils.js index fa6e682..278b86f 100644 --- a/src/app/utils.js +++ b/src/app/utils.js @@ -128,6 +128,10 @@ export function isDeadInteractiveSession(session) { ); } +export function canDeleteInteractiveWorkspace(session) { + return session?.adapter === "runtime-v1"; +} + export function isTerminalReadyInteractiveSession(session) { const lifecycleReady = session && @@ -170,7 +174,12 @@ export function interactiveSessionStatus(session) { return { label: humanStatus(session.workPhase), tone: "live" }; } if (session.status === "failed") return { label: "Failed", tone: "failed" }; - if (session.status === "stopping") return { label: "Stopping", tone: "provisioning" }; + if (session.status === "stopping") { + return { + label: canDeleteInteractiveWorkspace(session) ? "Deleting" : "Stopping", + tone: "provisioning", + }; + } if (session.status === "stopped" || session.status === "expired") { return { label: "Stopped", tone: "stopped" }; } diff --git a/src/index.ts b/src/index.ts index 432347d..8136d68 100644 --- a/src/index.ts +++ b/src/index.ts @@ -112,6 +112,7 @@ import { runtimeAdapterStopOutcome, runtimeAdapterTerminalFailureStatus, runtimeAdapterTerminalOriginMatches, + runtimeAdapterWorkspaceIdConflict, runtimeAdapterWorkspaceUrl, resolveCreateAfterStopRace, safeDesktopUrl, @@ -3680,7 +3681,7 @@ async function reconcileExternalInteractiveSession( return; } if (row.adapter !== runtimeAdapterName || !row.adapter_workspace_id) return; - const inspected = await inspectRuntimeAdapterWorkspace(env, row); + const inspected = await inspectRuntimeAdapterWorkspace(env, row, claimAt); const completedAt = Math.max(Date.now(), claimAt); const completionVersion = Math.max(completedAt, row.updated_at + 1); const requestedTerminalStatus = @@ -4899,7 +4900,7 @@ async function stopGitHubActionsSession( now: number, ): Promise { const revision = Math.max(now, session.updatedAt + 1); - const message = "GitHub Actions session canceled from Crabfleet"; + const message = "GitHub Actions terminal session ended from Crabfleet; workflow run not canceled"; const db = database(env); const expectedOwner = sql` id = ${session.id} @@ -4930,9 +4931,9 @@ async function stopGitHubActionsSession( control_requested_at: null, control_granted_at: null, control_expires_at: null, - work_state: "canceled", - work_phase: "canceled", - completion_reason: "stopped from Crabfleet", + work_state: "", + work_phase: "session_ended", + completion_reason: "Crabfleet terminal session ended; workflow run not canceled", last_event: message, updated_at: revision, }) @@ -8150,7 +8151,6 @@ async function interactiveSessionVnc(env: RuntimeEnv, user: User, id: string): P runtimeAdapterDesktopUrl(controlPlane, session.adapterWorkspaceId), { method: "POST", - body: JSON.stringify({}), }, ); responseBody = await readRuntimeAdapterResponseBody(response); @@ -11164,7 +11164,9 @@ async function provisionWithRuntimeAdapter( env: RuntimeEnv, session: InteractiveProvisionRequest, _agentToken?: string, + reconciliationOwner?: RuntimeAdapterCreateAttemptFence, ): Promise { + const replayingPendingCreate = reconciliationOwner !== undefined; const namespace = normalizeAdapterNamespace(env.CRABBOX_RUNTIME_ADAPTER_NAMESPACE ?? ""); const adapterWorkspaceId = session.adapterWorkspaceId ? normalizeAdapterWorkspaceId(session.adapterWorkspaceId) === session.adapterWorkspaceId @@ -11174,6 +11176,9 @@ async function provisionWithRuntimeAdapter( ? namespacedAdapterWorkspaceId(namespace, session.id) : null; if (!adapterWorkspaceId) { + if (replayingPendingCreate) { + throw new Error("runtime adapter create replay blocked: persisted workspace id is invalid"); + } return failedProvision( "runtime adapter provision failed: persisted workspace id or valid namespace is required", ); @@ -11195,6 +11200,14 @@ async function provisionWithRuntimeAdapter( const ttlSeconds = persistedRuntimeAdapterSeconds(session.adapterTtlSeconds); const idleTimeoutSeconds = persistedRuntimeAdapterSeconds(session.adapterIdleTimeoutSeconds); if (!requestedCapabilities || !ttlSeconds || !idleTimeoutSeconds) { + if (replayingPendingCreate) { + return ambiguousRuntimeAdapterProvision( + session, + adapterWorkspaceId, + requestedCapabilities ?? fallbackCapabilities, + "runtime adapter create replay blocked: persisted create settings are incomplete", + ); + } return releaseFailedRuntimeAdapterProvision( env, session.id, @@ -11240,6 +11253,14 @@ async function provisionWithRuntimeAdapter( }, ); if (!createPayloadJson) { + if (replayingPendingCreate) { + return ambiguousRuntimeAdapterProvision( + session, + adapterWorkspaceId, + requestedCapabilities, + "runtime adapter create replay blocked: persisted create payload is invalid", + ); + } return releaseFailedRuntimeAdapterProvision( env, session.id, @@ -11251,18 +11272,18 @@ async function provisionWithRuntimeAdapter( ), ); } - if ( - !(await stageRuntimeAdapterProvision( - env, - session, - baseUrl, - adapterWorkspaceId, - requestedCapabilities, - ttlSeconds, - idleTimeoutSeconds, - createPayloadJson, - )) - ) { + const createAttempt = await stageRuntimeAdapterProvision( + env, + session, + baseUrl, + adapterWorkspaceId, + requestedCapabilities, + ttlSeconds, + idleTimeoutSeconds, + createPayloadJson, + reconciliationOwner, + ); + if (!createAttempt) { return unresolvedRuntimeAdapterProvision( session, adapterWorkspaceId, @@ -11302,7 +11323,21 @@ async function provisionWithRuntimeAdapter( `HTTP ${response.status}`, [adapterWorkspaceId], ); - if (definitiveRuntimeAdapterCreateFailure(response.status)) { + if (runtimeAdapterWorkspaceIdConflict(response.status, responseBody)) { + const conflictResult = await failRuntimeAdapterWorkspaceIdConflict( + env, + session, + baseUrl, + adapterWorkspaceId, + createPayloadJson, + requestedCapabilities, + createAttempt, + `runtime adapter provision failed: ${responseMessage}`, + ); + if (conflictResult) return conflictResult; + throw conflict("runtime adapter workspace conflict response is stale"); + } + if (!replayingPendingCreate && definitiveRuntimeAdapterCreateFailure(response.status)) { return releaseFailedRuntimeAdapterProvision( env, session.id, @@ -11314,11 +11349,14 @@ async function provisionWithRuntimeAdapter( ), ); } + const messagePrefix = replayingPendingCreate + ? "runtime adapter create replay pending" + : "runtime adapter create outcome unknown"; return ambiguousRuntimeAdapterProvision( session, adapterWorkspaceId, requestedCapabilities, - `runtime adapter create outcome unknown: ${responseMessage}`, + `${messagePrefix}: ${responseMessage}`, ); } const parsed = parseAdapterWorkspaceResult(responseBody, { @@ -11357,6 +11395,13 @@ function persistedRuntimeAdapterSeconds(value: number | null | undefined): numbe return typeof value === "number" && Number.isInteger(value) && value > 0 ? value : null; } +type RuntimeAdapterCreateAttemptFence = { + status: InteractiveSessionStatus; + updatedAt: number; + lastReconciledAt: number | null; + terminalStatus: "failed" | null; +}; + async function stageRuntimeAdapterProvision( env: RuntimeEnv, session: InteractiveProvisionRequest, @@ -11366,8 +11411,10 @@ async function stageRuntimeAdapterProvision( ttlSeconds: number, idleTimeoutSeconds: number, createPayloadJson: string, -): Promise { - const staged = await database(env) + reconciliationOwner?: RuntimeAdapterCreateAttemptFence, +): Promise { + const stageAt = Date.now(); + let stage = database(env) .updateTable("interactive_sessions") .set({ adapter: runtimeAdapterName, @@ -11380,12 +11427,37 @@ async function stageRuntimeAdapterProvision( adapter_create_payload_json: createPayloadJson, adapter_create_pending: 1, reconcile_error: "runtime adapter create pending", + ...(reconciliationOwner + ? {} + : { updated_at: sql`MAX(updated_at + 1, ${stageAt})` }), }) .where("id", "=", session.id) .where("adapter_control_plane", "=", adapterControlPlane) - .where("status", "in", ["provisioning", "pending_adapter"]) + .where("adapter_workspace_id", "=", adapterWorkspaceId); + if (reconciliationOwner) { + stage = stage + .where("status", "=", reconciliationOwner.status) + .where("updated_at", "=", reconciliationOwner.updatedAt); + stage = reconciliationOwner.lastReconciledAt === null + ? stage.where("last_reconciled_at", "is", null) + : stage.where("last_reconciled_at", "=", reconciliationOwner.lastReconciledAt); + stage = reconciliationOwner.terminalStatus === null + ? stage.where("terminal_status", "is", null) + : stage.where("terminal_status", "=", reconciliationOwner.terminalStatus); + } else { + stage = stage.where("status", "in", ["provisioning", "pending_adapter"]); + } + const staged = await stage + .returning(["status", "updated_at", "last_reconciled_at", "terminal_status"]) .executeTakeFirst(); - return (staged.numUpdatedRows ?? 0n) > 0n; + return staged + ? { + status: staged.status, + updatedAt: staged.updated_at, + lastReconciledAt: staged.last_reconciled_at, + terminalStatus: staged.terminal_status, + } + : null; } function ambiguousRuntimeAdapterProvision( @@ -11446,6 +11518,109 @@ function unresolvedRuntimeAdapterProvision( }; } +async function failRuntimeAdapterWorkspaceIdConflict( + env: RuntimeEnv, + session: Pick, + adapterControlPlane: string, + adapterWorkspaceId: string, + createPayloadJson: string, + capabilities: RuntimeCapabilities, + createAttempt: RuntimeAdapterCreateAttemptFence, + message: string, +): Promise { + const now = Date.now(); + const failureMessage = clean(message, 500); + const lastReconciledOwner = createAttempt.lastReconciledAt === null + ? sql`last_reconciled_at IS NULL` + : sql`last_reconciled_at = ${createAttempt.lastReconciledAt}`; + const terminalStatusOwner = createAttempt.terminalStatus === null + ? sql`terminal_status IS NULL` + : sql`terminal_status = ${createAttempt.terminalStatus}`; + const expectedOwner = sql` + id = ${session.id} + AND adapter = ${runtimeAdapterName} + AND adapter_workspace_id = ${adapterWorkspaceId} + AND adapter_control_plane = ${adapterControlPlane} + AND adapter_create_payload_json = ${createPayloadJson} + AND adapter_requested_capabilities_json = ${JSON.stringify(capabilities)} + AND adapter_create_pending = 1 + AND status = ${createAttempt.status} + AND updated_at = ${createAttempt.updatedAt} + AND ${lastReconciledOwner} + AND ${terminalStatusOwner} + `; + const db = database(env); + const event = sql` + INSERT INTO interactive_session_events (session_id, actor, message, created_at) + SELECT ${session.id}, 'system', ${failureMessage}, ${now} + FROM interactive_sessions + WHERE ${expectedOwner} + `; + const detach = db + .updateTable("interactive_sessions") + .set({ + status: "failed", + adapter: null, + adapter_workspace_id: null, + adapter_control_plane: null, + provider_resource_id: null, + adapter_ttl_seconds: null, + adapter_idle_timeout_seconds: null, + adapter_requested_capabilities_json: null, + adapter_create_payload_json: null, + adapter_create_pending: 0, + lease_id: null, + attach_url: null, + vnc_url: null, + expires_at: null, + last_reconciled_at: now, + reconcile_error: failureMessage, + terminal_status: null, + terminal_failure_reason: failureMessage, + terminal_finalize_pending: 1, + stopped_at: now, + agent_token_hash: null, + controller: null, + control_requested_by: null, + control_requested_at: null, + control_granted_at: null, + control_expires_at: null, + last_event: failureMessage, + updated_at: sql`MAX(updated_at + 1, ${now})`, + }) + .where(expectedOwner) + .returning("updated_at"); + const results = await env.DB.batch<{ updated_at: number }>( + [event, detach].map((query) => { + const compiled = query.compile(db); + return env.DB.prepare(compiled.sql).bind(...compiled.parameters); + }), + ); + if (!results.at(-1)?.results.length) return null; + await archiveInteractiveSessionLogs(env, session.id, now).catch(() => undefined); + await finalizeTerminalInteractiveSession(env, session.id, "failed", now).catch(() => undefined); + return { + status: "failed", + leaseId: null, + attachUrl: null, + attachUrlPresent: true, + vncUrl: null, + message: failureMessage, + adapter: null, + profile: session.profile, + adapterWorkspaceId: null, + providerResourceId: null, + capabilities, + capabilitiesPresent: true, + expiresAt: null, + expiresAtPresent: true, + reconciledAt: now, + reconcileError: failureMessage, + terminalStatus: null, + createPending: false, + }; +} + async function releaseFailedRuntimeAdapterProvision( env: RuntimeEnv, sessionId: string, @@ -11658,6 +11833,7 @@ function runtimeAdapterRecord(session: InteractiveSessionRow): AdapterProvisionR async function inspectRuntimeAdapterWorkspace( env: RuntimeEnv, session: InteractiveSessionRow, + reconciliationClaimAt: number, ): Promise { const adapterWorkspaceId = session.adapter_workspace_id; const providerResourceId = session.provider_resource_id; @@ -11669,7 +11845,20 @@ async function inspectRuntimeAdapterWorkspace( session.adapter_control_plane, ); if (session.status === "stopping") { - return reconcileStoppingRuntimeAdapterWorkspace(env, session); + return reconcileStoppingRuntimeAdapterWorkspace(env, session, reconciliationClaimAt); + } + if (shouldReplayRuntimeAdapterCreate(session.status, session.adapter_create_pending === 1)) { + return provisionWithRuntimeAdapter( + env, + runtimeAdapterReplayRequest(runtimeAdapterRecord(session)), + undefined, + { + status: session.status, + updatedAt: session.updated_at, + lastReconciledAt: reconciliationClaimAt, + terminalStatus: session.terminal_status, + }, + ); } const response = await runtimeAdapterFetch( env, @@ -11678,12 +11867,6 @@ async function inspectRuntimeAdapterWorkspace( ); const responseBody = await readRuntimeAdapterResponseBody(response); if (response.status === 404) { - if (shouldReplayRuntimeAdapterCreate(session.status, session.adapter_create_pending === 1)) { - return provisionWithRuntimeAdapter( - env, - runtimeAdapterReplayRequest(runtimeAdapterRecord(session)), - ); - } return { status: "expired", leaseId: null, @@ -11733,14 +11916,38 @@ async function inspectRuntimeAdapterWorkspace( async function reconcileStoppingRuntimeAdapterWorkspace( env: RuntimeEnv, session: InteractiveSessionRow, + reconciliationClaimAt: number, ): Promise { const adapterWorkspaceId = session.adapter_workspace_id; if (!adapterWorkspaceId) throw new Error("runtime adapter workspace reference is incomplete"); let replayMessage: string | null = null; if (session.adapter_create_pending === 1) { - const replay = await replayStoppingRuntimeAdapterCreate(env, session); + const replay = await replayStoppingRuntimeAdapterCreate( + env, + session, + reconciliationClaimAt, + ); replayMessage = replay.message; + if (replay.terminalResult) return replay.terminalResult; + if (!replay.resolved) { + return { + status: "stopping", + leaseId: null, + attachUrl: null, + attachUrlPresent: true, + vncUrl: null, + message: replay.message, + adapter: runtimeAdapterName, + profile: session.profile, + adapterWorkspaceId, + providerResourceId: session.provider_resource_id, + reconciledAt: Date.now(), + reconcileError: replay.message, + terminalStatus: session.terminal_status, + createPending: true, + }; + } } let release: RuntimeAdapterStopResult; @@ -11815,11 +12022,13 @@ async function reconcileStoppingRuntimeAdapterWorkspace( type StoppingRuntimeAdapterReplay = { message: string; resolved: boolean; + terminalResult?: InteractiveProvisionResult; }; async function replayStoppingRuntimeAdapterCreate( env: RuntimeEnv, session: InteractiveSessionRow, + reconciliationClaimAt: number, ): Promise { const adapterWorkspaceId = session.adapter_workspace_id; const replay = runtimeAdapterReplayRequest(runtimeAdapterRecord(session)); @@ -11875,7 +12084,8 @@ async function replayStoppingRuntimeAdapterCreate( .where("adapter_idle_timeout_seconds", "=", idleTimeoutSeconds) .where("adapter_create_pending", "=", 1) .where("status", "=", "stopping") - .where("updated_at", "=", session.updated_at); + .where("updated_at", "=", session.updated_at) + .where("last_reconciled_at", "=", reconciliationClaimAt); ownership = session.terminal_status ? ownership.where("terminal_status", "=", session.terminal_status) : ownership.where("terminal_status", "is", null); @@ -11909,33 +12119,52 @@ async function replayStoppingRuntimeAdapterCreate( resolved: false, }; } - let message: string; if (!response.ok) { const responseMessage = redactedAdapterResponseMessage( responseBody, `HTTP ${response.status}`, [adapterWorkspaceId], ); - if (!definitiveRuntimeAdapterCreateFailure(response.status)) { - return { - message: `runtime adapter create replay pending: ${responseMessage}`, - resolved: false, - }; - } - message = `runtime adapter create replay resolved: ${responseMessage}`; - } else { - const parsed = parseAdapterWorkspaceResult(responseBody, { - workspaceId: adapterWorkspaceId, - profile: session.profile, - }); - if (!parsed || !adapterWorkspaceIdMatches(parsed, adapterWorkspaceId)) { - return { - message: "runtime adapter create replay pending: invalid workspace identity", - resolved: false, - }; + if (runtimeAdapterWorkspaceIdConflict(response.status, responseBody)) { + const terminalResult = await failRuntimeAdapterWorkspaceIdConflict( + env, + session, + controlPlane, + adapterWorkspaceId, + createPayloadJson, + requestedCapabilities, + { + status: "stopping", + updatedAt: session.updated_at, + lastReconciledAt: reconciliationClaimAt, + terminalStatus: session.terminal_status, + }, + `runtime adapter create replay failed: ${responseMessage}`, + ); + if (!terminalResult) { + return { + message: "runtime adapter create replay deferred: conflict response is stale", + resolved: false, + }; + } + return { message: terminalResult.message, resolved: true, terminalResult }; } - message = `runtime adapter create replay resolved: ${parsed.status}`; + return { + message: `runtime adapter create replay pending: ${responseMessage}`, + resolved: false, + }; } + const parsed = parseAdapterWorkspaceResult(responseBody, { + workspaceId: adapterWorkspaceId, + profile: session.profile, + }); + if (!parsed || !adapterWorkspaceIdMatches(parsed, adapterWorkspaceId)) { + return { + message: "runtime adapter create replay pending: invalid workspace identity", + resolved: false, + }; + } + const message = `runtime adapter create replay resolved: ${parsed.status}`; const resolvedAt = Date.now(); const terminalStatusOwner = session.terminal_status @@ -11953,6 +12182,7 @@ async function replayStoppingRuntimeAdapterCreate( AND adapter_create_pending = 1 AND status = 'stopping' AND updated_at = ${session.updated_at} + AND last_reconciled_at = ${reconciliationClaimAt} AND ${terminalStatusOwner} `; const db = database(env); @@ -12029,11 +12259,23 @@ async function stopRuntimeAdapterWorkspaceForSession( sessionId: string, adapterWorkspaceId: string, ): Promise { - const controlPlane = await registeredRuntimeAdapterControlPlaneForSession( + const registration = await database(env) + .selectFrom("interactive_sessions") + .select(["adapter_control_plane", "adapter_create_pending"]) + .where("id", "=", sessionId) + .where("adapter", "=", runtimeAdapterName) + .where("adapter_workspace_id", "=", adapterWorkspaceId) + .executeTakeFirst(); + const controlPlane = requireRegisteredRuntimeAdapterControlPlane( env, - sessionId, - adapterWorkspaceId, + registration?.adapter_control_plane, ); + if (registration?.adapter_create_pending !== 0) { + return { + status: "stopping", + message: "runtime adapter stop waiting for create resolution", + }; + } return stopRuntimeAdapterWorkspace(env, controlPlane, adapterWorkspaceId); } diff --git a/src/runtime-adapter.ts b/src/runtime-adapter.ts index eb3f529..daf20e9 100644 --- a/src/runtime-adapter.ts +++ b/src/runtime-adapter.ts @@ -373,6 +373,13 @@ export function definitiveRuntimeAdapterCreateFailure(status: number): boolean { return status >= 400 && status < 500 && ![408, 409, 423, 425, 429].includes(status); } +export function runtimeAdapterWorkspaceIdConflict(status: number, value: unknown): boolean { + if (status !== 409) return false; + const body = objectValue(value); + const error = objectValue(body.error); + return error.code === "workspace_id_conflict"; +} + export function effectiveAdapterCapabilities( result: Pick< AdapterWorkspaceResult, diff --git a/tests/app-utils.test.ts b/tests/app-utils.test.ts index e977a11..e6f525f 100644 --- a/tests/app-utils.test.ts +++ b/tests/app-utils.test.ts @@ -1,6 +1,7 @@ import assert from "node:assert/strict"; import { test } from "node:test"; import { + canDeleteInteractiveWorkspace, humanStatus, isActiveRun, isDeadInteractiveSession, @@ -135,6 +136,7 @@ test("interactive lifecycle helpers keep UI and terminal state aligned", () => { }; const provisioning = { kind: "interactive", status: "pending_adapter" }; const stopping = { kind: "interactive", status: "stopping" }; + const deleting = { ...stopping, adapter: "runtime-v1" }; const failed = { kind: "interactive", status: "failed" }; assert.equal(isActiveRun(live), true); @@ -174,6 +176,12 @@ test("interactive lifecycle helpers keep UI and terminal state aligned", () => { label: "Stopping", tone: "provisioning", }); + assert.deepEqual(interactiveSessionStatus(deleting), { + label: "Deleting", + tone: "provisioning", + }); + assert.equal(canDeleteInteractiveWorkspace(deleting), true); + assert.equal(canDeleteInteractiveWorkspace({ ...stopping, leaseId: "clawfleet:legacy" }), false); assert.equal(isDeadInteractiveSession(failed), true); assert.deepEqual(interactiveSessionStatus(failed), { label: "Failed", tone: "failed" }); assert.equal(humanStatus("pending_adapter"), "Pending Adapter"); diff --git a/tests/html-dialogs.test.ts b/tests/html-dialogs.test.ts index de2fd91..e5deaa7 100644 --- a/tests/html-dialogs.test.ts +++ b/tests/html-dialogs.test.ts @@ -23,3 +23,23 @@ test("fleet terminal affordances require attachable session state", async () => assert.match(source, /\{attachable \? \(/); assert.match(source, /cli=\{totals\.attachable \?\? props\.cli\}/); }); + +test("workspace deletion is explicit and available from Fleet", async () => { + const app = await readFile(new URL("../src/app/main.jsx", import.meta.url), "utf8"); + const fleet = await readFile(new URL("../src/app/fleet.jsx", import.meta.url), "utf8"); + + assert.match( + app, + /deletesWorkspace[\s\S]*"Delete Crabbox workspace\?"[\s\S]*"Stop Crabbox session\?"/, + ); + assert.match(app, /"End GitHub Actions terminal session\?"/); + assert.match(app, /It does not cancel the GitHub Actions workflow run/); + assert.match(app, /endsWorkflowSession[\s\S]*"End session"/); + assert.match(app, /legacy backend does not expose provider deletion/); + assert.match(app, /action: \(\) => interactiveSessionAction\(id, "stop"\)/); + assert.doesNotMatch(app, /This stops the terminal/); + assert.match(fleet, /!String\(session\.id\)\.startsWith\("LOCAL-"\)/); + assert.match(fleet, /canManage && actionable/); + assert.match(fleet, /const endLabel = ending/); + assert.match(fleet, /deleteInteractiveSession\(session\.id\)/); +}); diff --git a/tests/runtime-adapter.test.ts b/tests/runtime-adapter.test.ts index 2ff5868..444b309 100644 --- a/tests/runtime-adapter.test.ts +++ b/tests/runtime-adapter.test.ts @@ -27,6 +27,7 @@ import { runtimeAdapterStopOutcome, runtimeAdapterTerminalFailureStatus, runtimeAdapterTerminalOriginMatches, + runtimeAdapterWorkspaceIdConflict, runtimeAdapterWorkspaceUrl, resolveCreateAfterStopRace, safeDesktopUrl, @@ -110,7 +111,16 @@ test("adapter workspace id stays distinct from provider resource id", () => { providerResourceId: "cloud/project/box-42", status: "ready", attachUrl: "wss://controller.example/terminal/is-101", - capabilities: { desktop: true, browser: true, code: true }, + capabilities: { + terminal: true, + takeover: false, + vnc: true, + desktop: true, + logs: false, + artifacts: false, + browser: true, + code: true, + }, expiresAt: "2026-06-12T12:00:00Z", message: "cloud/project/box-42 is ready as fleet-a-is-101", }); @@ -122,7 +132,7 @@ test("adapter workspace id stays distinct from provider resource id", () => { assert.equal(result?.capabilities?.terminal, true); assert.equal(result?.capabilities?.desktop, true); assert.equal(result?.capabilities?.vnc, true); - assert.equal(result?.terminalCapabilityInferred, true); + assert.equal(result?.terminalCapabilityInferred, false); assert.equal(result?.message, "[workspace] is ready as [workspace]"); assert.deepEqual( effectiveAdapterCapabilities( @@ -137,11 +147,11 @@ test("adapter workspace id stays distinct from provider resource id", () => { ), { terminal: true, - takeover: true, + takeover: false, vnc: true, desktop: true, - logs: true, - artifacts: true, + logs: false, + artifacts: false, }, ); @@ -153,6 +163,14 @@ test("adapter workspace id stays distinct from provider resource id", () => { }); assert.equal(explicitlyDisabled?.capabilities?.terminal, false); assert.equal(explicitlyDisabled?.terminalCapabilityInferred, false); + assert.equal( + effectiveAdapterCapabilities( + explicitlyDisabled!, + { ...clearedAdapterCapabilities, terminal: true, takeover: true, artifacts: true }, + true, + )?.terminal, + false, + ); const explicitlyNull = parseAdapterWorkspaceResult({ id: "fleet-a-is-102-null", @@ -851,6 +869,41 @@ test("runtime adapter requests reject redirects with edge-supported fetch semant assert.doesNotMatch(fetchSource, /redirect: "error"/); }); +test("pending runtime adapter creates replay before any inspect", async () => { + const source = await readFile(new URL("../src/index.ts", import.meta.url), "utf8"); + const inspectStart = source.indexOf("async function inspectRuntimeAdapterWorkspace"); + const inspectEnd = source.indexOf( + "async function reconcileStoppingRuntimeAdapterWorkspace", + inspectStart, + ); + const inspectSource = source.slice(inspectStart, inspectEnd); + const provisionStart = source.indexOf("async function provisionWithRuntimeAdapter"); + const provisionEnd = source.indexOf("function persistedRuntimeAdapterSeconds", provisionStart); + const provisionSource = source.slice(provisionStart, provisionEnd); + const replayIndex = inspectSource.indexOf( + "shouldReplayRuntimeAdapterCreate(session.status, session.adapter_create_pending === 1)", + ); + const inspectFetchIndex = inspectSource.indexOf("const response = await runtimeAdapterFetch("); + const missingIndex = inspectSource.indexOf("if (response.status === 404)"); + const missingSource = inspectSource.slice(missingIndex); + + assert.ok(replayIndex >= 0); + assert.ok(inspectFetchIndex >= 0); + assert.ok(replayIndex < inspectFetchIndex); + assert.match( + inspectSource, + /runtimeAdapterReplayRequest\(runtimeAdapterRecord\(session\)\)/, + ); + assert.ok(missingIndex >= 0); + assert.doesNotMatch(missingSource, /provisionWithRuntimeAdapter/); + assert.match(provisionSource, /const replayingPendingCreate = reconciliationOwner !== undefined/); + assert.match( + provisionSource, + /!replayingPendingCreate && definitiveRuntimeAdapterCreateFailure\(response\.status\)/, + ); + assert.match(provisionSource, /runtime adapter create replay blocked/); +}); + test("stopping create replay owns the exact persisted lifecycle", async () => { const source = await readFile(new URL("../src/index.ts", import.meta.url), "utf8"); const reconcileStart = source.indexOf("async function reconcileStoppingRuntimeAdapterWorkspace"); @@ -858,14 +911,22 @@ test("stopping create replay owns the exact persisted lifecycle", async () => { const replayEnd = source.indexOf("async function stopRuntimeAdapterWorkspace(", replayStart); const reconcileSource = source.slice(reconcileStart, replayStart); const replaySource = source.slice(replayStart, replayEnd); + const unresolvedIndex = reconcileSource.indexOf("if (!replay.resolved)"); + const deleteIndex = reconcileSource.indexOf("stopRuntimeAdapterWorkspace("); - assert.match(reconcileSource, /replayStoppingRuntimeAdapterCreate\(env, session\)/); + assert.match(reconcileSource, /replayStoppingRuntimeAdapterCreate\([\s\S]*env,[\s\S]*session,/); assert.doesNotMatch(reconcileSource, /provisionWithRuntimeAdapter/); + assert.ok(unresolvedIndex >= 0); + assert.ok(deleteIndex >= 0); + assert.ok(unresolvedIndex < deleteIndex); + assert.match(reconcileSource, /reconcileError: replay\.message/); + assert.match(reconcileSource, /createPending: true/); assert.match(replaySource, /AND adapter_control_plane = \$\{controlPlane\}/); assert.match(replaySource, /AND adapter_create_payload_json = \$\{createPayloadJson\}/); assert.match(replaySource, /AND adapter_create_pending = 1/); assert.match(replaySource, /AND status = 'stopping'/); assert.match(replaySource, /AND updated_at = \$\{session\.updated_at\}/); + assert.match(replaySource, /AND last_reconciled_at = \$\{reconciliationClaimAt\}/); assert.match(replaySource, /runtimeAdapterCollectionUrl\(controlPlane\)/); assert.match(replaySource, /idempotency-key": adapterWorkspaceId/); assert.match(replaySource, /adapter_create_pending: 0/); @@ -874,6 +935,26 @@ test("stopping create replay owns the exact persisted lifecycle", async () => { assert.match(replaySource, /env\.DB\.batch/); }); +test("every session-bound adapter delete waits for create ambiguity to clear", async () => { + const source = await readFile(new URL("../src/index.ts", import.meta.url), "utf8"); + const releaseStart = source.indexOf("async function stopRuntimeAdapterWorkspaceForSession"); + const releaseEnd = source.indexOf("async function runtimeAdapterFetch", releaseStart); + const releaseSource = source.slice(releaseStart, releaseEnd); + const pendingGateIndex = releaseSource.indexOf( + "if (registration?.adapter_create_pending !== 0)", + ); + const providerDeleteIndex = releaseSource.indexOf( + "stopRuntimeAdapterWorkspace(env, controlPlane, adapterWorkspaceId)", + ); + + assert.match(releaseSource, /select\(\["adapter_control_plane", "adapter_create_pending"\]\)/); + assert.ok(pendingGateIndex >= 0); + assert.ok(providerDeleteIndex >= 0); + assert.ok(pendingGateIndex < providerDeleteIndex); + assert.match(releaseSource, /status: "stopping"/); + assert.match(releaseSource, /runtime adapter stop waiting for create resolution/); +}); + test("stateless Sandbox provision hook acquires durable standalone ownership", async () => { const source = await readFile(new URL("../src/index.ts", import.meta.url), "utf8"); const migration = await readFile( @@ -1267,8 +1348,9 @@ test("Sandbox cleanup and legacy stops use durable terminal transitions", async assert.match(scheduledSource, /completeLegacyInteractiveSessionStop/); assert.match(completeSource, /if \(owner\.runtime === githubActionsRuntime\) return false/); assert.match(completeSource, /async function stopGitHubActionsSession/); - assert.match(completeSource, /work_state: "canceled"/); - assert.match(completeSource, /completion_reason: "stopped from Crabfleet"/); + assert.match(completeSource, /work_state: ""/); + assert.match(completeSource, /work_phase: "session_ended"/); + assert.match(completeSource, /workflow run not canceled/); assert.match(stopSource, /session\.runtime === githubActionsRuntime/); assert.match(stopSource, /stopGitHubActionsSession\(env, session, userActor, now\)/); assert.match(stopSource, /interactive session lifecycle changed; retry stop/); @@ -1610,6 +1692,73 @@ test("non-retryable adapter client errors do not enter ambiguous replay", () => assert.equal(definitiveRuntimeAdapterCreateFailure(503), false); }); +test("explicit workspace id conflicts are distinct from retryable 409 responses", () => { + assert.equal( + runtimeAdapterWorkspaceIdConflict(409, { + error: { code: "workspace_id_conflict", message: "workspace id is already owned" }, + }), + true, + ); + assert.equal(runtimeAdapterWorkspaceIdConflict(409, { error: { code: "busy" } }), false); + assert.equal( + runtimeAdapterWorkspaceIdConflict(422, { error: { code: "workspace_id_conflict" } }), + false, + ); +}); + +test("workspace id conflicts detach without adopting or deleting the existing workspace", async () => { + const source = await readFile(new URL("../src/index.ts", import.meta.url), "utf8"); + const createStart = source.indexOf("async function provisionWithRuntimeAdapter"); + const createEnd = source.indexOf("function persistedRuntimeAdapterSeconds", createStart); + const createSource = source.slice(createStart, createEnd); + const conflictStart = source.indexOf("async function failRuntimeAdapterWorkspaceIdConflict"); + const conflictEnd = source.indexOf( + "async function releaseFailedRuntimeAdapterProvision", + conflictStart, + ); + const conflictSource = source.slice(conflictStart, conflictEnd); + const stageStart = source.indexOf("async function stageRuntimeAdapterProvision"); + const stageEnd = source.indexOf("function ambiguousRuntimeAdapterProvision", stageStart); + const stageSource = source.slice(stageStart, stageEnd); + const stoppingReplayStart = source.indexOf("async function replayStoppingRuntimeAdapterCreate"); + const stoppingReplayEnd = source.indexOf( + "async function stopRuntimeAdapterWorkspace(", + stoppingReplayStart, + ); + const stoppingReplaySource = source.slice(stoppingReplayStart, stoppingReplayEnd); + + assert.ok( + createSource.indexOf("runtimeAdapterWorkspaceIdConflict") < + createSource.indexOf("definitiveRuntimeAdapterCreateFailure"), + ); + assert.match(createSource, /failRuntimeAdapterWorkspaceIdConflict/); + assert.match(createSource, /throw conflict\("runtime adapter workspace conflict response is stale"\)/); + assert.match(stageSource, /updated_at: sql`MAX\(updated_at \+ 1, \$\{stageAt\}\)`/); + assert.match( + stageSource, + /returning\(\["status", "updated_at", "last_reconciled_at", "terminal_status"\]\)/, + ); + assert.match(stageSource, /last_reconciled_at", "=", reconciliationOwner\.lastReconciledAt/); + assert.match(conflictSource, /adapter: null/); + assert.match(conflictSource, /adapter_workspace_id: null/); + assert.match(conflictSource, /adapter_control_plane: null/); + assert.match(conflictSource, /adapter_create_payload_json: null/); + assert.match(conflictSource, /adapter_create_pending: 0/); + assert.match(conflictSource, /AND adapter_create_pending = 1/); + assert.match(conflictSource, /AND status = \$\{createAttempt\.status\}/); + assert.match(conflictSource, /AND updated_at = \$\{createAttempt\.updatedAt\}/); + assert.match(conflictSource, /AND \$\{lastReconciledOwner\}/); + assert.match(conflictSource, /AND \$\{terminalStatusOwner\}/); + assert.match(conflictSource, /if \(!results\.at\(-1\)\?\.results\.length\) return null/); + assert.match(conflictSource, /terminal_finalize_pending: 1/); + assert.match(conflictSource, /env\.DB\.batch/); + assert.match(conflictSource, /finalizeTerminalInteractiveSession\(env, session\.id, "failed", now\)/); + assert.doesNotMatch(conflictSource, /stopRuntimeAdapterWorkspace/); + assert.match(stoppingReplaySource, /runtimeAdapterWorkspaceIdConflict/); + assert.match(stoppingReplaySource, /terminalResult/); + assert.doesNotMatch(stoppingReplaySource, /definitiveRuntimeAdapterCreateFailure/); +}); + test("definitive adapter create errors retain a redacted provider reason before release", async () => { const source = await readFile(new URL("../src/index.ts", import.meta.url), "utf8"); const createStart = source.indexOf("async function provisionWithRuntimeAdapter"); @@ -1817,6 +1966,7 @@ test("desktop mint revalidates current ownership before redirect", async () => { assert.match(vncSource, /current\.capabilities\.desktop/); assert.match(vncSource, /canControlInteractiveSession/); assert.match(vncSource, /desktop authorization changed; retry/); + assert.doesNotMatch(vncSource, /body:\s*JSON\.stringify\(\{\}\)/); }); test("runtime adapter terminals use the server-side adapter bearer", async () => {