From 810f5b0088a0101131ad97746ef74e206f7c6083 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 12 Jun 2026 17:14:11 -0700 Subject: [PATCH 1/2] feat: add runtime adapter lifecycle --- CHANGELOG.md | 13 + README.md | 29 +- cmd/crabbox-ssh-gateway/main.go | 85 +- cmd/crabbox-ssh-gateway/main_test.go | 51 + cmd/crabfleet/main.go | 182 +- cmd/crabfleet/main_test.go | 79 + docs/admin.md | 4 +- docs/api.md | 60 +- docs/architecture.md | 21 + docs/quickstart.md | 2 +- docs/runs.md | 10 +- docs/spec-v2.md | 4 + migrations/0020_runtime_adapter_lifecycle.sql | 13 + migrations/0021_runtime_adapter_hardening.sql | 91 + migrations/0022_credential_policy_cleanup.sql | 174 + migrations/0023_standalone_sandbox_expiry.sql | 28 + src/app/fleet.jsx | 48 +- src/app/main.jsx | 73 +- src/app/utils.js | 51 +- src/bounded-response.ts | 48 + src/credential-policy-fence.ts | 117 + src/d1-execution.ts | 65 + src/fleet-state.ts | 121 +- src/index.ts | 8816 ++++++++++++++--- src/oauth.ts | 115 +- src/repo-selection.ts | 3 + src/runtime-adapter.ts | 890 ++ src/session-archive.ts | 49 + src/session-id.ts | 10 + src/terminal-authorization.ts | 27 + src/terminal-finalization.ts | 42 + src/terminal-target.ts | 18 + src/url-security.ts | 33 + tests/app-utils.test.ts | 51 + tests/bounded-response.test.ts | 32 + tests/credential-policy-fence.test.ts | 290 + tests/d1-execution.test.ts | 70 + tests/fleet-state.test.ts | 158 + tests/html-dialogs.test.ts | 10 + tests/oauth.test.ts | 128 +- tests/repo-selection.test.ts | 16 + tests/runtime-adapter.test.ts | 1922 ++++ tests/session-archive.test.ts | 24 + tests/session-id.test.ts | 271 + tests/terminal-authorization.test.ts | 42 + tests/terminal-finalization.test.ts | 120 + tests/terminal-target.test.ts | 43 + tests/url-security.test.ts | 34 + worker-configuration.d.ts | 15 + wrangler.jsonc | 4 + 50 files changed, 13186 insertions(+), 1416 deletions(-) create mode 100644 migrations/0020_runtime_adapter_lifecycle.sql create mode 100644 migrations/0021_runtime_adapter_hardening.sql create mode 100644 migrations/0022_credential_policy_cleanup.sql create mode 100644 migrations/0023_standalone_sandbox_expiry.sql create mode 100644 src/bounded-response.ts create mode 100644 src/credential-policy-fence.ts create mode 100644 src/d1-execution.ts create mode 100644 src/repo-selection.ts create mode 100644 src/runtime-adapter.ts create mode 100644 src/session-archive.ts create mode 100644 src/session-id.ts create mode 100644 src/terminal-authorization.ts create mode 100644 src/terminal-finalization.ts create mode 100644 src/terminal-target.ts create mode 100644 src/url-security.ts create mode 100644 tests/bounded-response.test.ts create mode 100644 tests/credential-policy-fence.test.ts create mode 100644 tests/d1-execution.test.ts create mode 100644 tests/repo-selection.test.ts create mode 100644 tests/runtime-adapter.test.ts create mode 100644 tests/session-archive.test.ts create mode 100644 tests/session-id.test.ts create mode 100644 tests/terminal-authorization.test.ts create mode 100644 tests/terminal-finalization.test.ts create mode 100644 tests/terminal-target.test.ts create mode 100644 tests/url-security.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 428ccbc..838c225 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,19 @@ ## Unreleased +- Add a tenant-namespaced versioned runtime lifecycle adapter with replayable idempotent create, monotonic workspace identities, CAS reconciliation, durable terminal finalization, confirmed provider release before failure/stop, presence-aware capability/expiry tracking, authenticated transient VNC redirects, and deployment-neutral configuration. +- Reconcile runtime lifecycles and every adapter's terminal archives on cron and direct access, preserve partial capability-object defaults while honoring authoritative lists and explicit terminal withdrawal, make PTY availability server-authoritative, preserve opaque signed terminal and desktop URLs byte-for-byte, retain adapter failure evidence through confirmed release and exact session-version archive finalization, preflight adapter credentials before session allocation, bind every external lifecycle to its immutable registered control plane, generation-fence managed and standalone Sandbox credential ownership across crashes and late requests, repair incomplete equal-count archives, run teardown only after an exact cleanup CAS, use unique concurrent archive attempts, and transactionally remove D1 archive pointers before best-effort R2 object cleanup. +- Harden adapter and terminal boundaries by redacting connection credentials from durable messages, requiring byte-exact grammar-valid workspace identity echoes, rejecting malformed non-null expiries and create-only `stopping` results, keeping recurring WebSocket authorization provider-free, and paging credential cleanup with durable fair-progress cursors while retaining Sandbox failure evidence. +- Fence ambiguous create replay during stop to the exact registered lifecycle, require immutable-request ownership claims before the stateless hook can provision a managed session ID, expose standalone Sandbox terminals through their own bearer-authenticated WebSocket route, atomically pair terminal events with archive-finalization markers, and prevent older equal-count session snapshots from replacing newer archive pointers. +- Require an exact durable lease or provision/refresh claim for every Sandbox credential-policy transition, atomically activate standalone owners with their matching policy generation, redact structured and header-form provider credentials, fence slow reconciliation by the original session revision and completion time, and reject adapter base URLs containing raw query or fragment delimiters. +- Atomically fence credential-policy cleanup against its durable owner and revalidate ownership before unregistering, keep versioned-adapter terminal credentials behind Worker-owned PTY routes, and rotate a fresh agent token into every managed Sandbox provision claim. +- Make Sandbox terminal intent monotonic under stop/failure races, fence initial provision completion against managed retries and refresh ownership without rejecting concurrent metadata writes, atomically version terminal metadata with its event and finalization marker, keep live credential-registration failures retryable, clean both sides of interrupted refreshes, expire standalone Sandboxes with authenticated stop, and exactly reserve managed session IDs from standalone use. +- Keep standalone Sandbox terminals behind a lifetime-authorized Worker WebSocket proxy and terminate their execution sessions during durable cleanup, fail closed legacy unfenced credential policies, isolate and persist per-owner cleanup failures, reserve managed IDs case-insensitively across upgrades, preserve SSH-link state across canonical OAuth redirects, reject lost runtime-stop claims, redact arbitrary escaped connection URLs, make terminal completion/release revisions monotonic, and retain single-read redacted adapter create/DELETE evidence through final archives. +- Reject stale same-generation credential-policy registrations, preflight and atomically stage failed managed Sandbox claims, require the provision bearer for standalone stop after backend removal, and backfill D1-only terminal archives when R2 is enabled later. +- Proactively generation-wrap migrated legacy Sandbox credential policies under a live durable lease before cleanup, with crash-safe cron retries that preserve unattended session credentials. +- Bound every runtime-adapter response stream, revalidate desktop authorization after minting, make legacy local stops atomic with scheduled crash recovery, and redact credentials before opaque provider identifiers. +- Recover active credential policies after a post-registration crash, redact provider identities from structured adapter errors, and propagate terminal dimensions through direct bridge and runner PTY routes without rewriting opaque adapter URLs. +- Support an optional authoritative `GITHUB_REDIRECT_URI` deployment binding with strict HTTPS callback validation, canonical-origin login handoff, and callback host/path enforcement while retaining safe request-origin defaults. - Replace native browser confirms and prompts with accessible Crabfleet dialogs for session cleanup, shutdown, and share-link fallback. - Sharpen the app visual system with flatter controls, tighter surfaces, and restrained overlay elevation. - Add Crabfleet session supervision metadata, owner/session tree listing, transcript retrieval, PTY messaging, and summary updates for Codex-spawned Codex sessions. diff --git a/README.md b/README.md index 7e3f387..418c570 100644 --- a/README.md +++ b/README.md @@ -26,12 +26,13 @@ Crabfleet gives OpenClaw maintainers a fleet dashboard where every Codex crabbox - **D1 + Kysely** for typed persistence: users, sessions, allowlists, repos, cards, events, run attempts, interactive sessions, diffs, and repo workflow evaluations. - **Ghostty WebAssembly** for the fullscreen attach grid and run log replay. - **Cloudflare Sandbox containers** for standalone interactive Codex CLI workspaces with live PTY attach. -- **Runtime adapter descriptors** for Container and Crabbox selection, capability display, interactive provision handoff, and guarded takeover. -- **Provision endpoint** at `/api/provision/interactive` that can use the built-in Sandbox backend or delegate to a generic runtime adapter or ClawFleet. +- **Runtime adapter descriptors** for Container and Crabbox selection, capability display, interactive lifecycle handoff, and guarded takeover. +- **Versioned lifecycle adapter** for idempotent external workspace creation, bounded status reconciliation, provider-backed stop, terminal attachment, and authenticated transient desktop connections. +- **Provision endpoint** at `/api/provision/interactive` that can use the built-in Sandbox backend or retain a legacy create-only adapter or ClawFleet integration, with durable ownership and a bearer-authenticated standalone PTY route. - **R2 session archives** for crabbox event NDJSON, transcripts, and summaries. - **GitHub API** for OAuth, org/team membership, and issue/PR previews across enabled repos. -Autonomous card execution, Crabbox VNC transport, Durable Object fanout, and merge automation are adapter targets, not faked in the current Worker. +Autonomous card execution, Durable Object fanout, and merge automation are adapter targets, not faked in the current Worker. External interactive terminal and desktop transport require a configured lifecycle adapter or bridge. The OpenClaw fleet/orchestrator backend should run on the Hetzner `openclaw-clawsweeper` host alongside ClawSweeper; the Worker stays the app/API front door. ## Quick Start @@ -164,12 +165,18 @@ The Crabbox namespace cutover intentionally has no old-name compatibility. Exist - `CRABBOX_BOOTSTRAP_TOKEN` – Optional owner break-glass token for setup/recovery - `GITHUB_CLIENT_ID` – GitHub OAuth app client ID (optional) - `GITHUB_CLIENT_SECRET` – GitHub OAuth app secret (optional) -- `GITHUB_REDIRECT_URI` – GitHub OAuth callback URL registered on the app (defaults to the request origin callback) +- `GITHUB_REDIRECT_URI` – Optional authoritative GitHub OAuth callback URL; when set it must be an absolute HTTPS URL with no credentials, query, or fragment and the exact `/auth/github/callback` path. Requests on another host restart login on this configured origin. When absent, the callback defaults to the HTTPS request origin (or literal-loopback HTTP for local development). - `GITHUB_ORG` – GitHub org for membership check (default: `openclaw`) - `GITHUB_TOKEN` – GitHub token for all enabled repo issue/PR previews and private repo `CRABBOX.md` refreshes (optional; public/default repo paths work without it) - `CRABBOX_TOKEN_ENCRYPTION_KEY` – Optional encryption key for per-session GitHub OAuth tokens; defaults to `GITHUB_CLIENT_SECRET` - `CRABBOX_INTERACTIVE_PROVISION_URL` – Optional adapter endpoint for standalone Codex CLI workspaces -- `CRABBOX_INTERACTIVE_PROVISION_TOKEN` – Optional bearer token sent to the interactive provision endpoint; required when backend URLs below are configured +- `CRABBOX_INTERACTIVE_PROVISION_TOKEN` – Optional bearer token sent to the interactive provision endpoint; required when backend URLs below are configured and always required to stop an existing standalone Sandbox +- `CRABBOX_STANDALONE_SANDBOX_TTL_SECONDS` – Optional built-in standalone Sandbox lifetime, default `14400`, bounded to 300–86400 seconds +- `CRABBOX_RUNTIME_ADAPTER_URL` – Optional base URL for the versioned workspace lifecycle adapter; takes precedence over the legacy create-only runtime provision URL and becomes immutable registration identity for each created lifecycle. Nested base paths are preserved; raw query or fragment delimiters are rejected. +- `CRABBOX_RUNTIME_ADAPTER_TOKEN` – Required bearer token for the versioned lifecycle adapter; sent only over HTTPS or literal loopback HTTP +- `CRABBOX_RUNTIME_ADAPTER_NAMESPACE` – Required stable tenant namespace when the versioned adapter is enabled; a DNS-safe label of at most 32 characters used in every workspace ID and idempotency key +- `CRABBOX_RUNTIME_ADAPTER_TTL_SECONDS` – Optional requested workspace TTL, default `14400` +- `CRABBOX_RUNTIME_ADAPTER_IDLE_SECONDS` – Optional requested workspace idle timeout, default `1800` - `CRABBOX_RUNTIME_PROVISION_URL` – Optional generic backend URL used by `/api/provision/interactive` - `CRABBOX_RUNTIME_PROVISION_TOKEN` – Optional bearer token sent to the generic runtime backend - `CRABBOX_CLOUDFLARE_RUNNER_URL` – Optional Crabbox Cloudflare container runner URL used by `/api/provision/interactive` @@ -178,7 +185,7 @@ The Crabbox namespace cutover intentionally has no old-name compatibility. Exist - `CRABBOX_CLOUDFLARE_RUNNER_WORKDIR` – Optional base workdir for provisioned sandboxes, default `/workspace/crabbox` - `CRABBOX_CLOUDFLARE_RUNNER_TTL_SECONDS` – Optional sandbox TTL, default `14400` - `CRABBOX_CLOUDFLARE_RUNNER_IDLE_SECONDS` – Optional idle timeout, default `1800` -- `CRABBOX_PTY_BRIDGE_URL` – Optional WebSocket PTY bridge URL/template for live Ghostty attach; supports `{id}`, `{leaseId}`, `{repo}`, `{branch}`, and `{runtime}` +- `CRABBOX_PTY_BRIDGE_URL` – Optional WebSocket PTY bridge URL/template for live Ghostty attach; requires WSS except literal loopback WS and supports `{id}`, `{leaseId}`, `{repo}`, `{branch}`, and `{runtime}` - `CRABBOX_PTY_BRIDGE_TOKEN` – Optional bearer token sent from Crabfleet to the PTY bridge - `CRABBOX_CLAWFLEET_URL` – Optional ClawFleet dashboard/API URL used by `/api/provision/interactive` for `crabbox` sessions - `CRABBOX_CLAWFLEET_TOKEN` – Optional bearer token sent to ClawFleet @@ -186,10 +193,20 @@ The Crabbox namespace cutover intentionally has no old-name compatibility. Exist - `CRABBOX_OPENCLAW_TOKEN` – Internal bearer token for OpenClaw/Discord service crabbox creation - `CRABFLEET_SSH_GATEWAY_TOKEN` / `CRABBOX_SSH_GATEWAY_TOKEN` – Shared bearer token for the Go SSH gateway internal API - `CRABFLEET_LOCAL_SANDBOX_BACKUPS` – Optional Cloudflare Sandbox checkpoint mode override; defaults to R2 binding uploads, set `0` for SDK presigned R2 uploads +- `CRABFLEET_LABEL` – Optional tenant label shown in the app, default `Crabfleet` +- `CRABFLEET_CANONICAL_URL` – Optional tenant app/API origin, default `https://crabfleet.openclaw.ai`; requires HTTPS except literal loopback HTTP +- `CRABFLEET_PRODUCT_URL` – Optional tenant product/docs origin, default `https://crabfleet.ai`; requires HTTPS except literal loopback HTTP +- `CRABFLEET_SSH_HOST` – Optional SSH command host shown in the app, default `crabd.sh` +- `CRABFLEET_PREFERRED_REPO` – Optional first/default enabled repo, default `openclaw/crabfleet` +- `CRABFLEET_DEFAULT_RUNTIME` – Optional interactive runtime default, `container` or `crabbox`; defaults to `container` +- `CRABFLEET_DEFAULT_PROFILE` – Optional opaque runtime-adapter profile, default `default` +- `CRABFLEET_DEV_LOGIN_ENABLED` – Explicit local-only development identity login gate; disabled unless exactly `true`, and still restricted to literal localhost requests - `OPENAI_API_KEY` – Required for built-in Cloudflare Sandbox Codex CLI sessions; injected by the Worker outbound path for Cloudflare Sandbox requests ### Verify Deployment +The app Worker includes a once-per-minute cron trigger for bounded runtime lifecycle and terminal-archive reconciliation. Keep the `triggers.crons` entry when deriving deployment configuration; direct session, PTY, and VNC access also performs CAS-guarded targeted refreshes. + ```bash curl -I https://crabfleet.openclaw.ai/healthz # Should return: 200 OK diff --git a/cmd/crabbox-ssh-gateway/main.go b/cmd/crabbox-ssh-gateway/main.go index c96a66d..d73c833 100644 --- a/cmd/crabbox-ssh-gateway/main.go +++ b/cmd/crabbox-ssh-gateway/main.go @@ -49,21 +49,28 @@ type stateResponse struct { } type interactiveSession struct { - ID string `json:"id"` - ParentSessionID string `json:"parentSessionId"` - RootSessionID string `json:"rootSessionId"` - Repo string `json:"repo"` - Branch string `json:"branch"` - Runtime string `json:"runtime"` - Status string `json:"status"` - Owner string `json:"owner"` - CreatedBy string `json:"createdBy"` - Purpose string `json:"purpose"` - Summary string `json:"summary"` - AttachURL string `json:"attachUrl"` - VNCURL string `json:"vncUrl"` - LastEvent string `json:"lastEvent"` - LogArchive logArchive `json:"logArchive"` + ID string `json:"id"` + ParentSessionID string `json:"parentSessionId"` + RootSessionID string `json:"rootSessionId"` + Repo string `json:"repo"` + Branch string `json:"branch"` + Runtime string `json:"runtime"` + Status string `json:"status"` + Owner string `json:"owner"` + CreatedBy string `json:"createdBy"` + Purpose string `json:"purpose"` + Summary string `json:"summary"` + Capabilities *sessionCapabilities `json:"capabilities"` + PtyAvailable *bool `json:"ptyAvailable"` + LeaseID string `json:"leaseId"` + AttachURL string `json:"attachUrl"` + VNCURL string `json:"vncUrl"` + LastEvent string `json:"lastEvent"` + LogArchive logArchive `json:"logArchive"` +} + +type sessionCapabilities struct { + Terminal bool `json:"terminal"` } type card struct { @@ -346,7 +353,10 @@ func runCommand(ctx context.Context, out io.ReadWriter, perms *ssh.Permissions, fmt.Fprintf(out, "error: %v\n", err) return 1 } - fmt.Fprintf(out, "session: %s\nrepo: %s\nstatus: %s\nattach: ssh crabfleet attach %s\n", session.ID, session.Repo, session.Status, session.ID) + fmt.Fprintf(out, "session: %s\nrepo: %s\nstatus: %s\n", session.ID, session.Repo, session.Status) + if attachable(session) { + fmt.Fprintf(out, "attach: ssh crabfleet attach %s\n", session.ID) + } if session.VNCURL != "" { fmt.Fprintf(out, "vnc: %s\n", terminalSafe(session.VNCURL)) } @@ -356,7 +366,7 @@ func runCommand(ctx context.Context, out io.ReadWriter, perms *ssh.Permissions, } return 0 } - if create.detach { + if create.detach || !attachable(session) { return 0 } return client.attach(ctx, auth.fingerprint, session.ID, out, pty) @@ -470,12 +480,51 @@ func runCommand(ctx context.Context, out io.ReadWriter, perms *ssh.Permissions, } } +func terminalCapable(session interactiveSession) bool { + return session.Capabilities == nil || session.Capabilities.Terminal +} + +func attachable(session interactiveSession) bool { + if !terminalCapable(session) || !ptyAttachable(session) { + return false + } + switch session.Status { + case "ready", "attached", "detached": + return true + default: + return false + } +} + +func ptyAttachable(session interactiveSession) bool { + if session.PtyAvailable != nil { + return *session.PtyAvailable + } + if strings.HasPrefix(session.LeaseID, "sandbox:") || strings.HasPrefix(session.LeaseID, "cloudflare:") { + return true + } + return strings.HasPrefix(session.AttachURL, "/api/interactive-sessions/") || validWebSocketAttachURL(session.AttachURL) +} + +func validWebSocketAttachURL(raw string) bool { + target, err := url.Parse(raw) + if err != nil || target.Host == "" || target.User != nil { + return false + } + if target.Scheme == "wss" { + return true + } + host := target.Hostname() + return target.Scheme == "ws" && (host == "localhost" || host == "127.0.0.1" || host == "::1") +} + func printHelp(out io.Writer, user user) { fmt.Fprintf(out, "Crabfleet SSH\nlogin: %s\nrole: %s\n\n", terminalSafe(displayUser(user)), terminalSafe(user.Role)) fmt.Fprintln(out, "commands:") fmt.Fprintln(out, " whoami") fmt.Fprintln(out, " list") fmt.Fprintln(out, " new [--repo owner/repo] [--branch main] [--runtime crabbox|container] [--parent id] [--purpose text] [--command codex] [--vnc] [prompt]") + fmt.Fprintln(out, " --runtime overrides the deployment default") fmt.Fprintln(out, " attach SESSION_ID") fmt.Fprintln(out, " vnc SESSION_ID") fmt.Fprintln(out, " logs SESSION_ID") @@ -745,7 +794,7 @@ func parseCreate(args []string, client *apiClient, fingerprint string) createArg var vnc bool fs.StringVar(&req.Repo, "repo", "", "repo") fs.StringVar(&req.Branch, "branch", "main", "branch") - fs.StringVar(&req.Runtime, "runtime", "crabbox", "runtime") + fs.StringVar(&req.Runtime, "runtime", "", "runtime override; defaults to deployment") fs.StringVar(&req.Command, "command", "", "command") fs.StringVar(&req.ParentSessionID, "parent", "", "parent session") fs.StringVar(&req.RootSessionID, "root", "", "root session") diff --git a/cmd/crabbox-ssh-gateway/main_test.go b/cmd/crabbox-ssh-gateway/main_test.go index b4354d7..ac269ad 100644 --- a/cmd/crabbox-ssh-gateway/main_test.go +++ b/cmd/crabbox-ssh-gateway/main_test.go @@ -76,6 +76,57 @@ func TestParseMessageKeepsNoEnterAndText(t *testing.T) { } } +func TestParseCreateLeavesRuntimeToDeploymentDefault(t *testing.T) { + create := parseCreate([]string{"--repo", "openclaw/crabfleet", "fix it"}, nil, "") + if create.request.Runtime != "" { + t.Fatalf("runtime = %q, want deployment default", create.request.Runtime) + } + + create = parseCreate( + []string{"--repo", "openclaw/crabfleet", "--runtime", "container", "fix it"}, + nil, + "", + ) + if create.request.Runtime != "container" { + t.Fatalf("runtime = %q, want explicit override", create.request.Runtime) + } +} + +func TestTerminalCapabilityWithdrawalSuppressesAttach(t *testing.T) { + if !terminalCapable(interactiveSession{}) { + t.Fatal("legacy session without capabilities should remain attachable") + } + if terminalCapable(interactiveSession{ + Capabilities: &sessionCapabilities{Terminal: false}, + }) { + t.Fatal("explicit terminal capability withdrawal should suppress attach") + } +} + +func TestCreateAutoAttachRequiresReadyResolvablePTY(t *testing.T) { + available := true + if attachable(interactiveSession{Status: "provisioning", PtyAvailable: &available}) { + t.Fatal("provisioning create must succeed without auto-attach") + } + available = false + if attachable(interactiveSession{Status: "ready", PtyAvailable: &available}) { + t.Fatal("ready session without a PTY route must not auto-attach") + } + available = true + if !attachable(interactiveSession{Status: "ready", PtyAvailable: &available}) { + t.Fatal("ready session with a PTY route should auto-attach") + } + if !attachable(interactiveSession{ + Status: "detached", + AttachURL: "/api/interactive-sessions/IS-1/pty", + }) { + t.Fatal("legacy API PTY routes should remain attachable") + } + if attachable(interactiveSession{Status: "ready", AttachURL: "ws://example.com/terminal"}) { + t.Fatal("insecure remote websocket should not auto-attach") + } +} + func TestPrintListShowsOwnersAndSessionTree(t *testing.T) { var out bytes.Buffer printList(&out, stateResponse{ diff --git a/cmd/crabfleet/main.go b/cmd/crabfleet/main.go index 340b5ba..7a2e3bb 100644 --- a/cmd/crabfleet/main.go +++ b/cmd/crabfleet/main.go @@ -63,7 +63,7 @@ type listCmd struct{} type newCmd struct { Repo string `help:"Repository to prepare, owner/repo."` Branch string `help:"Git branch to checkout." default:"main"` - Runtime string `help:"Runtime backend." enum:"crabbox,container" default:"crabbox"` + Runtime *string `help:"Runtime backend override; omit to use the deployment default." enum:"crabbox,container"` Command string `help:"Command to run after checkout." default:"codex --yolo"` Parent string `help:"Parent crabbox session id."` Root string `help:"Root crabbox session id."` @@ -151,22 +151,28 @@ type user struct { } type interactiveSession struct { - ID string `json:"id"` - ParentSessionID string `json:"parentSessionId"` - RootSessionID string `json:"rootSessionId"` - Repo string `json:"repo"` - Branch string `json:"branch"` - Runtime string `json:"runtime"` - Status string `json:"status"` - Owner string `json:"owner"` - CreatedBy string `json:"createdBy"` - Purpose string `json:"purpose"` - Summary string `json:"summary"` - LeaseID string `json:"leaseId"` - AttachURL string `json:"attachUrl"` - VNCURL string `json:"vncUrl"` - LastEvent string `json:"lastEvent"` - LogArchive logArchive `json:"logArchive"` + ID string `json:"id"` + ParentSessionID string `json:"parentSessionId"` + RootSessionID string `json:"rootSessionId"` + Repo string `json:"repo"` + Branch string `json:"branch"` + Runtime string `json:"runtime"` + Status string `json:"status"` + Owner string `json:"owner"` + CreatedBy string `json:"createdBy"` + Purpose string `json:"purpose"` + Summary string `json:"summary"` + Capabilities *sessionCapabilities `json:"capabilities"` + PtyAvailable *bool `json:"ptyAvailable"` + LeaseID string `json:"leaseId"` + AttachURL string `json:"attachUrl"` + VNCURL string `json:"vncUrl"` + LastEvent string `json:"lastEvent"` + LogArchive logArchive `json:"logArchive"` +} + +type sessionCapabilities struct { + Terminal bool `json:"terminal"` } type createSessionRequest struct { @@ -299,59 +305,13 @@ func (listCmd) Run(app *cli, api *apiClient) error { } func (cmd newCmd) Run(app *cli, api *apiClient) error { - prompt := strings.Join(cmd.Prompt, " ") - parent := cmd.Parent - if parent == "" { - parent = app.AgentID - } - root := cmd.Root - if root == "" { - root = os.Getenv("CRABFLEET_ROOT_SESSION_ID") - } - req := createSessionRequest{ - Repo: cmd.Repo, - Branch: cmd.Branch, - Runtime: cmd.Runtime, - Command: cmd.Command, - Prompt: prompt, - ParentSessionID: parent, - RootSessionID: root, - Purpose: cmd.Purpose, - Summary: cmd.Summary, - } + req := cmd.sessionRequest(app) session, err := api.createSession(context.Background(), req) if err != nil { if app.NoInput || app.JSON { return err } - args := []string{"new", "--branch", cmd.Branch, "--runtime", cmd.Runtime} - if cmd.Repo != "" { - args = append(args, "--repo", cmd.Repo) - } - if cmd.Command != "codex --yolo" { - args = append(args, "--command", cmd.Command) - } - if parent != "" { - args = append(args, "--parent", parent) - } - if root != "" { - args = append(args, "--root", root) - } - if cmd.Purpose != "" { - args = append(args, "--purpose", cmd.Purpose) - } - if cmd.Summary != "" { - args = append(args, "--summary", cmd.Summary) - } - if cmd.Detach { - args = append(args, "--detach") - } - if cmd.VNC { - args = append(args, "--vnc") - } - if prompt != "" { - args = append(args, prompt) - } + args := cmd.sshCreateArgs(req) if cmd.VNC { output, captureErr := runSSHCommandOutput(app, args...) if output != "" { @@ -380,7 +340,9 @@ func (cmd newCmd) Run(app *cli, api *apiClient) error { if session.Summary != "" { fmt.Fprintf(os.Stdout, "summary: %s\n", terminalSafe(session.Summary)) } - fmt.Fprintf(os.Stdout, "attach: crabfleet attach %s\n", session.ID) + if attachable(session) { + fmt.Fprintf(os.Stdout, "attach: crabfleet attach %s\n", session.ID) + } if session.VNCURL != "" { fmt.Fprintf(os.Stdout, "vnc: %s\n", session.VNCURL) } @@ -393,6 +355,68 @@ func (cmd newCmd) Run(app *cli, api *apiClient) error { return nil } +func (cmd newCmd) sessionRequest(app *cli) createSessionRequest { + prompt := strings.Join(cmd.Prompt, " ") + parent := cmd.Parent + if parent == "" { + parent = app.AgentID + } + root := cmd.Root + if root == "" { + root = os.Getenv("CRABFLEET_ROOT_SESSION_ID") + } + runtime := "" + if cmd.Runtime != nil { + runtime = *cmd.Runtime + } + return createSessionRequest{ + Repo: cmd.Repo, + Branch: cmd.Branch, + Runtime: runtime, + Command: cmd.Command, + Prompt: prompt, + ParentSessionID: parent, + RootSessionID: root, + Purpose: cmd.Purpose, + Summary: cmd.Summary, + } +} + +func (cmd newCmd) sshCreateArgs(req createSessionRequest) []string { + args := []string{"new", "--branch", req.Branch} + if req.Runtime != "" { + args = append(args, "--runtime", req.Runtime) + } + if req.Repo != "" { + args = append(args, "--repo", req.Repo) + } + if req.Command != "codex --yolo" { + args = append(args, "--command", req.Command) + } + if req.ParentSessionID != "" { + args = append(args, "--parent", req.ParentSessionID) + } + if req.RootSessionID != "" { + args = append(args, "--root", req.RootSessionID) + } + if req.Purpose != "" { + args = append(args, "--purpose", req.Purpose) + } + if req.Summary != "" { + args = append(args, "--summary", req.Summary) + } + if cmd.Detach { + args = append(args, "--detach") + } + if cmd.VNC { + args = append(args, "--vnc") + } + if req.Prompt != "" { + args = append(args, req.Prompt) + } + return args +} + func (cmd attachCmd) Run(app *cli, _ *apiClient) error { return runSSH(app, "attach", cmd.ID) } @@ -1118,6 +1142,9 @@ func terminalSafe(value string) string { } func attachable(session interactiveSession) bool { + if !terminalCapable(session) { + return false + } if !ptyAttachable(session) { return false } @@ -1129,13 +1156,30 @@ func attachable(session interactiveSession) bool { } } +func terminalCapable(session interactiveSession) bool { + return session.Capabilities == nil || session.Capabilities.Terminal +} + func ptyAttachable(session interactiveSession) bool { + if session.PtyAvailable != nil { + return *session.PtyAvailable + } if strings.HasPrefix(session.LeaseID, "sandbox:") || strings.HasPrefix(session.LeaseID, "cloudflare:") { return true } - return strings.HasPrefix(session.AttachURL, "/api/interactive-sessions/") || - strings.HasPrefix(session.AttachURL, "ws://") || - strings.HasPrefix(session.AttachURL, "wss://") + return strings.HasPrefix(session.AttachURL, "/api/interactive-sessions/") || validWebSocketAttachURL(session.AttachURL) +} + +func validWebSocketAttachURL(raw string) bool { + target, err := url.Parse(raw) + if err != nil || target.Host == "" || target.User != nil { + return false + } + if target.Scheme == "wss" { + return true + } + host := target.Hostname() + return target.Scheme == "ws" && (host == "localhost" || host == "127.0.0.1" || host == "::1") } func isTerminal(file *os.File) bool { diff --git a/cmd/crabfleet/main_test.go b/cmd/crabfleet/main_test.go index 418ccd2..a27fc64 100644 --- a/cmd/crabfleet/main_test.go +++ b/cmd/crabfleet/main_test.go @@ -2,8 +2,11 @@ package main import ( "bytes" + "encoding/json" "strings" "testing" + + "github.com/alecthomas/kong" ) func TestVersionIsSet(t *testing.T) { @@ -40,6 +43,60 @@ func TestFirstLineSkipsBlankLines(t *testing.T) { } } +func TestNewRuntimeOverrideIsOptional(t *testing.T) { + t.Setenv("CRABFLEET_ROOT_SESSION_ID", "") + parse := func(args ...string) cli { + var app cli + parser, err := kong.New( + &app, + kong.Name("crabfleet"), + kong.Vars{"version": version}, + ) + if err != nil { + t.Fatal(err) + } + if _, err := parser.Parse(args); err != nil { + t.Fatal(err) + } + return app + } + + app := parse("new") + cmd := app.New + req := cmd.sessionRequest(&cli{}) + if req.Runtime != "" { + t.Fatalf("runtime = %q, want deployment default", req.Runtime) + } + encoded, err := json.Marshal(req) + if err != nil { + t.Fatal(err) + } + if bytes.Contains(encoded, []byte(`"runtime"`)) { + t.Fatalf("omitted runtime was serialized: %s", encoded) + } + for _, arg := range cmd.sshCreateArgs(req) { + if arg == "--runtime" { + t.Fatal("SSH fallback forced a runtime override") + } + } + + cmd = parse("new", "--runtime", "container").New + req = cmd.sessionRequest(&cli{}) + if req.Runtime != "container" { + t.Fatalf("runtime = %q, want explicit override", req.Runtime) + } + args := cmd.sshCreateArgs(req) + found := false + for index := 0; index+1 < len(args); index++ { + if args[index] == "--runtime" && args[index+1] == "container" { + found = true + } + } + if !found { + t.Fatalf("explicit runtime missing from SSH fallback: %q", args) + } +} + func TestAttachableRequiresReadySessionWithAttachURL(t *testing.T) { if !attachable(interactiveSession{Status: "ready", AttachURL: "/api/interactive-sessions/IS-1/pty"}) { t.Fatal("ready session with sandbox attach URL should be attachable") @@ -50,9 +107,31 @@ func TestAttachableRequiresReadySessionWithAttachURL(t *testing.T) { if attachable(interactiveSession{Status: "ready", AttachURL: "https://example.com/console"}) { t.Fatal("http console URL should not be SSH attachable") } + if attachable(interactiveSession{Status: "ready", AttachURL: "ws://example.com/terminal"}) { + t.Fatal("insecure remote websocket should not be attachable") + } + if !attachable(interactiveSession{Status: "ready", AttachURL: "ws://127.0.0.1:9000/terminal"}) { + t.Fatal("loopback websocket should be attachable") + } if !attachable(interactiveSession{Status: "ready", LeaseID: "sandbox:test"}) { t.Fatal("sandbox lease should be attachable") } + if attachable(interactiveSession{ + Status: "ready", + LeaseID: "sandbox:test", + AttachURL: "/api/interactive-sessions/IS-1/pty", + Capabilities: &sessionCapabilities{Terminal: false}, + }) { + t.Fatal("session with withdrawn terminal capability should not be attachable") + } + available := false + if attachable(interactiveSession{ + Status: "ready", + LeaseID: "sandbox:test", + PtyAvailable: &available, + }) { + t.Fatal("server PTY availability should be authoritative") + } } func TestPrintFleetShowsOwnerSessionTreeAndSummaries(t *testing.T) { diff --git a/docs/admin.md b/docs/admin.md index eaa2a4b..86fd993 100644 --- a/docs/admin.md +++ b/docs/admin.md @@ -319,13 +319,13 @@ Recommended for production. **Setup:** 1. Create GitHub OAuth app in your org -2. Callback URL: the Worker `GITHUB_REDIRECT_URI` value, currently `https://crabfleet.openclaw.ai/auth/github/callback` +2. Callback URL: the optional Worker `GITHUB_REDIRECT_URI` value, or the request-origin `/auth/github/callback` URL when the binding is absent 3. Scopes: `read:user`, `read:org` 4. Add secrets to Cloudflare Worker: - `GITHUB_CLIENT_ID` - `GITHUB_CLIENT_SECRET` - `GITHUB_TOKEN` for all enabled repo previews and private repo `CRABBOX.md` refreshes (optional; public/default repo paths work without it) -5. Set `GITHUB_REDIRECT_URI` and `GITHUB_ORG` vars (`GITHUB_ORG` defaults to `openclaw`) +5. Set `GITHUB_ORG` (`openclaw` by default). For deployments with aliases, proxies, or a fixed OAuth app callback, set `GITHUB_REDIRECT_URI` to the authoritative absolute HTTPS URL with the exact `/auth/github/callback` path and no credentials, query, or fragment. **Session lifetime:** diff --git a/docs/api.md b/docs/api.md index 79d3e53..7ac2e78 100644 --- a/docs/api.md +++ b/docs/api.md @@ -38,10 +38,18 @@ Returns available login methods without requiring a session. "auth": { "github": true, "token": true + }, + "deployment": { + "label": "Crabfleet", + "canonicalUrl": "https://crabfleet.openclaw.ai", + "productUrl": "https://crabfleet.ai", + "sshHost": "crabd.sh" } } ``` +The unauthenticated response exposes branding and SSH connection fields only. Preferred repository, default runtime, adapter profile, and other routing configuration are returned after authentication through `/api/state`. + ### POST /api/login/token ```json @@ -56,10 +64,14 @@ Returns the bootstrap owner user and sets `crabbox_session`. Starts GitHub OAuth with `read:user read:org repo`. +When `GITHUB_REDIRECT_URI` is configured, that validated HTTPS callback is authoritative for both authorization and token exchange. Login and SSH-link requests received on another origin redirect to the configured origin before any host-only state cookie is created, preserving the pending SSH code through callback. Without the binding, Crabfleet uses the request-origin callback; insecure non-loopback HTTP origins are rejected. + ### GET /auth/github/callback Completes OAuth, verifies active org membership, applies the allowlist, stores the user, and redirects to `/app`. +With `GITHUB_REDIRECT_URI` configured, callback requests whose origin or path does not exactly match the configured callback are rejected before token exchange. + ## Session Endpoints ### POST /api/logout @@ -240,16 +252,38 @@ Provision hook used by `CRABBOX_INTERACTIVE_PROVISION_URL`. It accepts the same Auth: - If `CRABBOX_INTERACTIVE_PROVISION_TOKEN` is set, callers must send `Authorization: Bearer `. -- The token is required when `CRABBOX_RUNTIME_PROVISION_URL`, `CRABBOX_CLOUDFLARE_RUNNER_URL`, or `CRABBOX_CLAWFLEET_URL` is configured; backend-enabled deployments fail closed without it. +- The token is required when `CRABBOX_RUNTIME_ADAPTER_URL`, `CRABBOX_RUNTIME_PROVISION_URL`, `CRABBOX_CLOUDFLARE_RUNNER_URL`, or `CRABBOX_CLAWFLEET_URL` is configured; backend-enabled deployments fail closed without it. Backends: -- `CRABBOX_RUNTIME_PROVISION_URL`: forwards the session payload to a generic runtime adapter. +- Versioned lifecycle adapters are deliberately excluded from this stateless hook. Create those workspaces through `POST /api/interactive-sessions`, which durably records ownership before calling the adapter. +- Direct built-in Sandbox calls without a managed interactive-session row acquire a durable standalone ownership fence before credential-policy registration. Standalone IDs cannot use the case-insensitive `IS-` managed-session namespace. Retries with the same ID must match the original immutable request; abandoned claims and failed provisions enter the same generation-fenced cleanup path as managed sessions. +- A request whose ID already belongs to a managed interactive session is rejected unless every immutable request field matches that row and the call wins an exact session-version ownership claim before allocating a Sandbox. Completion commits through the immutable lease, claim, agent-token, and status ownership fence while monotonically advancing the session version, so an intervening metadata edit does not discard the non-replayable result. +- `CRABBOX_RUNTIME_PROVISION_URL`: forwards the session payload to a legacy create-only runtime adapter. - `CRABBOX_CLOUDFLARE_RUNNER_URL`: creates a Crabbox Cloudflare container sandbox and returns its lease reference. - `CRABBOX_CLAWFLEET_URL`: creates a ClawFleet OpenClaw instance and returns console/noVNC links. - ClawFleet handles `crabbox` sessions only; use `CRABBOX_RUNTIME_PROVISION_URL` or `CRABBOX_CLOUDFLARE_RUNNER_URL` for `container` sessions. - If neither backend is configured, returns `pending_adapter` with a message that the route is live. +For a successful direct built-in Sandbox provision, `attachUrl` is an absolute `wss://` URL under `/api/provision/interactive/:id/pty`, and `expiresAt` is bounded by `CRABBOX_STANDALONE_SANDBOX_TTL_SECONDS` (default four hours, maximum one day). Connect with the same `Authorization: Bearer ` header used for provisioning. The Worker validates the unexpired standalone owner and exact active credential-policy generation, strips the bearer before opening the Sandbox terminal, proxies the WebSocket while periodically revalidating that ownership, and closes both peers after stop, expiry, or policy revocation. It never routes the connection through `interactive_sessions`. `POST /api/provision/interactive/:id/stop` always requires that configured bearer, even if runtime backend bindings were removed after creation, and atomically moves the exact owner plus every matching policy into durable cleanup; expiry follows the same path from cron and PTY access, and cleanup terminates the Sandbox terminal execution session before deleting its owner row. + +#### Versioned runtime adapter + +Crabfleet authenticates every adapter request with `Authorization: Bearer CRABBOX_RUNTIME_ADAPTER_TOKEN`. Adapter URLs must use HTTPS, except literal loopback HTTP for same-host deployments. The original base URL may include a nested path, whose semantics are preserved, but any raw `?` or `#` delimiter is rejected even when its query or fragment is empty. Authenticated adapter requests reject redirects so the bearer token cannot cross origins. The canonical control-plane base URL is persisted with the lifecycle registration before create. Replay, inspect, desktop connection, and delete fail closed unless the current configured URL exactly matches that registered identity, so a configuration change cannot redirect an existing workspace ID or turn a 404 from another origin into release proof. + +- `POST /v1/workspaces`: idempotent create. Crabfleet persists the deterministic adapter identity, TTL, idle timeout, requested capabilities, and exact serialized create payload before the request, then sends the same namespaced DNS-safe lowercase `id` and `Idempotency-Key`, plus repo, branch, runtime, opaque profile, command, prompt, ownership/lineage, and lifecycle settings. A definitive non-2xx response is read once, sanitized, and durably recorded as the failure reason before provider release begins. After an ambiguous result, a bounded reconciliation pass retries only that immutable payload and key when inspect returns 404; later edits to session metadata do not alter it. +- `GET /v1/workspaces/:id`: inspect current status, capabilities, terminal URL, expiry, and provider resource identity. Status-only responses preserve previously stored capabilities and expiry; explicit `null` clears those fields. Active external sessions are reconciled in bounded batches; state responses wait only for a short foreground budget while remaining work continues in the Worker background. +- `DELETE /v1/workspaces/:id`: stop/release. Crabfleet enters `stopping` before calling the adapter and marks the session stopped only after `204`, `404`, or a valid exact-ID terminal response confirms release; malformed successful bodies remain `stopping`. Plain-text and malformed-JSON responses are read once and sanitized before their evidence is retained. An explicit stop whose ownership claim loses returns success only when the exact workspace is already stopping or terminal; otherwise it returns a lifecycle conflict. +- `POST /v1/workspaces/:id/connections/desktop`: mint a current transient desktop URL. `expiresAt` is optional; when present it must be in the future and no more than 15 minutes away. Accepted HTTPS URLs are treated as opaque signed connection material and redirected byte-for-byte without URL normalization. After minting, Crabfleet re-reads the exact current session status, control grant, capabilities, and registered adapter identity before redirecting; a concurrent stop, revocation, capability withdrawal, or lifecycle replacement discards the URL and denies access. + +`CRABBOX_RUNTIME_ADAPTER_NAMESPACE` is required and must remain stable for the deployment. It prevents workspace and idempotency collisions when an adapter serves more than one Crabfleet tenant. The adapter workspace `id` is an immutable lifecycle route key and remains separate from an opaque `providerResourceId`; the provider identity is never interpreted as a legacy lease or sandbox ID. Create, inspect, and stop responses must echo the byte-exact requested DNS-safe `id`; whitespace normalization is not accepted. Responses use `status`, `id`, optional `providerResourceId`, `attachUrl`, `capabilities`, `expiresAt`, and `message`. Only a literal `null` clears a previously stored expiry; a malformed non-null timestamp invalidates the response. A terminal URL implies terminal capability only when the response omits a terminal capability; an explicit `terminal: false` wins. Supported status values include `provisioning`, `ready`, `stopping`, `stopped`, `expired`, and `failed`. Create-only legacy adapters cannot return `stopping`, because they do not own a later reconciliation lifecycle. + +Every create, inspect, delete, and desktop response body is consumed through one 64 KiB bounded stream reader before JSON or text parsing. Declared or chunked oversized bodies are cancelled and fail safely: ambiguous create remains reconcilable, delete remains pending, inspect retries later, and desktop access is denied. + +Adapter messages are untrusted display text. Crabfleet removes raw and slash-escaped HTTP/WebSocket connection URLs directly from arbitrary message/detail text; Bearer and Basic credentials; authorization, cookie, and API-key headers; and sensitive assignments such as quoted JSON, colon fields, `token`, `ticket`, `access_token`, passwords, signatures, and secrets before replacing opaque provider identifiers and storing text in `lastEvent`, events, terminal failure evidence, or archives. For non-successful or malformed responses, opaque `providerResourceId`, `provider_resource_id`, `leaseId`, and `lease_id` values are collected from body, workspace, and error envelopes before redaction. Sanitizing credential structure first prevents an identifier such as `token` from hiding `token=secret`. + +An adapter-reported `failed` workspace is not locally terminal until Crabfleet calls DELETE and confirms release. Crabfleet durably clears create ambiguity and records the requested failed terminal state and original failure reason before awaiting DELETE, so reconciliation cannot replay the create or replace useful failure evidence with a generic release message. Asynchronous or uncertain release remains `stopping`; reconciliation records `failed` only after the workspace is gone. A stop racing an ambiguous create also remains `stopping`: every reconciliation pass uses a dedicated replay path fenced to the exact `stopping` row, pending marker, registered control plane, immutable payload, settings, and session version, then issues DELETE before recording the requested terminal state. Generic provisioning cannot restage that row. After confirmed release, Crabfleet re-reads and compare-and-swaps the current ambiguity marker and terminal intent: a cleared marker terminalizes immediately, while a still-pending create remains `stopping`. A valid exact-ID create response, including `provisioning`, or a definitive create rejection resolves that ambiguity. + ### GET /api/terminal/ws Viewer+, or public shared-link token for read-only sessions. Same-origin multiplex WebSocket endpoint used by the Ghostty WASM session grid. One browser socket can subscribe to multiple interactive sessions, receive PTY output frames, resize terminals, and send input only when the current user has control. @@ -275,12 +309,16 @@ Supported browser actions: - `Stop`: close the upstream subscription. - `Ping`: keepalive, answered with `Pong`. -Server messages include `Welcome`, `Output`, `Event`, `Error`, `ControlRevoked`, and `Pong`. Shared-link viewers can subscribe and scroll output, but input frames are rejected unless an owner/maintainer grants writable control. +Server messages include `Welcome`, `Output`, `Event`, `Error`, `ControlRevoked`, and `Pong`. Shared-link viewers can subscribe and scroll output, but input frames are rejected unless an owner/maintainer grants writable control. Subscriptions require the current `terminal` capability; withdrawing it prevents new attaches, closes existing terminal sockets on the next authorization check, suppresses raw attach URLs and attachable state from app, API, fleet, CLI, and SSH responses, and removes Fleet terminal/SSH affordances. Recurring and per-input authorization use short-lived D1 snapshots only; throttled subscription reconciliation runs independently and never blocks an input frame on provider I/O. ### POST /api/interactive-sessions/:id/clipboard Viewer+ with writable terminal control. Uploads a browser clipboard image/file body into the controlled Cloudflare Sandbox workspace and returns `{ path, name, mediaType, byteCount }`. The browser then pastes the returned path into the PTY. Max body size: 10 MiB. Non-Sandbox PTY backends do not expose file paste. +### GET /api/interactive-sessions/:id/vnc + +Viewer+ with writable session control. For `runtime-v1`, Crabfleet authenticates the browser session, asks the adapter to mint a current desktop connection, validates its HTTPS URL and optional bounded expiry, and issues a no-store redirect. Versioned-adapter desktop URLs are never persisted in D1 or returned by fleet state. API and CLI session views expose an absolute canonical Crabfleet browser URL for this cookie-authenticated route; the SSH gateway does not mint or receive the underlying adapter URL. Legacy adapters retain their existing validated absolute VNC URL behavior for browser and CLI clients. + ### GET /api/interactive-sessions/:id/pty Viewer+. Legacy single-session WebSocket endpoint. Crabfleet authenticates the browser session, verifies the interactive session is still attachable, verifies terminal control, then proxies PTY bytes to the configured runner. Owners and maintainers have control by default; other viewers require an approved control request. @@ -288,10 +326,10 @@ Viewer+. Legacy single-session WebSocket endpoint. Crabfleet authenticates the b Target resolution: - `CRABBOX_PTY_BRIDGE_URL`: explicit bridge WebSocket URL/template. Templates support `{id}`, `{leaseId}`, `{repo}`, `{branch}`, and `{runtime}`. Crabfleet appends `sessionId`, `leaseId`, `repo`, `branch`, `runtime`, and `command` query parameters. -- `attachUrl`: if the provision adapter returned a `ws://` or `wss://` URL, Crabfleet proxies to it. +- Provider terminal connection: if the provision adapter returned a `wss://` URL, or literal loopback `ws://` URL, Crabfleet retains it server-side and proxies to it unchanged, including its path and signed query string. - `CRABBOX_CLOUDFLARE_RUNNER_URL`: for `cloudflare:` leases, Crabfleet proxies to `/v1/sandboxes/:sandbox/pty` on the runner. -If `CRABBOX_PTY_BRIDGE_TOKEN` or `CRABBOX_CLOUDFLARE_RUNNER_TOKEN` is set, Crabfleet sends it as a bearer token only to the upstream bridge/runner. The browser never receives runner credentials. +Both multiplex and legacy direct PTY routes append terminal `cols` and `rows` only to configured bridge and Cloudflare runner endpoints, never to an adapter `attachUrl`. If `CRABBOX_PTY_BRIDGE_TOKEN` or `CRABBOX_CLOUDFLARE_RUNNER_TOKEN` is set, Crabfleet sends it as a bearer token only to the upstream bridge/runner. The browser never receives runner credentials. ### POST /api/interactive-sessions @@ -302,6 +340,7 @@ Maintainer+. Creates a standalone Codex CLI workspace request. "repo": "openclaw/crabfleet", "branch": "main", "runtime": "container", + "profile": "default", "command": "codex", "prompt": "Investigate flaky release CI", "parentSessionId": "IS-100", @@ -315,7 +354,8 @@ Fields: - `repo`: required, enabled repo. - `branch`: optional, default `main`. -- `runtime`: optional `crabbox` or `container`, default `container`. +- `runtime`: optional `crabbox` or `container`; omission uses `CRABFLEET_DEFAULT_RUNTIME`, which defaults to `container`. +- `profile`: optional opaque adapter profile, defaulted by `CRABFLEET_DEFAULT_PROFILE`. - `command`: optional, default `codex`. - `prompt`: optional initial context note. - `parentSessionId`: optional parent session for supervision trees. @@ -323,9 +363,11 @@ Fields: - `purpose`: optional short mission label. - `summary`: optional list/closeout summary. -If `CRABBOX_INTERACTIVE_PROVISION_URL` is configured, the Worker posts the request to that adapter and records returned `status`, `leaseId`, `attachUrl`, `vncUrl`, and `message`. Without an adapter the session is stored as `pending_adapter`. +If `CRABBOX_RUNTIME_ADAPTER_URL` is configured, the Worker creates and reconciles the versioned adapter workspace and records its lifecycle identity, status, capabilities, expiry, and terminal connection. Otherwise `CRABBOX_INTERACTIVE_PROVISION_URL` retains the legacy create-only behavior. Without an adapter the session is stored as `pending_adapter`. + +Session responses include `ptyAvailable`, the authenticated Worker's authoritative answer for whether the current terminal capability, lifecycle state, and configured Sandbox/bridge/runner route can resolve a PTY connection. A controllable `runtime-v1` session exposes only the Worker-owned `/api/interactive-sessions/:id/pty` route in `attachUrl`; the signed provider connection remains server-side even for owners and controllers. -Built-in Sandbox sessions receive `CRABFLEET_SESSION_ID`, `CRABFLEET_PARENT_SESSION_ID`, `CRABFLEET_ROOT_SESSION_ID`, `CRABFLEET_AGENT_TOKEN`, and `CRABFLEET_API_URL`. The agent token can call the `/api/agent/*` endpoints below for same-owner session discovery, child creation, transcripts, and summary updates. +Built-in Sandbox sessions receive `CRABFLEET_SESSION_ID`, `CRABFLEET_PARENT_SESSION_ID`, `CRABFLEET_ROOT_SESSION_ID`, `CRABFLEET_AGENT_TOKEN`, and `CRABFLEET_API_URL`. The managed provision hook rotates a fresh agent token in the same durable claim that owns provisioning, then injects that exact token into the Sandbox. The agent token can call the `/api/agent/*` endpoints below for same-owner session discovery, child creation, transcripts, and summary updates. ### GET /api/interactive-sessions/:id/transcript @@ -355,7 +397,7 @@ Actions: - `revoke_control`: owner/maintainer, revoke active delegated control. - `enable_multiplayer`: session creator, prefix submitted terminal prompts with the actor. - `disable_multiplayer`: session creator, stop prefixing submitted terminal prompts with the actor. -- `stop`: owner/maintainer, mark stopped. +- `stop`: owner/maintainer, stop the provider workspace first, then mark stopped; asynchronous releases remain `stopping` until reconciliation confirms completion. Response: diff --git a/docs/architecture.md b/docs/architecture.md index 8049447..4c8fe9a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -60,6 +60,27 @@ When a card is claimed, Crabfleet records a runtime descriptor: The UI and API both use capabilities. Takeover is visible and accepted only for an active run whose descriptor advertises takeover. +Interactive sessions can use the versioned external lifecycle adapter configured by `CRABBOX_RUNTIME_ADAPTER_URL`. The public contract is provider-neutral: + +- `POST /v1/workspaces` creates an idempotent workspace from a stable tenant-namespaced DNS-safe adapter ID, repo/ref, opaque profile, command, ownership, TTL, and requested capabilities. Crabfleet persists the identity, canonical control-plane registration, and complete immutable lifecycle snapshot before the network request so timeouts remain recoverable without moving the workspace ID between providers. +- `GET /v1/workspaces/:id` reconciles status, terminal connection, adapter capabilities, expiry, and the separate opaque provider resource ID. +- `DELETE /v1/workspaces/:id` releases the provider workspace before Crabfleet marks the session stopped. +- `POST /v1/workspaces/:id/connections/desktop` mints a current desktop connection after Crabfleet authorization. + +Active external sessions are reconciled with compare-and-swap updates so a stale inspect cannot overwrite a concurrent stop. The reconciliation claim and completion both fence the original session revision, and changed state commits a completion-time revision strictly newer than the snapshot; slow provider I/O therefore cannot regress `updated_at` or overwrite a concurrent same-status edit. Credential-cleanup completion and confirmed runtime release use the same ownership CAS and `MAX(updated_at + 1, now)` rule, so terminal archive versions cannot move backward. A durable create-ambiguity marker prevents a stop from becoming terminal while an idempotent request outcome is unknown; a parsed response clears the marker, while ambiguous outcomes replay the exact serialized original request and issue DELETE on every stopping pass until release is confirmed. Replay during `stopping` has its own exact-row path fenced by the pending marker, registered control plane, immutable payload/settings, requested terminal state, and session version; it does not pass through generic provision staging. Definitive create failures read the provider response once, redact it through the shared sanitizer, durably enter `stopping`, clear create ambiguity, and record failed terminal intent plus the actionable failure reason before the DELETE request. All adapter bodies use one 64 KiB bounded stream reader before parsing, including chunked responses, so oversized provider output cannot consume unbounded Worker memory or starve reconciliation. Redaction removes credential and URL structure before substituting opaque provider identifiers, preventing identifier text from masking a secret-bearing field. Redacted provider messages from successful, pending, and failed DELETE responses are persisted while release is pending and retained as events in the final archive. After confirmed release, Crabfleet re-reads and compare-and-swaps the current marker and terminal intent, preventing a pre-DELETE snapshot from either losing failed intent or leaving a resolved create stuck in `stopping`; the original reason remains in the terminal event, API state, and archive. Every terminal event insert and finalization-marker update share one D1 batch, including the finalizer's idempotent synthetic event; summary, sharing, multiplayer, and control mutations include their row update in that same batch. Legacy local stop records its request event, stopped event, terminal state, and finalization marker in one exact-owner D1 batch; cron and targeted reconciliation recover pre-existing or interrupted `stopping` rows. The winning terminal transition forces the final archive, R2 transcript, and summary; equal-count archive replacement is monotonic by mutable session version, while concurrent writers use unique object keys and delete only keys proven not to be the committed archive. Status-only inspection preserves omitted capability, expiry, and terminal fields, while explicit clears remain explicit, including explicit terminal-capability withdrawal despite a terminal URL. Raw terminal URLs, attachability, and UI terminal/SSH affordances are redacted from every outward session shape while terminal capability is false. Known signed connection URLs are also removed from JSON and common slash-escaped provider-message representations before persistence or display. Adapter failures enter provider release first and become locally failed only after release confirmation. State reads with an `ExecutionContext` have a short reconciliation budget and hand at most one three-workspace wave to the Worker background, keeping worst-case adapter timeouts inside the platform lifetime; callers without a context await completion. D1 stores the immutable adapter workspace ID and optional opaque provider resource ID separately, plus the immutable payload and TTL/idle/capability snapshot, create-ambiguity marker, current capabilities, expiry, desired terminal status, last-reconcile, and reconcile-error state. Provider resource IDs never enter legacy lease parsing. Authenticated adapter calls reject redirects. Versioned-adapter terminal and VNC bearer URLs are validated without normalization and accepted values retain their exact signed bytes. Terminal URLs remain server-side and API clients receive only the authenticated Worker PTY route; VNC URLs are never persisted, and browser, CLI, and SSH views receive an absolute canonical Crabfleet browser route which authenticates control before redirecting to the transient adapter URL. After desktop mint, Crabfleet re-reads the exact session status, control grant, capability, and adapter identity before releasing the transient redirect, so concurrent revocation discards the URL. Legacy adapters keep their existing absolute URL contract. + +The versioned adapter is reachable only through the durable interactive-session lifecycle, never the stateless provision hook. Existing lifecycle operations resolve through the persisted canonical control-plane identity and fail closed if the live deployment binding differs or disappears. An explicit stop that loses its initial compare-and-swap succeeds only after rereading the exact workspace in `stopping` or terminal state; a concurrent active mutation instead returns conflict. Terminal attach and recurring socket grants require the adapter's current `terminal` capability; withdrawing it closes active sockets. Direct adapter `attachUrl` values are opaque signed connection material retained server-side and passed through multiplex and legacy direct PTY paths byte-for-byte; only Crabfleet-owned bridge and runner endpoints receive `cols` and `rows` query parameters. A PTY transport failure leaves the lifecycle workspace detached and retryable rather than terminalizing it without provider release. + +Interactive session numbers come from a persistent monotonic sequence, so cleanup never reuses an adapter route or idempotency key. `CRABBOX_RUNTIME_ADAPTER_NAMESPACE` must also be unique and stable for each tenant sharing an adapter. + +Deployment identity is runtime configuration rather than an adapter concern. The API returns the configured label, canonical/product URLs, SSH host, preferred repo, default runtime, and opaque default profile so the same public Worker and UI can front different private adapters without source changes. + +The public `/api/auth` bootstrap exposes only label, canonical/product URLs, and SSH host. Preferred repo, runtime/profile defaults, and adapter routing remain in authenticated state. + +Runtime lifecycle reconciliation runs from the Worker cron every minute and from bounded, CAS-claimed targeted refreshes on direct session, PTY, and VNC access. Active provider inspection remains scoped to the versioned adapter, while pending terminal archives for every adapter share the same retryable finalizer. Archive metadata includes the mutable session version; pending finalization and D1 deletion proceed only when event count, terminal status, failure reason, summary, and session version all match. Enabling `SESSION_LOGS` later requeues finalized D1-only archives with null object keys, so the same finalizer backfills R2 before cleanup. WebSocket input and recurring permission checks read only cached D1 authorization state; each subscription schedules provider reconciliation separately, with one throttled request in flight, so provider latency cannot block the multiplex frame queue. Fleet polling remains an opportunistic accelerator rather than the lifecycle clock; terminal archive markers retry even when provider credentials are unavailable. Sandbox credential-policy registrations and deletions use a durable D1 outbox. Every registration begin, renewal, activation, and reference repair proves either the exact currently stored Sandbox lease or a live durable initial/refresh/standalone claim; there is no unfenced registration path. Initial and managed provision claims fence the current session revision before external effects; required bindings and token-encryption material are checked before managed claim/token rotation, and every later non-ready result atomically stages the exact claim for terminal cleanup. Their non-replayable completion instead fences the immutable lease, claim, agent-token hash, and status ownership while advancing the mutable session version monotonically, so concurrent metadata writes survive. A winning managed retry atomically adopts its new lease and retires the original Sandbox policy. Stop and failure cleanup uses an exact current or stored refresh fence, stages every current/refresh Sandbox policy in the same batch, and merges terminal intent with `failed > expired > stopped` precedence so a lower-priority race cannot erase failure evidence. Managed sessions and direct stateless Sandbox provisions first acquire durable ownership claims; standalone IDs are excluded from the managed `IS-` namespace. Standalone activation bumps all matching active policy-generation rows and activates the owner in one D1 batch, with a bounded expiry. Its PTY route terminates at a Worker WebSocket proxy that periodically rechecks the exact owner revision, expiry, lease, and active policy generation; stop, expiry, or revocation closes both peers. Standalone stop always requires the provision bearer, including after backend bindings are removed. Authenticated stop, cron expiry, and expired PTY access atomically stage that exact owner and its policy cleanup, and the retryable cleanup destroys the terminal execution session before deleting the owner. Registration claims and generations are committed before any policy POST and renewed before each lookup. Same-generation Durable Object writes may renew the current claim monotonically or replace it only with a later-expiring claim, preventing a delayed abandoned POST from overwriting the newer policy. A registration error on an expected live current lease clears its claim into a retryable state instead of staging cleanup that the live owner forbids; once ownership is gone, the same transition stages cleanup. Cleanup waits for live claims, then atomically stores a generation tombstone before deleting the matching policy. Whether a late POST or cleanup reaches the Durable Object first, the tombstone prevents credential resurrection, including when a Worker dies after POST but before its D1 completion update. Cleanup discovery and alias normalization use bounded high-water pages with persisted row and group cursors; deletion retries are ordered by oldest attempt with deterministic ties, so a large or continuously growing backlog cannot keep an invocation from reaching tombstone work or starve older rows. Session or standalone-owner cleanup transitions and their policy transitions share one D1 batch, and the unregister claim revalidates that neither a current lease/refresh nor a live standalone owner still expects that Sandbox; losing the owner CAS therefore cannot tombstone a live policy. Cron retries partial or failed cleanup idempotently and finalizes only after every policy reference is gone. Dead-session cleanup captures the archive keys, then claims, revalidates, and deletes the event, archive, and session rows in one D1 batch. R2 objects are deleted only after that commit, so an object-delete failure can leak unreferenced objects but cannot leave a surviving D1 row pointing at deleted keys. + +Credential injection fails closed unless the generation-wrapped Durable Object policy matches the complete active D1 generation and an exact live managed or standalone owner; raw legacy records and expired standalone policies are never served. If the Worker dies after the Durable Object accepts a current registration but before D1 activation, reconciliation verifies every lookup alias and the exact live owner, then promotes the matching expired registration claim to active before cleanup scanning. A lookup or ownership error defers cleanup for that pass. Raw records discovered during upgrade remain retained but unavailable while cron promotes their migrated D1 generation under an exact current-lease registration claim. An early raw lookup synchronously invokes that same repair and retries once, so an unattended session need not wait for cron. Each alias is wrapped transactionally in the Durable Object, successful completion replaces the legacy D1 generation, and an interrupted pass resumes idempotently after its claim expires. Cleanup accepts the old wrapped generation when stop races promotion, preventing a stranded Durable Object record. Standalone terminal-destruction failures remain on that owner's cleanup row with a monotonic retry revision, while other owners, runtime-adapter reconciliation, and terminal archives continue. The managed ID allocator compares standalone reservations case-insensitively, and the upgrade migration advances its sequence beyond every numeric standalone reservation before allocating again. + Current selection order: 1. Explicit card runtime `container` or `crabbox` diff --git a/docs/quickstart.md b/docs/quickstart.md index 5cbc4ba..e40d975 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -80,7 +80,7 @@ Click New crabbox or use the CLI: crabfleet new --repo openclaw/crabfleet "fix the failing check" ``` -Crabbox is the default runtime so terminal and WebVNC affordances appear as soon as the provision adapter returns links. +The CLI omits `runtime` unless `--runtime` is passed, so the deployment chooses via `CRABFLEET_DEFAULT_RUNTIME` (`container` when unset). Set a deployment to `crabbox` when its configured adapter provides terminal and WebVNC capabilities. ## 6. Create a Card diff --git a/docs/runs.md b/docs/runs.md index 5f30bb8..b7ed5f1 100644 --- a/docs/runs.md +++ b/docs/runs.md @@ -94,15 +94,19 @@ The Take over action records `controlIntent = "takeover"` and operator only for ## Interactive CLI Sessions -Maintainers can create a standalone Codex CLI session without making a board card. The Worker stores the requested repo, branch, runtime, command, owner, attach/VNC URLs, status, and event log in D1. The default runtime is `container` so production opens a Worker-owned Cloudflare Sandbox Codex terminal without requiring a separate crabbox adapter. +Maintainers can create a standalone Codex CLI session without making a board card. The Worker stores the requested repo, branch, runtime, command, owner, attach/VNC URLs, status, and event log in D1. `CRABFLEET_DEFAULT_RUNTIME` selects the deployment default (`container` when unset); the CLI and SSH gateway leave runtime unspecified unless the operator passes `--runtime`. Interactive sessions also store `parentSessionId`, `rootSessionId`, `createdBy`, `purpose`, and `summary`. Built-in Sandbox sessions export `CRABFLEET_SESSION_ID`, `CRABFLEET_PARENT_SESSION_ID`, `CRABFLEET_ROOT_SESSION_ID`, `CRABFLEET_AGENT_TOKEN`, and `CRABFLEET_API_URL`; the Go CLI uses those values to list sibling/child sessions, create children, send PTY messages, fetch transcripts, and update summaries without an SSH key. -Session events are mirrored into the `SESSION_LOGS` R2 binding when configured. Crabfleet writes NDJSON, Markdown transcript, and summary objects under `orgs/openclaw/interactive-sessions//`, while D1 keeps the compact event list and archive keys for the app, CLI, and SSH gateway. +Adapter capability arrays are authoritative: omitting `terminal`, `pty`, or `ssh` withdraws terminal access. A valid WSS (or literal-loopback WS) terminal URL implies terminal access only when capabilities are omitted entirely or an object omits all terminal-related keys. `ptyAvailable` additionally requires a ready lifecycle state and a resolvable configured Sandbox, bridge, direct WebSocket, or Cloudflare runner route. + +Session events are mirrored into the `SESSION_LOGS` R2 binding when configured. Crabfleet writes NDJSON, Markdown transcript, and summary objects under `orgs/openclaw/interactive-sessions//`, while D1 keeps the compact event list and archive keys for the app, CLI, and SSH gateway. If the binding is enabled after D1-only terminal archives were finalized, cron and targeted reconciliation requeue their null-key snapshots and backfill the objects before cleanup. Cleanup transactionally removes the finalized D1 session, events, and archive pointers before best-effort R2 deletion, so a partial object-delete failure is an unreferenced leak rather than a dangling archive reference. Stops for local legacy sessions atomically commit the request event, stopped event, terminal state, and finalization marker; cron and targeted access repair older `stopping` rows left by interrupted deployments. + +Sandbox credential policies have a separate durable cleanup lifecycle. Registration commits a generation and expiring claim in D1 before any external POST. If the Durable Object accepted every alias before the Worker crashed, reconciliation verifies that matching generation and the exact live owner, clears the expired D1 claim, and promotes the group to active before cleanup scanning; transient lookup or ownership failures defer cleanup. The upgrade migration seeds active legacy policies for proactive repair: cron claims each exact live lease, atomically generation-wraps every retained raw Durable Object policy, and activates all lookup aliases. A raw lookup also runs this fenced repair synchronously and retries once, avoiding a credential gap before the first cron pass. A crash before D1 completion leaves an expiring repair claim; the next pass resumes the same generation idempotently, while stop can still stage cleanup. Raw records remain unserved but retained until this repair or authorized cleanup. Credential injection rechecks that complete active generation and its exact D1 owner, so raw legacy Durable Object records, expired standalone policies, and orphaned generations fail closed. A registration error for an expected live current lease clears into a retryable registration state; an owner transition instead stages that generation for cleanup. Stop, expiry, provisioning failure, and superseded-resource cleanup atomically pair the durable owner transition with policy staging, revoke the session agent token and terminal control, terminate standalone terminal execution sessions, wait out live registration claims, and revalidate that no live owner still expects the Sandbox before persisting a matching generation tombstone; this makes both lost owner CAS operations and late POSTs harmless across Worker termination. Bounded persisted scan/group cursors keep large cleanup backlogs fair. Failed or partial deletes remain `stopping` and retry from cron until every recorded policy lookup is gone, then enter normal terminal archive finalization with the original failure reason intact. A standalone terminal-destruction failure is recorded on that owner and retried without blocking other cleanup owners, runtime-adapter reconciliation, or terminal archives. If `CRABBOX_INTERACTIVE_PROVISION_URL` is not set, new sessions stay `pending_adapter` and remain visible in the Ghostty grid. If it is set, Crabfleet posts the session request to that endpoint with optional bearer auth from `CRABBOX_INTERACTIVE_PROVISION_TOKEN`; the response can set `status`, `leaseId`, `attachUrl`, `vncUrl`, and `message`. -Crabfleet also ships a built-in provision hook at `/api/provision/interactive`. Point `CRABBOX_INTERACTIVE_PROVISION_URL` at that route to use Worker-side backend selection. Set `CRABBOX_INTERACTIVE_PROVISION_TOKEN` for backend-enabled deployments; the route fails closed without it when a backend is configured. The route delegates to `CRABBOX_RUNTIME_PROVISION_URL` when set, creates a Cloudflare Container sandbox for `container` sessions through `CRABBOX_CLOUDFLARE_RUNNER_URL` when configured, or creates a ClawFleet OpenClaw instance for `crabbox` sessions through `CRABBOX_CLAWFLEET_URL`; without a matching backend it returns `pending_adapter` with a clear setup message. +Crabfleet also ships a built-in provision hook at `/api/provision/interactive`. Point `CRABBOX_INTERACTIVE_PROVISION_URL` at that route to use Worker-side backend selection. Set `CRABBOX_INTERACTIVE_PROVISION_TOKEN` for backend-enabled deployments; the route fails closed without it when a backend is configured. Direct built-in standalone Sandboxes reject the reserved `IS-` namespace, expire after the bounded `CRABBOX_STANDALONE_SANDBOX_TTL_SECONDS`, and can be stopped through the bearer-authenticated `/api/provision/interactive/:id/stop` route. The route delegates to `CRABBOX_RUNTIME_PROVISION_URL` when set, creates a Cloudflare Container sandbox for `container` sessions through `CRABBOX_CLOUDFLARE_RUNNER_URL` when configured, or creates a ClawFleet OpenClaw instance for `crabbox` sessions through `CRABBOX_CLAWFLEET_URL`; without a matching backend it returns `pending_adapter` with a clear setup message. Cloudflare runner configuration: diff --git a/docs/spec-v2.md b/docs/spec-v2.md index cd3cd79..71845aa 100644 --- a/docs/spec-v2.md +++ b/docs/spec-v2.md @@ -75,6 +75,7 @@ Fleet state shape: "ready": 2, "attached": 0, "detached": 0, + "stopping": 0, "stopped": 1, "expired": 0, "failed": 1 @@ -111,6 +112,8 @@ Per-session summary: - `policy.hasGithubToken` - `policy.openAIBaseUrlHost` +`attachable` is true only for terminal-capable `ready`, `attached`, or `detached` sessions with current control and a resolvable Sandbox, configured bridge, valid WSS/literal-loopback WS URL, or configured Cloudflare runner route. + ## Security The fleet endpoint must never return raw secrets. @@ -185,6 +188,7 @@ Session states: - `ready`: attachable. - `attached`: attachable and observed attached. - `detached`: attachable but not currently attached. +- `stopping`: external adapter release requested but not yet confirmed. - `stopped`: intentionally stopped. - `expired`: lease expired. - `failed`: provision/run failed. diff --git a/migrations/0020_runtime_adapter_lifecycle.sql b/migrations/0020_runtime_adapter_lifecycle.sql new file mode 100644 index 0000000..9734e3f --- /dev/null +++ b/migrations/0020_runtime_adapter_lifecycle.sql @@ -0,0 +1,13 @@ +ALTER TABLE interactive_sessions ADD COLUMN adapter TEXT; +ALTER TABLE interactive_sessions ADD COLUMN profile TEXT NOT NULL DEFAULT 'default'; +ALTER TABLE interactive_sessions ADD COLUMN adapter_workspace_id TEXT; +ALTER TABLE interactive_sessions ADD COLUMN adapter_control_plane TEXT; +ALTER TABLE interactive_sessions ADD COLUMN provider_resource_id TEXT; +ALTER TABLE interactive_sessions ADD COLUMN capabilities_json TEXT NOT NULL DEFAULT '{}'; +ALTER TABLE interactive_sessions ADD COLUMN expires_at INTEGER; +ALTER TABLE interactive_sessions ADD COLUMN last_reconciled_at INTEGER; +ALTER TABLE interactive_sessions ADD COLUMN reconcile_error TEXT; +ALTER TABLE interactive_sessions ADD COLUMN terminal_status TEXT; + +CREATE INDEX IF NOT EXISTS idx_interactive_sessions_adapter_status + ON interactive_sessions(adapter, status, last_reconciled_at); diff --git a/migrations/0021_runtime_adapter_hardening.sql b/migrations/0021_runtime_adapter_hardening.sql new file mode 100644 index 0000000..48078fe --- /dev/null +++ b/migrations/0021_runtime_adapter_hardening.sql @@ -0,0 +1,91 @@ +CREATE TABLE IF NOT EXISTS id_sequences ( + name TEXT PRIMARY KEY, + last_id INTEGER NOT NULL +); + +INSERT OR IGNORE INTO id_sequences(name, last_id) +VALUES ('interactive_sessions', 100); + +UPDATE id_sequences +SET last_id = MAX( + last_id, + COALESCE( + (SELECT MAX(CAST(substr(id, 4) AS INTEGER)) FROM interactive_sessions WHERE id LIKE 'IS-%'), + 100 + ) +) +WHERE name = 'interactive_sessions'; + +ALTER TABLE interactive_sessions ADD COLUMN adapter_ttl_seconds INTEGER; +ALTER TABLE interactive_sessions ADD COLUMN adapter_idle_timeout_seconds INTEGER; +ALTER TABLE interactive_sessions ADD COLUMN adapter_requested_capabilities_json TEXT; +ALTER TABLE interactive_sessions ADD COLUMN adapter_create_payload_json TEXT; +ALTER TABLE interactive_sessions ADD COLUMN adapter_create_pending INTEGER NOT NULL DEFAULT 0; +ALTER TABLE interactive_sessions ADD COLUMN terminal_finalize_pending INTEGER NOT NULL DEFAULT 0; +ALTER TABLE interactive_sessions ADD COLUMN terminal_failure_reason TEXT; +ALTER TABLE interactive_session_log_archives ADD COLUMN session_updated_at INTEGER; + +UPDATE interactive_sessions +SET + adapter_ttl_seconds = COALESCE(adapter_ttl_seconds, 14400), + adapter_idle_timeout_seconds = COALESCE(adapter_idle_timeout_seconds, 1800), + adapter_requested_capabilities_json = COALESCE( + adapter_requested_capabilities_json, + CASE runtime + WHEN 'crabbox' THEN '{"terminal":true,"takeover":true,"vnc":true,"desktop":true,"logs":true,"artifacts":true}' + ELSE '{"terminal":true,"takeover":false,"vnc":false,"desktop":false,"logs":true,"artifacts":true}' + END + ), + adapter_create_payload_json = COALESCE( + adapter_create_payload_json, + json_object( + 'id', adapter_workspace_id, + 'parentSessionId', parent_session_id, + 'rootSessionId', COALESCE(root_session_id, id), + 'repo', repo, + 'branch', branch, + 'runtime', runtime, + 'profile', profile, + 'command', command, + 'prompt', prompt, + 'purpose', purpose, + 'summary', summary, + 'owner', owner, + 'createdBy', created_by, + 'ttlSeconds', COALESCE(adapter_ttl_seconds, 14400), + 'idleTimeoutSeconds', COALESCE(adapter_idle_timeout_seconds, 1800), + 'capabilities', json_object( + 'desktop', + CASE runtime WHEN 'crabbox' THEN json('true') ELSE json('false') END + ) + ) + ), + adapter_create_pending = CASE + WHEN status IN ('provisioning', 'pending_adapter') THEN 1 + ELSE adapter_create_pending + END +WHERE adapter = 'runtime-v1'; + +UPDATE interactive_sessions +SET + provider_resource_id = COALESCE(provider_resource_id, lease_id), + lease_id = NULL +WHERE adapter = 'runtime-v1'; + +UPDATE interactive_sessions +SET + status = 'stopping', + terminal_status = 'failed', + terminal_failure_reason = COALESCE( + terminal_failure_reason, + reconcile_error, + last_event, + 'interactive workspace failed after release' + ) +WHERE adapter = 'runtime-v1' + AND status = 'failed' + AND adapter_workspace_id IS NOT NULL; + +UPDATE interactive_sessions +SET terminal_finalize_pending = 1 +WHERE status IN ('stopped', 'expired', 'failed'); diff --git a/migrations/0022_credential_policy_cleanup.sql b/migrations/0022_credential_policy_cleanup.sql new file mode 100644 index 0000000..8d84c48 --- /dev/null +++ b/migrations/0022_credential_policy_cleanup.sql @@ -0,0 +1,174 @@ +ALTER TABLE interactive_sessions ADD COLUMN credential_cleanup_terminal_status TEXT + CHECK ( + credential_cleanup_terminal_status IS NULL + OR credential_cleanup_terminal_status IN ('stopped', 'expired', 'failed') + ); + +ALTER TABLE interactive_sessions ADD COLUMN sandbox_refresh_sandbox_id TEXT; +ALTER TABLE interactive_sessions ADD COLUMN sandbox_refresh_claim TEXT; +ALTER TABLE interactive_sessions ADD COLUMN sandbox_refresh_claim_expires_at INTEGER; + +CREATE TABLE IF NOT EXISTS standalone_sandbox_provisions ( + id TEXT PRIMARY KEY, + request_hash TEXT NOT NULL, + sandbox_id TEXT NOT NULL UNIQUE, + state TEXT NOT NULL CHECK (state IN ('provisioning', 'active', 'cleanup_pending')), + ownership_claim TEXT, + ownership_claim_expires_at INTEGER, + lease_id TEXT, + attach_url TEXT, + vnc_url TEXT, + message TEXT NOT NULL, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + CHECK ( + (ownership_claim IS NULL AND ownership_claim_expires_at IS NULL) + OR (ownership_claim IS NOT NULL AND ownership_claim_expires_at IS NOT NULL) + ) +); + +CREATE INDEX IF NOT EXISTS idx_standalone_sandbox_provision_state + ON standalone_sandbox_provisions(state, ownership_claim_expires_at, updated_at); + +CREATE TABLE IF NOT EXISTS interactive_session_credential_policies ( + session_id TEXT NOT NULL, + sandbox_id TEXT NOT NULL, + lookup_id TEXT NOT NULL, + state TEXT NOT NULL CHECK (state IN ('registering', 'active', 'cleanup_pending')), + registration_generation TEXT NOT NULL, + registration_claim TEXT, + registration_claim_expires_at INTEGER, + attempt_count INTEGER NOT NULL DEFAULT 0, + last_attempt_at INTEGER, + last_error TEXT, + cleanup_claim TEXT, + cleanup_claim_expires_at INTEGER, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY (session_id, sandbox_id, lookup_id), + CHECK ( + (registration_claim IS NULL AND registration_claim_expires_at IS NULL) + OR (registration_claim IS NOT NULL AND registration_claim_expires_at IS NOT NULL) + ), + CHECK ( + registration_claim IS NULL + OR state IN ('registering', 'cleanup_pending') + ) +); + +CREATE INDEX IF NOT EXISTS idx_interactive_policy_cleanup + ON interactive_session_credential_policies(state, cleanup_claim_expires_at, last_attempt_at); + +CREATE INDEX IF NOT EXISTS idx_interactive_policy_session + ON interactive_session_credential_policies(session_id, state); + +CREATE INDEX IF NOT EXISTS idx_interactive_policy_registration + ON interactive_session_credential_policies(state, registration_claim_expires_at); + +CREATE INDEX IF NOT EXISTS idx_interactive_policy_cleanup_groups + ON interactive_session_credential_policies(state, session_id, sandbox_id); + +CREATE INDEX IF NOT EXISTS idx_interactive_policy_fair_cleanup + ON interactive_session_credential_policies( + state, + COALESCE(last_attempt_at, created_at), + session_id, + sandbox_id, + lookup_id + ); + +CREATE INDEX IF NOT EXISTS idx_interactive_session_credential_completion + ON interactive_sessions(status, credential_cleanup_terminal_status, stopped_at, id); + +CREATE TABLE IF NOT EXISTS credential_policy_reconcile_state ( + id INTEGER PRIMARY KEY CHECK (id = 1), + last_rowid INTEGER NOT NULL DEFAULT 0, + scan_max_rowid INTEGER NOT NULL DEFAULT 0, + group_session_id TEXT NOT NULL DEFAULT '', + group_sandbox_id TEXT NOT NULL DEFAULT '', + group_max_session_id TEXT NOT NULL DEFAULT '', + group_max_sandbox_id TEXT NOT NULL DEFAULT '', + updated_at INTEGER NOT NULL +); + +INSERT OR IGNORE INTO credential_policy_reconcile_state ( + id, + last_rowid, + scan_max_rowid, + group_session_id, + group_sandbox_id, + group_max_session_id, + group_max_sandbox_id, + updated_at +) +VALUES (1, 0, 0, '', '', '', '', 0); + +INSERT OR IGNORE INTO interactive_session_credential_policies ( + session_id, + sandbox_id, + lookup_id, + state, + registration_generation, + registration_claim, + registration_claim_expires_at, + created_at, + updated_at +) +SELECT + id, + CASE + WHEN instr(substr(lease_id, 9), ':') > 0 + THEN substr(lease_id, 9, instr(substr(lease_id, 9), ':') - 1) + ELSE substr(lease_id, 9) + END, + CASE + WHEN instr(substr(lease_id, 9), ':') > 0 + THEN substr(lease_id, 9, instr(substr(lease_id, 9), ':') - 1) + ELSE substr(lease_id, 9) + END, + CASE + WHEN status IN ('stopping', 'stopped', 'expired', 'failed') THEN 'cleanup_pending' + ELSE 'active' + END, + 'legacy:' || id || ':' || CASE + WHEN instr(substr(lease_id, 9), ':') > 0 + THEN substr(lease_id, 9, instr(substr(lease_id, 9), ':') - 1) + ELSE substr(lease_id, 9) + END, + NULL, + NULL, + COALESCE(updated_at, 0), + COALESCE(updated_at, 0) +FROM interactive_sessions +WHERE (adapter IS NULL OR adapter != 'runtime-v1') + AND lease_id LIKE 'sandbox:%'; + +UPDATE interactive_sessions +SET + status = 'stopping', + credential_cleanup_terminal_status = CASE + WHEN status = 'stopping' THEN 'stopped' + WHEN status IN ('stopped', 'expired', 'failed') THEN status + ELSE credential_cleanup_terminal_status + END, + terminal_failure_reason = CASE + WHEN status = 'failed' THEN COALESCE( + NULLIF(terminal_failure_reason, ''), + NULLIF(reconcile_error, ''), + NULLIF(last_event, ''), + 'interactive workspace failed during credential cleanup' + ) + ELSE terminal_failure_reason + END, + terminal_finalize_pending = 0, + agent_token_hash = NULL, + attach_url = NULL, + vnc_url = NULL, + controller = NULL, + control_requested_by = NULL, + control_requested_at = NULL, + control_granted_at = NULL, + control_expires_at = NULL +WHERE (adapter IS NULL OR adapter != 'runtime-v1') + AND lease_id LIKE 'sandbox:%' + AND status IN ('stopping', 'stopped', 'expired', 'failed'); diff --git a/migrations/0023_standalone_sandbox_expiry.sql b/migrations/0023_standalone_sandbox_expiry.sql new file mode 100644 index 0000000..767bb14 --- /dev/null +++ b/migrations/0023_standalone_sandbox_expiry.sql @@ -0,0 +1,28 @@ +ALTER TABLE standalone_sandbox_provisions ADD COLUMN expires_at INTEGER; + +UPDATE standalone_sandbox_provisions +SET expires_at = CASE + WHEN lower(id) GLOB 'is-[0-9]*' + AND substr(lower(id), 4) NOT GLOB '*[^0-9]*' + THEN 0 + ELSE created_at + 14400000 +END +WHERE state IN ('provisioning', 'active'); + +UPDATE id_sequences +SET last_id = MAX( + last_id, + COALESCE( + ( + SELECT MAX(CAST(substr(lower(id), 4) AS INTEGER)) + FROM standalone_sandbox_provisions + WHERE lower(id) GLOB 'is-[0-9]*' + AND substr(lower(id), 4) NOT GLOB '*[^0-9]*' + ), + 100 + ) +) +WHERE name = 'interactive_sessions'; + +CREATE INDEX IF NOT EXISTS idx_standalone_sandbox_provision_expiry + ON standalone_sandbox_provisions(state, expires_at, updated_at); diff --git a/src/app/fleet.jsx b/src/app/fleet.jsx index 4b85d34..dfe6f62 100644 --- a/src/app/fleet.jsx +++ b/src/app/fleet.jsx @@ -3,15 +3,18 @@ import { canMaintain, elapsed, interactiveSessionStatus, + isFleetSessionAttachable, isTerminalReadyInteractiveSession, runCapabilities, sessionLogsUrl, } from "./utils.js"; const productDomain = "crabfleet.openclaw.ai"; -const sshHost = "crabd.sh"; export function FleetPage(props) { + const deployment = props.state.deployment || {}; + const sshHost = deployment.sshHost || "crabd.sh"; + const preferredRepo = deployment.preferredRepo || "openclaw/crabfleet"; const sessions = props.state.interactiveSessions || []; const fleet = props.state.fleet; const totals = fleet?.totals || {}; @@ -32,7 +35,7 @@ export function FleetPage(props) {
-
OPENCLAW / FLEET COMMAND
+
{deployment.label || "Crabfleet"} / FLEET COMMAND

{readyCount} crabboxes live

@@ -74,12 +77,14 @@ export function FleetPage(props) { queue: props.queue, review: props.review, }} - cli={props.cli} + cli={totals.attachable ?? props.cli} />
@@ -114,6 +119,7 @@ export function FleetPage(props) { key={session.id} session={{ ...session, fleet: fleetSessionsById.get(session.id) }} openSessionGrid={props.openSessionGrid} + sshHost={sshHost} /> ))} @@ -126,7 +132,7 @@ export function FleetPage(props) {
No crabboxes on the board

Create one from SSH, the Go CLI, or the app.

- +
); } -function FleetBox({ session, openSessionGrid }) { +function FleetBox({ session, openSessionGrid, sshHost }) { const capabilities = runCapabilities(session); + const attachable = isFleetSessionAttachable(session); const archiveCount = session.logArchive?.eventCount || session.logs?.length || 0; const fleetPolicy = session.fleet?.policy; const status = interactiveSessionStatus(session); const seen = session.lastSeenAt || session.updatedAt || session.createdAt; + const desktopEligible = !["stopping", "stopped", "expired", "failed"].includes(session.status); return (
@@ -280,19 +286,23 @@ function FleetBox({ session, openSessionGrid }) { {fleetPolicy?.present ? {fleetPolicy.allowedHostCount} egress : null}

{session.lastEvent || "Waiting for crabbox"}

-
- - ssh {sshHost} attach {session.id} - - {seen ? `seen ${elapsed(seen)}` : "no heartbeat"} -
+ {attachable ? ( +
+ + ssh {sshHost} attach {session.id} + + {seen ? `seen ${elapsed(seen)}` : "no heartbeat"} +
+ ) : null}
- + {attachable ? ( + + ) : null} {session.vncUrl ? ( - ) : capabilities.vnc ? ( + ) : desktopEligible && session.adapter === "runtime-v1" && capabilities.vnc ? (
-

OpenClaw crabboxes, SSH-first.

+

Managed crabboxes, SSH-first.

@@ -2511,8 +2543,9 @@ function PolicyBox({ disabled, state, updatePolicy }) { ); } -function WorkflowBox({ disabled, workflows, refreshWorkflow }) { - const [repo, setRepo] = useState(preferredRepo); +function WorkflowBox({ disabled, workflows, refreshWorkflow, preferred = preferredRepo }) { + const [repo, setRepo] = useState(preferred); + useEffect(() => setRepo(preferred), [preferred]); return (

Workflows

@@ -2520,7 +2553,7 @@ function WorkflowBox({ disabled, workflows, refreshWorkflow }) { setRepo(event.currentTarget.value)} diff --git a/src/app/utils.js b/src/app/utils.js index 1cbe9c6..1d62400 100644 --- a/src/app/utils.js +++ b/src/app/utils.js @@ -24,10 +24,10 @@ export function canOwn(user) { return roleRank(user?.role) >= roleRank("owner"); } -export function preferredRepos(repos = []) { +export function preferredRepos(repos = [], preferred = preferredRepo) { return [...repos].sort((left, right) => { - if (left === preferredRepo) return -1; - if (right === preferredRepo) return 1; + if (left === preferred) return -1; + if (right === preferred) return 1; return left.localeCompare(right); }); } @@ -65,9 +65,12 @@ export function runRuntime(session) { } export function runCapabilities(session) { - if (session.kind === "interactive") { + if (session.kind !== "interactive" && session.run?.capabilities) { + return session.run.capabilities; + } + if (session.kind === "interactive" || session.capabilities) { const crabbox = session.runtime === "crabbox"; - return { + const defaults = { terminal: true, takeover: false, vnc: crabbox, @@ -75,8 +78,8 @@ export function runCapabilities(session) { logs: true, artifacts: false, }; + return { ...defaults, ...session.capabilities }; } - if (session.run?.capabilities) return session.run.capabilities; const crabbox = runRuntime(session) === "crabbox"; return { terminal: true, @@ -119,11 +122,21 @@ export function isDeadInteractiveSession(session) { } export function isTerminalReadyInteractiveSession(session) { - return ( + const lifecycleReady = session && (session.kind === undefined || session.kind === "interactive") && - terminalReadyInteractiveStatuses.has(session.status) - ); + terminalReadyInteractiveStatuses.has(session.status) && + (session.capabilities?.terminal ?? runCapabilities(session).terminal); + if (!lifecycleReady) return false; + if (session.sharedReadOnly === true) return true; + if (session.canControl === true) return session.ptyAvailable === true; + return true; +} + +export function isFleetSessionAttachable(session) { + if (!runCapabilities(session).terminal) return false; + if (typeof session?.fleet?.attachable === "boolean") return session.fleet.attachable; + return isTerminalReadyInteractiveSession(session) && Boolean(session.attachUrl); } export function humanStatus(value) { @@ -140,6 +153,7 @@ export function interactiveSessionStatus(session) { return { label: "Unavailable", tone: "failed" }; } if (session.status === "failed") return { label: "Failed", tone: "failed" }; + if (session.status === "stopping") return { label: "Stopping", tone: "provisioning" }; if (session.status === "stopped" || session.status === "expired") { return { label: "Stopped", tone: "stopped" }; } @@ -283,12 +297,22 @@ export function optimisticInteractiveSession(data, owner) { const repo = String(data.get("repo") || preferredRepo); const branch = String(data.get("branch") || "main"); const runtime = String(data.get("runtime") || "container"); + const profile = String(data.get("profile") || "default"); const runtimeLabel = runtime === "crabbox" ? "Crabbox" : "Cloudflare Sandbox"; return { id: `LOCAL-${now}`, repo, branch, runtime, + profile, + capabilities: { + terminal: true, + takeover: false, + vnc: runtime === "crabbox", + desktop: runtime === "crabbox", + logs: true, + artifacts: false, + }, command: interactiveCommand(data.get("command")), prompt: String(data.get("prompt") || ""), owner: owner || "local", @@ -335,6 +359,15 @@ export function linkedInteractiveSessionPlaceholder(id, options = {}) { repo: "Codex session", branch: "", runtime: "crabbox", + profile: "default", + capabilities: { + terminal: true, + takeover: false, + vnc: true, + desktop: true, + logs: true, + artifacts: false, + }, command: "codex", prompt: "", owner: "unknown", diff --git a/src/bounded-response.ts b/src/bounded-response.ts new file mode 100644 index 0000000..b8b1cd9 --- /dev/null +++ b/src/bounded-response.ts @@ -0,0 +1,48 @@ +export const runtimeAdapterResponseBodyLimitBytes = 64 * 1024; + +export class ResponseBodyLimitError extends Error { + constructor(limit: number) { + super(`response body exceeds ${limit} bytes`); + this.name = "ResponseBodyLimitError"; + } +} + +export async function readBoundedResponseText( + response: Response, + limit = runtimeAdapterResponseBodyLimitBytes, +): Promise { + if (!Number.isSafeInteger(limit) || limit < 0) throw new Error("invalid response body limit"); + const declaredLength = response.headers.get("content-length"); + if (/^\d+$/u.test(declaredLength ?? "") && Number(declaredLength) > limit) { + await response.body?.cancel().catch(() => undefined); + throw new ResponseBodyLimitError(limit); + } + if (!response.body) return ""; + + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let total = 0; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + if (!value?.byteLength) continue; + if (total + value.byteLength > limit) { + await reader.cancel().catch(() => undefined); + throw new ResponseBodyLimitError(limit); + } + chunks.push(value); + total += value.byteLength; + } + } finally { + reader.releaseLock(); + } + + const body = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + body.set(chunk, offset); + offset += chunk.byteLength; + } + return new TextDecoder().decode(body); +} diff --git a/src/credential-policy-fence.ts b/src/credential-policy-fence.ts new file mode 100644 index 0000000..ba5584d --- /dev/null +++ b/src/credential-policy-fence.ts @@ -0,0 +1,117 @@ +export type CredentialPolicyGenerationRecord = { + generation: string; + registrationClaim: string; + registrationExpiresAt: number; + policy: T; +}; + +export type CredentialPolicyGenerationTombstone = { + generation: string; + sessionId: string; + tombstonedAt: number; +}; + +export type CredentialPolicyLegacyMigration = { + generation: string; + registrationClaim: string; + registrationExpiresAt: number; + sessionId: string; +}; + +export function credentialPolicyRegistrationAccepted( + current: CredentialPolicyGenerationRecord | undefined, + tombstone: CredentialPolicyGenerationTombstone | undefined, + incoming: CredentialPolicyGenerationRecord, + now: number, +): boolean { + if (incoming.registrationExpiresAt <= now) return false; + if (tombstone?.generation === incoming.generation) { + return false; + } + if (!current) return true; + if ( + current.generation !== incoming.generation || + current.policy.sessionId !== incoming.policy.sessionId + ) { + return false; + } + if (current.registrationClaim === incoming.registrationClaim) { + return incoming.registrationExpiresAt >= current.registrationExpiresAt; + } + return incoming.registrationExpiresAt > current.registrationExpiresAt; +} + +export function credentialPolicyCleanupMatches( + current: CredentialPolicyGenerationRecord | undefined, + generation: string, + sessionId: string, +): boolean { + return Boolean( + current && current.generation === generation && current.policy.sessionId === sessionId, + ); +} + +export function migratedCredentialPolicyRecord( + current: CredentialPolicyGenerationRecord | undefined, + legacy: T | undefined, + tombstone: CredentialPolicyGenerationTombstone | undefined, + migration: CredentialPolicyLegacyMigration, + now: number, +): CredentialPolicyGenerationRecord | undefined { + if (migration.registrationExpiresAt <= now || tombstone?.generation === migration.generation) { + return undefined; + } + const policy = current?.policy ?? legacy; + if (!policy || policy.sessionId !== migration.sessionId) return undefined; + const incoming = { + generation: migration.generation, + registrationClaim: migration.registrationClaim, + registrationExpiresAt: migration.registrationExpiresAt, + policy, + }; + if (!current) return incoming; + if ( + current.generation !== migration.generation && + (!current.generation.startsWith("legacy:") || + migration.registrationExpiresAt <= current.registrationExpiresAt) + ) { + return undefined; + } + if ( + current.generation === migration.generation && + !credentialPolicyRegistrationAccepted(current, tombstone, incoming, now) + ) { + return undefined; + } + return incoming; +} + +export function credentialPolicyMigrationCleanupMatches( + current: CredentialPolicyGenerationRecord | undefined, + generation: string, + sessionId: string, +): boolean { + return Boolean( + current && + current.generation !== generation && + current.generation.startsWith("legacy:") && + current.policy.sessionId === sessionId, + ); +} + +export function credentialPolicySandboxIsExpected( + leaseSandboxId: string | null, + policySandboxId: string, + refreshSandboxId: string | null, + refreshClaim: string | null, + refreshClaimExpiresAt: number | null, + now: number, +): boolean { + if (leaseSandboxId === policySandboxId) return true; + return Boolean( + refreshSandboxId === policySandboxId && + refreshClaim && + refreshClaimExpiresAt !== null && + refreshClaimExpiresAt > now, + ); +} diff --git a/src/d1-execution.ts b/src/d1-execution.ts new file mode 100644 index 0000000..b57e725 --- /dev/null +++ b/src/d1-execution.ts @@ -0,0 +1,65 @@ +import type { CompiledQuery, DatabaseConnection, QueryResult } from "kysely"; + +type D1ExecutionMeta = { + changes?: number; + last_row_id?: number; +}; + +type D1ExecutionResult = { + results?: R[]; + meta: D1ExecutionMeta; +}; + +export type D1StatementExecution = { + all(): Promise>; + run(): Promise>; +}; + +export type D1Execution = { + rows: R[]; + changes?: number; + lastRowId?: number; +}; + +export class D1Connection implements DatabaseConnection { + private readonly d1: D1Database; + + constructor(d1: D1Database) { + this.d1 = d1; + } + + async executeQuery(compiledQuery: CompiledQuery): Promise> { + const statement = this.d1.prepare(compiledQuery.sql).bind(...compiledQuery.parameters); + const result = await executeD1Statement(statement, compiledQuery.sql); + const queryResult: QueryResult = { rows: result.rows }; + if (typeof result.changes === "number") { + Object.assign(queryResult, { numAffectedRows: BigInt(result.changes) }); + } + if (typeof result.lastRowId === "number") { + Object.assign(queryResult, { insertId: BigInt(result.lastRowId) }); + } + return queryResult; + } + + async *streamQuery(compiledQuery: CompiledQuery): AsyncIterableIterator> { + yield await this.executeQuery(compiledQuery); + } +} + +export async function executeD1Statement( + statement: D1StatementExecution, + sqlText: string, +): Promise> { + const returnsRows = d1QueryReturnsRows(sqlText); + const result = returnsRows ? await statement.all() : await statement.run(); + return { + rows: returnsRows ? ((result.results ?? []) as R[]) : [], + ...(typeof result.meta.changes === "number" ? { changes: result.meta.changes } : {}), + ...(typeof result.meta.last_row_id === "number" ? { lastRowId: result.meta.last_row_id } : {}), + }; +} + +export function d1QueryReturnsRows(sqlText: string): boolean { + const normalized = sqlText.trim(); + return /^(?:select|with|pragma)\b/i.test(normalized) || /\breturning\b/i.test(normalized); +} diff --git a/src/fleet-state.ts b/src/fleet-state.ts index 4fa032a..443ca75 100644 --- a/src/fleet-state.ts +++ b/src/fleet-state.ts @@ -4,6 +4,7 @@ export type FleetStatus = | "ready" | "attached" | "detached" + | "stopping" | "stopped" | "expired" | "failed"; @@ -17,6 +18,7 @@ export type FleetSessionInput = { repo: string; branch: string; runtime: FleetRuntime; + adapter?: string | null; owner: string; createdBy?: string; purpose?: string; @@ -25,6 +27,9 @@ export type FleetSessionInput = { leaseId: string | null; attachUrl: string | null; vncUrl: string | null; + capabilities?: { vnc?: boolean; desktop?: boolean; terminal?: boolean }; + canControl?: boolean; + ptyAvailable?: boolean; lastEvent: string; createdAt: number; updatedAt: number; @@ -54,6 +59,9 @@ export type FleetStateOptions = { generatedAt: number; productUrl: string; registryAvailable?: boolean; + sandboxAvailable?: boolean | undefined; + ptyBridgeUrl?: string | null | undefined; + cloudflareRunnerUrl?: string | null | undefined; }; export type FleetSessionSummary = { @@ -116,18 +124,34 @@ export type FleetState = { sessions: FleetSessionSummary[]; }; +export type PtyRouteKind = "sandbox" | "bridge" | "attach" | "cloudflare"; + +export type PtyRouteSession = { + adapter?: string | null; + leaseId: string | null; + attachUrl: string | null; +}; + +export type PtyRouteConfig = { + sandboxAvailable?: boolean | undefined; + bridgeUrl?: string | null | undefined; + cloudflareRunnerUrl?: string | null | undefined; +}; + const allStatuses: FleetStatus[] = [ "provisioning", "pending_adapter", "ready", "attached", "detached", + "stopping", "stopped", "expired", "failed", ]; -const inactiveStatuses = new Set(["stopped", "expired", "failed"]); +const inactiveStatuses = new Set(["stopping", "stopped", "expired", "failed"]); +const ptyReadyStatuses = new Set(["ready", "attached", "detached"]); export function buildFleetState( sessions: FleetSessionInput[], @@ -139,7 +163,9 @@ export function buildFleetState( if (!policiesBySession.has(policy.sessionId)) policiesBySession.set(policy.sessionId, policy); } const sessionSummaries = sessions - .map((session) => fleetSessionSummary(session, policiesBySession.get(session.id) ?? null)) + .map((session) => + fleetSessionSummary(session, policiesBySession.get(session.id) ?? null, options), + ) .sort((a, b) => b.updatedAt - a.updatedAt || a.id.localeCompare(b.id)); const byStatus = Object.fromEntries(allStatuses.map((status) => [status, 0])) as Record< @@ -182,9 +208,16 @@ export function buildFleetState( export function fleetSessionSummary( session: FleetSessionInput, policy: FleetSandboxPolicySummary | null, + options: Pick< + FleetStateOptions, + "sandboxAvailable" | "ptyBridgeUrl" | "cloudflareRunnerUrl" + > = {}, ): FleetSessionSummary { const sandboxId = sandboxIdFromLeaseId(session.leaseId); const archived = Boolean(session.logArchive?.eventCount); + const terminalCapable = + session.capabilities?.terminal === true || + (session.adapter !== "runtime-v1" && session.capabilities?.terminal !== false); return { id: session.id, parentSessionId: session.parentSessionId ?? null, @@ -198,8 +231,25 @@ export function fleetSessionSummary( summary: session.summary ?? session.purpose ?? session.lastEvent, status: session.status, active: !inactiveStatuses.has(session.status), - attachable: Boolean(session.attachUrl) && !inactiveStatuses.has(session.status), - vnc: Boolean(session.vncUrl), + attachable: + terminalCapable && + session.canControl !== false && + (session.ptyAvailable ?? + Boolean( + ptyRouteKind(session, { + sandboxAvailable: options.sandboxAvailable, + bridgeUrl: options.ptyBridgeUrl, + cloudflareRunnerUrl: options.cloudflareRunnerUrl, + }), + )) && + ptyReadyStatuses.has(session.status), + vnc: + !inactiveStatuses.has(session.status) && + Boolean( + session.vncUrl || + (session.adapter === "runtime-v1" && + (session.capabilities?.vnc || session.capabilities?.desktop)), + ), archived, logEvents: session.logArchive?.eventCount ?? session.logs?.length ?? 0, leaseId: session.leaseId, @@ -225,3 +275,66 @@ export function sandboxIdFromLeaseId(leaseId: string | null | undefined): string const [sandboxId] = leaseId.slice("sandbox:".length).split(":"); return sandboxId || null; } + +export function ptyRouteKind( + session: PtyRouteSession, + config: PtyRouteConfig, +): PtyRouteKind | null { + const leaseId = session.adapter === "runtime-v1" ? null : session.leaseId; + if (config.sandboxAvailable && leaseId?.startsWith("sandbox:")) return "sandbox"; + if (configuredBridgeWebSocketUrl(config.bridgeUrl)) return "bridge"; + if (safePtyWebSocketUrl(session.attachUrl)) return "attach"; + if (leaseId?.startsWith("cloudflare:") && safePtyHttpUrl(config.cloudflareRunnerUrl ?? null)) { + return "cloudflare"; + } + return null; +} + +function configuredBridgeWebSocketUrl(value: string | null | undefined): string | null { + const candidate = String(value ?? "") + .trim() + .replaceAll(/\{(?:id|leaseId|repo|branch|runtime)\}/g, "route-value"); + if (!candidate) return null; + try { + const url = new URL(candidate); + if (url.protocol === "https:") url.protocol = "wss:"; + if (url.protocol === "http:") url.protocol = "ws:"; + return safePtyWebSocketUrl(url.toString()); + } catch { + return null; + } +} + +function safePtyWebSocketUrl(value: string | null | undefined): string | null { + return safePtyUrl(value, "wss:", "ws:"); +} + +function safePtyHttpUrl(value: string | null | undefined): string | null { + return safePtyUrl(value, "https:", "http:"); +} + +function safePtyUrl( + value: string | null | undefined, + secureProtocol: string, + loopbackProtocol: string, +): string | null { + if (!value) return null; + try { + const url = new URL(value); + if (url.username || url.password) return null; + if (url.protocol === secureProtocol) return url.toString(); + if (url.protocol !== loopbackProtocol || !isPtyLoopbackHostname(url.hostname)) return null; + return url.toString(); + } catch { + return null; + } +} + +function isPtyLoopbackHostname(hostname: string): boolean { + return ( + hostname === "localhost" || + hostname === "127.0.0.1" || + hostname === "::1" || + hostname === "[::1]" + ); +} diff --git a/src/index.ts b/src/index.ts index 2fe1663..f77cdd3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -10,7 +10,9 @@ import { type Dialect, type Driver, type Generated, - type QueryResult, + type RawBuilder, + type Selectable, + type UpdateObject, } from "kysely"; import { ContainerProxy, @@ -36,9 +38,20 @@ import { terminalSubmittedLine, type TerminalInputState, } from "./terminal-multiplayer"; -import { buildFleetState, type FleetSandboxPolicySummary, type FleetState } from "./fleet-state"; +import { + buildFleetState, + ptyRouteKind, + type FleetSandboxPolicySummary, + type FleetState, + type PtyRouteKind, +} from "./fleet-state"; import { githubRequestCanUseRepoCredential, matchesAnyHost } from "./sandbox-security"; -import { githubOAuthRedirectUri } from "./oauth"; +import { + githubOAuthCallbackRequestMatches, + githubOAuthCanonicalLoginUrl, + githubOAuthCanonicalSshLinkUrl, + githubOAuthRedirectUri, +} from "./oauth"; import { APP_HTML, GHOSTTY_BROWSER_EXTERNAL_JS, @@ -57,11 +70,78 @@ import { canonicalAppRedirect, productHostResponse, } from "./canonical-host"; +import { + adapterFailureReleaseState, + adapterWorkspaceIdMatches, + clearedAdapterCapabilities, + createOnlyAdapterStatus, + definitiveRuntimeAdapterCreateFailure, + effectiveAdapterCapabilities, + currentAdapterDesktopConnection, + legacyLeaseIdForAdapter, + namespacedAdapterWorkspaceId, + normalizeAdapterNamespace, + normalizeAdapterWorkspaceId, + parseAdapterWorkspaceResult, + redactedAdapterMessage, + redactedAdapterResponseMessage, + runtimeAdapterCreatePayload, + runtimeAdapterCollectionUrl, + runtimeAdapterControlPlaneIdentity, + runtimeAdapterBrowserVncUrl, + runtimeAdapterDesktopUrl, + runtimeAdapterName, + runtimeAdapterReplayRequest, + retainedRuntimeAdapterFailureMessage, + runtimeAdapterStopOutcome, + runtimeAdapterTerminalFailureStatus, + runtimeAdapterWorkspaceUrl, + resolveCreateAfterStopRace, + safeDesktopUrl, + safeWebSocketUrl, + shouldReplayRuntimeAdapterCreate, + validatedRuntimeAdapterCreatePayloadJson, + type AdapterWorkspaceResult, +} from "./runtime-adapter"; +import { allocateInteractiveSessionIdSql, formatInteractiveSessionId } from "./session-id"; +import { configuredHttpOrigin, developmentIdentityEnabled } from "./url-security"; +import { D1Connection } from "./d1-execution"; +import { preferredEnabledRepo } from "./repo-selection"; +import { completeTerminalFinalization } from "./terminal-finalization"; +import { sizedTerminalTargetUrl } from "./terminal-target"; +import { cachedBooleanGrant } from "./terminal-authorization"; +import { obsoleteSessionArchiveObjectKeys, sessionArchiveAttemptKeys } from "./session-archive"; +import { readBoundedResponseText } from "./bounded-response"; +import { + credentialPolicyCleanupMatches, + credentialPolicyMigrationCleanupMatches, + credentialPolicyRegistrationAccepted, + credentialPolicySandboxIsExpected, + migratedCredentialPolicyRecord, + type CredentialPolicyGenerationRecord, + type CredentialPolicyGenerationTombstone, + type CredentialPolicyLegacyMigration, +} from "./credential-policy-fence"; type Role = "viewer" | "maintainer" | "owner"; const defaultInteractiveCommand = "codex --yolo"; +type DeploymentConfig = { + label: string; + canonicalUrl: string; + productUrl: string; + sshHost: string; + preferredRepo: string; + defaultRuntime: "crabbox" | "container"; + defaultProfile: string; +}; + +type PublicDeploymentConfig = Pick< + DeploymentConfig, + "label" | "canonicalUrl" | "productUrl" | "sshHost" +>; + type RuntimeEnv = Env & { DB: D1Database; BACKUP_BUCKET?: R2Bucket; @@ -76,8 +156,14 @@ type RuntimeEnv = Env & { GITHUB_ORG?: string; CRABBOX_INTERACTIVE_PROVISION_URL?: string; CRABBOX_INTERACTIVE_PROVISION_TOKEN?: string; + CRABBOX_STANDALONE_SANDBOX_TTL_SECONDS?: string; CRABBOX_RUNTIME_PROVISION_URL?: string; CRABBOX_RUNTIME_PROVISION_TOKEN?: string; + CRABBOX_RUNTIME_ADAPTER_URL?: string; + CRABBOX_RUNTIME_ADAPTER_TOKEN?: string; + CRABBOX_RUNTIME_ADAPTER_NAMESPACE?: string; + CRABBOX_RUNTIME_ADAPTER_TTL_SECONDS?: string; + CRABBOX_RUNTIME_ADAPTER_IDLE_SECONDS?: string; CRABBOX_CLOUDFLARE_RUNNER_URL?: string; CRABBOX_CLOUDFLARE_RUNNER_TOKEN?: string; CRABBOX_CLOUDFLARE_RUNNER_INSTANCE_TYPE?: string; @@ -96,6 +182,14 @@ type RuntimeEnv = Env & { BACKUP_BUCKET_NAME?: string; CLOUDFLARE_ACCOUNT_ID?: string; CRABFLEET_LOCAL_SANDBOX_BACKUPS?: string; + CRABFLEET_LABEL?: string; + CRABFLEET_CANONICAL_URL?: string; + CRABFLEET_PRODUCT_URL?: string; + CRABFLEET_SSH_HOST?: string; + CRABFLEET_PREFERRED_REPO?: string; + CRABFLEET_DEFAULT_RUNTIME?: string; + CRABFLEET_DEFAULT_PROFILE?: string; + CRABFLEET_DEV_LOGIN_ENABLED?: string; OPENAI_API_KEY?: string; OPENAI_BASE_URL?: string; OPENAI_ORG_ID?: string; @@ -272,6 +366,7 @@ type RunAttempt = { leaseId: string | null; attachUrl: string | null; vncUrl: string | null; + ptyAvailable: boolean; selectionReason: string | null; capabilities: RuntimeCapabilities; operator: string | null; @@ -289,6 +384,7 @@ type InteractiveSessionStatus = | "ready" | "attached" | "detached" + | "stopping" | "stopped" | "expired" | "failed"; @@ -300,6 +396,14 @@ type InteractiveSession = { repo: string; branch: string; runtime: "crabbox" | "container"; + adapter: string | null; + profile: string; + adapterWorkspaceId: string | null; + providerResourceId: string | null; + capabilities: RuntimeCapabilities; + expiresAt: number | null; + lastReconciledAt: number | null; + reconcileError: string | null; command: string; prompt: string; purpose: string; @@ -328,6 +432,7 @@ type InteractiveSession = { canChangeMultiplayer?: boolean; canRequestControl?: boolean; sharedReadOnly?: boolean; + ptyAvailable?: boolean; logs: string[]; logArchive: InteractiveSessionLogArchive | null; }; @@ -360,13 +465,55 @@ type SandboxRuntimeSession = (InteractiveProvisionRequest | InteractiveSession) githubToken?: string; }; +type SandboxLease = { + sandboxId: string; + terminalSessionId: string; +}; + +type SandboxLeaseRefreshFence = { + claim: string; + expiresAt: number; + refreshLeaseId: string | null; + sandboxId: string; +}; + +type SandboxCurrentLeaseFence = { + leaseId: string; + sandboxId: string; +}; + +type StandaloneSandboxProvisionFence = { + claim: string; + provisionId: string; + sandboxId: string; +}; + +type SandboxManagedOwnershipFence = SandboxCurrentLeaseFence | SandboxLeaseRefreshFence; + +type SandboxTerminalCleanupOwnership = { + fence: SandboxManagedOwnershipFence; + sandboxIds: string[]; + terminalLeaseId: string; +}; + +type SandboxCredentialPolicyOwnershipFence = + | SandboxManagedOwnershipFence + | StandaloneSandboxProvisionFence; + type InteractiveProvisionRequest = { id: string; + adapterWorkspaceId?: string | null; + adapterControlPlane?: string | null; + adapterTtlSeconds?: number | null; + adapterIdleTimeoutSeconds?: number | null; + adapterRequestedCapabilities?: RuntimeCapabilities | null; + adapterCreatePayloadJson?: string | null; parentSessionId: string | null; rootSessionId: string | null; repo: string; branch: string; runtime: "crabbox" | "container"; + profile: string; command: string; prompt: string; purpose: string; @@ -380,12 +527,26 @@ type InteractiveProvisionResult = { status: InteractiveSessionStatus; leaseId: string | null; attachUrl: string | null; + attachUrlPresent?: boolean; vncUrl: string | null; message: string; + adapter?: string | null; + profile?: string; + adapterWorkspaceId?: string | null; + providerResourceId?: string | null; + capabilities?: RuntimeCapabilities | null; + capabilitiesPresent?: boolean; + expiresAt?: number | null; + expiresAtPresent?: boolean; + reconciledAt?: number | null; + reconcileError?: string | null; + terminalStatus?: "failed" | null; + createPending?: boolean; }; type SandboxCredentialPolicy = { allowedHosts: string[]; + expiresAt?: number; githubCredentialSource?: "none" | "session" | "worker"; githubRepo: string; githubRepoNodeId?: string; @@ -397,6 +558,18 @@ type SandboxCredentialPolicy = { sessionId: string; }; +type StoredSandboxCredentialPolicy = CredentialPolicyGenerationRecord; + +type SandboxCredentialPolicyLegacyMigration = CredentialPolicyLegacyMigration & { + sandboxIds: string[]; +}; + +type SandboxCredentialPolicyRegistration = { + generation: string; + claim: string; + lookupIds: string[]; +}; + type SandboxFleetPolicyResult = { available: boolean; policies: FleetSandboxPolicySummary[]; @@ -436,6 +609,16 @@ type TerminalUpstream = { markConnected: () => Promise; }; +type StandaloneSandboxTerminalOwnership = { + provisionId: string; + requestHash: string; + sandboxId: string; + leaseId: string; + expiresAt: number; + updatedAt: number; + policyGeneration: string; +}; + type SandboxExecutionSession = Awaited>; type SandboxSessionTarget = Pick; @@ -559,6 +742,27 @@ type InteractiveSessionTable = { repo: string; branch: string; runtime: "crabbox" | "container"; + adapter: string | null; + profile: string; + adapter_workspace_id: string | null; + adapter_control_plane: string | null; + provider_resource_id: string | null; + capabilities_json: string; + expires_at: number | null; + last_reconciled_at: number | null; + reconcile_error: string | null; + terminal_status: "failed" | null; + terminal_failure_reason: Generated; + adapter_ttl_seconds: number | null; + adapter_idle_timeout_seconds: number | null; + adapter_requested_capabilities_json: string | null; + adapter_create_payload_json: string | null; + adapter_create_pending: number; + terminal_finalize_pending: Generated; + credential_cleanup_terminal_status: Generated<"stopped" | "expired" | "failed" | null>; + sandbox_refresh_sandbox_id: Generated; + sandbox_refresh_claim: Generated; + sandbox_refresh_claim_expires_at: Generated; command: string; prompt: string; purpose: string; @@ -586,6 +790,8 @@ type InteractiveSessionTable = { agent_token_hash: string | null; }; +type InteractiveSessionRow = Selectable; + type RepoWorkflowTable = { repo: string; status: WorkflowStatus; @@ -617,6 +823,7 @@ type InteractiveSessionEventTable = { type InteractiveSessionLogArchiveTable = { session_id: string; event_count: number; + session_updated_at: number | null; events_key: string | null; transcript_key: string | null; summary_key: string | null; @@ -624,6 +831,50 @@ type InteractiveSessionLogArchiveTable = { updated_at: number; }; +type InteractiveSessionCredentialPolicyTable = { + session_id: string; + sandbox_id: string; + lookup_id: string; + state: "registering" | "active" | "cleanup_pending"; + registration_generation: string; + registration_claim: string | null; + registration_claim_expires_at: number | null; + attempt_count: Generated; + last_attempt_at: number | null; + last_error: string | null; + cleanup_claim: string | null; + cleanup_claim_expires_at: number | null; + created_at: number; + updated_at: number; +}; + +type CredentialPolicyReconcileStateTable = { + id: number; + last_rowid: number; + scan_max_rowid: number; + group_session_id: string; + group_sandbox_id: string; + group_max_session_id: string; + group_max_sandbox_id: string; + updated_at: number; +}; + +type StandaloneSandboxProvisionTable = { + id: string; + request_hash: string; + sandbox_id: string; + state: "provisioning" | "active" | "cleanup_pending"; + ownership_claim: string | null; + ownership_claim_expires_at: number | null; + lease_id: string | null; + attach_url: string | null; + vnc_url: string | null; + expires_at: Generated; + message: string; + created_at: number; + updated_at: number; +}; + type AuditEventTable = { id: Generated; actor: string; @@ -653,6 +904,11 @@ type SshLinkCodeTable = { created_at: number; }; +type IdSequenceTable = { + name: string; + last_id: number; +}; + type Database = { settings: SettingsTable; allow_entries: AllowEntryTable; @@ -664,15 +920,19 @@ type Database = { interactive_sessions: InteractiveSessionTable; interactive_session_events: InteractiveSessionEventTable; interactive_session_log_archives: InteractiveSessionLogArchiveTable; + interactive_session_credential_policies: InteractiveSessionCredentialPolicyTable; + credential_policy_reconcile_state: CredentialPolicyReconcileStateTable; + standalone_sandbox_provisions: StandaloneSandboxProvisionTable; repo_workflows: RepoWorkflowTable; events: EventTable; audit_events: AuditEventTable; ssh_keys: SshKeyTable; ssh_link_codes: SshLinkCodeTable; + id_sequences: IdSequenceTable; }; type CompilableQuery = { - compile(): CompiledQuery; + compile(executorProvider: Kysely): CompiledQuery; }; const encoder = new TextEncoder(); @@ -690,16 +950,6 @@ const preferredRepo = "openclaw/crabfleet"; const sandboxLeasePrefix = "sandbox:"; const sandboxLeaseProfile = "autostart-v4"; const activeRunStatuses: readonly RunStatus[] = ["queued", "leasing", "running"]; -const interactiveSessionStatuses: readonly InteractiveSessionStatus[] = [ - "provisioning", - "pending_adapter", - "ready", - "attached", - "detached", - "stopped", - "expired", - "failed", -]; const deadInteractiveSessionStatuses: readonly InteractiveSessionStatus[] = [ "stopped", "expired", @@ -710,6 +960,19 @@ const runtimeOptions = ["auto", "container", "crabbox"] as const; const mergePolicyOptions = ["open_pr", "merge_when_green", "fix_until_green_and_merge"] as const; const defaultStallMs = 5 * 60 * 1000; const workflowCacheMs = 60 * 60 * 1000; +const runtimeAdapterReconcileIntervalMs = 15_000; +const runtimeAdapterReconcileLimit = 3; +const runtimeAdapterReconcileConcurrency = 3; +const runtimeAdapterReconcileForegroundBudgetMs = 750; +const terminalCleanupDeletePending = 2; +const credentialPolicyCleanupLimit = 8; +const credentialPolicyScanLimit = 32; +const credentialPolicyCleanupClaimMs = 30_000; +const credentialPolicyRegistrationClaimMs = 60_000; +const credentialPolicyProvisioningStaleMs = 15 * 60_000; +const credentialPolicyLegacyGenerationPrefix = "legacy:"; +const credentialPolicyLegacyRepairClaimPrefix = "legacy-repair:"; +const standaloneSandboxDefaultTtlSeconds = 14_400; const containerCapabilities: RuntimeCapabilities = { terminal: true, takeover: false, @@ -726,6 +989,41 @@ const crabboxCapabilities: RuntimeCapabilities = { logs: true, artifacts: true, }; +function runtimeAdapterCreateSettings( + env: RuntimeEnv, + runtime: "crabbox" | "container", +): { + ttlSeconds: number; + idleTimeoutSeconds: number; + capabilities: RuntimeCapabilities; +} { + return { + ttlSeconds: clampedSeconds(env.CRABBOX_RUNTIME_ADAPTER_TTL_SECONDS, 14_400), + idleTimeoutSeconds: clampedSeconds(env.CRABBOX_RUNTIME_ADAPTER_IDLE_SECONDS, 1_800), + capabilities: runtime === "crabbox" ? crabboxCapabilities : containerCapabilities, + }; +} + +function deploymentConfig(env: RuntimeEnv): DeploymentConfig { + return { + label: clean(env.CRABFLEET_LABEL, 80) || "Crabfleet", + canonicalUrl: configuredHttpOrigin(env.CRABFLEET_CANONICAL_URL, appCanonicalOrigin), + productUrl: configuredHttpOrigin(env.CRABFLEET_PRODUCT_URL, "https://crabfleet.ai"), + sshHost: clean(env.CRABFLEET_SSH_HOST, 240) || "crabd.sh", + preferredRepo: normalizeRepo(env.CRABFLEET_PREFERRED_REPO) || preferredRepo, + defaultRuntime: oneOf( + env.CRABFLEET_DEFAULT_RUNTIME, + ["crabbox", "container"] as const, + "container", + ), + defaultProfile: clean(env.CRABFLEET_DEFAULT_PROFILE, 120) || "default", + }; +} + +function publicDeploymentConfig(env: RuntimeEnv): PublicDeploymentConfig { + const { label, canonicalUrl, productUrl, sshHost } = deploymentConfig(env); + return { label, canonicalUrl, productUrl, sshHost }; +} class D1Dialect implements Dialect { constructor(private readonly d1: D1Database) {} @@ -773,51 +1071,20 @@ class D1Driver implements Driver { async destroy(): Promise {} } -class D1Connection implements DatabaseConnection { - constructor(private readonly d1: D1Database) {} - - async executeQuery(compiledQuery: CompiledQuery): Promise> { - const statement = this.d1.prepare(compiledQuery.sql).bind(...compiledQuery.parameters); - if (isReadQuery(compiledQuery.sql)) { - const result = await statement.all(); - return { rows: result.results ?? [] }; - } - - const result = await statement.run(); - const changes = result.meta.changes; - const lastRowId = result.meta.last_row_id; - const queryResult: QueryResult = { rows: [] }; - if (typeof changes === "number") { - Object.assign(queryResult, { numAffectedRows: BigInt(changes) }); - } - if (typeof lastRowId === "number") { - Object.assign(queryResult, { insertId: BigInt(lastRowId) }); - } - return queryResult; - } - - async *streamQuery(compiledQuery: CompiledQuery): AsyncIterableIterator> { - yield await this.executeQuery(compiledQuery); - } -} - function database(env: RuntimeEnv): Kysely { return new Kysely({ dialect: new D1Dialect(env.DB) }); } async function executeBatch(env: RuntimeEnv, queries: readonly CompilableQuery[]): Promise { + const db = database(env); await env.DB.batch( queries.map((query) => { - const compiled = query.compile(); + const compiled = query.compile(db); return env.DB.prepare(compiled.sql).bind(...compiled.parameters); }), ); } -function isReadQuery(sqlText: string): boolean { - return /^(?:select|with|pragma)\b/i.test(sqlText.trim()); -} - const defaultSandboxEgressHosts = [ "api.github.com", "api.openai.com", @@ -837,6 +1104,8 @@ const defaultSandboxEgressHosts = [ const sandboxControlObjectName = "__session-control"; const sandboxPolicyKey = (sandboxId: string) => `sandbox:${sandboxId}`; +const sandboxPolicyTombstoneKey = (sandboxId: string, generation: string) => + `sandbox-tombstone:${encodeURIComponent(sandboxId)}:${encodeURIComponent(generation)}`; const sandboxCheckpointListKey = (sessionId: string) => `checkpoints:${sessionId}`; const sandboxCheckpointKey = (sessionId: string, checkpointId: string) => `checkpoint:${sessionId}:${checkpointId}`; @@ -889,11 +1158,99 @@ async function sandboxCredentialPolicy( ): Promise { const stub = sandboxControlStub(env); if (!stub) return null; - const response = await stub.fetch( - `https://crabfleet.internal/api/session-control/egress/${encodeURIComponent(sandboxId)}`, - ); + const policyUrl = `https://crabfleet.internal/api/session-control/egress/${encodeURIComponent(sandboxId)}`; + let response = await stub.fetch(policyUrl); + if (response.status === 409) { + const legacy = (await response.json().catch(() => null)) as { sessionId?: unknown } | null; + const legacySessionId = clean(legacy?.sessionId, 120); + if (legacySessionId) { + await repairLegacySandboxCredentialPolicyBatch(env, Date.now(), legacySessionId); + response = await stub.fetch(policyUrl); + } + } if (!response.ok) return null; - return (await response.json()) as SandboxCredentialPolicy; + const generation = response.headers.get("x-crabfleet-policy-generation"); + const policy = (await response.json().catch(() => null)) as SandboxCredentialPolicy | null; + if ( + !generation || + !policy || + !(await sandboxCredentialPolicyHasDurableOwner(env, sandboxId, generation, policy, Date.now())) + ) { + return null; + } + return policy; +} + +async function sandboxCredentialPolicyHasDurableOwner( + env: RuntimeEnv, + lookupId: string, + generation: string, + policy: SandboxCredentialPolicy, + now: number, +): Promise { + if ( + !generation || + policy.sandboxId !== lookupId || + typeof policy.sessionId !== "string" || + !policy.sessionId || + !Array.isArray(policy.allowedHosts) || + typeof policy.githubRepo !== "string" || + typeof policy.owner !== "string" || + (policy.expiresAt !== undefined && + (!Number.isFinite(policy.expiresAt) || policy.expiresAt <= now)) + ) { + return false; + } + const db = database(env); + const refs = await db + .selectFrom("interactive_session_credential_policies") + .select("sandbox_id") + .where("session_id", "=", policy.sessionId) + .where("lookup_id", "=", lookupId) + .where("state", "=", "active") + .where("registration_generation", "=", generation) + .where("registration_claim", "is", null) + .execute(); + if (refs.length !== 1) return false; + const canonicalSandboxId = refs[0]?.sandbox_id; + if ( + !canonicalSandboxId || + (await activeSandboxCredentialPolicyGeneration(env, policy.sessionId, canonicalSandboxId)) !== + generation + ) { + return false; + } + const standalone = await db + .selectFrom("standalone_sandbox_provisions") + .select(["state", "ownership_claim", "ownership_claim_expires_at", "expires_at"]) + .where("id", "=", policy.sessionId) + .where("sandbox_id", "=", canonicalSandboxId) + .executeTakeFirst(); + if (standalone) { + const ownerActive = + standalone.state === "active" || + (standalone.state === "provisioning" && + Boolean(standalone.ownership_claim) && + (standalone.ownership_claim_expires_at ?? 0) > now); + return Boolean( + ownerActive && + policy.expiresAt !== undefined && + policy.expiresAt === standalone.expires_at && + policy.expiresAt > now, + ); + } + const owner = await sql<{ expected: number }>` + SELECT CASE + WHEN ${sandboxCredentialPolicyCleanupAuthorizedCondition( + policy.sessionId, + canonicalSandboxId, + now, + )} + THEN 0 + ELSE 1 + END AS expected + `.execute(db); + return owner.rows[0]?.expected === 1; } async function sandboxOutbound( @@ -904,6 +1261,9 @@ async function sandboxOutbound( const url = new URL(request.url); const host = url.hostname.toLowerCase(); const policy = await sandboxCredentialPolicy(env, context.containerId); + if (policy?.expiresAt !== undefined && policy.expiresAt <= Date.now()) { + return new Response("Crabfleet standalone Sandbox credentials expired.\n", { status: 403 }); + } const openAIHost = policy?.openAIBaseUrl ? new URL(policy.openAIBaseUrl).hostname.toLowerCase() : "api.openai.com"; @@ -964,33 +1324,159 @@ export class SessionControlDO extends DurableObject { const url = new URL(request.url); try { if (request.method === "POST" && url.pathname === "/api/session-control/register") { - const policy = (await request.json()) as SandboxCredentialPolicy; - await this.ctx.storage.put(sandboxPolicyKey(policy.sandboxId), policy); + const registration = (await request.json()) as StoredSandboxCredentialPolicy; + if (!validSandboxCredentialPolicyRegistration(registration)) { + return json({ error: "invalid credential policy registration" }, { status: 400 }); + } + const outcome = await this.ctx.storage.transaction(async (transaction) => { + const [stored, tombstone] = await Promise.all([ + transaction.get( + sandboxPolicyKey(registration.policy.sandboxId), + ), + transaction.get( + sandboxPolicyTombstoneKey(registration.policy.sandboxId, registration.generation), + ), + ]); + const current = storedSandboxCredentialPolicy(stored); + const legacy = legacySandboxCredentialPolicy(stored); + if (legacy && legacy.sessionId !== registration.policy.sessionId) return "conflict"; + if (!credentialPolicyRegistrationAccepted(current, tombstone, registration, Date.now())) { + return tombstone ? "tombstoned" : "conflict"; + } + await transaction.put(sandboxPolicyKey(registration.policy.sandboxId), registration); + return "stored"; + }); + if (outcome !== "stored") { + return json({ error: `credential policy generation ${outcome}` }, { status: 409 }); + } + return json({ ok: true }); + } + + if (request.method === "POST" && url.pathname === "/api/session-control/migrate-legacy") { + const migration = (await request.json()) as SandboxCredentialPolicyLegacyMigration; + if (!validSandboxCredentialPolicyLegacyMigration(migration)) { + return json({ error: "invalid legacy credential policy migration" }, { status: 400 }); + } + const outcome = await this.ctx.storage.transaction(async (transaction) => { + const records = await Promise.all( + migration.sandboxIds.map(async (sandboxId) => ({ + sandboxId, + stored: await transaction.get< + StoredSandboxCredentialPolicy | SandboxCredentialPolicy + >(sandboxPolicyKey(sandboxId)), + tombstone: await transaction.get( + sandboxPolicyTombstoneKey(sandboxId, migration.generation), + ), + })), + ); + const sourcePolicy = records + .map( + ({ stored }) => + storedSandboxCredentialPolicy(stored)?.policy ?? + legacySandboxCredentialPolicy(stored), + ) + .find((policy) => policy?.sessionId === migration.sessionId); + if (!sourcePolicy) return "conflict"; + const migratedRecords: Array<{ + sandboxId: string; + policy: StoredSandboxCredentialPolicy; + }> = []; + for (const record of records) { + const current = storedSandboxCredentialPolicy(record.stored); + const legacy = + legacySandboxCredentialPolicy(record.stored) ?? + (!current ? { ...sourcePolicy, sandboxId: record.sandboxId } : undefined); + const migrated = migratedCredentialPolicyRecord( + current, + legacy, + record.tombstone, + migration, + Date.now(), + ); + if (!migrated || migrated.policy.sandboxId !== record.sandboxId) { + return record.tombstone ? "tombstoned" : "conflict"; + } + migratedRecords.push({ sandboxId: record.sandboxId, policy: migrated }); + } + for (const record of migratedRecords) { + await transaction.put(sandboxPolicyKey(record.sandboxId), record.policy); + } + return "stored"; + }); + if (outcome !== "stored") { + return json({ error: `legacy credential policy migration ${outcome}` }, { status: 409 }); + } return json({ ok: true }); } if (request.method === "GET" && url.pathname === "/api/session-control/policies") { - const entries = await this.ctx.storage.list({ + const entries = await this.ctx.storage.list< + StoredSandboxCredentialPolicy | SandboxCredentialPolicy + >({ prefix: "sandbox:", }); - const policies = dedupeSandboxPolicies([...entries.values()]).map((policy) => - redactSandboxPolicy(policy, Boolean(this.env.GITHUB_TOKEN)), - ); + const policies = dedupeSandboxPolicies( + [...entries.values()] + .map((stored) => sandboxCredentialPolicyFromStorage(stored)) + .filter((policy): policy is SandboxCredentialPolicy => Boolean(policy)), + ).map((policy) => redactSandboxPolicy(policy, Boolean(this.env.GITHUB_TOKEN))); return json({ policies }); } const egressMatch = url.pathname.match(/^\/api\/session-control\/egress\/([^/]+)$/); if (request.method === "GET" && egressMatch) { const sandboxId = decodeURIComponent(egressMatch[1] ?? ""); - const policy = await this.ctx.storage.get( - sandboxPolicyKey(sandboxId), - ); - return policy ? json(policy) : json({ error: "not found" }, { status: 404 }); + const key = sandboxPolicyKey(sandboxId); + const stored = await this.ctx.storage.get< + StoredSandboxCredentialPolicy | SandboxCredentialPolicy + >(key); + const current = storedSandboxCredentialPolicy(stored); + const legacy = legacySandboxCredentialPolicy(stored); + const policy = sandboxCredentialPolicyFromStorage(stored); + if (!current || !policy) { + if (legacy) { + return json( + { error: "legacy credential policy migration required", sessionId: legacy.sessionId }, + { status: 409 }, + ); + } + return json({ error: "not found" }, { status: 404 }); + } + return json(policy, { + headers: { "x-crabfleet-policy-generation": current.generation }, + }); } const sandboxMatch = url.pathname.match(/^\/api\/session-control\/sandbox\/([^/]+)$/); if (request.method === "DELETE" && sandboxMatch) { - await this.ctx.storage.delete(sandboxPolicyKey(decodeURIComponent(sandboxMatch[1] ?? ""))); + const sandboxId = decodeURIComponent(sandboxMatch[1] ?? ""); + const tombstone = (await request.json()) as CredentialPolicyGenerationTombstone; + if (!validCredentialPolicyTombstone(tombstone)) { + return json({ error: "invalid credential policy tombstone" }, { status: 400 }); + } + await this.ctx.storage.transaction(async (transaction) => { + const key = sandboxPolicyKey(sandboxId); + const stored = await transaction.get< + StoredSandboxCredentialPolicy | SandboxCredentialPolicy + >(key); + const current = storedSandboxCredentialPolicy(stored); + const legacy = legacySandboxCredentialPolicy(stored); + await transaction.put( + sandboxPolicyTombstoneKey(sandboxId, tombstone.generation), + tombstone, + ); + if ( + credentialPolicyCleanupMatches(current, tombstone.generation, tombstone.sessionId) || + credentialPolicyMigrationCleanupMatches( + current, + tombstone.generation, + tombstone.sessionId, + ) || + legacy?.sessionId === tombstone.sessionId + ) { + await transaction.delete(key); + } + }); return json({ ok: true }); } @@ -1037,6 +1523,99 @@ export class SessionControlDO extends DurableObject { } } +function storedSandboxCredentialPolicy( + value: StoredSandboxCredentialPolicy | SandboxCredentialPolicy | undefined, +): StoredSandboxCredentialPolicy | undefined { + if ( + value && + "policy" in value && + typeof value.generation === "string" && + typeof value.registrationClaim === "string" && + typeof value.registrationExpiresAt === "number" + ) { + return value; + } + return undefined; +} + +function legacySandboxCredentialPolicy( + value: StoredSandboxCredentialPolicy | SandboxCredentialPolicy | undefined, +): SandboxCredentialPolicy | undefined { + return value && !("policy" in value) ? value : undefined; +} + +function sandboxCredentialPolicyFromStorage( + value: StoredSandboxCredentialPolicy | SandboxCredentialPolicy | undefined, +): SandboxCredentialPolicy | undefined { + const current = storedSandboxCredentialPolicy(value); + if (!current) return undefined; + if ( + current.policy.expiresAt !== undefined && + (!Number.isFinite(current.policy.expiresAt) || current.policy.expiresAt <= Date.now()) + ) { + return undefined; + } + return current.policy; +} + +function validSandboxCredentialPolicyRegistration(value: StoredSandboxCredentialPolicy): boolean { + return Boolean( + value && + typeof value.generation === "string" && + value.generation.length > 0 && + value.generation.length <= 200 && + typeof value.registrationClaim === "string" && + value.registrationClaim.length > 0 && + value.registrationClaim.length <= 200 && + Number.isFinite(value.registrationExpiresAt) && + value.policy && + typeof value.policy.sandboxId === "string" && + typeof value.policy.sessionId === "string" && + (value.policy.expiresAt === undefined || + (Number.isFinite(value.policy.expiresAt) && value.policy.expiresAt > Date.now())), + ); +} + +function validSandboxCredentialPolicyLegacyMigration( + value: SandboxCredentialPolicyLegacyMigration, +): boolean { + return Boolean( + value && + typeof value.generation === "string" && + value.generation.length > 0 && + value.generation.length <= 200 && + !value.generation.startsWith(credentialPolicyLegacyGenerationPrefix) && + typeof value.registrationClaim === "string" && + value.registrationClaim.startsWith(credentialPolicyLegacyRepairClaimPrefix) && + value.registrationClaim.length <= 200 && + Number.isFinite(value.registrationExpiresAt) && + Array.isArray(value.sandboxIds) && + value.sandboxIds.length > 0 && + value.sandboxIds.length <= 8 && + new Set(value.sandboxIds).size === value.sandboxIds.length && + value.sandboxIds.every( + (sandboxId) => + typeof sandboxId === "string" && sandboxId.length > 0 && sandboxId.length <= 200, + ) && + typeof value.sessionId === "string" && + value.sessionId.length > 0 && + value.sessionId.length <= 200, + ); +} + +function validCredentialPolicyTombstone(value: CredentialPolicyGenerationTombstone): boolean { + return Boolean( + value && + typeof value.generation === "string" && + value.generation.length > 0 && + value.generation.length <= 200 && + typeof value.sessionId === "string" && + value.sessionId.length > 0 && + value.sessionId.length <= 200 && + Number.isFinite(value.tombstonedAt), + ); +} + function dedupeSandboxPolicies(policies: SandboxCredentialPolicy[]): SandboxCredentialPolicy[] { const seen = new Set(); const result: SandboxCredentialPolicy[] = []; @@ -1097,7 +1676,7 @@ async function readSandboxFleetPolicies(env: RuntimeEnv): Promise { + async fetch(request: Request, env: RuntimeEnv, context: ExecutionContext): Promise { const url = new URL(request.url); try { @@ -1180,7 +1759,7 @@ export default { } if (url.pathname.startsWith("/api/")) { - return await api(request, env); + return await api(request, env, context); } if ( @@ -1210,9 +1789,24 @@ export default { return json({ error: message }, { status: Number.isFinite(status) ? status : 500 }); } }, + async scheduled( + _controller: ScheduledController, + env: RuntimeEnv, + context: ExecutionContext, + ): Promise { + context.waitUntil( + reconcileInteractiveSessionLifecycleBatch(env, Date.now()).catch((error) => { + console.error("scheduled interactive session reconciliation failed", error); + }), + ); + }, } satisfies ExportedHandler; -async function api(request: Request, env: RuntimeEnv): Promise { +async function api( + request: Request, + env: RuntimeEnv, + context: ExecutionContext, +): Promise { const url = new URL(request.url); if (request.method === "POST" && url.pathname === "/api/login/token") { @@ -1228,7 +1822,31 @@ async function api(request: Request, env: RuntimeEnv): Promise { } if (request.method === "GET" && url.pathname === "/api/auth") { - return json({ auth: authMethods(env, request) }); + return json({ auth: authMethods(env, request), deployment: publicDeploymentConfig(env) }); + } + + const standaloneProvisionPtyMatch = url.pathname.match( + /^\/api\/provision\/interactive\/([^/]+)\/pty$/, + ); + if (request.method === "GET" && standaloneProvisionPtyMatch) { + return standaloneSandboxPty( + request, + env, + decodeURIComponent(standaloneProvisionPtyMatch[1] ?? ""), + ); + } + + const standaloneProvisionStopMatch = url.pathname.match( + /^\/api\/provision\/interactive\/([^/]+)\/stop$/, + ); + if (request.method === "POST" && standaloneProvisionStopMatch) { + return json( + await stopStandaloneSandboxProvision( + request, + env, + decodeURIComponent(standaloneProvisionStopMatch[1] ?? ""), + ), + ); } if (request.method === "POST" && url.pathname === "/api/provision/interactive") { @@ -1259,7 +1877,7 @@ async function api(request: Request, env: RuntimeEnv): Promise { if (request.method === "GET" && sshInteractiveReadMatch) { const user = await requireSshGatewayUser(request, env); requireRole(user, "viewer"); - const session = await readInteractiveSession( + const session = await readFreshInteractiveSession( env, decodeURIComponent(sshInteractiveReadMatch[1] ?? ""), ); @@ -1272,7 +1890,7 @@ async function api(request: Request, env: RuntimeEnv): Promise { ); if (request.method === "GET" && agentInteractiveReadMatch) { const { user } = await requireAgentSession(request, env); - const session = await readInteractiveSession( + const session = await readFreshInteractiveSession( env, decodeURIComponent(agentInteractiveReadMatch[1] ?? ""), ); @@ -1465,12 +2083,12 @@ async function api(request: Request, env: RuntimeEnv): Promise { } if (request.method === "GET" && url.pathname === "/api/state") { - return json(await readState(request, env, user)); + return json(await readState(request, env, user, context)); } if (request.method === "GET" && url.pathname === "/api/fleet") { requireRole(user, "viewer"); - return json({ fleet: await readFleetState(env, user) }); + return json({ fleet: await readFleetState(env, user, undefined, context) }); } if (request.method === "GET" && url.pathname === "/api/github/refs") { @@ -1491,7 +2109,7 @@ async function api(request: Request, env: RuntimeEnv): Promise { const interactiveSessionReadMatch = url.pathname.match(/^\/api\/interactive-sessions\/([^/]+)$/); if (request.method === "GET" && interactiveSessionReadMatch) { requireRole(user, "viewer"); - const session = await readInteractiveSession( + const session = await readFreshInteractiveSession( env, decodeURIComponent(interactiveSessionReadMatch[1] ?? ""), ); @@ -1554,6 +2172,18 @@ async function api(request: Request, env: RuntimeEnv): Promise { ); } + const interactiveSessionVncMatch = url.pathname.match( + /^\/api\/interactive-sessions\/([^/]+)\/vnc$/, + ); + if (request.method === "GET" && interactiveSessionVncMatch) { + requireRole(user, "viewer"); + return interactiveSessionVnc( + env, + user, + decodeURIComponent(interactiveSessionVncMatch[1] ?? ""), + ); + } + const interactiveSessionCheckpointsMatch = url.pathname.match( /^\/api\/interactive-sessions\/([^/]+)\/checkpoints$/, ); @@ -1711,7 +2341,7 @@ async function tokenLogin(request: Request, env: RuntimeEnv): Promise } async function devIdentityLogin(request: Request, env: RuntimeEnv): Promise { - if (!isLocalDevRequest(request)) return json({ error: "not found" }, { status: 404 }); + if (!devIdentityEnabled(env, request)) return json({ error: "not found" }, { status: 404 }); const body = await readJson<{ id?: string; name?: string; role?: string }>(request); const id = devIdentityId(body.id); @@ -1741,6 +2371,10 @@ async function githubLogin(request: Request, env: RuntimeEnv): Promise const url = new URL(request.url); const redirectUri = githubOAuthRedirectUri(url, env.GITHUB_REDIRECT_URI); + const canonicalLoginUrl = githubOAuthCanonicalLoginUrl(url, env.GITHUB_REDIRECT_URI); + if (canonicalLoginUrl) { + return redirect(canonicalLoginUrl, { "cache-control": "no-store" }); + } const state = crypto.randomUUID(); const target = new URL("https://github.com/login/oauth/authorize"); target.searchParams.set("client_id", env.GITHUB_CLIENT_ID); @@ -1759,13 +2393,21 @@ async function githubCallback(request: Request, env: RuntimeEnv): Promise { + const canonicalLinkUrl = githubOAuthCanonicalSshLinkUrl( + request.url, + code, + env.GITHUB_REDIRECT_URI, + ); + if (canonicalLinkUrl) { + return redirect(canonicalLinkUrl, { "cache-control": "no-store" }); + } const codeHash = await sha256(code); const row = await database(env) .selectFrom("ssh_link_codes") @@ -2004,7 +2654,7 @@ async function requireUser(request: Request, env: RuntimeEnv): Promise { } if (user.subject.startsWith("dev:")) { - if (!isLocalDevRequest(request)) { + if (!devIdentityEnabled(env, request)) { await db.deleteFrom("sessions").where("token_hash", "=", tokenHash).execute(); throw unauthorized(); } @@ -2060,11 +2710,14 @@ async function sshAuth(request: Request, env: RuntimeEnv): Promise(request); if (!normalizeRepo(body.repo)) { - const repo = await database(env) + const preferred = deploymentConfig(env).preferredRepo; + const repos = await database(env) .selectFrom("repos") .select("repo") .where("enabled", "=", 1) .orderBy("repo") - .executeTakeFirst(); - body.repo = repo?.repo ?? preferredRepo; + .execute(); + const selectedRepo = preferredEnabledRepo( + repos.map((repo) => repo.repo), + preferred, + ); + if (selectedRepo) body.repo = selectedRepo; } const result = await createInteractiveSessionFromInput(env, user, body, githubToken); await audit(env, user, `ssh interactive session created ${result.session.id}`, Date.now()); @@ -2169,6 +2828,7 @@ async function agentCreateInteractiveSession( repo?: string; branch?: string; runtime?: string; + profile?: string; command?: string; prompt?: string; parentSessionId?: string; @@ -2176,7 +2836,7 @@ async function agentCreateInteractiveSession( purpose?: string; summary?: string; }>(request); - if (!normalizeRepo(body.repo)) body.repo = parent.repo || preferredRepo; + if (!normalizeRepo(body.repo)) body.repo = parent.repo || deploymentConfig(env).preferredRepo; const result = await createInteractiveSessionFromInput(env, user, body, undefined, { createdBy: `session:${parent.id}`, owner: parent.owner, @@ -2201,6 +2861,7 @@ async function openClawCreateCrabbox( repo?: string; branch?: string; runtime?: string; + profile?: string; command?: string; prompt?: string; owner?: string; @@ -2279,7 +2940,7 @@ async function requireAgentSession( throw unauthorized(); } const session = interactiveSession(row, []); - if (deadInteractiveSessionStatuses.includes(session.status)) { + if (session.status === "stopping" || deadInteractiveSessionStatuses.includes(session.status)) { throw forbidden("agent session is not active"); } return { @@ -2421,57 +3082,492 @@ function sshGatewayTokens(env: RuntimeEnv): string[] { ); } -async function readState( - request: Request, +async function reconcileExternalInteractiveSessions( env: RuntimeEnv, - user: User, -): Promise> { - await reconcileStalledRuns(env, Date.now()); - const db = database(env); - const [settings, allow, repos, cards, interactiveSessions, workflows] = await Promise.all([ - readSettings(env), - user.role === "owner" - ? db.selectFrom("allow_entries").select(["value", "role"]).orderBy("value").execute() - : Promise.resolve([]), - db.selectFrom("repos").select("repo").where("enabled", "=", 1).orderBy("repo").execute(), - readCards(env), - readInteractiveSessions(env, user), - user.role === "owner" ? readWorkflowSummaries(env) : Promise.resolve([]), + now: number, + context?: ExecutionContext, +): Promise { + const reconciliation = reconcileInteractiveSessionLifecycleBatch(env, now).catch((error) => { + console.error("interactive session reconciliation failed", error); + }); + if (!context) { + await reconciliation; + return; + } + context.waitUntil(reconciliation); + await Promise.race([ + reconciliation, + new Promise((resolve) => setTimeout(resolve, runtimeAdapterReconcileForegroundBudgetMs)), ]); - const repoNames = sortRepos(repos.map((row) => row.repo)); - const fleet = await readFleetState(env, user, interactiveSessions); +} - return { - user, - auth: authMethods(env, request), - org: settings.org ?? "OpenClaw", - cap: numberSetting(settings.cap, 20), - retention: settings.retention ?? "30", - merge: settings.merge ?? "guarded", - allow, - repos: repoNames, - workflows, - cards, - interactiveSessions, - fleet, - }; +async function reconcileInteractiveSessionLifecycleBatch( + env: RuntimeEnv, + now: number, +): Promise { + await reconcileCredentialPolicyCleanupBatch(env, now); + await reconcileLegacyStoppingInteractiveSessionBatch(env, now); + await reconcileExternalInteractiveSessionBatch(env, now); } -async function readFleetState( +async function reconcileLegacyStoppingInteractiveSessionBatch( env: RuntimeEnv, - user: User, - sessions?: InteractiveSession[], -): Promise { - const [interactiveSessions, policyResult] = await Promise.all([ + now: number, + sessionId?: string, +): Promise { + let query = database(env) + .selectFrom("interactive_sessions") + .selectAll() + .where("status", "=", "stopping") + .where((expression) => + expression.or([ + expression("adapter", "is", null), + expression("adapter", "!=", runtimeAdapterName), + ]), + ) + .where("credential_cleanup_terminal_status", "is", null) + .where(sql`lease_id IS NULL OR lease_id NOT LIKE ${`${sandboxLeasePrefix}%`}`) + .orderBy("updated_at", "asc") + .limit(runtimeAdapterReconcileLimit); + if (sessionId) query = query.where("id", "=", sessionId); + const candidates = await query.execute(); + await mapWithConcurrency(candidates, runtimeAdapterReconcileConcurrency, async (session) => { + await completeLegacyInteractiveSessionStop( + env, + { + id: session.id, + status: session.status, + adapter: session.adapter, + leaseId: session.lease_id, + updatedAt: session.updated_at, + }, + "system", + now, + ).catch((error) => { + console.error(`legacy interactive session stop recovery failed for ${session.id}`, error); + }); + }); +} + +async function requeueTerminalArchiveObjectBackfill( + env: RuntimeEnv, + sessionId?: string, +): Promise { + if (!env.SESSION_LOGS) return; + const sessionFilter = sessionId ? sql`AND session.id = ${sessionId}` : sql``; + const limit = sessionId ? 1 : runtimeAdapterReconcileLimit * 2; + await sql` + UPDATE interactive_sessions + SET terminal_finalize_pending = 1, + last_reconciled_at = NULL + WHERE id IN ( + SELECT session.id + FROM interactive_sessions AS session + JOIN interactive_session_log_archives AS archive + ON archive.session_id = session.id + WHERE session.status IN ('stopped', 'expired', 'failed') + AND session.terminal_finalize_pending = 0 + AND ( + archive.events_key IS NULL + OR archive.transcript_key IS NULL + OR archive.summary_key IS NULL + ) + ${sessionFilter} + ORDER BY session.updated_at ASC, session.id ASC + LIMIT ${limit} + ) + `.execute(database(env)); +} + +async function reconcileExternalInteractiveSessionBatch( + env: RuntimeEnv, + now: number, +): Promise { + await requeueTerminalArchiveObjectBackfill(env); + const providerConfigured = runtimeAdapterProviderConfigured(env); + const activeStatuses: InteractiveSessionStatus[] = [ + "provisioning", + "pending_adapter", + "ready", + "attached", + "detached", + "stopping", + ]; + const terminalStatuses: InteractiveSessionStatus[] = ["stopped", "expired", "failed"]; + const rows = await database(env) + .selectFrom("interactive_sessions") + .selectAll() + .where((expression) => + providerConfigured + ? expression.or([ + expression.and([ + expression("status", "in", terminalStatuses), + expression("terminal_finalize_pending", "=", 1), + ]), + expression.and([ + expression("adapter", "=", runtimeAdapterName), + expression("status", "in", activeStatuses), + ]), + ]) + : expression.and([ + expression("status", "in", terminalStatuses), + expression("terminal_finalize_pending", "=", 1), + ]), + ) + .orderBy("last_reconciled_at", "asc") + .limit(runtimeAdapterReconcileLimit * 2) + .execute(); + const due = rows + .filter( + (row) => + !row.last_reconciled_at || + now - row.last_reconciled_at >= runtimeAdapterReconcileIntervalMs, + ) + .slice(0, runtimeAdapterReconcileLimit); + await mapWithConcurrency(due, runtimeAdapterReconcileConcurrency, async (row) => { + await reconcileExternalInteractiveSession(env, row, now); + }); +} + +async function reconcileExternalInteractiveSessionById( + env: RuntimeEnv, + id: string, + now = Date.now(), +): Promise { + await reconcileCredentialPolicyCleanupBatch(env, now, id); + await reconcileLegacyStoppingInteractiveSessionBatch(env, now, id); + await requeueTerminalArchiveObjectBackfill(env, id); + const row = await database(env) + .selectFrom("interactive_sessions") + .selectAll() + .where("id", "=", id) + .executeTakeFirst(); + if (!row) return; + const terminalFinalizationPending = + row.terminal_finalize_pending === 1 && + (row.status === "stopped" || row.status === "expired" || row.status === "failed"); + const providerConfigured = runtimeAdapterProviderConfigured(env); + const active = [ + "provisioning", + "pending_adapter", + "ready", + "attached", + "detached", + "stopping", + ].includes(row.status); + if ( + !terminalFinalizationPending && + (row.adapter !== runtimeAdapterName || !providerConfigured || !active) + ) { + return; + } + if (row.last_reconciled_at && now - row.last_reconciled_at < runtimeAdapterReconcileIntervalMs) { + return; + } + await reconcileExternalInteractiveSession(env, row, now); +} + +async function reconcileExternalInteractiveSession( + env: RuntimeEnv, + row: InteractiveSessionRow, + now: number, +): Promise { + const terminalFinalizationStatus: "stopped" | "expired" | "failed" | null = + row.terminal_finalize_pending === 1 && + (row.status === "stopped" || row.status === "expired" || row.status === "failed") + ? row.status + : null; + if ( + !terminalFinalizationStatus && + (row.adapter !== runtimeAdapterName || !row.adapter_workspace_id) + ) { + return; + } + const claimAt = Math.max(now, Date.now(), (row.last_reconciled_at ?? 0) + 1); + let claim = database(env) + .updateTable("interactive_sessions") + .set({ last_reconciled_at: claimAt }) + .where("id", "=", row.id) + .where("status", "=", row.status) + .where("updated_at", "=", row.updated_at); + claim = row.last_reconciled_at + ? claim.where("last_reconciled_at", "=", row.last_reconciled_at) + : claim.where("last_reconciled_at", "is", null); + const claimed = await claim.executeTakeFirst(); + if ((claimed.numUpdatedRows ?? 0n) === 0n) return; + + try { + if (terminalFinalizationStatus) { + await finalizeTerminalInteractiveSession( + env, + row.id, + terminalFinalizationStatus, + row.stopped_at ?? now, + ); + return; + } + if (row.adapter !== runtimeAdapterName || !row.adapter_workspace_id) return; + const inspected = await inspectRuntimeAdapterWorkspace(env, row); + const completedAt = Math.max(Date.now(), claimAt); + const completionVersion = Math.max(completedAt, row.updated_at + 1); + const requestedTerminalStatus = + inspected.terminalStatus === undefined ? row.terminal_status : inspected.terminalStatus; + const status = reconciledInteractiveStatus( + row.status, + inspected.status, + requestedTerminalStatus, + ); + const inactive = ["stopping", "stopped", "expired", "failed"].includes(status); + const terminalStatus = ["stopped", "expired", "failed"].includes(status) + ? null + : requestedTerminalStatus; + const terminal = inactive + ? null + : inspected.attachUrlPresent + ? inspected.attachUrl + : row.attach_url; + const capabilities = inspected.capabilities + ? JSON.stringify(inspected.capabilities) + : inspected.capabilitiesPresent + ? JSON.stringify(clearedAdapterCapabilities) + : row.capabilities_json; + const expiresAt = inspected.expiresAtPresent ? (inspected.expiresAt ?? null) : row.expires_at; + const createPending = + inspected.createPending === undefined + ? row.adapter_create_pending + : inspected.createPending + ? 1 + : 0; + const stateChanged = + status !== row.status || + terminal !== row.attach_url || + capabilities !== row.capabilities_json || + (inspected.providerResourceId ?? row.provider_resource_id) !== row.provider_resource_id || + expiresAt !== row.expires_at || + terminalStatus !== row.terminal_status || + createPending !== row.adapter_create_pending || + (inspected.reconcileError ?? null) !== row.reconcile_error; + const messageChanged = inspected.message !== row.last_event; + const expectedOwner = sql` + id = ${row.id} + AND adapter = ${runtimeAdapterName} + AND status = ${row.status} + AND updated_at = ${row.updated_at} + AND last_reconciled_at = ${claimAt} + `; + const db = database(env); + const update = db + .updateTable("interactive_sessions") + .set({ + status, + lease_id: null, + provider_resource_id: inspected.providerResourceId ?? row.provider_resource_id, + attach_url: terminal, + // Connection-bearing desktop URLs are never persisted. + vnc_url: null, + capabilities_json: capabilities, + expires_at: expiresAt, + last_reconciled_at: completedAt, + reconcile_error: inspected.reconcileError ?? null, + terminal_status: terminalStatus, + adapter_create_pending: createPending, + terminal_finalize_pending: ["stopped", "expired", "failed"].includes(status) + ? 1 + : row.terminal_finalize_pending, + ...(inactive + ? { + agent_token_hash: null, + controller: null, + control_requested_by: null, + control_requested_at: null, + control_granted_at: null, + control_expires_at: null, + } + : {}), + stopped_at: ["stopped", "expired", "failed"].includes(status) + ? (row.stopped_at ?? completedAt) + : row.stopped_at, + ...(stateChanged || messageChanged + ? { updated_at: completionVersion, last_event: inspected.message } + : {}), + }) + .where(expectedOwner) + .returning("updated_at"); + const queries: CompilableQuery[] = []; + if (stateChanged || messageChanged) { + queries.push(sql` + INSERT INTO interactive_session_events (session_id, actor, message, created_at) + SELECT ${row.id}, 'system', ${clean(inspected.message, 1000)}, ${completedAt} + FROM interactive_sessions + WHERE ${expectedOwner} + `); + } + queries.push(update); + const results = await env.DB.batch<{ updated_at: number }>( + queries.map((query) => { + const compiled = query.compile(db); + return env.DB.prepare(compiled.sql).bind(...compiled.parameters); + }), + ); + if (!results.at(-1)?.results.length) { + const current = await readInteractiveSession(env, row.id); + if (current && ["stopped", "expired", "failed"].includes(current.status)) { + await finalizeTerminalInteractiveSession( + env, + current.id, + current.status as "stopped" | "expired" | "failed", + current.stoppedAt ?? now, + ).catch(() => undefined); + return; + } + const currentAdapterProvision = Boolean( + current && + current.adapter === runtimeAdapterName && + current.adapterWorkspaceId === inspected.adapterWorkspaceId && + ["provisioning", "pending_adapter", "ready", "attached", "detached"].includes( + current.status, + ), + ); + if (!currentAdapterProvision && inspected.adapterWorkspaceId) { + await stopSupersededRuntimeAdapterProvision( + env, + row.id, + inspected.adapterWorkspaceId, + inspected.createPending === true, + Date.now(), + ); + } + return; + } + if (stateChanged || messageChanged) { + await archiveInteractiveSessionLogs(env, row.id, completedAt).catch(() => undefined); + } + if ( + status !== row.status && + (status === "stopped" || status === "expired" || status === "failed") + ) { + await finalizeTerminalInteractiveSession( + env, + row.id, + status, + row.stopped_at ?? completedAt, + ).catch(() => undefined); + } + } catch (error) { + const failedAt = Math.max(Date.now(), claimAt); + await database(env) + .updateTable("interactive_sessions") + .set({ + last_reconciled_at: failedAt, + reconcile_error: safeProviderError( + error, + [row.adapter_workspace_id, row.provider_resource_id], + [row.attach_url], + ), + updated_at: Math.max(failedAt, row.updated_at + 1), + }) + .where("id", "=", row.id) + .where("status", "=", row.status) + .where("updated_at", "=", row.updated_at) + .where("last_reconciled_at", "=", claimAt) + .execute(); + } +} + +function reconciledInteractiveStatus( + current: InteractiveSessionStatus, + next: InteractiveSessionStatus, + terminalStatus: "failed" | null, +): InteractiveSessionStatus { + if (current === "stopping") { + if (["stopped", "expired", "failed"].includes(next)) return terminalStatus ?? next; + return "stopping"; + } + if ((current === "attached" || current === "detached") && next === "ready") return current; + return next; +} + +async function mapWithConcurrency( + values: T[], + concurrency: number, + operation: (value: T) => Promise, +): Promise { + let cursor = 0; + const worker = async () => { + while (cursor < values.length) { + const index = cursor; + cursor += 1; + await operation(values[index] as T); + } + }; + await Promise.all( + Array.from({ length: Math.min(Math.max(1, concurrency), values.length) }, () => worker()), + ); +} + +async function readState( + request: Request, + env: RuntimeEnv, + user: User, + context?: ExecutionContext, +): Promise> { + await reconcileStalledRuns(env, Date.now()); + await reconcileExternalInteractiveSessions(env, Date.now(), context); + const db = database(env); + const [settings, allow, repos, cards, interactiveSessions, workflows] = await Promise.all([ + readSettings(env), + user.role === "owner" + ? db.selectFrom("allow_entries").select(["value", "role"]).orderBy("value").execute() + : Promise.resolve([]), + db.selectFrom("repos").select("repo").where("enabled", "=", 1).orderBy("repo").execute(), + readCards(env), + readInteractiveSessions(env, user), + user.role === "owner" ? readWorkflowSummaries(env) : Promise.resolve([]), + ]); + const repoNames = sortRepos( + repos.map((row) => row.repo), + deploymentConfig(env).preferredRepo, + ); + const fleet = await readFleetState(env, user, interactiveSessions); + + return { + user, + auth: authMethods(env, request), + deployment: deploymentConfig(env), + org: settings.org ?? "OpenClaw", + cap: numberSetting(settings.cap, 20), + retention: settings.retention ?? "30", + merge: settings.merge ?? "guarded", + allow, + repos: repoNames, + workflows, + cards, + interactiveSessions, + fleet, + }; +} + +async function readFleetState( + env: RuntimeEnv, + user: User, + sessions?: InteractiveSession[], + context?: ExecutionContext, +): Promise { + const deployment = deploymentConfig(env); + if (!sessions) await reconcileExternalInteractiveSessions(env, Date.now(), context); + const [interactiveSessions, policyResult] = await Promise.all([ sessions ? Promise.resolve(sessions) : readInteractiveSessions(env, user), readSandboxFleetPolicies(env), ]); return buildFleetState(interactiveSessions, policyResult.policies, { - canonicalUrl: appCanonicalOrigin, - productUrl: "https://crabfleet.ai", + canonicalUrl: deployment.canonicalUrl, + productUrl: deployment.productUrl, defaultEgressHosts: defaultSandboxEgressHosts, generatedAt: Date.now(), registryAvailable: policyResult.available, + sandboxAvailable: Boolean(env.SANDBOX), + ptyBridgeUrl: env.CRABBOX_PTY_BRIDGE_URL, + cloudflareRunnerUrl: env.CRABBOX_CLOUDFLARE_RUNNER_URL, }); } @@ -2505,6 +3601,7 @@ async function createInteractiveSessionFromInput( repo?: string; branch?: string; runtime?: string; + profile?: string; command?: string; prompt?: string; parentSessionId?: string; @@ -2524,9 +3621,12 @@ async function createInteractiveSessionFromInput( if (!repo) throw badRequest("repo is required"); await requireRepo(env, repo); const branch = clean(body.branch, 120) || "main"; - const runtime = oneOf(body.runtime, ["crabbox", "container"], "container") as + const deployment = deploymentConfig(env); + const runtime = oneOf(body.runtime, ["crabbox", "container"], deployment.defaultRuntime) as | "crabbox" | "container"; + requireRuntimeAdapterCreatePreflight(env, runtime); + const profile = clean(body.profile, 120) || deployment.defaultProfile; const command = interactiveCommand(body.command); const prompt = clean(body.prompt, 4000); const purpose = interactiveSessionPurpose(body.purpose, prompt, repo, branch, command); @@ -2545,6 +3645,49 @@ async function createInteractiveSessionFromInput( const id = await nextInteractiveSessionId(env); const rootSessionId = lineage.rootSessionId ?? id; const agentToken = newAgentToken(); + const initialAgentTokenHash = await sha256(agentToken); + const initialSandboxLease = runtime === "container" && env.SANDBOX ? newSandboxLease(id) : null; + const initialSandboxOwnership: SandboxCurrentLeaseFence | null = initialSandboxLease + ? { + leaseId: sandboxLeaseId(initialSandboxLease), + sandboxId: initialSandboxLease.sandboxId, + } + : null; + const adapterWorkspaceId = initialRuntimeAdapterWorkspaceId(env, runtime, id); + const adapterControlPlane = adapterWorkspaceId + ? configuredRuntimeAdapterControlPlane(env) + : null; + const adapterSettings = adapterWorkspaceId ? runtimeAdapterCreateSettings(env, runtime) : null; + const adapterCreatePayload = + adapterWorkspaceId && adapterSettings + ? runtimeAdapterCreatePayload( + { + namespace: normalizeAdapterNamespace( + env.CRABBOX_RUNTIME_ADAPTER_NAMESPACE ?? "", + ) as string, + id, + parentSessionId: lineage.parentSessionId, + rootSessionId, + repo, + branch, + runtime, + profile, + command, + prompt, + purpose, + summary, + owner, + createdBy, + ttlSeconds: adapterSettings.ttlSeconds, + idleTimeoutSeconds: adapterSettings.idleTimeoutSeconds, + desktop: adapterSettings.capabilities.desktop, + }, + adapterWorkspaceId, + ) + : null; + const adapterCreatePayloadJson = adapterCreatePayload + ? JSON.stringify(adapterCreatePayload) + : null; try { await db .insertInto("interactive_sessions") @@ -2555,6 +3698,25 @@ async function createInteractiveSessionFromInput( repo, branch, runtime, + adapter: adapterWorkspaceId ? runtimeAdapterName : null, + profile, + adapter_workspace_id: adapterWorkspaceId, + adapter_control_plane: adapterControlPlane, + provider_resource_id: null, + capabilities_json: JSON.stringify( + runtime === "crabbox" ? crabboxCapabilities : containerCapabilities, + ), + expires_at: null, + last_reconciled_at: adapterWorkspaceId ? now : null, + reconcile_error: adapterWorkspaceId ? "runtime adapter create pending" : null, + terminal_status: null, + adapter_ttl_seconds: adapterSettings?.ttlSeconds ?? null, + adapter_idle_timeout_seconds: adapterSettings?.idleTimeoutSeconds ?? null, + adapter_requested_capabilities_json: adapterSettings + ? JSON.stringify(adapterSettings.capabilities) + : null, + adapter_create_payload_json: adapterCreatePayloadJson, + adapter_create_pending: adapterWorkspaceId ? 1 : 0, command, prompt, purpose, @@ -2562,7 +3724,7 @@ async function createInteractiveSessionFromInput( owner, created_by: createdBy, status: "provisioning", - lease_id: null, + lease_id: initialSandboxOwnership?.leaseId ?? null, attach_url: null, vnc_url: null, last_event: "interactive workspace requested", @@ -2579,7 +3741,7 @@ async function createInteractiveSessionFromInput( control_granted_at: null, control_expires_at: null, multiplayer_mode: 0, - agent_token_hash: await sha256(agentToken), + agent_token_hash: initialAgentTokenHash, }) .execute(); await appendInteractiveSessionEvent(env, id, user, "interactive workspace requested", now); @@ -2587,11 +3749,22 @@ async function createInteractiveSessionFromInput( env, { id, + ...(adapterWorkspaceId ? { adapterWorkspaceId } : {}), + ...(adapterControlPlane ? { adapterControlPlane } : {}), + ...(adapterSettings + ? { + adapterTtlSeconds: adapterSettings.ttlSeconds, + adapterIdleTimeoutSeconds: adapterSettings.idleTimeoutSeconds, + adapterRequestedCapabilities: adapterSettings.capabilities, + adapterCreatePayloadJson, + } + : {}), parentSessionId: lineage.parentSessionId, rootSessionId, repo, branch, runtime, + profile, command, prompt, purpose, @@ -2601,30 +3774,122 @@ async function createInteractiveSessionFromInput( ...(githubToken ? { githubToken } : {}), }, agentToken, + initialSandboxLease && initialSandboxOwnership + ? { lease: initialSandboxLease, ownership: initialSandboxOwnership } + : undefined, ); if (provisioned) { - await db + const initialTerminalStatus: "stopped" | "expired" | "failed" | null = + provisioned.status === "stopped" || + provisioned.status === "expired" || + provisioned.status === "failed" + ? provisioned.status + : null; + const terminalAt = provisioned.reconciledAt ?? now + 1; + const completionVersionFloor = Math.max(terminalAt, now + 1); + const provisionUpdate = await db .updateTable("interactive_sessions") .set({ status: provisioned.status, - lease_id: provisioned.leaseId, - attach_url: provisioned.attachUrl, - vnc_url: provisioned.vncUrl, + lease_id: provisioned.adapter === runtimeAdapterName ? null : provisioned.leaseId, + attach_url: initialTerminalStatus ? null : provisioned.attachUrl, + // Versioned adapter desktop URLs are minted on demand and never persisted. + vnc_url: provisioned.adapter === runtimeAdapterName ? null : provisioned.vncUrl, + adapter: provisioned.adapter ?? null, + profile: provisioned.profile ?? profile, + adapter_workspace_id: provisioned.adapterWorkspaceId ?? null, + provider_resource_id: provisioned.providerResourceId ?? null, + capabilities_json: JSON.stringify( + provisioned.capabilities ?? + (runtime === "crabbox" ? crabboxCapabilities : containerCapabilities), + ), + expires_at: provisioned.expiresAt ?? null, + last_reconciled_at: provisioned.reconciledAt ?? null, + reconcile_error: provisioned.reconcileError ?? null, + terminal_status: initialTerminalStatus ? null : (provisioned.terminalStatus ?? null), + adapter_create_pending: initialTerminalStatus ? 0 : provisioned.createPending ? 1 : 0, + terminal_finalize_pending: initialTerminalStatus ? 1 : 0, + ...(initialTerminalStatus + ? { + stopped_at: terminalAt, + agent_token_hash: null, + controller: null, + control_requested_by: null, + control_requested_at: null, + control_granted_at: null, + control_expires_at: null, + } + : {}), last_event: provisioned.message, - updated_at: now + 1, + updated_at: sql`MAX(updated_at + 1, ${completionVersionFloor})`, }) .where("id", "=", id) - .execute(); + .where("status", "in", ["provisioning", "pending_adapter"]) + .where(sql`lease_id IS ${initialSandboxOwnership?.leaseId ?? null}`) + .where("agent_token_hash", "=", initialAgentTokenHash) + .where("sandbox_refresh_sandbox_id", "is", null) + .where("sandbox_refresh_claim", "is", null) + .where("sandbox_refresh_claim_expires_at", "is", null) + .executeTakeFirst(); + if ((provisionUpdate.numUpdatedRows ?? 0n) === 0n) { + let current = await readInteractiveSession(env, id); + const currentAdapterProvision = Boolean( + current && + current.adapter === runtimeAdapterName && + current.adapterWorkspaceId === provisioned.adapterWorkspaceId && + ["provisioning", "pending_adapter", "ready", "attached", "detached"].includes( + current.status, + ), + ); + if ( + !currentAdapterProvision && + provisioned.adapter === runtimeAdapterName && + provisioned.adapterWorkspaceId + ) { + await stopSupersededRuntimeAdapterProvision( + env, + id, + provisioned.adapterWorkspaceId, + provisioned.createPending === true, + Date.now(), + ); + } + if ( + provisioned.adapter !== runtimeAdapterName && + provisioned.leaseId?.startsWith(sandboxLeasePrefix) + ) { + await queueSandboxCredentialPolicyCleanup( + env, + id, + sandboxLeaseInfo({ id, leaseId: provisioned.leaseId }).sandboxId, + ); + await reconcileCredentialPolicyCleanupBatch(env, Date.now(), id); + current = await readInteractiveSession(env, id); + } + if (!current) throw new Error("interactive session disappeared during provisioning"); + return { session: decorateInteractiveSession(current, user, env) }; + } await appendInteractiveSessionEvent(env, id, user, provisioned.message, now + 1); + if (initialTerminalStatus) { + await finalizeTerminalInteractiveSession( + env, + id, + initialTerminalStatus, + terminalAt, + ).catch(() => undefined); + } } else { await db .updateTable("interactive_sessions") .set({ status: "pending_adapter", last_event: "waiting for interactive runtime adapter", - updated_at: now + 1, + updated_at: sql`MAX(updated_at + 1, ${now + 1})`, }) .where("id", "=", id) + .where("status", "=", "provisioning") + .where(sql`lease_id IS ${initialSandboxOwnership?.leaseId ?? null}`) + .where("agent_token_hash", "=", initialAgentTokenHash) .execute(); await appendInteractiveSessionEvent( env, @@ -2654,6 +3919,252 @@ async function createInteractiveSessionFromInput( throw new Error("failed to allocate interactive session id"); } +function initialRuntimeAdapterWorkspaceId( + env: RuntimeEnv, + runtime: "crabbox" | "container", + sessionId: string, +): string | null { + if (!env.CRABBOX_RUNTIME_ADAPTER_URL || (runtime === "container" && env.SANDBOX)) return null; + const namespace = normalizeAdapterNamespace(env.CRABBOX_RUNTIME_ADAPTER_NAMESPACE ?? ""); + if (!namespace) { + throw serviceUnavailable( + "runtime adapter namespace is required and must be a DNS-safe label of at most 32 characters", + ); + } + const adapterWorkspaceId = namespacedAdapterWorkspaceId(namespace, sessionId); + if (!adapterWorkspaceId) throw serviceUnavailable("runtime adapter workspace id is invalid"); + return adapterWorkspaceId; +} + +function requireRuntimeAdapterCreatePreflight( + env: RuntimeEnv, + runtime: "crabbox" | "container", +): void { + if (!env.CRABBOX_RUNTIME_ADAPTER_URL || (runtime === "container" && env.SANDBOX)) return; + if (!configuredRuntimeAdapterControlPlane(env)) { + throw serviceUnavailable("runtime adapter URL must use HTTPS or literal loopback HTTP"); + } + if (!runtimeAdapterToken(env)) { + throw serviceUnavailable("runtime adapter token is not configured"); + } +} + +function configuredRuntimeAdapterControlPlane(env: RuntimeEnv): string | null { + return runtimeAdapterControlPlaneIdentity(env.CRABBOX_RUNTIME_ADAPTER_URL); +} + +function requireRegisteredRuntimeAdapterControlPlane( + env: RuntimeEnv, + registeredControlPlane: string | null | undefined, +): string { + if (!registeredControlPlane) { + throw new Error("runtime adapter control-plane registration is missing"); + } + const configuredControlPlane = configuredRuntimeAdapterControlPlane(env); + if (!configuredControlPlane) { + throw new Error("runtime adapter control plane is unavailable"); + } + if (configuredControlPlane !== registeredControlPlane) { + throw new Error("runtime adapter control plane differs from workspace registration"); + } + if (!runtimeAdapterToken(env)) throw new Error("runtime adapter token is not configured"); + return registeredControlPlane; +} + +async function registeredRuntimeAdapterControlPlaneForSession( + env: RuntimeEnv, + sessionId: string, + adapterWorkspaceId: string, +): Promise { + const registration = await database(env) + .selectFrom("interactive_sessions") + .select("adapter_control_plane") + .where("id", "=", sessionId) + .where("adapter", "=", runtimeAdapterName) + .where("adapter_workspace_id", "=", adapterWorkspaceId) + .executeTakeFirst(); + return requireRegisteredRuntimeAdapterControlPlane(env, registration?.adapter_control_plane); +} + +async function stopSupersededRuntimeAdapterProvision( + env: RuntimeEnv, + sessionId: string, + adapterWorkspaceId: string, + createPending: boolean, + now: number, +): Promise { + if (!createPending) { + await clearRuntimeAdapterCreatePending(env, sessionId, adapterWorkspaceId); + } + try { + const release = await stopRuntimeAdapterWorkspaceForSession(env, sessionId, adapterWorkspaceId); + if (release.status === "stopped") { + await recordConfirmedRuntimeAdapterRelease( + env, + sessionId, + adapterWorkspaceId, + now, + release.message, + ); + return; + } + await persistRuntimeAdapterStopEvidence( + env, + sessionId, + adapterWorkspaceId, + release.message, + now, + null, + ); + } catch (error) { + const message = safeProviderError(error, [adapterWorkspaceId]); + const pendingMessage = `superseded runtime adapter stop pending: ${message}`; + await persistRuntimeAdapterStopEvidence( + env, + sessionId, + adapterWorkspaceId, + pendingMessage, + now, + message, + ); + } +} + +async function recordConfirmedRuntimeAdapterRelease( + env: RuntimeEnv, + sessionId: string, + adapterWorkspaceId: string, + now: number, + releaseMessage?: string, +): Promise<"stopping" | "stopped" | "failed" | null> { + for (let attempt = 0; attempt < 3; attempt += 1) { + const lifecycle = await database(env) + .selectFrom("interactive_sessions") + .select([ + "adapter_create_pending", + "terminal_status", + "terminal_failure_reason", + "reconcile_error", + "last_event", + "updated_at", + ]) + .where("id", "=", sessionId) + .where("adapter", "=", runtimeAdapterName) + .where("adapter_workspace_id", "=", adapterWorkspaceId) + .where("status", "=", "stopping") + .executeTakeFirst(); + if (!lifecycle) return null; + + const resolved = resolveCreateAfterStopRace( + lifecycle.adapter_create_pending === 1, + lifecycle.terminal_status, + ); + const failureMessage = retainedRuntimeAdapterFailureMessage( + lifecycle.terminal_failure_reason, + lifecycle.reconcile_error, + lifecycle.last_event, + ); + const retainedReleaseMessage = clean(releaseMessage, 500) || null; + const values = + resolved.status === "stopping" + ? ({ + adapter_create_pending: 1, + last_reconciled_at: now, + updated_at: sql`MAX(updated_at + 1, ${now})`, + last_event: + retainedReleaseMessage ?? "runtime adapter stop waiting for create resolution", + } as const) + : ({ + status: resolved.status, + lease_id: null, + attach_url: null, + vnc_url: null, + terminal_status: resolved.terminalStatus, + terminal_failure_reason: resolved.status === "failed" ? failureMessage : null, + adapter_create_pending: 0, + terminal_finalize_pending: 1, + last_reconciled_at: now, + reconcile_error: resolved.status === "failed" ? failureMessage : null, + stopped_at: now, + agent_token_hash: null, + controller: null, + control_requested_by: null, + control_requested_at: null, + control_granted_at: null, + control_expires_at: null, + updated_at: sql`MAX(updated_at + 1, ${now})`, + last_event: + resolved.status === "failed" + ? failureMessage + : (retainedReleaseMessage ?? "interactive workspace stopped"), + } as const); + const terminalStatusOwner = lifecycle.terminal_status + ? sql`terminal_status = ${lifecycle.terminal_status}` + : sql`terminal_status IS NULL`; + const expectedOwner = sql` + id = ${sessionId} + AND adapter = ${runtimeAdapterName} + AND adapter_workspace_id = ${adapterWorkspaceId} + AND status = 'stopping' + AND adapter_create_pending = ${lifecycle.adapter_create_pending} + AND updated_at = ${lifecycle.updated_at} + AND ${terminalStatusOwner} + `; + const db = database(env); + const update = db + .updateTable("interactive_sessions") + .set(values) + .where(expectedOwner) + .returning("updated_at"); + const recordReleaseEvent = + retainedReleaseMessage && + (resolved.status !== "stopping" || lifecycle.last_event !== retainedReleaseMessage); + const queries: CompilableQuery[] = []; + if (recordReleaseEvent) { + queries.push(sql` + INSERT INTO interactive_session_events (session_id, actor, message, created_at) + SELECT ${sessionId}, 'system', ${retainedReleaseMessage}, ${now} + FROM interactive_sessions + WHERE ${expectedOwner} + `); + } + queries.push(update); + const results = await env.DB.batch<{ updated_at: number }>( + queries.map((query) => { + const compiled = query.compile(db); + return env.DB.prepare(compiled.sql).bind(...compiled.parameters); + }), + ); + if (results.at(-1)?.results.length) { + if (recordReleaseEvent) { + await archiveInteractiveSessionLogs(env, sessionId, now).catch(() => undefined); + } + if (resolved.status === "stopped" || resolved.status === "failed") { + await finalizeTerminalInteractiveSession(env, sessionId, resolved.status, now).catch( + () => undefined, + ); + } + return resolved.status; + } + } + return null; +} + +async function clearRuntimeAdapterCreatePending( + env: RuntimeEnv, + sessionId: string, + adapterWorkspaceId: string, +): Promise { + await database(env) + .updateTable("interactive_sessions") + .set({ adapter_create_pending: 0 }) + .where("id", "=", sessionId) + .where("adapter", "=", runtimeAdapterName) + .where("adapter_workspace_id", "=", adapterWorkspaceId) + .where("status", "=", "stopping") + .execute(); +} + async function resolveInteractiveSessionLineage( env: RuntimeEnv, user: User, @@ -2709,36 +4220,292 @@ async function cleanupInteractiveSessions( let query = db .selectFrom("interactive_sessions") .selectAll() - .where("status", "in", deadInteractiveSessionStatuses); + .where("status", "in", deadInteractiveSessionStatuses) + .where("terminal_finalize_pending", "=", 0).where(sql` + NOT EXISTS ( + SELECT 1 + FROM interactive_session_credential_policies AS policy + WHERE policy.session_id = interactive_sessions.id + ) + `).where(sql` + COALESCE( + ( + SELECT event_count + FROM interactive_session_log_archives AS archive + WHERE archive.session_id = interactive_sessions.id + ), + -1 + ) >= ( + SELECT count(*) + FROM interactive_session_events AS event + WHERE event.session_id = interactive_sessions.id + ) + `).where(sql` + EXISTS ( + SELECT 1 + FROM interactive_session_log_archives AS archive + WHERE archive.session_id = interactive_sessions.id + AND archive.session_updated_at = interactive_sessions.updated_at + ) + `).where(sql` + ${env.SESSION_LOGS ? 1 : 0} = 0 + OR EXISTS ( + SELECT 1 + FROM interactive_session_log_archives AS archive + WHERE archive.session_id = interactive_sessions.id + AND archive.events_key IS NOT NULL + AND archive.transcript_key IS NOT NULL + AND archive.summary_key IS NOT NULL + ) + `); if (ids.length) query = query.where("id", "in", ids); - const removedSessions = (await query.execute()) - .map((row) => interactiveSession(row, [])) - .filter((session) => canManageInteractiveSession(user, session)); - const removedIds = removedSessions.map((session) => session.id); - if (removedIds.length) { + const candidates = (await query.execute()).filter((row) => + canManageInteractiveSession(user, interactiveSession(row, [])), + ); + const removedIds = ( await Promise.all( - removedSessions.map((session) => unregisterInteractiveSessionCredentialPolicy(env, session)), - ); - const archives = await db - .selectFrom("interactive_session_log_archives") - .selectAll() - .where("session_id", "in", removedIds) - .execute(); - await Promise.all(archives.map((archive) => cleanupSessionLogArchiveObjects(env, archive))); - await db - .deleteFrom("interactive_session_log_archives") - .where("session_id", "in", removedIds) - .execute(); - await db - .deleteFrom("interactive_session_events") - .where("session_id", "in", removedIds) - .execute(); - await db.deleteFrom("interactive_sessions").where("id", "in", removedIds).execute(); + candidates.map(async (row) => { + const archive = await db + .selectFrom("interactive_session_log_archives") + .selectAll() + .where("session_id", "=", row.id) + .executeTakeFirst(); + const removed = await deleteFinalizedInteractiveSession(env, row, archive); + if (!removed) return null; + await cleanupSessionLogArchiveObjects(env, archive).catch((error) => { + console.error(`session archive object cleanup leaked for ${row.id}`, error); + }); + return row.id; + }), + ) + ).filter((id): id is string => Boolean(id)); + if (removedIds.length) { await audit(env, user, `interactive sessions cleaned ${removedIds.join(",")}`, Date.now()); } return { state: await readState(request, env, user), removedIds }; } +async function deleteFinalizedInteractiveSession( + env: RuntimeEnv, + row: InteractiveSessionRow, + archive: Selectable | undefined, +): Promise { + const db = database(env); + const claimToken = `cleanup:${crypto.randomUUID()}`; + const finalClaim = db + .updateTable("interactive_sessions") + .set({ + terminal_finalize_pending: terminalCleanupDeletePending, + reconcile_error: claimToken, + }) + .where("id", "=", row.id) + .where("status", "=", row.status) + .where("updated_at", "=", row.updated_at) + .where("terminal_finalize_pending", "=", 0).where(sql` + NOT EXISTS ( + SELECT 1 + FROM interactive_session_credential_policies + WHERE session_id = ${row.id} + ) + `).where(sql` + ${archive ? 1 : 0} = 1 + AND EXISTS ( + SELECT 1 + FROM interactive_session_log_archives + WHERE session_id = ${row.id} + AND event_count = ${archive?.event_count ?? -1} + AND session_updated_at IS ${archive?.session_updated_at ?? null} + AND session_updated_at = ${row.updated_at} + AND events_key IS ${archive?.events_key ?? null} + AND transcript_key IS ${archive?.transcript_key ?? null} + AND summary_key IS ${archive?.summary_key ?? null} + AND archived_at = ${archive?.archived_at ?? -1} + AND updated_at = ${archive?.updated_at ?? -1} + ) + `).where(sql` + COALESCE( + ( + SELECT event_count + FROM interactive_session_log_archives + WHERE session_id = ${row.id} + ), + -1 + ) >= ( + SELECT count(*) + FROM interactive_session_events + WHERE session_id = ${row.id} + ) + `).where(sql` + ${env.SESSION_LOGS ? 1 : 0} = 0 + OR EXISTS ( + SELECT 1 + FROM interactive_session_log_archives + WHERE session_id = ${row.id} + AND events_key IS NOT NULL + AND transcript_key IS NOT NULL + AND summary_key IS NOT NULL + ) + `); + const ownsFinalClaim = sql`EXISTS ( + SELECT 1 + FROM interactive_sessions + WHERE id = ${row.id} + AND status = ${row.status} + AND updated_at = ${row.updated_at} + AND terminal_finalize_pending = ${terminalCleanupDeletePending} + AND reconcile_error = ${claimToken} + )`; + // D1 batches are transactional, so no event can interleave between the claim and row deletes. + await executeBatch(env, [ + finalClaim, + db + .deleteFrom("interactive_session_events") + .where("session_id", "=", row.id) + .where(ownsFinalClaim), + db + .deleteFrom("interactive_session_log_archives") + .where("session_id", "=", row.id) + .where(ownsFinalClaim), + db + .deleteFrom("interactive_sessions") + .where("id", "=", row.id) + .where("status", "=", row.status) + .where("updated_at", "=", row.updated_at) + .where("terminal_finalize_pending", "=", terminalCleanupDeletePending) + .where("reconcile_error", "=", claimToken), + ]); + const current = await db + .selectFrom("interactive_sessions") + .select("id") + .where("id", "=", row.id) + .executeTakeFirst(); + return !current; +} + +async function mutateInteractiveSessionMetadataAtomically( + env: RuntimeEnv, + session: Pick, + user: User, + message: string, + values: UpdateObject, + now = Date.now(), +): Promise { + const db = database(env); + const eventMessage = clean(message, 1000); + const revision = Math.max(now, session.updatedAt + 1); + const expectedOwner = sql` + id = ${session.id} + AND status = ${session.status} + AND updated_at = ${session.updatedAt} + `; + const eventQuery = sql` + INSERT INTO interactive_session_events (session_id, actor, message, created_at) + SELECT ${session.id}, ${actor(user)}, ${eventMessage}, ${now} + FROM interactive_sessions + WHERE ${expectedOwner} + `; + const updateQuery = db + .updateTable("interactive_sessions") + .set({ + ...values, + terminal_finalize_pending: sql`CASE + WHEN status IN ('stopped', 'expired', 'failed') THEN 1 + ELSE terminal_finalize_pending + END`, + updated_at: revision, + last_event: eventMessage, + }) + .where(expectedOwner) + .returning("updated_at"); + const results = await env.DB.batch<{ updated_at: number }>( + [eventQuery, updateQuery].map((query) => { + const compiled = query.compile(db); + return env.DB.prepare(compiled.sql).bind(...compiled.parameters); + }), + ); + if (!results.at(-1)?.results.some((row) => row.updated_at === revision)) { + throw conflict("interactive session lifecycle changed; retry metadata update"); + } + await archiveInteractiveSessionLogs(env, session.id, now).catch(() => undefined); +} + +type LegacyInteractiveSessionStopOwner = { + id: string; + status: InteractiveSessionStatus; + adapter: string | null; + leaseId: string | null; + updatedAt: number; +}; + +async function completeLegacyInteractiveSessionStop( + env: RuntimeEnv, + owner: LegacyInteractiveSessionStopOwner, + eventActor: string, + now: number, +): Promise { + const db = database(env); + const revision = Math.max(now, owner.updatedAt + 1); + const actorName = clean(eventActor, 120) || "system"; + const expectedOwner = sql` + id = ${owner.id} + AND status = ${owner.status} + AND updated_at = ${owner.updatedAt} + AND adapter IS ${owner.adapter} + AND lease_id IS ${owner.leaseId} + AND (adapter IS NULL OR adapter != ${runtimeAdapterName}) + AND (lease_id IS NULL OR lease_id NOT LIKE ${`${sandboxLeasePrefix}%`}) + AND credential_cleanup_terminal_status IS NULL + `; + const requestedMessage = "interactive workspace stop requested"; + const finalMessage = "interactive workspace stopped"; + const requestedEvent = sql` + INSERT INTO interactive_session_events (session_id, actor, message, created_at) + SELECT ${owner.id}, ${actorName}, ${requestedMessage}, ${now} + FROM interactive_sessions + WHERE ${expectedOwner} + `; + const stoppedEvent = sql` + INSERT INTO interactive_session_events (session_id, actor, message, created_at) + SELECT ${owner.id}, ${actorName}, ${finalMessage}, ${now} + FROM interactive_sessions + WHERE ${expectedOwner} + `; + const stop = db + .updateTable("interactive_sessions") + .set({ + status: "stopped", + stopped_at: sql`COALESCE(stopped_at, ${now})`, + reconcile_error: null, + terminal_status: null, + adapter_create_pending: 0, + terminal_finalize_pending: 1, + agent_token_hash: null, + attach_url: null, + vnc_url: null, + controller: null, + control_requested_by: null, + control_requested_at: null, + control_granted_at: null, + control_expires_at: null, + updated_at: revision, + last_event: finalMessage, + }) + .where(expectedOwner) + .returning("updated_at"); + const results = await env.DB.batch<{ updated_at: number }>( + [requestedEvent, stoppedEvent, stop].map((query) => { + const compiled = query.compile(db); + return env.DB.prepare(compiled.sql).bind(...compiled.parameters); + }), + ); + const stopped = results.at(-1)?.results.some((row) => row.updated_at === revision) ?? false; + if (stopped) { + await archiveInteractiveSessionLogs(env, owner.id, now).catch(() => undefined); + await finalizeTerminalInteractiveSession(env, owner.id, "stopped", now).catch(() => undefined); + } + return stopped; +} + async function mutateInteractiveSession( request: Request, env: RuntimeEnv, @@ -2746,16 +4513,19 @@ async function mutateInteractiveSession( id: string, action: string, ): Promise<{ session: InteractiveSession; shareUrl?: string }> { - const session = await readInteractiveSession(env, id); + const session = await readFreshInteractiveSession(env, id); if (!session) throw notFound("interactive session not found"); const now = Date.now(); const userActor = actor(user); const canManage = canManageInteractiveSession(user, session); if (action === "attach") { + if (!session.capabilities.terminal) { + throw badRequest("session does not advertise terminal access"); + } if (!canControlInteractiveSession(user, session, now, canGrantDelegatedControl(env, session))) { throw forbidden("terminal control has not been granted"); } - if (["expired", "failed", "stopped"].includes(session.status)) { + if (["stopping", "expired", "failed", "stopped"].includes(session.status)) { throw badRequest(`session is ${session.status}`); } const nextStatus = @@ -2766,17 +4536,21 @@ async function mutateInteractiveSession( : session.status === "provisioning" ? "attach requested; workspace provisioning" : "interactive terminal attached"; - await database(env) + const attached = await database(env) .updateTable("interactive_sessions") .set({ status: nextStatus, last_seen_at: now, - updated_at: now, + updated_at: sql`MAX(updated_at + 1, ${now})`, last_event: message, }) .where("id", "=", id) - .where("status", "!=", "stopped") - .execute(); + .where("status", "=", session.status) + .where("updated_at", "=", session.updatedAt) + .executeTakeFirst(); + if ((attached.numUpdatedRows ?? 0n) === 0n) { + throw conflict("interactive session lifecycle changed; retry attach"); + } await appendInteractiveSessionEvent(env, id, user, message, now); return { session: decorateInteractiveSession( @@ -2792,18 +4566,18 @@ async function mutateInteractiveSession( const token = shareToken(); const tokenHash = await sha256(token); const preview = token.slice(0, 8); - await database(env) - .updateTable("interactive_sessions") - .set({ + await mutateInteractiveSessionMetadataAtomically( + env, + session, + user, + "read-only share link enabled", + { share_mode: "link_read", share_token_hash: tokenHash, share_token_preview: preview, - updated_at: now, - last_event: "read-only share link enabled", - }) - .where("id", "=", id) - .execute(); - await appendInteractiveSessionEvent(env, id, user, "read-only share link enabled", now); + }, + now, + ); await audit(env, user, `interactive session share enabled ${id}`, now); return { session: decorateInteractiveSession( @@ -2817,9 +4591,12 @@ async function mutateInteractiveSession( if (action === "disable_share") { if (!canManage) throw forbidden("only the session owner or maintainer can disable sharing"); - await database(env) - .updateTable("interactive_sessions") - .set({ + await mutateInteractiveSessionMetadataAtomically( + env, + session, + user, + "session sharing disabled", + { share_mode: "private", share_token_hash: null, share_token_preview: null, @@ -2828,12 +4605,9 @@ async function mutateInteractiveSession( controller: null, control_granted_at: null, control_expires_at: null, - updated_at: now, - last_event: "session sharing disabled", - }) - .where("id", "=", id) - .execute(); - await appendInteractiveSessionEvent(env, id, user, "session sharing disabled", now); + }, + now, + ); await audit(env, user, `interactive session share disabled ${id}`, now); return { session: decorateInteractiveSession( @@ -2850,16 +4624,14 @@ async function mutateInteractiveSession( } const enabled = action === "enable_multiplayer"; const message = enabled ? "multiplayer mode enabled" : "multiplayer mode disabled"; - await database(env) - .updateTable("interactive_sessions") - .set({ - multiplayer_mode: enabled ? 1 : 0, - updated_at: now, - last_event: message, - }) - .where("id", "=", id) - .execute(); - await appendInteractiveSessionEvent(env, id, user, message, now); + await mutateInteractiveSessionMetadataAtomically( + env, + session, + user, + message, + { multiplayer_mode: enabled ? 1 : 0 }, + now, + ); await audit( env, user, @@ -2882,24 +4654,19 @@ async function mutateInteractiveSession( if (canControlInteractiveSession(user, session, now, canGrantDelegatedControl(env, session))) { return { session: decorateInteractiveSession(session, user, env) }; } - if (["expired", "failed", "stopped"].includes(session.status)) { + if (["stopping", "expired", "failed", "stopped"].includes(session.status)) { throw badRequest(`session is ${session.status}`); } - await database(env) - .updateTable("interactive_sessions") - .set({ - control_requested_by: userActor, - control_requested_at: now, - updated_at: now, - last_event: `${userActor} requested terminal control`, - }) - .where("id", "=", id) - .execute(); - await appendInteractiveSessionEvent( + const message = `${userActor} requested terminal control`; + await mutateInteractiveSessionMetadataAtomically( env, - id, + session, user, - `${userActor} requested terminal control`, + message, + { + control_requested_by: userActor, + control_requested_at: now, + }, now, ); return { @@ -2918,24 +4685,19 @@ async function mutateInteractiveSession( throw badRequest("delegated terminal control requires a revocable PTY bridge"); } const expires = now + 30 * 60 * 1000; - await database(env) - .updateTable("interactive_sessions") - .set({ + const message = `control granted to ${session.controlRequestedBy}`; + await mutateInteractiveSessionMetadataAtomically( + env, + session, + user, + message, + { controller: session.controlRequestedBy, control_granted_at: now, control_expires_at: expires, control_requested_by: null, control_requested_at: null, - updated_at: now, - last_event: `control granted to ${session.controlRequestedBy}`, - }) - .where("id", "=", id) - .execute(); - await appendInteractiveSessionEvent( - env, - id, - user, - `control granted to ${session.controlRequestedBy}`, + }, now, ); await audit( @@ -2956,23 +4718,18 @@ async function mutateInteractiveSession( if (action === "deny_control") { if (!canManage) throw forbidden("only the session owner or maintainer can deny control"); const requester = session.controlRequestedBy; - await database(env) - .updateTable("interactive_sessions") - .set({ - control_requested_by: null, - control_requested_at: null, - updated_at: now, - last_event: requester - ? `control request denied for ${requester}` - : "control request denied", - }) - .where("id", "=", id) - .execute(); - await appendInteractiveSessionEvent( + const message = requester + ? `control request denied for ${requester}` + : "control request denied"; + await mutateInteractiveSessionMetadataAtomically( env, - id, + session, user, - requester ? `control request denied for ${requester}` : "control request denied", + message, + { + control_requested_by: null, + control_requested_at: null, + }, now, ); return { @@ -2986,18 +4743,18 @@ async function mutateInteractiveSession( if (action === "revoke_control") { if (!canManage) throw forbidden("only the session owner or maintainer can revoke control"); - await database(env) - .updateTable("interactive_sessions") - .set({ + await mutateInteractiveSessionMetadataAtomically( + env, + session, + user, + "terminal control revoked", + { controller: null, control_granted_at: null, control_expires_at: null, - updated_at: now, - last_event: "terminal control revoked", - }) - .where("id", "=", id) - .execute(); - await appendInteractiveSessionEvent(env, id, user, "terminal control revoked", now); + }, + now, + ); await audit(env, user, `interactive session control revoked ${id}`, now); return { session: decorateInteractiveSession( @@ -3010,23 +4767,187 @@ async function mutateInteractiveSession( if (action === "stop") { if (!canManage) throw forbidden("only the session owner or maintainer can stop"); - await unregisterInteractiveSessionCredentialPolicy(env, session); - await database(env) - .updateTable("interactive_sessions") - .set({ - status: "stopped", - stopped_at: now, - controller: null, - control_requested_by: null, - control_requested_at: null, - updated_at: now, - last_event: "interactive workspace stopped", - }) - .where("id", "=", id) - .where("status", "!=", "stopped") - .execute(); - await appendInteractiveSessionEvent(env, id, user, "interactive workspace stopped", now); - await archiveInteractiveSessionLogs(env, id, now, { force: true }).catch(() => undefined); + if (["stopped", "expired", "failed"].includes(session.status)) { + if (isSandboxInteractiveSession(session)) { + const staged = await stageTerminalCredentialPolicyCleanupById( + env, + session.id, + session.status as "stopped" | "expired" | "failed", + "sandbox credential cleanup pending", + now, + ); + if (!staged) throw conflict("interactive session lifecycle changed; retry stop"); + await reconcileCredentialPolicyCleanupBatch(env, now, session.id); + const current = await readInteractiveSession(env, session.id); + if (current) return { session: decorateInteractiveSession(current, user, env) }; + } + await finalizeTerminalInteractiveSession( + env, + session.id, + session.status as "stopped" | "expired" | "failed", + session.stoppedAt ?? now, + ).catch(() => undefined); + return { session: decorateInteractiveSession(session, user, env) }; + } + if (session.adapter === runtimeAdapterName) { + if (!session.adapterWorkspaceId) { + throw serviceUnavailable("runtime adapter workspace reference is incomplete"); + } + const stopClaimRevision = Math.max(now, session.updatedAt + 1); + const stopClaim = await database(env) + .updateTable("interactive_sessions") + .set({ + status: "stopping", + lease_id: null, + updated_at: stopClaimRevision, + last_event: "runtime adapter stop requested", + reconcile_error: null, + agent_token_hash: null, + attach_url: null, + vnc_url: null, + controller: null, + control_requested_by: null, + control_requested_at: null, + control_granted_at: null, + control_expires_at: null, + }) + .where("id", "=", id) + .where("status", "=", session.status) + .where("updated_at", "=", session.updatedAt) + .executeTakeFirst(); + if ((stopClaim.numUpdatedRows ?? 0n) === 0n) { + const current = await readInteractiveSession(env, id); + if ( + !current || + current.adapter !== runtimeAdapterName || + current.adapterWorkspaceId !== session.adapterWorkspaceId || + !["stopping", "stopped", "expired", "failed"].includes(current.status) + ) { + throw conflict("interactive session lifecycle changed; retry stop"); + } + return { + session: decorateInteractiveSession(current, user, env), + }; + } + await appendInteractiveSessionEvent(env, id, user, "runtime adapter stop requested", now); + let adapterStop: RuntimeAdapterStopResult; + try { + adapterStop = await stopRuntimeAdapterWorkspaceForSession( + env, + session.id, + session.adapterWorkspaceId, + ); + } catch (error) { + const message = safeProviderError(error, [session.adapterWorkspaceId]); + const pendingMessage = `runtime adapter stop pending: ${message}`; + await persistRuntimeAdapterStopEvidence( + env, + id, + session.adapterWorkspaceId, + pendingMessage, + now, + message, + actor(user), + ); + throw serviceUnavailable(`runtime adapter stop failed: ${message}`); + } + if (adapterStop.status === "stopping") { + const lifecycle = await database(env) + .selectFrom("interactive_sessions") + .select("adapter_create_pending") + .where("id", "=", id) + .where("adapter", "=", runtimeAdapterName) + .where("adapter_workspace_id", "=", session.adapterWorkspaceId) + .where("status", "=", "stopping") + .executeTakeFirst(); + const message = lifecycle?.adapter_create_pending + ? `${adapterStop.message}; runtime adapter stop waiting for create resolution` + : adapterStop.message; + await persistRuntimeAdapterStopEvidence( + env, + id, + session.adapterWorkspaceId, + message, + now, + null, + actor(user), + ); + return { + session: decorateInteractiveSession( + (await readInteractiveSession(env, id)) as InteractiveSession, + user, + env, + ), + }; + } + const resolved = await recordConfirmedRuntimeAdapterRelease( + env, + id, + session.adapterWorkspaceId, + Date.now(), + adapterStop.message, + ); + if (resolved === "failed" || resolved === "stopped") { + await audit(env, user, `interactive session stopped ${id}`, Date.now()); + } + return { + session: decorateInteractiveSession( + (await readInteractiveSession(env, id)) as InteractiveSession, + user, + env, + ), + }; + } + if (isSandboxInteractiveSession(session)) { + const message = "interactive workspace stop waiting for credential cleanup"; + const staged = await stageTerminalCredentialPolicyCleanupById( + env, + session.id, + "stopped", + message, + now, + ); + if (!staged) { + const current = await readInteractiveSession(env, id); + if (!current) throw notFound("interactive session not found"); + const terminalIntent = await database(env) + .selectFrom("interactive_sessions") + .select("credential_cleanup_terminal_status") + .where("id", "=", id) + .where("status", "=", "stopping") + .executeTakeFirst(); + if (terminalIntent?.credential_cleanup_terminal_status) { + return { session: decorateInteractiveSession(current, user, env) }; + } + if (["stopped", "expired", "failed"].includes(current.status)) { + return { session: decorateInteractiveSession(current, user, env) }; + } + throw conflict("interactive session lifecycle changed; retry stop"); + } + await appendInteractiveSessionEvent( + env, + id, + user, + "interactive workspace stop requested", + now, + ); + await reconcileCredentialPolicyCleanupBatch(env, now, id); + return { + session: decorateInteractiveSession( + (await readInteractiveSession(env, id)) as InteractiveSession, + user, + env, + ), + }; + } + if (!(await completeLegacyInteractiveSessionStop(env, session, actor(user), now))) { + const current = await readInteractiveSession(env, id); + if (!current) throw notFound("interactive session not found"); + if (!["stopped", "expired", "failed"].includes(current.status)) { + throw conflict("interactive session lifecycle changed; retry stop"); + } + return { session: decorateInteractiveSession(current, user, env) }; + } await audit(env, user, `interactive session stopped ${id}`, now); return { session: decorateInteractiveSession( @@ -3040,82 +4961,1426 @@ async function mutateInteractiveSession( throw badRequest("unknown action"); } -async function unregisterSandboxCredentialPolicy( +async function unregisterSandboxCredentialPolicyLookup( env: RuntimeEnv, - sandboxId: string, + lookupId: string, + generation: string, + sessionId: string, ): Promise { const stub = sandboxControlStub(env); - if (!stub) return; - await Promise.all( - sandboxLookupIds(env, sandboxId).map((lookupId) => - stub.fetch( - `https://crabfleet.internal/api/session-control/sandbox/${encodeURIComponent(lookupId)}`, - { method: "DELETE" }, + if (!stub) throw serviceUnavailable("sandbox credential policy cleanup is unavailable"); + let response: Response; + try { + response = await stub.fetch( + `https://crabfleet.internal/api/session-control/sandbox/${encodeURIComponent(lookupId)}`, + { + method: "DELETE", + body: JSON.stringify({ generation, sessionId, tombstonedAt: Date.now() }), + headers: { "content-type": "application/json" }, + }, + ); + } catch { + throw serviceUnavailable("sandbox credential policy cleanup failed"); + } + if (!response.ok) { + throw serviceUnavailable("sandbox credential policy cleanup failed"); + } +} + +function sandboxCredentialPolicyRefQueries( + env: RuntimeEnv, + sessionId: string, + sandboxId: string, + state: "registering" | "active" | "cleanup_pending", + generation: string, + now: number, + authorizationCondition: RawBuilder, +): CompilableQuery[] { + return sandboxLookupIds(env, sandboxId).map( + (lookupId) => sql` + INSERT INTO interactive_session_credential_policies ( + session_id, + sandbox_id, + lookup_id, + state, + registration_generation, + registration_claim, + registration_claim_expires_at, + attempt_count, + last_attempt_at, + last_error, + cleanup_claim, + cleanup_claim_expires_at, + created_at, + updated_at + ) SELECT + ${sessionId}, + ${sandboxId}, + ${lookupId}, + ${state}, + ${generation}, + NULL, + NULL, + 0, + NULL, + NULL, + NULL, + NULL, + ${now}, + ${now} + WHERE ${authorizationCondition} + ON CONFLICT(session_id, sandbox_id, lookup_id) DO UPDATE SET + state = CASE + WHEN interactive_session_credential_policies.state = 'cleanup_pending' + OR excluded.state = 'cleanup_pending' + THEN 'cleanup_pending' + WHEN interactive_session_credential_policies.registration_claim IS NOT NULL + THEN interactive_session_credential_policies.state + ELSE excluded.state + END, + last_error = CASE + WHEN interactive_session_credential_policies.state = 'cleanup_pending' + OR interactive_session_credential_policies.registration_claim IS NOT NULL + OR excluded.state = 'cleanup_pending' + THEN interactive_session_credential_policies.last_error + ELSE NULL + END, + cleanup_claim = CASE + WHEN interactive_session_credential_policies.state = 'cleanup_pending' + OR interactive_session_credential_policies.registration_claim IS NOT NULL + OR excluded.state = 'cleanup_pending' + THEN interactive_session_credential_policies.cleanup_claim + ELSE NULL + END, + cleanup_claim_expires_at = CASE + WHEN interactive_session_credential_policies.state = 'cleanup_pending' + OR interactive_session_credential_policies.registration_claim IS NOT NULL + OR excluded.state = 'cleanup_pending' + THEN interactive_session_credential_policies.cleanup_claim_expires_at + ELSE NULL + END, + updated_at = excluded.updated_at + `, + ); +} + +function sandboxCredentialPolicyCleanupAuthorizedCondition( + sessionId: string, + sandboxId: string, + now: number, +): RawBuilder { + const leasePrefix = `${sandboxLeasePrefix}${sandboxId}`; + return sql` + NOT EXISTS ( + SELECT 1 + FROM standalone_sandbox_provisions AS owner + WHERE owner.id = ${sessionId} + AND owner.sandbox_id = ${sandboxId} + AND ( + owner.state = 'active' + OR ( + owner.state = 'provisioning' + AND owner.ownership_claim IS NOT NULL + AND owner.ownership_claim_expires_at > ${now} + ) + ) + ) + AND NOT EXISTS ( + SELECT 1 + FROM interactive_sessions AS session + WHERE session.id = ${sessionId} + AND (session.adapter IS NULL OR session.adapter != ${runtimeAdapterName}) + AND session.status IN ('provisioning', 'pending_adapter', 'ready', 'attached', 'detached') + AND session.credential_cleanup_terminal_status IS NULL + AND session.agent_token_hash IS NOT NULL + AND ( + ( + session.lease_id IS NOT NULL + AND substr(session.lease_id, 1, ${leasePrefix.length}) = ${leasePrefix} + AND ( + length(session.lease_id) = ${leasePrefix.length} + OR substr(session.lease_id, ${leasePrefix.length + 1}, 1) = ':' + ) + ) + OR ( + session.sandbox_refresh_sandbox_id = ${sandboxId} + AND session.sandbox_refresh_claim IS NOT NULL + AND session.sandbox_refresh_claim_expires_at > ${now} + ) + ) + ) + `; +} + +async function sandboxCredentialPolicyGeneration( + env: RuntimeEnv, + sessionId: string, + sandboxId: string, +): Promise { + return ( + (await existingSandboxCredentialPolicyGeneration(env, sessionId, sandboxId)) ?? + `generation:${crypto.randomUUID()}` + ); +} + +async function existingSandboxCredentialPolicyGeneration( + env: RuntimeEnv, + sessionId: string, + sandboxId: string, +): Promise { + const existing = await database(env) + .selectFrom("interactive_session_credential_policies") + .select("registration_generation") + .where("session_id", "=", sessionId) + .where("sandbox_id", "=", sandboxId) + .orderBy("lookup_id", "asc") + .executeTakeFirst(); + return existing?.registration_generation ?? null; +} + +function activeSandboxCredentialPolicyCondition( + env: RuntimeEnv, + sessionId: string, + sandboxId: string, + generation: string, + updatedAt?: number, +): RawBuilder { + const lookupIds = sandboxLookupIds(env, sandboxId); + const updatedAtCondition = + updatedAt === undefined ? sql`1 = 1` : sql`updated_at = ${updatedAt}`; + return sql` + ( + SELECT count(DISTINCT lookup_id) + FROM interactive_session_credential_policies + WHERE session_id = ${sessionId} + AND sandbox_id = ${sandboxId} + AND lookup_id IN (${sql.join(lookupIds)}) + AND state = 'active' + AND registration_generation = ${generation} + AND registration_claim IS NULL + AND ${updatedAtCondition} + ) = ${lookupIds.length} + AND NOT EXISTS ( + SELECT 1 + FROM interactive_session_credential_policies + WHERE session_id = ${sessionId} + AND sandbox_id = ${sandboxId} + AND ( + state != 'active' + OR registration_generation != ${generation} + OR registration_claim IS NOT NULL + OR NOT (${updatedAtCondition}) + ) + ) + `; +} + +async function activeSandboxCredentialPolicyGeneration( + env: RuntimeEnv, + sessionId: string, + sandboxId: string, +): Promise { + const rows = await database(env) + .selectFrom("interactive_session_credential_policies") + .select(["lookup_id", "state", "registration_generation", "registration_claim"]) + .where("session_id", "=", sessionId) + .where("sandbox_id", "=", sandboxId) + .execute(); + const expected = sandboxLookupIds(env, sandboxId); + const generation = rows[0]?.registration_generation; + if ( + !generation || + !expected.every((lookupId) => + rows.some( + (row) => + row.lookup_id === lookupId && + row.state === "active" && + row.registration_generation === generation && + row.registration_claim === null, ), + ) || + rows.some( + (row) => + row.state !== "active" || + row.registration_generation !== generation || + row.registration_claim !== null, + ) + ) { + return null; + } + return generation; +} + +async function queueSandboxCredentialPolicyCleanup( + env: RuntimeEnv, + sessionId: string, + sandboxId: string, + now = Date.now(), +): Promise { + const generation = await sandboxCredentialPolicyGeneration(env, sessionId, sandboxId); + await executeBatch( + env, + sandboxCredentialPolicyRefQueries( + env, + sessionId, + sandboxId, + "cleanup_pending", + generation, + now, + sandboxCredentialPolicyCleanupAuthorizedCondition(sessionId, sandboxId, now), ), ); } -async function unregisterInteractiveSessionCredentialPolicy( +function sandboxTerminalCleanupOwnership( + session: Pick< + InteractiveSessionRow, + | "id" + | "lease_id" + | "sandbox_refresh_sandbox_id" + | "sandbox_refresh_claim" + | "sandbox_refresh_claim_expires_at" + >, +): SandboxTerminalCleanupOwnership | null { + if (!session.lease_id?.startsWith(sandboxLeasePrefix)) return null; + const terminalLeaseId = sandboxLeaseWithoutRefresh(session.lease_id); + let currentSandboxId: string; + try { + currentSandboxId = sandboxLeaseInfo({ id: session.id, leaseId: terminalLeaseId }).sandboxId; + } catch { + return null; + } + const refreshValues = [ + session.sandbox_refresh_sandbox_id, + session.sandbox_refresh_claim, + session.sandbox_refresh_claim_expires_at, + ]; + const refreshPresent = refreshValues.some((value) => value !== null); + if (refreshPresent && refreshValues.some((value) => value === null)) return null; + if ( + session.sandbox_refresh_sandbox_id && + session.sandbox_refresh_claim && + session.sandbox_refresh_claim_expires_at !== null + ) { + return { + fence: { + claim: session.sandbox_refresh_claim, + expiresAt: session.sandbox_refresh_claim_expires_at, + refreshLeaseId: session.lease_id, + sandboxId: session.sandbox_refresh_sandbox_id, + }, + sandboxIds: [...new Set([currentSandboxId, session.sandbox_refresh_sandbox_id])], + terminalLeaseId, + }; + } + return { + fence: { leaseId: session.lease_id, sandboxId: currentSandboxId }, + sandboxIds: [currentSandboxId], + terminalLeaseId, + }; +} + +function sandboxManagedOwnershipFencesMatch( + left: SandboxManagedOwnershipFence, + right: SandboxManagedOwnershipFence, +): boolean { + if ("leaseId" in left || "leaseId" in right) { + return ( + "leaseId" in left && + "leaseId" in right && + left.leaseId === right.leaseId && + left.sandboxId === right.sandboxId + ); + } + return ( + left.claim === right.claim && + left.expiresAt === right.expiresAt && + left.refreshLeaseId === right.refreshLeaseId && + left.sandboxId === right.sandboxId + ); +} + +function terminalCleanupIntentRank(status: "stopped" | "expired" | "failed"): number { + return status === "failed" ? 3 : status === "expired" ? 2 : 1; +} + +async function stageTerminalCredentialPolicyCleanup( env: RuntimeEnv, - session: Pick, -): Promise { - if (session.leaseId?.startsWith(sandboxLeasePrefix)) { - await unregisterSandboxCredentialPolicy(env, sandboxLeaseInfo(session).sandboxId); + session: InteractiveSessionRow, + terminalStatus: "stopped" | "expired" | "failed", + message: string, + now: number, + failureReason?: string, + requiredFence?: SandboxManagedOwnershipFence, +): Promise { + const ownership = sandboxTerminalCleanupOwnership(session); + if ( + !ownership || + (requiredFence && !sandboxManagedOwnershipFencesMatch(ownership.fence, requiredFence)) + ) { + return false; } + const db = database(env); + const stageRevision = Math.max(now, session.updated_at + 1); + const generations = await Promise.all( + ownership.sandboxIds.map(async (sandboxId) => ({ + generation: await sandboxCredentialPolicyGeneration(env, session.id, sandboxId), + sandboxId, + })), + ); + const cleanupIntent = sql<"stopped" | "expired" | "failed">`CASE + WHEN credential_cleanup_terminal_status = 'failed' OR ${terminalStatus} = 'failed' + THEN 'failed' + WHEN credential_cleanup_terminal_status = 'expired' OR ${terminalStatus} = 'expired' + THEN 'expired' + ELSE 'stopped' + END`; + const failureFallback = + failureReason ?? + (terminalStatus === "failed" + ? message + : "interactive workspace failed during credential cleanup"); + const failureEvidence = sql`CASE + WHEN credential_cleanup_terminal_status = 'failed' THEN COALESCE( + NULLIF(terminal_failure_reason, ''), + NULLIF(reconcile_error, ''), + NULLIF(last_event, ''), + ${failureFallback} + ) + WHEN ${terminalStatus} = 'failed' THEN COALESCE( + NULLIF(terminal_failure_reason, ''), + NULLIF(${failureFallback}, ''), + NULLIF(reconcile_error, ''), + NULLIF(last_event, ''), + 'interactive workspace failed during credential cleanup' + ) + ELSE terminal_failure_reason + END`; + const sessionTransition = db + .updateTable("interactive_sessions") + .set({ + status: "stopping", + lease_id: ownership.terminalLeaseId, + credential_cleanup_terminal_status: cleanupIntent, + terminal_finalize_pending: 0, + sandbox_refresh_sandbox_id: null, + sandbox_refresh_claim: null, + sandbox_refresh_claim_expires_at: null, + agent_token_hash: null, + attach_url: null, + vnc_url: null, + controller: null, + control_requested_by: null, + control_requested_at: null, + control_granted_at: null, + control_expires_at: null, + reconcile_error: sql`CASE + WHEN credential_cleanup_terminal_status = 'failed' OR ${terminalStatus} = 'failed' + THEN COALESCE(${failureEvidence}, ${message}) + ELSE ${message} + END`, + terminal_failure_reason: failureEvidence, + terminal_status: null, + adapter_create_pending: 0, + stopped_at: sql`COALESCE(stopped_at, ${now})`, + updated_at: stageRevision, + last_event: message, + }) + .where("id", "=", session.id) + .where("updated_at", "=", session.updated_at) + .where((expression) => + expression.or([ + expression("adapter", "is", null), + expression("adapter", "!=", runtimeAdapterName), + ]), + ) + .where(sandboxManagedStoredOwnershipCondition(ownership.fence)); + const policyTransitions = generations.flatMap(({ generation, sandboxId }) => [ + ...sandboxCredentialPolicyRefQueries( + env, + session.id, + sandboxId, + "cleanup_pending", + generation, + stageRevision, + sandboxCredentialPolicyCleanupAuthorizedCondition(session.id, sandboxId, stageRevision), + ), + db + .updateTable("interactive_session_credential_policies") + .set({ + state: "cleanup_pending", + updated_at: stageRevision, + }) + .where("session_id", "=", session.id) + .where("sandbox_id", "=", sandboxId) + .where( + sandboxCredentialPolicyCleanupAuthorizedCondition(session.id, sandboxId, stageRevision), + ), + ]); + await executeBatch(env, [sessionTransition, ...policyTransitions]); + const staged = await db + .selectFrom("interactive_sessions") + .select([ + "status", + "credential_cleanup_terminal_status", + "terminal_failure_reason", + "updated_at", + ]) + .where("id", "=", session.id) + .executeTakeFirst(); + return Boolean( + staged?.status === "stopping" && + staged.updated_at === stageRevision && + staged.credential_cleanup_terminal_status && + terminalCleanupIntentRank(staged.credential_cleanup_terminal_status) >= + terminalCleanupIntentRank(terminalStatus) && + (staged.credential_cleanup_terminal_status !== "failed" || staged.terminal_failure_reason), + ); } -async function interactiveTerminalHub( - request: Request, +async function stageTerminalCredentialPolicyCleanupById( env: RuntimeEnv, - user: User | null, -): Promise { - if (request.headers.get("upgrade")?.toLowerCase() !== "websocket") { - throw badRequest("websocket upgrade required"); + sessionId: string, + terminalStatus: "stopped" | "expired" | "failed", + message: string, + now: number, + failureReason?: string, + requiredFence?: SandboxManagedOwnershipFence, +): Promise { + for (let attempt = 0; attempt < 3; attempt += 1) { + const session = await database(env) + .selectFrom("interactive_sessions") + .selectAll() + .where("id", "=", sessionId) + .executeTakeFirst(); + if (!session) return false; + if ( + session.status === "stopping" && + session.credential_cleanup_terminal_status && + terminalCleanupIntentRank(session.credential_cleanup_terminal_status) >= + terminalCleanupIntentRank(terminalStatus) && + (session.credential_cleanup_terminal_status !== "failed" || session.terminal_failure_reason) + ) { + return true; + } + if ( + await stageTerminalCredentialPolicyCleanup( + env, + session, + terminalStatus, + message, + now, + failureReason, + session.status === "stopping" && session.credential_cleanup_terminal_status + ? undefined + : requiredFence, + ) + ) { + return true; + } } - if (!user && !(await canOpenAnonymousTerminalHub(request, env))) { - throw unauthorized(); + return false; +} + +type CredentialPolicyScanRow = { + scan_rowid: number; + session_id: string; + sandbox_id: string; + lookup_id: string; + policy_state: "registering" | "active"; + registration_generation: string; + registration_claim: string | null; + registration_claim_expires_at: number | null; + policy_updated_at: number; + matched_session_id: string | null; + session_adapter: string | null; + session_status: InteractiveSessionStatus | null; + session_lease_id: string | null; + credential_cleanup_terminal_status: "stopped" | "expired" | "failed" | null; + session_sandbox_refresh_sandbox_id: string | null; + session_sandbox_refresh_claim: string | null; + session_sandbox_refresh_claim_expires_at: number | null; + session_agent_token_hash: string | null; + session_updated_at: number | null; + matched_standalone_id: string | null; + standalone_state: "provisioning" | "active" | "cleanup_pending" | null; + standalone_claim: string | null; + standalone_claim_expires_at: number | null; + standalone_updated_at: number | null; +}; + +async function scanCredentialPolicyCleanupPage( + env: RuntimeEnv, + now: number, + sessionId?: string, +): Promise { + const db = database(env); + const state = sessionId + ? null + : await db + .selectFrom("credential_policy_reconcile_state") + .select(["last_rowid", "scan_max_rowid"]) + .where("id", "=", 1) + .executeTakeFirst(); + const originalCursor = state?.last_rowid ?? 0; + const originalMaxRowid = state?.scan_max_rowid ?? 0; + let cursor = sessionId ? 0 : originalCursor; + let maxRowid = sessionId + ? Number.MAX_SAFE_INTEGER + : originalMaxRowid || (await maximumCredentialPolicyRowid(db)); + let rows = await readCredentialPolicyScanPage(db, cursor, maxRowid, sessionId); + if (!sessionId && rows.length === 0 && (cursor > 0 || maxRowid > 0)) { + cursor = 0; + maxRowid = await maximumCredentialPolicyRowid(db); + rows = await readCredentialPolicyScanPage(db, cursor, maxRowid); + } + const attemptedRepairs = new Set(); + const repairedRegistrations = new Set(); + const deferredRegistrations = new Set(); + for (const row of rows) { + const registrationKey = `${row.session_id}\u0000${row.sandbox_id}\u0000${row.registration_generation}`; + if (attemptedRepairs.has(registrationKey)) continue; + attemptedRepairs.add(registrationKey); + try { + if (await repairActiveSandboxCredentialPolicyRegistration(env, row, now)) { + repairedRegistrations.add(registrationKey); + } + } catch (error) { + deferredRegistrations.add(registrationKey); + console.error("active sandbox credential policy repair failed", error); + } } + const candidates = rows.filter( + (row) => + !repairedRegistrations.has( + `${row.session_id}\u0000${row.sandbox_id}\u0000${row.registration_generation}`, + ) && + !deferredRegistrations.has( + `${row.session_id}\u0000${row.sandbox_id}\u0000${row.registration_generation}`, + ) && + credentialPolicyScanRequiresCleanup(row, now), + ); + for (const row of candidates) { + const transitionRevision = Math.max( + now, + row.policy_updated_at + 1, + (row.session_updated_at ?? 0) + 1, + (row.standalone_updated_at ?? 0) + 1, + ); + let policyTransition = db + .updateTable("interactive_session_credential_policies") + .set({ state: "cleanup_pending", updated_at: transitionRevision }) + .where("session_id", "=", row.session_id) + .where("sandbox_id", "=", row.sandbox_id) + .where("lookup_id", "=", row.lookup_id) + .where("state", "=", row.policy_state) + .where("registration_generation", "=", row.registration_generation) + .where("updated_at", "=", row.policy_updated_at) + .where( + sandboxCredentialPolicyCleanupAuthorizedCondition(row.session_id, row.sandbox_id, now), + ); + policyTransition = row.registration_claim + ? policyTransition.where("registration_claim", "=", row.registration_claim) + : policyTransition.where("registration_claim", "is", null); + policyTransition = + row.registration_claim_expires_at === null + ? policyTransition.where("registration_claim_expires_at", "is", null) + : policyTransition.where( + "registration_claim_expires_at", + "=", + row.registration_claim_expires_at, + ); + if (row.matched_standalone_id) { + if (!row.standalone_state || row.standalone_updated_at === null) continue; + let ownerTransition = db + .updateTable("standalone_sandbox_provisions") + .set({ + state: "cleanup_pending", + ownership_claim: null, + ownership_claim_expires_at: null, + updated_at: transitionRevision, + }) + .where("id", "=", row.matched_standalone_id) + .where("sandbox_id", "=", row.sandbox_id) + .where("state", "=", row.standalone_state) + .where("updated_at", "=", row.standalone_updated_at); + ownerTransition = row.standalone_claim + ? ownerTransition.where("ownership_claim", "=", row.standalone_claim) + : ownerTransition.where("ownership_claim", "is", null); + ownerTransition = + row.standalone_claim_expires_at === null + ? ownerTransition.where("ownership_claim_expires_at", "is", null) + : ownerTransition.where( + "ownership_claim_expires_at", + "=", + row.standalone_claim_expires_at, + ); + await executeBatch(env, [ownerTransition, policyTransition]); + continue; + } + if (!row.matched_session_id || row.session_adapter === runtimeAdapterName) { + await executeBatch(env, [policyTransition]); + continue; + } + const sessionTransition = sql` + UPDATE interactive_sessions + SET terminal_failure_reason = CASE + WHEN credential_cleanup_terminal_status = 'failed' + OR status = 'failed' + OR ( + credential_cleanup_terminal_status IS NULL + AND status NOT IN ('stopping', 'stopped', 'expired') + ) + THEN COALESCE( + NULLIF(terminal_failure_reason, ''), + NULLIF(reconcile_error, ''), + NULLIF(last_event, ''), + 'sandbox credential registration cleanup failed' + ) + ELSE terminal_failure_reason + END, + status = 'stopping', + credential_cleanup_terminal_status = CASE + WHEN credential_cleanup_terminal_status = 'failed' OR status = 'failed' + THEN 'failed' + WHEN credential_cleanup_terminal_status = 'expired' OR status = 'expired' + THEN 'expired' + WHEN credential_cleanup_terminal_status = 'stopped' + OR status IN ('stopping', 'stopped') + THEN 'stopped' + ELSE 'failed' + END, + terminal_status = NULL, + adapter_create_pending = 0, + terminal_finalize_pending = 0, + sandbox_refresh_sandbox_id = NULL, + sandbox_refresh_claim = NULL, + sandbox_refresh_claim_expires_at = NULL, + agent_token_hash = NULL, + attach_url = NULL, + vnc_url = NULL, + controller = NULL, + control_requested_by = NULL, + control_requested_at = NULL, + control_granted_at = NULL, + control_expires_at = NULL, + reconcile_error = CASE + WHEN credential_cleanup_terminal_status = 'failed' + OR status = 'failed' + OR ( + credential_cleanup_terminal_status IS NULL + AND status NOT IN ('stopping', 'stopped', 'expired') + ) + THEN COALESCE( + NULLIF(terminal_failure_reason, ''), + NULLIF(reconcile_error, ''), + NULLIF(last_event, ''), + 'sandbox credential registration cleanup failed' + ) + ELSE 'sandbox credential registration cleanup pending' + END, + stopped_at = COALESCE(stopped_at, ${now}), + updated_at = ${transitionRevision}, + last_event = 'sandbox credential registration cleanup pending' + WHERE id = ${row.matched_session_id} + AND adapter IS ${row.session_adapter} + AND status IS ${row.session_status} + AND lease_id IS ${row.session_lease_id} + AND credential_cleanup_terminal_status IS ${row.credential_cleanup_terminal_status} + AND sandbox_refresh_sandbox_id IS ${row.session_sandbox_refresh_sandbox_id} + AND sandbox_refresh_claim IS ${row.session_sandbox_refresh_claim} + AND sandbox_refresh_claim_expires_at IS ${row.session_sandbox_refresh_claim_expires_at} + AND agent_token_hash IS ${row.session_agent_token_hash} + AND updated_at IS ${row.session_updated_at} + `; + await executeBatch(env, [sessionTransition, policyTransition]); + } + if (!sessionId) { + const nextCursor = rows.at(-1)?.scan_rowid ?? 0; + await sql` + UPDATE credential_policy_reconcile_state + SET last_rowid = ${nextCursor}, scan_max_rowid = ${maxRowid}, updated_at = ${now} + WHERE id = 1 + AND last_rowid = ${originalCursor} + AND scan_max_rowid = ${originalMaxRowid} + `.execute(db); + } +} + +async function readCredentialPolicyScanPage( + db: Kysely, + cursor: number, + maxRowid: number, + sessionId?: string, +): Promise { + const sessionFilter = sessionId ? sql`AND policy.session_id = ${sessionId}` : sql``; + const result = await sql` + SELECT + policy.rowid AS scan_rowid, + policy.session_id, + policy.sandbox_id, + policy.lookup_id, + policy.state AS policy_state, + policy.registration_generation, + policy.registration_claim, + policy.registration_claim_expires_at, + policy.updated_at AS policy_updated_at, + session.id AS matched_session_id, + session.adapter AS session_adapter, + session.status AS session_status, + session.lease_id AS session_lease_id, + session.credential_cleanup_terminal_status, + session.sandbox_refresh_sandbox_id AS session_sandbox_refresh_sandbox_id, + session.sandbox_refresh_claim AS session_sandbox_refresh_claim, + session.sandbox_refresh_claim_expires_at AS session_sandbox_refresh_claim_expires_at, + session.agent_token_hash AS session_agent_token_hash, + session.updated_at AS session_updated_at, + standalone.id AS matched_standalone_id, + standalone.state AS standalone_state, + standalone.ownership_claim AS standalone_claim, + standalone.ownership_claim_expires_at AS standalone_claim_expires_at, + standalone.updated_at AS standalone_updated_at + FROM interactive_session_credential_policies AS policy + LEFT JOIN interactive_sessions AS session ON session.id = policy.session_id + LEFT JOIN standalone_sandbox_provisions AS standalone + ON standalone.id = policy.session_id + AND standalone.sandbox_id = policy.sandbox_id + WHERE policy.rowid > ${cursor} + AND policy.rowid <= ${maxRowid} + AND policy.state IN ('registering', 'active') + ${sessionFilter} + ORDER BY policy.rowid ASC + LIMIT ${credentialPolicyScanLimit} + `.execute(db); + return result.rows; +} - const pair = new WebSocketPair(); - const client = pair[0]; - const server = pair[1]; - const subscriptions = new Map(); - const pendingSubscriptions = new Map(); - let queue = Promise.resolve(); - let hubClosed = false; +async function maximumCredentialPolicyRowid(db: Kysely): Promise { + const result = await sql<{ max_rowid: number }>` + SELECT COALESCE(MAX(rowid), 0) AS max_rowid + FROM interactive_session_credential_policies + `.execute(db); + return result.rows[0]?.max_rowid ?? 0; +} - server.accept(); - sendTerminalJson(server, TerminalMessageType.Welcome, "", { - ok: true, - version: 1, - multiplex: true, - }); +async function repairActiveSandboxCredentialPolicyRegistration( + env: RuntimeEnv, + row: CredentialPolicyScanRow, + now: number, +): Promise { + if ( + row.policy_state !== "registering" || + (row.registration_claim !== null && + (row.registration_claim_expires_at ?? Number.NEGATIVE_INFINITY) > now) + ) { + return false; + } + const ownershipFence = credentialPolicyScanOwnershipFence(row, now); + if (!ownershipFence) return false; + if (!(await sandboxCredentialPolicyExists(env, row.sandbox_id, row.registration_generation))) { + return false; + } + const repaired = await recordSandboxCredentialPolicyRefs( + env, + row.session_id, + row.sandbox_id, + "active", + row.registration_generation, + ownershipFence, + now, + ); + if (!repaired) { + throw new Error("active sandbox credential policy repair lost durable ownership"); + } + return true; +} - const closeSubscription = (id: string, code = 1000, reason = "unsubscribed") => { - const subscription = subscriptions.get(id); - if (!subscription) return; - subscriptions.delete(id); - subscription.markClosing(reason); - if (subscription.viewCheck !== null) clearInterval(subscription.viewCheck); - if (subscription.upstream.readyState < WebSocket.CLOSING) { - subscription.upstream.close(code, reason); +function credentialPolicyScanOwnershipFence( + row: CredentialPolicyScanRow, + now: number, +): SandboxCredentialPolicyOwnershipFence | null { + if ( + row.matched_standalone_id === row.session_id && + row.standalone_state === "provisioning" && + row.standalone_claim && + (row.standalone_claim_expires_at ?? Number.NEGATIVE_INFINITY) > now + ) { + return { + claim: row.standalone_claim, + provisionId: row.matched_standalone_id, + sandboxId: row.sandbox_id, + }; + } + if (!row.matched_session_id || !row.session_lease_id) return null; + try { + const lease = sandboxLeaseInfo({ + id: row.matched_session_id, + adapter: row.session_adapter, + leaseId: row.session_lease_id, + }); + if (lease.sandboxId === row.sandbox_id) { + return { leaseId: row.session_lease_id, sandboxId: row.sandbox_id }; } - }; - - const closeAll = (code = 1000, reason = "client closed") => { - for (const id of subscriptions.keys()) closeSubscription(id, code, reason); - }; + } catch { + return null; + } + if ( + row.session_sandbox_refresh_sandbox_id === row.sandbox_id && + row.session_sandbox_refresh_claim && + (row.session_sandbox_refresh_claim_expires_at ?? Number.NEGATIVE_INFINITY) > now + ) { + return { + claim: row.session_sandbox_refresh_claim, + expiresAt: row.session_sandbox_refresh_claim_expires_at as number, + refreshLeaseId: row.session_lease_id, + sandboxId: row.sandbox_id, + }; + } + return null; +} - server.addEventListener("message", (event) => { - queue = queue - .catch(() => undefined) - .then(async () => { - const data = await webSocketMessageData(event.data); - const bytes = - typeof data === "string" ? encoder.encode(data) : new Uint8Array(data.slice(0)); - const frame = decodeTerminalFrame(bytes); - if (!frame) { +function credentialPolicyScanRequiresCleanup(row: CredentialPolicyScanRow, now: number): boolean { + const registrationAbandoned = + row.policy_state === "registering" && + (row.registration_claim === null || + (row.registration_claim_expires_at ?? Number.NEGATIVE_INFINITY) <= now); + if (row.matched_standalone_id) { + if (row.standalone_state === "active") return false; + if (row.standalone_state === "provisioning") { + return (row.standalone_claim_expires_at ?? Number.NEGATIVE_INFINITY) <= now; + } + return true; + } + if (!row.matched_session_id || row.session_adapter === runtimeAdapterName) return true; + if ( + row.credential_cleanup_terminal_status !== null || + row.session_status === "stopping" || + row.session_status === "stopped" || + row.session_status === "expired" || + row.session_status === "failed" + ) { + return true; + } + const leaseSandboxId = row.session_lease_id?.startsWith(sandboxLeasePrefix) + ? (row.session_lease_id.slice(sandboxLeasePrefix.length).split(":", 1)[0] ?? null) + : null; + const sandboxExpected = credentialPolicySandboxIsExpected( + leaseSandboxId, + row.sandbox_id, + row.session_sandbox_refresh_sandbox_id, + row.session_sandbox_refresh_claim, + row.session_sandbox_refresh_claim_expires_at, + now, + ); + if ( + row.session_agent_token_hash !== null && + sandboxExpected && + (row.session_status === "ready" || + row.session_status === "attached" || + row.session_status === "detached") + ) { + return false; + } + if (registrationAbandoned) return true; + if ( + row.policy_state === "active" && + (row.session_status === "provisioning" || row.session_status === "pending_adapter") && + row.policy_updated_at <= now - credentialPolicyProvisioningStaleMs && + (row.session_updated_at ?? Number.NEGATIVE_INFINITY) <= + now - credentialPolicyProvisioningStaleMs + ) { + return true; + } + if ( + row.policy_state === "active" && + (row.session_status === "ready" || + row.session_status === "attached" || + row.session_status === "detached") + ) { + return true; + } + return false; +} + +async function normalizeCredentialPolicyCleanupGroups( + env: RuntimeEnv, + now: number, + sessionId?: string, +): Promise { + const db = database(env); + const state = sessionId + ? null + : await db + .selectFrom("credential_policy_reconcile_state") + .select([ + "group_session_id", + "group_sandbox_id", + "group_max_session_id", + "group_max_sandbox_id", + ]) + .where("id", "=", 1) + .executeTakeFirst(); + const originalSessionCursor = state?.group_session_id ?? ""; + const originalSandboxCursor = state?.group_sandbox_id ?? ""; + const originalMaxSession = state?.group_max_session_id ?? ""; + const originalMaxSandbox = state?.group_max_sandbox_id ?? ""; + let maximum = sessionId + ? { session_id: sessionId, sandbox_id: "\uffff" } + : originalMaxSession + ? { session_id: originalMaxSession, sandbox_id: originalMaxSandbox } + : await maximumCredentialPolicyCleanupGroup(db); + let groups = await readCredentialPolicyCleanupGroups( + db, + sessionId ? "" : originalSessionCursor, + sessionId ? "" : originalSandboxCursor, + maximum, + sessionId, + ); + if (!sessionId && groups.length === 0 && maximum) { + maximum = await maximumCredentialPolicyCleanupGroup(db); + groups = await readCredentialPolicyCleanupGroups(db, "", "", maximum); + } + for (const group of groups) { + await queueSandboxCredentialPolicyCleanup(env, group.session_id, group.sandbox_id, now); + } + if (!sessionId) { + const last = groups.at(-1); + await db + .updateTable("credential_policy_reconcile_state") + .set({ + group_session_id: last?.session_id ?? "", + group_sandbox_id: last?.sandbox_id ?? "", + group_max_session_id: maximum?.session_id ?? "", + group_max_sandbox_id: maximum?.sandbox_id ?? "", + updated_at: now, + }) + .where("id", "=", 1) + .where("group_session_id", "=", originalSessionCursor) + .where("group_sandbox_id", "=", originalSandboxCursor) + .where("group_max_session_id", "=", originalMaxSession) + .where("group_max_sandbox_id", "=", originalMaxSandbox) + .execute(); + } +} + +async function readCredentialPolicyCleanupGroups( + db: Kysely, + sessionCursor: string, + sandboxCursor: string, + maximum: { session_id: string; sandbox_id: string } | null, + sessionId?: string, +): Promise> { + let query = db + .selectFrom("interactive_session_credential_policies") + .select(["session_id", "sandbox_id"]) + .distinct() + .where("state", "=", "cleanup_pending") + .where((expression) => + expression.or([ + expression("session_id", ">", sessionCursor), + expression.and([ + expression("session_id", "=", sessionCursor), + expression("sandbox_id", ">", sandboxCursor), + ]), + ]), + ) + .where((expression) => + maximum + ? expression.or([ + expression("session_id", "<", maximum.session_id), + expression.and([ + expression("session_id", "=", maximum.session_id), + expression("sandbox_id", "<=", maximum.sandbox_id), + ]), + ]) + : expression("session_id", "=", ""), + ) + .orderBy("session_id", "asc") + .orderBy("sandbox_id", "asc") + .limit(credentialPolicyCleanupLimit); + if (sessionId) query = query.where("session_id", "=", sessionId); + return query.execute(); +} + +async function maximumCredentialPolicyCleanupGroup( + db: Kysely, +): Promise<{ session_id: string; sandbox_id: string } | null> { + return ( + (await db + .selectFrom("interactive_session_credential_policies") + .select(["session_id", "sandbox_id"]) + .where("state", "=", "cleanup_pending") + .orderBy("session_id", "desc") + .orderBy("sandbox_id", "desc") + .executeTakeFirst()) ?? null + ); +} + +async function reconcileCredentialPolicyCleanupBatch( + env: RuntimeEnv, + now: number, + sessionId?: string, +): Promise { + await repairLegacySandboxCredentialPolicyBatch(env, now, sessionId).catch((error) => { + console.error("legacy sandbox credential policy repair batch failed", error); + }); + await expireStandaloneSandboxProvisions(env, now, sessionId).catch((error) => { + console.error("standalone Sandbox expiry failed", error); + }); + await scanCredentialPolicyCleanupPage(env, now, sessionId).catch((error) => { + console.error("credential policy cleanup scan failed", error); + }); + await normalizeCredentialPolicyCleanupGroups(env, now, sessionId).catch((error) => { + console.error("credential policy cleanup group normalization failed", error); + }); + let query = database(env) + .selectFrom("interactive_session_credential_policies") + .selectAll() + .where("state", "=", "cleanup_pending") + .where((expression) => + expression.or([ + expression("cleanup_claim", "is", null), + expression("cleanup_claim_expires_at", "<", now), + ]), + ) + .where(sql` + NOT EXISTS ( + SELECT 1 + FROM interactive_session_credential_policies AS registration + WHERE registration.session_id = interactive_session_credential_policies.session_id + AND registration.sandbox_id = interactive_session_credential_policies.sandbox_id + AND registration.registration_claim IS NOT NULL + AND registration.registration_claim_expires_at > ${now} + ) + `) + .orderBy(sql`COALESCE(last_attempt_at, created_at)`, "asc") + .orderBy("session_id", "asc") + .orderBy("sandbox_id", "asc") + .orderBy("lookup_id", "asc") + .limit(credentialPolicyCleanupLimit); + if (sessionId) query = query.where("session_id", "=", sessionId); + const policies = await query.execute(); + await mapWithConcurrency(policies, 3, async (policy) => { + await reconcileCredentialPolicyCleanup(env, policy, now); + }); + let completedSessions = database(env) + .selectFrom("interactive_sessions") + .select("id") + .where("status", "in", ["stopping", "stopped", "expired", "failed"]) + .where("credential_cleanup_terminal_status", "is not", null) + .where(sql` + NOT EXISTS ( + SELECT 1 + FROM interactive_session_credential_policies AS policy + WHERE policy.session_id = interactive_sessions.id + ) + `) + .orderBy("stopped_at", "asc") + .orderBy("id", "asc") + .limit(credentialPolicyCleanupLimit); + if (sessionId) completedSessions = completedSessions.where("id", "=", sessionId); + await mapWithConcurrency(await completedSessions.execute(), 3, async (session) => { + await completeCredentialPolicyCleanupSession(env, session.id, Date.now()); + }); + let standaloneCleanup = database(env) + .selectFrom("standalone_sandbox_provisions") + .select(["id", "sandbox_id"]) + .where("state", "=", "cleanup_pending") + .orderBy("updated_at", "asc") + .limit(credentialPolicyCleanupLimit); + if (sessionId) standaloneCleanup = standaloneCleanup.where("id", "=", sessionId); + await mapWithConcurrency(await standaloneCleanup.execute(), 3, async (owner) => { + await completeStandaloneSandboxProvisionCleanupSafely(env, owner.id, owner.sandbox_id); + }); +} + +async function reconcileCredentialPolicyCleanup( + env: RuntimeEnv, + policy: Selectable, + now: number, +): Promise { + const claim = crypto.randomUUID(); + const claimed = await sql` + UPDATE interactive_session_credential_policies + SET cleanup_claim = ${claim}, + cleanup_claim_expires_at = ${now + credentialPolicyCleanupClaimMs}, + attempt_count = attempt_count + 1, + last_attempt_at = ${now}, + updated_at = ${now} + WHERE session_id = ${policy.session_id} + AND sandbox_id = ${policy.sandbox_id} + AND lookup_id = ${policy.lookup_id} + AND registration_generation = ${policy.registration_generation} + AND state = 'cleanup_pending' + AND (cleanup_claim IS NULL OR cleanup_claim_expires_at < ${now}) + AND ${sandboxCredentialPolicyCleanupAuthorizedCondition( + policy.session_id, + policy.sandbox_id, + now, + )} + AND NOT EXISTS ( + SELECT 1 + FROM interactive_session_credential_policies AS registration + WHERE registration.session_id = interactive_session_credential_policies.session_id + AND registration.sandbox_id = interactive_session_credential_policies.sandbox_id + AND registration.registration_claim IS NOT NULL + AND registration.registration_claim_expires_at > ${now} + ) + `.execute(database(env)); + if ((claimed.numAffectedRows ?? 0n) === 0n) return; + try { + await unregisterSandboxCredentialPolicyLookup( + env, + policy.lookup_id, + policy.registration_generation, + policy.session_id, + ); + } catch (error) { + await database(env) + .updateTable("interactive_session_credential_policies") + .set({ + last_error: clean(error instanceof Error ? error.message : String(error), 500), + cleanup_claim: null, + cleanup_claim_expires_at: null, + updated_at: Date.now(), + }) + .where("session_id", "=", policy.session_id) + .where("sandbox_id", "=", policy.sandbox_id) + .where("lookup_id", "=", policy.lookup_id) + .where("registration_generation", "=", policy.registration_generation) + .where("cleanup_claim", "=", claim) + .execute(); + return; + } + await database(env) + .deleteFrom("interactive_session_credential_policies") + .where("session_id", "=", policy.session_id) + .where("sandbox_id", "=", policy.sandbox_id) + .where("lookup_id", "=", policy.lookup_id) + .where("registration_generation", "=", policy.registration_generation) + .where("cleanup_claim", "=", claim) + .execute(); + await completeCredentialPolicyCleanupSession(env, policy.session_id, Date.now()); + await completeStandaloneSandboxProvisionCleanupSafely(env, policy.session_id, policy.sandbox_id); +} + +async function completeStandaloneSandboxProvisionCleanupSafely( + env: RuntimeEnv, + provisionId: string, + sandboxId: string, +): Promise { + try { + await completeStandaloneSandboxProvisionCleanup(env, provisionId, sandboxId); + } catch (error) { + const now = Date.now(); + const message = `standalone Sandbox cleanup pending: ${safeProviderError(error, [ + provisionId, + sandboxId, + ])}`; + try { + await database(env) + .updateTable("standalone_sandbox_provisions") + .set({ + message, + updated_at: sql`MAX(updated_at + 1, ${now})`, + }) + .where("id", "=", provisionId) + .where("sandbox_id", "=", sandboxId) + .where("state", "=", "cleanup_pending") + .execute(); + } catch (persistError) { + console.error("standalone Sandbox cleanup failure persistence failed", persistError); + } + } +} + +async function completeStandaloneSandboxProvisionCleanup( + env: RuntimeEnv, + provisionId: string, + sandboxId: string, +): Promise { + const db = database(env); + const owner = await db + .selectFrom("standalone_sandbox_provisions") + .selectAll() + .where("id", "=", provisionId) + .where("sandbox_id", "=", sandboxId) + .where("state", "=", "cleanup_pending") + .executeTakeFirst(); + if (!owner) return; + if (owner.lease_id) { + if (!env.SANDBOX) throw serviceUnavailable("Sandbox binding is not configured"); + if (!isCurrentSandboxLease(owner.lease_id)) { + throw serviceUnavailable("standalone Sandbox cleanup lease is invalid"); + } + const lease = sandboxLeaseInfo({ id: owner.id, leaseId: owner.lease_id }); + if (lease.sandboxId !== owner.sandbox_id) { + throw serviceUnavailable("standalone Sandbox cleanup ownership is inconsistent"); + } + try { + await getSandbox(env.SANDBOX, owner.sandbox_id).deleteSession(lease.terminalSessionId); + } catch (error) { + if (!isSandboxSessionAlreadyGone(error, lease.terminalSessionId)) throw error; + } + } + await db + .deleteFrom("standalone_sandbox_provisions") + .where("id", "=", provisionId) + .where("sandbox_id", "=", sandboxId) + .where("request_hash", "=", owner.request_hash) + .where("state", "=", "cleanup_pending") + .where("updated_at", "=", owner.updated_at) + .where(sql`lease_id IS ${owner.lease_id}`) + .where(sql`expires_at IS ${owner.expires_at}`) + .where(sql` + NOT EXISTS ( + SELECT 1 + FROM interactive_session_credential_policies AS policy + WHERE policy.session_id = ${provisionId} + AND policy.sandbox_id = ${sandboxId} + ) + `) + .execute(); +} + +async function completeCredentialPolicyCleanupSession( + env: RuntimeEnv, + sessionId: string, + now: number, +): Promise { + const db = database(env); + const remaining = await db + .selectFrom("interactive_session_credential_policies") + .select(({ fn }) => fn.countAll().as("count")) + .where("session_id", "=", sessionId) + .executeTakeFirst(); + if (Number(remaining?.count ?? 0) > 0) return; + const session = await db + .selectFrom("interactive_sessions") + .select([ + "status", + "credential_cleanup_terminal_status", + "terminal_failure_reason", + "reconcile_error", + "last_event", + "stopped_at", + "updated_at", + ]) + .where("id", "=", sessionId) + .executeTakeFirst(); + const terminalStatus = session?.credential_cleanup_terminal_status; + if (!session || !terminalStatus) return; + const failureMessage = retainedRuntimeAdapterFailureMessage( + session.terminal_failure_reason, + session.reconcile_error, + session.last_event, + ); + const updated = await db + .updateTable("interactive_sessions") + .set({ + status: terminalStatus, + credential_cleanup_terminal_status: null, + terminal_status: null, + adapter_create_pending: 0, + terminal_finalize_pending: 1, + sandbox_refresh_sandbox_id: null, + sandbox_refresh_claim: null, + sandbox_refresh_claim_expires_at: null, + terminal_failure_reason: terminalStatus === "failed" ? failureMessage : null, + reconcile_error: terminalStatus === "failed" ? failureMessage : null, + stopped_at: session.stopped_at ?? now, + updated_at: sql`MAX(updated_at + 1, ${now})`, + last_event: + terminalStatus === "failed" + ? failureMessage + : terminalStatus === "expired" + ? "interactive workspace expired after credential cleanup" + : "interactive workspace stopped after credential cleanup", + }) + .where("id", "=", sessionId) + .where("status", "=", session.status) + .where("updated_at", "=", session.updated_at) + .where("credential_cleanup_terminal_status", "=", terminalStatus) + .where(sql` + NOT EXISTS ( + SELECT 1 + FROM interactive_session_credential_policies + WHERE session_id = ${sessionId} + ) + `) + .executeTakeFirst(); + if ((updated.numUpdatedRows ?? 0n) === 0n) return; + await finalizeTerminalInteractiveSession( + env, + sessionId, + terminalStatus, + session.stopped_at ?? now, + ).catch(() => undefined); +} + +function legacyInteractiveSessionLeaseId( + session: Pick, +): string | null { + return legacyLeaseIdForAdapter(session.adapter, session.leaseId); +} + +function isSandboxInteractiveSession( + session: Pick, +): boolean { + return legacyInteractiveSessionLeaseId(session)?.startsWith(sandboxLeasePrefix) === true; +} + +async function interactiveTerminalHub( + request: Request, + env: RuntimeEnv, + user: User | null, +): Promise { + if (request.headers.get("upgrade")?.toLowerCase() !== "websocket") { + throw badRequest("websocket upgrade required"); + } + if (!user && !(await canOpenAnonymousTerminalHub(request, env))) { + throw unauthorized(); + } + + const pair = new WebSocketPair(); + const client = pair[0]; + const server = pair[1]; + const subscriptions = new Map(); + const pendingSubscriptions = new Map(); + let queue = Promise.resolve(); + let hubClosed = false; + + server.accept(); + sendTerminalJson(server, TerminalMessageType.Welcome, "", { + ok: true, + version: 1, + multiplex: true, + }); + + const closeSubscription = (id: string, code = 1000, reason = "unsubscribed") => { + const subscription = subscriptions.get(id); + if (!subscription) return; + subscriptions.delete(id); + subscription.markClosing(reason); + if (subscription.viewCheck !== null) clearInterval(subscription.viewCheck); + if (subscription.upstream.readyState < WebSocket.CLOSING) { + subscription.upstream.close(code, reason); + } + }; + + const closeAll = (code = 1000, reason = "client closed") => { + for (const id of subscriptions.keys()) closeSubscription(id, code, reason); + }; + + server.addEventListener("message", (event) => { + queue = queue + .catch(() => undefined) + .then(async () => { + const data = await webSocketMessageData(event.data); + const bytes = + typeof data === "string" ? encoder.encode(data) : new Uint8Array(data.slice(0)); + const frame = decodeTerminalFrame(bytes); + if (!frame) { sendTerminalJson(server, TerminalMessageType.Error, "", { error: "invalid frame" }); return; } @@ -3251,7 +6516,7 @@ async function writeTerminalClipboardFile( rawName: unknown, rawMediaType: unknown, ): Promise<{ path: string; name: string; mediaType: string; byteCount: number }> { - if (!session.leaseId?.startsWith(sandboxLeasePrefix) || !env.SANDBOX) { + if (!isSandboxInteractiveSession(session) || !env.SANDBOX) { throw serviceUnavailable("clipboard file paste requires a Cloudflare Sandbox session"); } if (!bytes.byteLength || bytes.byteLength > terminalClipboardMaxBytes) { @@ -3306,19 +6571,25 @@ async function subscribeTerminalHubSession( return; } - const session = await readInteractiveSession(env, id); + const session = await readFreshInteractiveSession(env, id); if (!session) { sendTerminalJson(client, TerminalMessageType.Error, id, { error: "interactive session not found", }); return; } - if (["expired", "failed", "stopped"].includes(session.status)) { + if (["stopping", "expired", "failed", "stopped"].includes(session.status)) { sendTerminalJson(client, TerminalMessageType.Error, id, { error: `session is ${session.status}`, }); return; } + if (!session.capabilities.terminal) { + sendTerminalJson(client, TerminalMessageType.Error, id, { + error: "session does not advertise terminal access", + }); + return; + } if (!(await canViewTerminalSession(request, env, user, session))) { sendTerminalJson(client, TerminalMessageType.Error, id, { error: "unauthorized" }); return; @@ -3328,6 +6599,7 @@ async function subscribeTerminalHubSession( const canInput = terminalInputGrant(env, user, session); const canInputNow = await canInput(); const canView = terminalViewGrant(request, env, user, session); + const reconcileSubscription = terminalSubscriptionReconciler(env, id); const cols = canInputNow ? terminalDimension(subscription.cols, 120) : 120; const rows = canInputNow ? terminalDimension(subscription.rows, 34) : 34; let closingReason: string | undefined; @@ -3350,10 +6622,15 @@ async function subscribeTerminalHubSession( rows, ); } catch (error) { - const message = `terminal unavailable: ${ - error instanceof Error ? clean(error.message, 180) : "terminal connection failed" - }`; - if (session.leaseId?.startsWith(sandboxLeasePrefix) && env.SANDBOX) { + const message = redactedAdapterMessage( + `terminal unavailable: ${ + error instanceof Error ? error.message : "terminal connection failed" + }`, + "failed", + [session.adapterWorkspaceId, session.providerResourceId], + [session.attachUrl], + ); + if (isSandboxInteractiveSession(session) && env.SANDBOX) { await markInteractiveTerminalDetached(env, user, id, Date.now(), message); } else { await markInteractiveTerminalUnavailable(env, user, id, Date.now(), message); @@ -3381,6 +6658,7 @@ async function subscribeTerminalHubSession( }); }; viewCheck = setInterval(() => { + reconcileSubscription(); void canView() .then((allowed) => { if (!allowed) revokeView(); @@ -3424,17 +6702,25 @@ async function subscribeTerminalHubSession( }); upstream.addEventListener("close", (event) => { const closeReason = consumeCloseReason(); + const safeUpstreamReason = event.reason + ? redactedAdapterMessage( + event.reason, + "detached", + [session.adapterWorkspaceId, session.providerResourceId], + [session.attachUrl], + ) + : ""; subscriptions.delete(id); if (viewCheck !== null) clearInterval(viewCheck); if (!isPassiveTerminalClose(closeReason)) { - const message = terminalCloseMessage(event.code, event.reason); + const message = terminalCloseMessage(event.code, safeUpstreamReason); void markInteractiveTerminalDetached(env, user, id, Date.now(), message); } if (client.readyState === WebSocket.OPEN) { sendTerminalJson(client, TerminalMessageType.Event, id, { type: "closed", code: event.code, - reason: closeReason || event.reason, + reason: closeReason || safeUpstreamReason, }); } }); @@ -3445,7 +6731,7 @@ async function subscribeTerminalHubSession( const message = "terminal unavailable: upstream terminal error"; if (!isPassiveTerminalClose(closeReason)) { const markTerminal = - session.leaseId?.startsWith(sandboxLeasePrefix) && env.SANDBOX + isSandboxInteractiveSession(session) && env.SANDBOX ? markInteractiveTerminalDetached : markInteractiveTerminalUnavailable; void markTerminal(env, user, id, Date.now(), message); @@ -3460,7 +6746,12 @@ async function subscribeTerminalHubSession( }); } catch (error) { sendTerminalJson(client, TerminalMessageType.Error, id, { - error: error instanceof Error ? clean(error.message, 180) : "terminal subscription failed", + error: redactedAdapterMessage( + error instanceof Error ? error.message : "terminal subscription failed", + "failed", + [session.adapterWorkspaceId, session.providerResourceId], + [session.attachUrl], + ), }); } } @@ -3473,8 +6764,12 @@ async function openInteractiveTerminalUpstream( cols: number, rows: number, ): Promise { + if (!session.capabilities.terminal) { + throw serviceUnavailable("session does not advertise terminal access"); + } const now = Date.now(); - if (session.leaseId?.startsWith(sandboxLeasePrefix) && env.SANDBOX) { + const routeKind = interactivePtyRouteKind(env, session); + if (routeKind === "sandbox" && env.SANDBOX) { const runtimeSession = await sandboxSessionWithGitHubToken(request, env, user, session); const sandboxSession = await ensureCurrentSandboxLease(request, env, user, runtimeSession); const lease = sandboxLeaseInfo(sandboxSession); @@ -3507,14 +6802,11 @@ async function openInteractiveTerminalUpstream( }; } - const target = interactiveTerminalTarget(env, session); + const target = interactiveTerminalTarget(env, session, routeKind); if (!target) throw serviceUnavailable("PTY bridge is not configured for this session"); - const upstreamResponse = await fetch( - addQuery(target.url, { cols: String(cols), rows: String(rows) }), - { - headers: interactiveTerminalHeaders(session, target.authorization), - }, - ); + const upstreamResponse = await fetch(sizedTerminalTargetUrl(target.url, routeKind, cols, rows), { + headers: interactiveTerminalHeaders(session, target.authorization), + }); const upstream = upstreamResponse.webSocket; if (!upstream || upstreamResponse.status !== 101) { throw serviceUnavailable(`PTY bridge HTTP ${upstreamResponse.status}`); @@ -3571,7 +6863,7 @@ async function markInteractiveTerminalDetached( .select("status") .where("id", "=", id) .executeTakeFirst(); - if (!existing || ["expired", "failed", "stopped"].includes(existing.status)) return; + if (!existing || ["stopping", "expired", "failed", "stopped"].includes(existing.status)) return; await database(env) .updateTable("interactive_sessions") .set({ @@ -3593,28 +6885,59 @@ async function markInteractiveTerminalUnavailable( ): Promise { const existing = await database(env) .selectFrom("interactive_sessions") - .select(["id", "lease_id", "status"]) + .selectAll() .where("id", "=", id) .executeTakeFirst(); if (!existing || ["expired", "failed", "stopped"].includes(existing.status)) return; + if (runtimeAdapterTerminalFailureStatus(existing.adapter) === "detached") { + if (existing.status === "stopping") return; + await markInteractiveTerminalDetached(env, user, id, now, message); + return; + } + const legacySession = { + adapter: existing.adapter, + leaseId: existing.lease_id, + }; + if (isSandboxInteractiveSession(legacySession)) { + const staged = await stageTerminalCredentialPolicyCleanupById( + env, + id, + "failed", + message, + now, + message, + ); + if (!staged) return; + await appendInteractiveSessionLog(env, id, user, message, now); + await reconcileCredentialPolicyCleanupBatch(env, now, id); + return; + } + if (existing.status === "stopping") return; const update = await database(env) .updateTable("interactive_sessions") .set({ status: "expired", - updated_at: now, + agent_token_hash: null, + attach_url: null, + vnc_url: null, + controller: null, + control_requested_by: null, + control_requested_at: null, + control_granted_at: null, + control_expires_at: null, + updated_at: sql`MAX(updated_at + 1, ${now})`, stopped_at: now, + terminal_finalize_pending: 1, last_event: message, }) .where("id", "=", id) - .where("status", "not in", ["expired", "failed", "stopped"]) + .where("status", "=", existing.status) + .where("updated_at", "=", existing.updated_at) .executeTakeFirst(); if ((update.numUpdatedRows ?? 0n) > 0n) { - await unregisterInteractiveSessionCredentialPolicy(env, { - id: existing.id, - leaseId: existing.lease_id, - }); + await appendInteractiveSessionLog(env, id, user, message, now); + await finalizeTerminalInteractiveSession(env, id, "expired", now).catch(() => undefined); } - await appendInteractiveSessionLog(env, id, user, message, now); } async function uploadInteractiveSessionClipboard( @@ -3628,7 +6951,7 @@ async function uploadInteractiveSessionClipboard( } const session = await readInteractiveSession(env, id); if (!session) throw notFound("interactive session not found"); - if (["expired", "failed", "stopped"].includes(session.status)) { + if (["stopping", "expired", "failed", "stopped"].includes(session.status)) { throw badRequest(`session is ${session.status}`); } const bytes = await readClipboardUploadBytes(request); @@ -3664,9 +6987,26 @@ function terminalInputGrant( user: User | null, session: InteractiveSession, ): () => Promise { - if (!user) return async () => false; - if (canManageInteractiveSession(user, session)) return async () => true; - return () => canControlInteractiveSessionById(env, user, session.id); + if (!user || !session.capabilities.terminal) return async () => false; + return cachedBooleanGrant(() => canControlInteractiveSessionById(env, user, session.id)); +} + +function terminalSubscriptionReconciler(env: RuntimeEnv, id: string): () => void { + let nextAt = Date.now() + runtimeAdapterReconcileIntervalMs; + let inFlight = false; + return () => { + const now = Date.now(); + if (inFlight || now < nextAt) return; + inFlight = true; + nextAt = now + runtimeAdapterReconcileIntervalMs; + void reconcileExternalInteractiveSessionById(env, id, now) + .catch((error) => { + console.error("terminal subscription reconciliation failed", error); + }) + .finally(() => { + inFlight = false; + }); + }; } function terminalViewGrant( @@ -3715,13 +7055,14 @@ async function isSharedSessionToken(env: RuntimeEnv, id: string, token: string): if (!token) return false; const row = await database(env) .selectFrom("interactive_sessions") - .select(["share_token_hash", "share_mode", "status"]) + .select(["share_token_hash", "share_mode", "status", "runtime", "capabilities_json"]) .where("id", "=", id) .where("share_mode", "=", "link_read") .executeTakeFirst(); return Boolean( row?.share_token_hash && - !["expired", "failed", "stopped"].includes(row.status) && + !["stopping", "expired", "failed", "stopped"].includes(row.status) && + runtimeCapabilities(row.runtime, row.capabilities_json).terminal && (await sha256(token)) === row.share_token_hash, ); } @@ -3769,42 +7110,63 @@ async function interactiveSessionPty( throw badRequest("websocket upgrade required"); } - const session = await readInteractiveSession(env, id); + const session = await readFreshInteractiveSession(env, id); if (!session) throw notFound("interactive session not found"); - if (["expired", "failed", "stopped"].includes(session.status)) { + if (["stopping", "expired", "failed", "stopped"].includes(session.status)) { throw badRequest(`session is ${session.status}`); } + if (!session.capabilities.terminal) { + throw badRequest("session does not advertise terminal access"); + } if ( !canControlInteractiveSession(user, session, Date.now(), canGrantDelegatedControl(env, session)) ) { throw forbidden("terminal control has not been granted"); } - const canManage = canManageInteractiveSession(user, session); - - if (session.leaseId?.startsWith(sandboxLeasePrefix) && env.SANDBOX) { + const routeKind = interactivePtyRouteKind(env, session); + if (routeKind === "sandbox" && env.SANDBOX) { return interactiveSandboxTerminal( request, env, user, session, - canManage ? undefined : () => canControlInteractiveSessionById(env, user, id), + terminalInputGrant(env, user, session), + terminalSubscriptionReconciler(env, id), ); } - const target = interactiveTerminalTarget(env, session); + const target = interactiveTerminalTarget(env, session, routeKind); if (!target) throw serviceUnavailable("PTY bridge is not configured for this session"); + const targetUrl = sizedTerminalTargetUrl( + target.url, + routeKind, + terminalSize(request, "cols", 120), + terminalSize(request, "rows", 34), + ); + if (!targetUrl) throw serviceUnavailable("PTY bridge URL is invalid"); const pair = new WebSocketPair(); const client = pair[0]; const server = pair[1]; let upstreamResponse: Response; try { - upstreamResponse = await fetch(target.url, { + upstreamResponse = await fetch(targetUrl, { headers: interactiveTerminalHeaders(session, target.authorization), }); } catch (error) { server.accept(); - server.close(1011, `PTY bridge failed: ${clean(String(error), 120)}`); + server.close( + 1011, + clean( + redactedAdapterMessage( + `PTY bridge failed: ${String(error)}`, + "failed", + [session.adapterWorkspaceId, session.providerResourceId], + [target.url, session.attachUrl], + ), + 120, + ), + ); return new Response(null, { status: 101, webSocket: client }); } const upstream = upstreamResponse.webSocket; @@ -3819,7 +7181,8 @@ async function interactiveSessionPty( bridgeWebSockets( server, upstream, - canManage ? undefined : () => canControlInteractiveSessionById(env, user, id), + terminalInputGrant(env, user, session), + terminalSubscriptionReconciler(env, id), ); const now = Date.now(); @@ -3829,11 +7192,11 @@ async function interactiveSessionPty( status: session.status === "ready" || session.status === "detached" ? "attached" : session.status, last_seen_at: now, - updated_at: now, + updated_at: sql`MAX(updated_at + 1, ${now})`, last_event: "PTY terminal connected", }) .where("id", "=", id) - .where("status", "!=", "stopped") + .where("status", "in", ["ready", "attached", "detached"]) .execute(); await appendInteractiveSessionEvent(env, id, user, "PTY terminal connected", now); @@ -3846,6 +7209,7 @@ async function interactiveSandboxTerminal( user: User, session: InteractiveSession, canSendLeft?: () => Promise, + reconcileSubscription?: () => void, ): Promise { if (!env.SANDBOX) throw serviceUnavailable("Sandbox binding is not configured"); const runtimeSession = await sandboxSessionWithGitHubToken(request, env, user, session); @@ -3886,7 +7250,7 @@ async function interactiveSandboxTerminal( Date.now(), "Cloudflare Sandbox terminal connected", ); - bridgeWebSockets(server, upstream, canSendLeft); + bridgeWebSockets(server, upstream, canSendLeft, reconcileSubscription); return new Response(null, { status: 101, webSocket: client }); } @@ -3903,7 +7267,7 @@ async function readInteractiveSessionDiagnostics( ) { throw forbidden("terminal control has not been granted"); } - if (!env.SANDBOX || !session.leaseId?.startsWith(sandboxLeasePrefix)) { + if (!env.SANDBOX || !isSandboxInteractiveSession(session)) { return { session: decoratedSession, diagnostics: { @@ -4161,10 +7525,10 @@ async function managedSandboxSession( if (!canManageInteractiveSession(user, session)) { throw forbidden("only the session owner or maintainer can manage checkpoints"); } - if (!env.SANDBOX || !session.leaseId?.startsWith(sandboxLeasePrefix)) { + if (!env.SANDBOX || !isSandboxInteractiveSession(session)) { throw badRequest("checkpoints require a Cloudflare Sandbox session"); } - if (["expired", "failed", "stopped"].includes(session.status)) { + if (["stopping", "expired", "failed", "stopped"].includes(session.status)) { throw badRequest(`session is ${session.status}`); } return session; @@ -4182,30 +7546,139 @@ function sandboxBackupAllowedHosts(env: RuntimeEnv): string[] { : []; } -function interactiveTerminalTarget( - env: RuntimeEnv, - session: InteractiveSession, -): InteractiveTerminalTarget | null { - if (env.CRABBOX_PTY_BRIDGE_URL) { - const url = interactiveBridgeUrl(env.CRABBOX_PTY_BRIDGE_URL, session); - if (!url) return null; - return { - url, - authorization: bearer(env.CRABBOX_PTY_BRIDGE_TOKEN), - }; +async function interactiveSessionVnc(env: RuntimeEnv, user: User, id: string): Promise { + const session = await readFreshInteractiveSession(env, id); + if (!session) throw notFound("interactive session not found"); + if (["stopping", "stopped", "expired", "failed"].includes(session.status)) { + throw badRequest(`session is ${session.status}`); } - - if (session.attachUrl && /^wss?:\/\//i.test(session.attachUrl)) { - return { url: session.attachUrl, authorization: null }; + const now = Date.now(); + const delegatedControl = canGrantDelegatedControl(env, session); + if (!canControlInteractiveSession(user, session, now, delegatedControl)) { + throw forbidden("terminal control has not been granted"); + } + let target: string; + if (session.adapter === runtimeAdapterName) { + if (!["ready", "attached", "detached"].includes(session.status)) { + throw badRequest(`session is ${session.status}`); + } + if (!session.capabilities.vnc && !session.capabilities.desktop) { + throw badRequest("session does not advertise desktop access"); + } + if (!session.adapterWorkspaceId) { + throw serviceUnavailable("runtime adapter workspace reference is incomplete"); + } + let controlPlane: string; + try { + controlPlane = await registeredRuntimeAdapterControlPlaneForSession( + env, + session.id, + session.adapterWorkspaceId, + ); + } catch (error) { + throw serviceUnavailable(clean(error instanceof Error ? error.message : String(error), 240)); + } + let response: Response; + let responseBody: unknown; + try { + response = await runtimeAdapterFetch( + env, + runtimeAdapterDesktopUrl(controlPlane, session.adapterWorkspaceId), + { + method: "POST", + body: JSON.stringify({}), + }, + ); + responseBody = await readRuntimeAdapterResponseBody(response); + } catch (error) { + throw serviceUnavailable( + `runtime adapter desktop connection failed: ${clean(String(error), 240)}`, + ); + } + if (!response.ok) { + throw serviceUnavailable(`runtime adapter desktop connection HTTP ${response.status}`); + } + const connection = currentAdapterDesktopConnection(responseBody, Date.now()); + if (!connection) throw serviceUnavailable("desktop connection has an invalid expiry"); + if (!(await currentRuntimeAdapterDesktopAccess(env, user, session, controlPlane))) { + throw forbidden("desktop authorization changed; retry"); + } + target = connection.url; + } else { + const legacyTarget = safeDesktopUrl(session.vncUrl); + if (!legacyTarget) throw badRequest("legacy desktop connection is not available"); + target = legacyTarget; + } + return new Response(null, { + status: 302, + headers: { + location: target, + "cache-control": "no-store", + "referrer-policy": "no-referrer", + }, + }); +} + +async function currentRuntimeAdapterDesktopAccess( + env: RuntimeEnv, + user: User, + expected: InteractiveSession, + controlPlane: string, +): Promise { + const currentRow = await database(env) + .selectFrom("interactive_sessions") + .selectAll() + .where("id", "=", expected.id) + .where("adapter", "=", runtimeAdapterName) + .where("adapter_workspace_id", "=", expected.adapterWorkspaceId) + .where("adapter_control_plane", "=", controlPlane) + .where(sql`provider_resource_id IS ${expected.providerResourceId}`) + .where("runtime", "=", expected.runtime) + .where("profile", "=", expected.profile) + .where("adapter_create_pending", "=", 0) + .where("status", "in", ["ready", "attached", "detached"]) + .executeTakeFirst(); + if (!currentRow) return false; + const current = interactiveSession(currentRow, []); + if (!current.capabilities.vnc && !current.capabilities.desktop) return false; + return canControlInteractiveSession( + user, + current, + Date.now(), + canGrantDelegatedControl(env, current), + ); +} + +function interactiveTerminalTarget( + env: RuntimeEnv, + session: InteractiveSession, + routeKind = interactivePtyRouteKind(env, session), +): InteractiveTerminalTarget | null { + if (routeKind === "bridge" && env.CRABBOX_PTY_BRIDGE_URL) { + const url = interactiveBridgeUrl(env.CRABBOX_PTY_BRIDGE_URL, session); + if (!url) return null; + return { + url, + authorization: bearer(env.CRABBOX_PTY_BRIDGE_TOKEN), + }; + } + + const attachUrl = routeKind === "attach" ? safeWebSocketUrl(session.attachUrl) : null; + if (attachUrl) { + return { url: attachUrl, authorization: null }; } - if (session.leaseId?.startsWith("cloudflare:") && env.CRABBOX_CLOUDFLARE_RUNNER_URL) { - const sandboxId = session.leaseId.slice("cloudflare:".length); + const leaseId = legacyInteractiveSessionLeaseId(session); + if ( + routeKind === "cloudflare" && + leaseId?.startsWith("cloudflare:") && + env.CRABBOX_CLOUDFLARE_RUNNER_URL + ) { + const sandboxId = leaseId.slice("cloudflare:".length); + const runnerUrl = safeDesktopUrl(env.CRABBOX_CLOUDFLARE_RUNNER_URL); + if (!runnerUrl) return null; const url = addQuery( - joinUrl( - env.CRABBOX_CLOUDFLARE_RUNNER_URL, - `/v1/sandboxes/${encodeURIComponent(sandboxId)}/pty`, - ), + joinUrl(runnerUrl, `/v1/sandboxes/${encodeURIComponent(sandboxId)}/pty`), terminalQuery(session), ); if (!url) return null; @@ -4218,10 +7691,22 @@ function interactiveTerminalTarget( return null; } +function interactivePtyRouteKind( + env: RuntimeEnv, + session: Pick, +): PtyRouteKind | null { + return ptyRouteKind(session, { + sandboxAvailable: Boolean(env.SANDBOX), + bridgeUrl: env.CRABBOX_PTY_BRIDGE_URL, + cloudflareRunnerUrl: env.CRABBOX_CLOUDFLARE_RUNNER_URL, + }); +} + function interactiveBridgeUrl(base: string, session: InteractiveSession): string { + const leaseId = legacyInteractiveSessionLeaseId(session) ?? ""; const replacements: Record = { id: session.id, - leaseId: session.leaseId ?? "", + leaseId, repo: session.repo, branch: session.branch, runtime: session.runtime, @@ -4230,16 +7715,17 @@ function interactiveBridgeUrl(base: string, session: InteractiveSession): string for (const [key, value] of Object.entries(replacements)) { url = url.replaceAll(`{${key}}`, encodeURIComponent(value)); } - return addQuery(httpToWebSocketUrl(url), terminalQuery(session)); + return safeWebSocketUrl(addQuery(httpToWebSocketUrl(url), terminalQuery(session))) ?? ""; } function terminalQuery(session: InteractiveSession): Record { return { sessionId: session.id, - leaseId: session.leaseId ?? "", + leaseId: legacyInteractiveSessionLeaseId(session) ?? "", repo: session.repo, branch: session.branch, runtime: session.runtime, + profile: session.profile, command: session.command, }; } @@ -4310,6 +7796,8 @@ function bridgeWebSockets( left: WebSocket, right: WebSocket, canSendLeft?: () => Promise, + reconcileSubscription?: () => void, + deniedReason = "terminal control revoked", ): void { let leftInputQueue = Promise.resolve(); let rightOutputQueue = Promise.resolve(); @@ -4325,12 +7813,13 @@ function bridgeWebSockets( leftCanSend = canSend; if (!canSend) { stopControlCheck(); - closePair(left, right, 1008, "terminal control revoked"); + closePair(left, right, 1008, deniedReason); return false; } return true; }; const scheduleControlCheck = () => { + reconcileSubscription?.(); if (controlCheckInFlight) return; controlCheckInFlight = verifyControl() .then(() => undefined) @@ -4351,7 +7840,7 @@ function bridgeWebSockets( .then(async () => { if (left.readyState !== WebSocket.OPEN || right.readyState !== WebSocket.OPEN) return; if (!leftCanSend || !(await verifyControl())) { - closePair(left, right, 1008, "terminal control revoked"); + closePair(left, right, 1008, deniedReason); return; } right.send(await webSocketMessageData(data)); @@ -4403,7 +7892,10 @@ async function webSocketMessageData(data: unknown): Promise { if (session.runtime === "container" && env.SANDBOX) { - return provisionWithSandbox(env, session, agentToken); + if (!sandboxProvision) { + return failedProvision("Cloudflare Sandbox durable ownership is missing"); + } + return provisionWithSandbox( + env, + session, + agentToken, + sandboxProvision.lease, + sandboxProvision.ownership, + ); + } + if (env.CRABBOX_RUNTIME_ADAPTER_URL) { + return provisionWithRuntimeAdapter(env, session, agentToken); } if (!env.CRABBOX_INTERACTIVE_PROVISION_URL) return null; - if (isBuiltInInteractiveProvisionUrl(env.CRABBOX_INTERACTIVE_PROVISION_URL)) { - return provisionInteractivePayload(env, session); + if (isBuiltInInteractiveProvisionUrl(env, env.CRABBOX_INTERACTIVE_PROVISION_URL)) { + return provisionInteractivePayload(env, session, agentToken); } let response: Response; try { @@ -4458,7 +7966,7 @@ async function provisionInteractiveSession( }; } const body = (await response.json().catch(() => ({}))) as Record; - const status = optionalOneOf(body.status, interactiveSessionStatuses); + const status = createOnlyAdapterStatus(body.status); if (!status) { return { status: "failed", @@ -4468,878 +7976,3449 @@ async function provisionInteractiveSession( message: "interactive provision failed: invalid adapter response", }; } - return { - status, - leaseId: clean(body.leaseId ?? body.lease_id, 240) || null, - attachUrl: clean(body.attachUrl ?? body.attach_url, 1000) || null, - vncUrl: clean(body.vncUrl ?? body.vnc_url, 1000) || null, - message: clean(body.message, 500) || `interactive workspace ${status}`, - }; + const leaseId = clean(body.leaseId ?? body.lease_id, 240) || null; + const attachUrl = clean(body.attachUrl ?? body.attach_url, 1000) || null; + const vncUrl = clean(body.vncUrl ?? body.vnc_url, 1000) || null; + return { + status, + leaseId, + attachUrl, + vncUrl, + message: redactedAdapterMessage( + clean(body.message, 500) || null, + status, + [leaseId], + [attachUrl, vncUrl], + ), + }; +} + +async function provisionInteractiveEndpoint( + request: Request, + env: RuntimeEnv, +): Promise { + authorizeProvisionEndpoint(request, env); + const session = await readJson>(request); + const id = clean(session.id, 120); + const repo = normalizeRepo(session.repo); + const branch = clean(session.branch, 120) || "main"; + const runtime = oneOf(session.runtime, ["crabbox", "container"], "container") as + | "crabbox" + | "container"; + const command = interactiveCommand(session.command); + const profile = clean(session.profile, 120) || deploymentConfig(env).defaultProfile; + const prompt = clean(session.prompt, 4000); + const purpose = interactiveSessionPurpose(session.purpose, prompt, repo, branch, command); + const summary = interactiveSessionSummary(session.summary, purpose, prompt); + const owner = clean(session.owner, 240); + const githubToken = clean(session.githubToken, 4000) || undefined; + if (!id || !repo || !owner) { + return failedProvision("interactive provision failed: invalid session request"); + } + const payload: InteractiveProvisionRequest = { + id, + repo, + branch, + runtime, + profile, + command, + prompt, + purpose, + summary, + owner, + createdBy: clean(session.createdBy, 240) || owner, + parentSessionId: clean(session.parentSessionId, 120) || null, + rootSessionId: clean(session.rootSessionId, 120) || id, + ...(githubToken ? { githubToken } : {}), + }; + const managed = await database(env) + .selectFrom("interactive_sessions") + .selectAll() + .where("id", "=", payload.id) + .executeTakeFirst(); + if (managed) { + if (payload.runtime !== "container" || !env.SANDBOX) { + return failedProvision( + "interactive provision failed: managed session id is not available to this backend", + ); + } + return provisionManagedSandboxEndpoint(env, payload, managed); + } + if (payload.runtime === "container" && env.SANDBOX) { + if (managedInteractiveSessionId(payload.id)) { + return failedProvision( + "interactive provision failed: standalone provision id uses the managed session namespace", + ); + } + return provisionStandaloneSandbox(env, payload); + } + return provisionInteractivePayload(env, payload); +} + +function managedSandboxProvisionPayloadMatches( + payload: InteractiveProvisionRequest, + session: InteractiveSessionRow, +): boolean { + return ( + payload.id === session.id && + payload.parentSessionId === session.parent_session_id && + payload.rootSessionId === (session.root_session_id ?? session.id) && + payload.repo === session.repo && + payload.branch === session.branch && + payload.runtime === session.runtime && + payload.profile === session.profile && + payload.command === session.command && + payload.prompt === session.prompt && + payload.purpose === session.purpose && + payload.summary === session.summary && + payload.owner === session.owner && + payload.createdBy === session.created_by + ); +} + +async function provisionManagedSandboxEndpoint( + env: RuntimeEnv, + payload: InteractiveProvisionRequest, + session: InteractiveSessionRow, +): Promise { + if ( + !managedSandboxProvisionPayloadMatches(payload, session) || + !["provisioning", "pending_adapter"].includes(session.status) || + session.adapter === runtimeAdapterName || + session.credential_cleanup_terminal_status !== null + ) { + return failedProvision( + "interactive provision failed: managed session request does not match durable ownership", + ); + } + const preflightError = sandboxProvisionPreflightError(env, payload); + if (preflightError) return failedProvision(preflightError); + const now = Date.now(); + const claimRevision = Math.max(now, session.updated_at + 1); + const agentToken = newAgentToken(); + const agentTokenHash = await sha256(agentToken); + const lease = newSandboxLease(payload.id); + const fence: SandboxLeaseRefreshFence = { + claim: `managed-provision:${crypto.randomUUID()}`, + expiresAt: now + credentialPolicyProvisioningStaleMs, + refreshLeaseId: session.lease_id, + sandboxId: lease.sandboxId, + }; + const claimed = await database(env) + .updateTable("interactive_sessions") + .set({ + sandbox_refresh_sandbox_id: fence.sandboxId, + sandbox_refresh_claim: fence.claim, + sandbox_refresh_claim_expires_at: fence.expiresAt, + agent_token_hash: agentTokenHash, + last_event: "managed Sandbox provision claimed", + updated_at: claimRevision, + }) + .where("id", "=", session.id) + .where("updated_at", "=", session.updated_at) + .where("status", "in", ["provisioning", "pending_adapter"]) + .where(sql`parent_session_id IS ${payload.parentSessionId}`) + .where(sql`COALESCE(root_session_id, id) = ${payload.rootSessionId}`) + .where("runtime", "=", payload.runtime) + .where("repo", "=", payload.repo) + .where("branch", "=", payload.branch) + .where("profile", "=", payload.profile) + .where("command", "=", payload.command) + .where("prompt", "=", payload.prompt) + .where("purpose", "=", payload.purpose) + .where("summary", "=", payload.summary) + .where("owner", "=", payload.owner) + .where("created_by", "=", payload.createdBy) + .where((expression) => + expression.or([ + expression("adapter", "is", null), + expression("adapter", "!=", runtimeAdapterName), + ]), + ) + .where("credential_cleanup_terminal_status", "is", null) + .where(sql`agent_token_hash IS ${session.agent_token_hash}`) + .where(sql`lease_id IS ${session.lease_id}`) + .where((expression) => + expression.or([ + expression("sandbox_refresh_claim", "is", null), + expression("sandbox_refresh_claim_expires_at", "<=", now), + ]), + ) + .executeTakeFirst(); + if ((claimed.numUpdatedRows ?? 0n) === 0n) { + return failedProvision("interactive provision failed: managed session claim was not acquired"); + } + + let provisioned: InteractiveProvisionResult; + try { + provisioned = await provisionWithSandbox(env, payload, agentToken, lease, fence); + } catch (error) { + const message = `Cloudflare Sandbox provision failed: ${safeProviderError(error)}`; + await stageFailedManagedSandboxProvision(env, session.id, fence, message, Date.now()); + return failedProvision(message); + } + if (provisioned.status !== "ready") { + await stageFailedManagedSandboxProvision( + env, + session.id, + fence, + provisioned.message, + Date.now(), + ); + return provisioned; + } + const expectedLeaseId = sandboxLeaseId(lease); + const previousSandboxId = session.lease_id?.startsWith(sandboxLeasePrefix) + ? sandboxLeaseInfo({ + id: session.id, + leaseId: sandboxLeaseWithoutRefresh(session.lease_id), + }).sandboxId + : null; + const finishedAt = Date.now(); + if (provisioned.leaseId !== expectedLeaseId) { + const message = "interactive provision failed: managed Sandbox lease mismatch"; + const staged = await stageTerminalCredentialPolicyCleanupById( + env, + session.id, + "failed", + message, + finishedAt, + message, + fence, + ); + if (!staged) { + return failedProvision("interactive provision failed: managed session ownership changed"); + } + await reconcileCredentialPolicyCleanupBatch(env, finishedAt, session.id); + return failedProvision(message); + } + const db = database(env); + const commitRevision = Math.max(finishedAt, claimRevision + 1); + const commitQueries: CompilableQuery[] = [ + db + .updateTable("interactive_sessions") + .set({ + status: "ready", + lease_id: expectedLeaseId, + attach_url: provisioned.attachUrl, + vnc_url: provisioned.vncUrl, + sandbox_refresh_sandbox_id: null, + sandbox_refresh_claim: null, + sandbox_refresh_claim_expires_at: null, + last_event: provisioned.message, + updated_at: sql`MAX(updated_at + 1, ${commitRevision})`, + }) + .where("id", "=", session.id) + .where("status", "in", ["provisioning", "pending_adapter"]) + .where(sql`lease_id IS ${fence.refreshLeaseId}`) + .where("sandbox_refresh_sandbox_id", "=", fence.sandboxId) + .where("sandbox_refresh_claim", "=", fence.claim) + .where("sandbox_refresh_claim_expires_at", "=", fence.expiresAt) + .where("sandbox_refresh_claim_expires_at", ">", finishedAt) + .where("agent_token_hash", "=", agentTokenHash), + ]; + if (previousSandboxId && previousSandboxId !== lease.sandboxId) { + commitQueries.push( + db + .updateTable("interactive_session_credential_policies") + .set({ + state: "cleanup_pending", + cleanup_claim: null, + cleanup_claim_expires_at: null, + updated_at: commitRevision, + }) + .where("session_id", "=", session.id) + .where("sandbox_id", "=", previousSandboxId).where(sql` + EXISTS ( + SELECT 1 + FROM interactive_sessions AS owner + WHERE owner.id = ${session.id} + AND owner.status = 'ready' + AND owner.lease_id = ${expectedLeaseId} + AND owner.agent_token_hash = ${agentTokenHash} + AND owner.credential_cleanup_terminal_status IS NULL + AND owner.sandbox_refresh_sandbox_id IS NULL + AND owner.sandbox_refresh_claim IS NULL + AND owner.sandbox_refresh_claim_expires_at IS NULL + ) + `), + ); + } + await executeBatch(env, commitQueries); + const current = await db + .selectFrom("interactive_sessions") + .select(["lease_id", "status", "sandbox_refresh_claim", "agent_token_hash"]) + .where("id", "=", session.id) + .executeTakeFirst(); + if ( + current?.lease_id === expectedLeaseId && + current.sandbox_refresh_claim === null && + current.agent_token_hash === agentTokenHash && + ["ready", "attached", "detached"].includes(current.status) + ) { + if (previousSandboxId && previousSandboxId !== lease.sandboxId) { + await reconcileCredentialPolicyCleanupBatch(env, commitRevision, session.id); + } + return provisioned; + } + await stageFailedManagedSandboxProvision( + env, + session.id, + fence, + "interactive provision failed: managed session ownership changed", + finishedAt, + ); + return failedProvision("interactive provision failed: managed session ownership changed"); +} + +async function provisionStandaloneSandbox( + env: RuntimeEnv, + payload: InteractiveProvisionRequest, +): Promise { + if (managedInteractiveSessionId(payload.id)) { + return failedProvision( + "interactive provision failed: standalone provision id uses the managed session namespace", + ); + } + const { githubToken: _githubToken, ...ownershipPayload } = payload; + const requestHash = await sha256(JSON.stringify(ownershipPayload)); + const db = database(env); + const now = Date.now(); + let previous = await db + .selectFrom("standalone_sandbox_provisions") + .selectAll() + .where("id", "=", payload.id) + .executeTakeFirst(); + if (previous && previous.request_hash !== requestHash) { + return failedProvision("interactive provision failed: provision id is already registered"); + } + if (previous?.state === "active") { + if (!previous.expires_at || previous.expires_at <= Date.now()) { + await stageStandaloneSandboxProvisionCleanup( + env, + previous, + "standalone Sandbox provision expired", + Date.now(), + ); + await reconcileCredentialPolicyCleanupBatch(env, Date.now(), payload.id); + return failedProvision("interactive provision failed: standalone Sandbox provision expired"); + } + return { + status: "ready", + leaseId: previous.lease_id, + attachUrl: previous.attach_url, + vncUrl: previous.vnc_url, + expiresAt: previous.expires_at, + expiresAtPresent: true, + message: previous.message, + }; + } + if (previous?.state === "cleanup_pending") { + return failedProvision("interactive provision failed: previous credential cleanup is pending"); + } + if ( + previous?.state === "provisioning" && + (previous.ownership_claim_expires_at ?? Number.NEGATIVE_INFINITY) <= now + ) { + const staged = await stageStandaloneSandboxProvisionCleanup( + env, + previous, + "abandoned standalone Sandbox provision cleanup", + now, + ); + if (!staged) { + return failedProvision("interactive provision failed: standalone ownership changed"); + } + await reconcileCredentialPolicyCleanupBatch(env, now, payload.id); + previous = await db + .selectFrom("standalone_sandbox_provisions") + .selectAll() + .where("id", "=", payload.id) + .executeTakeFirst(); + if (previous) { + return failedProvision( + "interactive provision failed: previous credential cleanup is pending", + ); + } + } + + const expiresAt = + now + + clampedSeconds(env.CRABBOX_STANDALONE_SANDBOX_TTL_SECONDS, standaloneSandboxDefaultTtlSeconds) * + 1000; + const lease = newSandboxLease(payload.id); + const claim = `standalone:${crypto.randomUUID()}`; + await sql` + INSERT INTO standalone_sandbox_provisions ( + id, + request_hash, + sandbox_id, + state, + ownership_claim, + ownership_claim_expires_at, + lease_id, + attach_url, + vnc_url, + expires_at, + message, + created_at, + updated_at + ) VALUES ( + ${payload.id}, + ${requestHash}, + ${lease.sandboxId}, + 'provisioning', + ${claim}, + ${now + credentialPolicyProvisioningStaleMs}, + ${sandboxLeaseId(lease)}, + NULL, + NULL, + ${expiresAt}, + 'standalone Sandbox provision started', + ${now}, + ${now} + ) + ON CONFLICT(id) DO UPDATE SET + sandbox_id = excluded.sandbox_id, + ownership_claim = excluded.ownership_claim, + ownership_claim_expires_at = excluded.ownership_claim_expires_at, + lease_id = excluded.lease_id, + attach_url = NULL, + vnc_url = NULL, + expires_at = excluded.expires_at, + message = excluded.message, + updated_at = excluded.updated_at + WHERE standalone_sandbox_provisions.request_hash = excluded.request_hash + AND standalone_sandbox_provisions.state = 'provisioning' + AND standalone_sandbox_provisions.ownership_claim_expires_at <= ${now} + `.execute(db); + const ownership = await db + .selectFrom("standalone_sandbox_provisions") + .select(["sandbox_id", "state", "ownership_claim", "ownership_claim_expires_at", "expires_at"]) + .where("id", "=", payload.id) + .executeTakeFirst(); + if ( + ownership?.sandbox_id !== lease.sandboxId || + ownership.state !== "provisioning" || + ownership.ownership_claim !== claim || + (ownership.ownership_claim_expires_at ?? 0) <= now || + ownership.expires_at !== expiresAt + ) { + return failedProvision("interactive provision failed: provision id is already in progress"); + } + if (previous?.sandbox_id && previous.sandbox_id !== lease.sandboxId) { + await queueSandboxCredentialPolicyCleanup(env, payload.id, previous.sandbox_id, now); + } + const fence: StandaloneSandboxProvisionFence = { + claim, + provisionId: payload.id, + sandboxId: lease.sandboxId, + }; + const result = await provisionWithSandbox(env, payload, undefined, lease, fence); + const finishedAt = Date.now(); + if (result.status !== "ready") { + await db + .updateTable("standalone_sandbox_provisions") + .set({ + state: "cleanup_pending", + ownership_claim: null, + ownership_claim_expires_at: null, + message: result.message, + updated_at: finishedAt, + }) + .where("id", "=", payload.id) + .where("sandbox_id", "=", lease.sandboxId) + .where("ownership_claim", "=", claim) + .where("expires_at", "=", expiresAt) + .execute(); + await queueSandboxCredentialPolicyCleanup(env, payload.id, lease.sandboxId, finishedAt); + await reconcileCredentialPolicyCleanupBatch(env, finishedAt, payload.id); + return result; + } + const policyGeneration = await activeSandboxCredentialPolicyGeneration( + env, + payload.id, + lease.sandboxId, + ); + const activationVersion = Math.max(Date.now(), finishedAt + 1); + if (policyGeneration) { + const ownerStillClaimed = sql`EXISTS ( + SELECT 1 + FROM standalone_sandbox_provisions AS owner + WHERE owner.id = ${payload.id} + AND owner.sandbox_id = ${lease.sandboxId} + AND owner.state = 'provisioning' + AND owner.ownership_claim = ${claim} + AND owner.ownership_claim_expires_at > ${activationVersion} + AND owner.expires_at = ${expiresAt} + AND owner.expires_at > ${activationVersion} + )`; + await executeBatch(env, [ + db + .updateTable("interactive_session_credential_policies") + .set({ updated_at: activationVersion }) + .where("session_id", "=", payload.id) + .where("sandbox_id", "=", lease.sandboxId) + .where("state", "=", "active") + .where("registration_generation", "=", policyGeneration) + .where("registration_claim", "is", null) + .where(ownerStillClaimed), + db + .updateTable("standalone_sandbox_provisions") + .set({ + state: "active", + ownership_claim: null, + ownership_claim_expires_at: null, + lease_id: result.leaseId, + attach_url: result.attachUrl, + vnc_url: result.vncUrl, + message: result.message, + updated_at: activationVersion, + }) + .where("id", "=", payload.id) + .where("sandbox_id", "=", lease.sandboxId) + .where("state", "=", "provisioning") + .where("ownership_claim", "=", claim) + .where("ownership_claim_expires_at", ">", activationVersion) + .where("expires_at", "=", expiresAt) + .where("expires_at", ">", activationVersion) + .where( + activeSandboxCredentialPolicyCondition( + env, + payload.id, + lease.sandboxId, + policyGeneration, + activationVersion, + ), + ), + ]); + } + const activated = await db + .selectFrom("standalone_sandbox_provisions") + .select(["state", "sandbox_id", "lease_id", "expires_at"]) + .where("id", "=", payload.id) + .executeTakeFirst(); + if ( + activated?.state !== "active" || + activated.sandbox_id !== lease.sandboxId || + activated.lease_id !== result.leaseId || + activated.expires_at !== expiresAt + ) { + await db + .updateTable("standalone_sandbox_provisions") + .set({ + state: "cleanup_pending", + ownership_claim: null, + ownership_claim_expires_at: null, + message: "standalone ownership claim expired", + updated_at: finishedAt, + }) + .where("id", "=", payload.id) + .where("sandbox_id", "=", lease.sandboxId) + .where("ownership_claim", "=", claim) + .where("expires_at", "=", expiresAt) + .execute(); + await queueSandboxCredentialPolicyCleanup(env, payload.id, lease.sandboxId, finishedAt); + await reconcileCredentialPolicyCleanupBatch(env, finishedAt, payload.id); + return failedProvision("interactive provision failed: standalone ownership claim expired"); + } + return { ...result, expiresAt, expiresAtPresent: true }; +} + +function managedInteractiveSessionId(id: string): boolean { + return /^is-[0-9]+$/i.test(id); +} + +async function stageStandaloneSandboxProvisionCleanup( + env: RuntimeEnv, + owner: Selectable, + message: string, + now: number, +): Promise { + if (owner.state === "cleanup_pending") return true; + const db = database(env); + const transitionRevision = Math.max(now, owner.updated_at + 1); + const generation = await sandboxCredentialPolicyGeneration(env, owner.id, owner.sandbox_id); + let ownerTransition = db + .updateTable("standalone_sandbox_provisions") + .set({ + state: "cleanup_pending", + ownership_claim: null, + ownership_claim_expires_at: null, + attach_url: null, + vnc_url: null, + message, + updated_at: transitionRevision, + }) + .where("id", "=", owner.id) + .where("request_hash", "=", owner.request_hash) + .where("sandbox_id", "=", owner.sandbox_id) + .where("state", "=", owner.state) + .where("updated_at", "=", owner.updated_at); + ownerTransition = owner.ownership_claim + ? ownerTransition.where("ownership_claim", "=", owner.ownership_claim) + : ownerTransition.where("ownership_claim", "is", null); + ownerTransition = + owner.ownership_claim_expires_at === null + ? ownerTransition.where("ownership_claim_expires_at", "is", null) + : ownerTransition.where("ownership_claim_expires_at", "=", owner.ownership_claim_expires_at); + ownerTransition = owner.lease_id + ? ownerTransition.where("lease_id", "=", owner.lease_id) + : ownerTransition.where("lease_id", "is", null); + ownerTransition = + owner.expires_at === null + ? ownerTransition.where("expires_at", "is", null) + : ownerTransition.where("expires_at", "=", owner.expires_at); + const cleanupAuthorized = sandboxCredentialPolicyCleanupAuthorizedCondition( + owner.id, + owner.sandbox_id, + transitionRevision, + ); + await executeBatch(env, [ + ownerTransition, + ...sandboxCredentialPolicyRefQueries( + env, + owner.id, + owner.sandbox_id, + "cleanup_pending", + generation, + transitionRevision, + cleanupAuthorized, + ), + db + .updateTable("interactive_session_credential_policies") + .set({ state: "cleanup_pending", updated_at: transitionRevision }) + .where("session_id", "=", owner.id) + .where("sandbox_id", "=", owner.sandbox_id) + .where(cleanupAuthorized), + ]); + const staged = await db + .selectFrom("standalone_sandbox_provisions") + .select(["state", "sandbox_id", "updated_at"]) + .where("id", "=", owner.id) + .executeTakeFirst(); + return Boolean( + staged?.state === "cleanup_pending" && + staged.sandbox_id === owner.sandbox_id && + staged.updated_at === transitionRevision, + ); +} + +async function expireStandaloneSandboxProvisions( + env: RuntimeEnv, + now: number, + provisionId?: string, +): Promise { + const idFilter = provisionId ? sql`AND id = ${provisionId}` : sql``; + const result = await sql>` + SELECT * + FROM standalone_sandbox_provisions + WHERE ( + (state = 'active' AND (expires_at IS NULL OR expires_at <= ${now})) + OR ( + state = 'provisioning' + AND ( + expires_at IS NULL + OR expires_at <= ${now} + OR ownership_claim_expires_at IS NULL + OR ownership_claim_expires_at <= ${now} + ) + ) + OR ( + state = 'active' + AND lower(id) GLOB 'is-[0-9]*' + AND substr(lower(id), 4) NOT GLOB '*[^0-9]*' + ) + ) + ${idFilter} + ORDER BY COALESCE(expires_at, 0) ASC, updated_at ASC, id ASC + LIMIT ${credentialPolicyCleanupLimit} + `.execute(database(env)); + await mapWithConcurrency(result.rows, 3, async (owner) => { + await stageStandaloneSandboxProvisionCleanup( + env, + owner, + managedInteractiveSessionId(owner.id) + ? "standalone provision used the reserved managed session namespace" + : "standalone Sandbox provision expired", + now, + ); + }); +} + +async function stopStandaloneSandboxProvision( + request: Request, + env: RuntimeEnv, + provisionId: string, +): Promise { + authorizeProvisionBearerToken(request, env); + const owner = await database(env) + .selectFrom("standalone_sandbox_provisions") + .selectAll() + .where("id", "=", provisionId) + .executeTakeFirst(); + if (!owner) throw notFound("standalone Sandbox provision not found"); + const now = Date.now(); + const staged = await stageStandaloneSandboxProvisionCleanup( + env, + owner, + "standalone Sandbox stop requested", + now, + ); + if (!staged) throw conflict("standalone Sandbox ownership changed; retry stop"); + await reconcileCredentialPolicyCleanupBatch(env, now, provisionId); + const remaining = await database(env) + .selectFrom("standalone_sandbox_provisions") + .select("state") + .where("id", "=", provisionId) + .executeTakeFirst(); + return { + status: remaining ? "stopping" : "stopped", + leaseId: null, + attachUrl: null, + vncUrl: null, + expiresAt: null, + expiresAtPresent: true, + message: remaining ? "standalone Sandbox cleanup pending" : "standalone Sandbox stopped", + }; +} + +function standaloneSandboxAttachUrl(env: RuntimeEnv, provisionId: string): string { + const url = new URL( + `/api/provision/interactive/${encodeURIComponent(provisionId)}/pty`, + deploymentConfig(env).canonicalUrl, + ); + url.protocol = url.protocol === "http:" ? "ws:" : "wss:"; + return url.toString(); +} + +async function standaloneSandboxPty( + request: Request, + env: RuntimeEnv, + provisionId: string, +): Promise { + authorizeProvisionEndpoint(request, env); + if (request.headers.get("upgrade")?.toLowerCase() !== "websocket") { + throw badRequest("websocket upgrade required"); + } + if (!env.SANDBOX) throw serviceUnavailable("Sandbox binding is not configured"); + const owner = await database(env) + .selectFrom("standalone_sandbox_provisions") + .selectAll() + .where("id", "=", provisionId) + .where("state", "=", "active") + .executeTakeFirst(); + if ( + !owner?.lease_id || + !isCurrentSandboxLease(owner.lease_id) || + !owner.expires_at || + owner.expires_at <= Date.now() || + managedInteractiveSessionId(provisionId) + ) { + if (owner) { + const now = Date.now(); + await stageStandaloneSandboxProvisionCleanup( + env, + owner, + managedInteractiveSessionId(provisionId) + ? "standalone provision used the reserved managed session namespace" + : "standalone Sandbox provision expired", + now, + ); + await reconcileCredentialPolicyCleanupBatch(env, now, provisionId); + } + throw notFound("standalone Sandbox provision not found"); + } + const lease = sandboxLeaseInfo({ id: provisionId, leaseId: owner.lease_id }); + if (lease.sandboxId !== owner.sandbox_id) { + throw serviceUnavailable("standalone Sandbox ownership is inconsistent"); + } + const policyGeneration = await activeSandboxCredentialPolicyGeneration( + env, + provisionId, + owner.sandbox_id, + ); + if (!policyGeneration) { + throw serviceUnavailable("standalone Sandbox credentials are unavailable"); + } + const terminalOwnership: StandaloneSandboxTerminalOwnership = { + provisionId, + requestHash: owner.request_hash, + sandboxId: owner.sandbox_id, + leaseId: owner.lease_id, + expiresAt: owner.expires_at, + updatedAt: owner.updated_at, + policyGeneration, + }; + const terminalGrant = standaloneSandboxTerminalGrant(env, terminalOwnership); + if (!(await terminalGrant())) { + throw serviceUnavailable("standalone Sandbox terminal authorization changed"); + } + const sandbox = getSandbox(env.SANDBOX, owner.sandbox_id); + let response: Response; + try { + const terminalSession = await sandbox.getSession(lease.terminalSessionId); + const terminalHeaders = new Headers(request.headers); + terminalHeaders.delete("authorization"); + terminalHeaders.delete("cookie"); + response = await terminalSession.terminal(new Request(request, { headers: terminalHeaders }), { + cols: terminalSize(request, "cols", 120), + rows: terminalSize(request, "rows", 34), + shell: sandboxTerminalShellPath(provisionId), + }); + } catch (error) { + throw serviceUnavailable(`standalone Sandbox terminal failed: ${safeProviderError(error)}`); + } + if (!response.webSocket || response.status !== 101) { + throw serviceUnavailable(`standalone Sandbox terminal HTTP ${response.status}`); + } + const pair = new WebSocketPair(); + const client = pair[0]; + const server = pair[1]; + server.accept(); + response.webSocket.accept(); + bridgeWebSockets( + server, + response.webSocket, + terminalGrant, + undefined, + "standalone Sandbox authorization revoked or expired", + ); + return new Response(null, { status: 101, webSocket: client }); +} + +function standaloneSandboxTerminalGrant( + env: RuntimeEnv, + ownership: StandaloneSandboxTerminalOwnership, +): () => Promise { + return cachedBooleanGrant(async () => { + const now = Date.now(); + const owner = await database(env) + .selectFrom("standalone_sandbox_provisions") + .select("id") + .where("id", "=", ownership.provisionId) + .where("request_hash", "=", ownership.requestHash) + .where("sandbox_id", "=", ownership.sandboxId) + .where("state", "=", "active") + .where("lease_id", "=", ownership.leaseId) + .where("expires_at", "=", ownership.expiresAt) + .where("expires_at", ">", now) + .where("updated_at", "=", ownership.updatedAt) + .where( + activeSandboxCredentialPolicyCondition( + env, + ownership.provisionId, + ownership.sandboxId, + ownership.policyGeneration, + ownership.updatedAt, + ), + ) + .executeTakeFirst(); + return Boolean(owner); + }); +} + +function isBuiltInInteractiveProvisionUrl(env: RuntimeEnv, value: string): boolean { + if (value === "/api/provision/interactive") return true; + try { + const url = new URL(value); + return ( + url.pathname === "/api/provision/interactive" && + (url.hostname === new URL(deploymentConfig(env).canonicalUrl).hostname || + url.hostname === appCanonicalHost || + appRedirectHosts.has(url.hostname)) + ); + } catch { + return false; + } +} + +async function provisionInteractivePayload( + env: RuntimeEnv, + payload: InteractiveProvisionRequest, + _agentToken?: string, +): Promise { + if (payload.runtime === "container" && env.SANDBOX) { + return failedProvision("Cloudflare Sandbox provision requires durable ownership"); + } + if (env.CRABBOX_RUNTIME_ADAPTER_URL) { + return failedProvision( + "versioned runtime adapter requires a durable interactive session lifecycle", + ); + } + if (env.CRABBOX_RUNTIME_PROVISION_URL) { + return forwardRuntimeProvision(env, payload); + } + if (payload.runtime === "container" && env.CRABBOX_CLOUDFLARE_RUNNER_URL) { + return provisionWithCloudflareRunner(env, payload); + } + if (payload.runtime === "crabbox" && env.CRABBOX_CLAWFLEET_URL) { + return provisionWithClawFleet(env, payload); + } + return { + status: "pending_adapter", + leaseId: null, + attachUrl: null, + attachUrlPresent: true, + vncUrl: null, + message: "provision route live; runtime backend not configured", + }; +} + +function authorizeProvisionEndpoint(request: Request, env: RuntimeEnv): void { + const hasBackend = Boolean( + env.SANDBOX || + env.CRABBOX_RUNTIME_ADAPTER_URL || + env.CRABBOX_RUNTIME_PROVISION_URL || + env.CRABBOX_CLOUDFLARE_RUNNER_URL || + env.CRABBOX_CLAWFLEET_URL, + ); + if (!env.CRABBOX_INTERACTIVE_PROVISION_TOKEN) { + if (hasBackend) { + throw serviceUnavailable("interactive provision token is not configured"); + } + return; + } + authorizeProvisionBearerToken(request, env); +} + +function authorizeProvisionBearerToken(request: Request, env: RuntimeEnv): void { + if (!env.CRABBOX_INTERACTIVE_PROVISION_TOKEN) { + throw serviceUnavailable("interactive provision token is not configured"); + } + const expected = `Bearer ${env.CRABBOX_INTERACTIVE_PROVISION_TOKEN}`; + if (request.headers.get("authorization") !== expected) throw unauthorized(); +} + +function sandboxProvisionPreflightError( + env: RuntimeEnv, + session: SandboxRuntimeSession, +): string | null { + if (!env.SANDBOX) return "Cloudflare Sandbox binding is not configured"; + if (!env.SESSION_CONTROL) return "SESSION_CONTROL Durable Object is not configured"; + if (!env.OPENAI_API_KEY) { + return "OPENAI_API_KEY is not configured for Cloudflare Sandbox Codex"; + } + if (session.githubToken && !env.CRABBOX_TOKEN_ENCRYPTION_KEY && !env.GITHUB_CLIENT_SECRET) { + return "CRABBOX_TOKEN_ENCRYPTION_KEY or GITHUB_CLIENT_SECRET is required for user GitHub tokens"; + } + return null; +} + +async function stageFailedManagedSandboxProvision( + env: RuntimeEnv, + sessionId: string, + ownershipFence: SandboxLeaseRefreshFence, + message: string, + now: number, +): Promise { + const staged = await stageTerminalCredentialPolicyCleanupById( + env, + sessionId, + "failed", + message, + now, + message, + ownershipFence, + ); + await reconcileCredentialPolicyCleanupBatch(env, now, sessionId); + return staged; +} + +async function provisionWithSandbox( + env: RuntimeEnv, + session: InteractiveProvisionRequest, + agentToken: string | undefined, + lease: SandboxLease, + ownershipFence: SandboxCredentialPolicyOwnershipFence, +): Promise { + try { + const preflightError = sandboxProvisionPreflightError(env, session); + if (preflightError) throw new Error(preflightError); + const workdir = sandboxWorkdir(session.id); + const sandbox = getSandbox(env.SANDBOX!, lease.sandboxId); + if (!("provisionId" in ownershipFence) && !agentToken) { + throw new Error("managed Sandbox agent token is unavailable"); + } + await registerSandboxCredentialPolicy(env, session, lease.sandboxId, ownershipFence); + await setupSandboxTerminalSession( + sandbox, + env, + session, + workdir, + lease.terminalSessionId, + agentToken, + ); + } catch (error) { + const message = safeProviderError(error); + const cleanupMessage = `Cloudflare Sandbox provision failed: ${message}`; + const failureAt = Date.now(); + if ("provisionId" in ownershipFence) { + await queueSandboxCredentialPolicyCleanup(env, session.id, lease.sandboxId, failureAt); + } else { + await stageTerminalCredentialPolicyCleanupById( + env, + session.id, + "failed", + cleanupMessage, + failureAt, + cleanupMessage, + ownershipFence, + ); + } + await reconcileCredentialPolicyCleanupBatch(env, Date.now(), session.id); + return { + status: "stopping", + leaseId: sandboxLeaseId(lease), + attachUrl: null, + vncUrl: null, + message: `${cleanupMessage}; credential cleanup pending`, + terminalStatus: "failed", + createPending: false, + }; + } + + return { + status: "ready", + leaseId: sandboxLeaseId(lease), + attachUrl: + "provisionId" in ownershipFence + ? standaloneSandboxAttachUrl(env, session.id) + : `/api/interactive-sessions/${encodeURIComponent(session.id)}/pty`, + vncUrl: null, + message: `Cloudflare Sandbox ready for ${session.repo}`, + }; +} + +function sandboxManagedOwnershipCondition( + ownershipFence: SandboxManagedOwnershipFence, + now: number, +): RawBuilder { + if ("leaseId" in ownershipFence) { + return sql` + lease_id = ${ownershipFence.leaseId} + AND sandbox_refresh_sandbox_id IS NULL + AND sandbox_refresh_claim IS NULL + AND sandbox_refresh_claim_expires_at IS NULL + `; + } + return sql` + lease_id IS ${ownershipFence.refreshLeaseId} + AND sandbox_refresh_sandbox_id = ${ownershipFence.sandboxId} + AND sandbox_refresh_claim = ${ownershipFence.claim} + AND sandbox_refresh_claim_expires_at > ${now} + `; +} + +function sandboxManagedStoredOwnershipCondition( + ownershipFence: SandboxManagedOwnershipFence, +): RawBuilder { + if ("leaseId" in ownershipFence) { + return sql` + lease_id = ${ownershipFence.leaseId} + AND sandbox_refresh_sandbox_id IS NULL + AND sandbox_refresh_claim IS NULL + AND sandbox_refresh_claim_expires_at IS NULL + `; + } + return sql` + lease_id IS ${ownershipFence.refreshLeaseId} + AND sandbox_refresh_sandbox_id = ${ownershipFence.sandboxId} + AND sandbox_refresh_claim = ${ownershipFence.claim} + AND sandbox_refresh_claim_expires_at = ${ownershipFence.expiresAt} + `; +} + +function sandboxCredentialPolicyOwnerCondition( + sessionId: string, + sandboxId: string, + ownershipFence: SandboxCredentialPolicyOwnershipFence, + now: number, +): RawBuilder { + if ("provisionId" in ownershipFence) { + return sql`EXISTS ( + SELECT 1 + FROM standalone_sandbox_provisions AS owner + WHERE owner.id = ${sessionId} + AND owner.id = ${ownershipFence.provisionId} + AND owner.sandbox_id = ${sandboxId} + AND owner.sandbox_id = ${ownershipFence.sandboxId} + AND owner.state = 'provisioning' + AND owner.ownership_claim = ${ownershipFence.claim} + AND owner.ownership_claim_expires_at > ${now} + )`; + } + return sql`EXISTS ( + SELECT 1 + FROM interactive_sessions + WHERE id = ${sessionId} + AND ${sandboxId} = ${ownershipFence.sandboxId} + AND (adapter IS NULL OR adapter != ${runtimeAdapterName}) + AND status IN ('provisioning', 'pending_adapter', 'ready', 'attached', 'detached') + AND credential_cleanup_terminal_status IS NULL + AND agent_token_hash IS NOT NULL + AND ${sandboxManagedOwnershipCondition(ownershipFence, now)} + )`; +} + +function sandboxCredentialPolicyRegistrationQueries( + sessionId: string, + sandboxId: string, + registration: SandboxCredentialPolicyRegistration, + registrationExpiresAt: number, + now: number, + ownershipFence: SandboxCredentialPolicyOwnershipFence, +): CompilableQuery[] { + return registration.lookupIds.map( + (lookupId) => sql` + INSERT INTO interactive_session_credential_policies ( + session_id, + sandbox_id, + lookup_id, + state, + registration_generation, + registration_claim, + registration_claim_expires_at, + attempt_count, + last_attempt_at, + last_error, + cleanup_claim, + cleanup_claim_expires_at, + created_at, + updated_at + ) + SELECT + ${sessionId}, + ${sandboxId}, + ${lookupId}, + 'registering', + ${registration.generation}, + ${registration.claim}, + ${registrationExpiresAt}, + 0, + NULL, + NULL, + NULL, + NULL, + ${now}, + ${now} + WHERE ${sandboxCredentialPolicyOwnerCondition(sessionId, sandboxId, ownershipFence, now)} + ON CONFLICT(session_id, sandbox_id, lookup_id) DO UPDATE SET + state = 'registering', + registration_generation = excluded.registration_generation, + registration_claim = excluded.registration_claim, + registration_claim_expires_at = excluded.registration_claim_expires_at, + last_error = NULL, + cleanup_claim = NULL, + cleanup_claim_expires_at = NULL, + updated_at = excluded.updated_at + WHERE interactive_session_credential_policies.state != 'cleanup_pending' + AND ( + interactive_session_credential_policies.registration_claim IS NULL + OR interactive_session_credential_policies.registration_claim_expires_at <= ${now} + ) + AND ${sandboxCredentialPolicyOwnerCondition(sessionId, sandboxId, ownershipFence, now)} + `, + ); +} + +async function beginSandboxCredentialPolicyRegistration( + env: RuntimeEnv, + sessionId: string, + sandboxId: string, + ownershipFence: SandboxCredentialPolicyOwnershipFence, +): Promise { + const db = database(env); + const lookupIds = sandboxLookupIds(env, sandboxId); + const existing = await db + .selectFrom("interactive_session_credential_policies") + .select("registration_generation") + .distinct() + .where("session_id", "=", sessionId) + .where("sandbox_id", "=", sandboxId) + .execute(); + if (existing.length > 1) { + throw new Error("sandbox credential policy generations are inconsistent"); + } + const registration = { + generation: existing[0]?.registration_generation ?? `generation:${crypto.randomUUID()}`, + claim: `registration:${crypto.randomUUID()}`, + lookupIds, + }; + const now = Date.now(); + const registrationExpiresAt = now + credentialPolicyRegistrationClaimMs; + await executeBatch( + env, + sandboxCredentialPolicyRegistrationQueries( + sessionId, + sandboxId, + registration, + registrationExpiresAt, + now, + ownershipFence, + ), + ); + const claimed = await db + .selectFrom("interactive_session_credential_policies") + .select([ + "lookup_id", + "state", + "registration_generation", + "registration_claim", + "registration_claim_expires_at", + ]) + .where("session_id", "=", sessionId) + .where("sandbox_id", "=", sandboxId) + .where("lookup_id", "in", lookupIds) + .execute(); + if ( + claimed.length !== lookupIds.length || + claimed.some( + (row) => + row.state !== "registering" || + row.registration_generation !== registration.generation || + row.registration_claim !== registration.claim || + row.registration_claim_expires_at !== registrationExpiresAt, + ) + ) { + await abandonSandboxCredentialPolicyRegistration( + env, + sessionId, + sandboxId, + registration, + "sandbox credential policy registration claim was not acquired", + ); + throw new Error("sandbox credential policy registration is unavailable"); + } + return registration; +} + +async function beginLegacySandboxCredentialPolicyRepair( + env: RuntimeEnv, + sessionId: string, + sandboxId: string, + ownershipFence: SandboxCurrentLeaseFence, +): Promise { + const db = database(env); + const lookupIds = sandboxLookupIds(env, sandboxId); + const existing = await db + .selectFrom("interactive_session_credential_policies") + .select(["registration_generation", "registration_claim"]) + .where("session_id", "=", sessionId) + .where("sandbox_id", "=", sandboxId) + .execute(); + const generations = [...new Set(existing.map((row) => row.registration_generation))]; + const existingGeneration = generations[0]; + if (!existingGeneration || generations.length !== 1) { + throw new Error("legacy sandbox credential policy generations are inconsistent"); + } + const resuming = existing.some((row) => + row.registration_claim?.startsWith(credentialPolicyLegacyRepairClaimPrefix), + ); + if (!existingGeneration?.startsWith(credentialPolicyLegacyGenerationPrefix) && !resuming) { + throw new Error("legacy sandbox credential policy repair is not pending"); + } + const registration: SandboxCredentialPolicyRegistration = { + generation: existingGeneration.startsWith(credentialPolicyLegacyGenerationPrefix) + ? `generation:${crypto.randomUUID()}` + : existingGeneration, + claim: `${credentialPolicyLegacyRepairClaimPrefix}${crypto.randomUUID()}`, + lookupIds, + }; + const now = Date.now(); + const registrationExpiresAt = now + credentialPolicyRegistrationClaimMs; + await executeBatch( + env, + sandboxCredentialPolicyRegistrationQueries( + sessionId, + sandboxId, + registration, + registrationExpiresAt, + now, + ownershipFence, + ), + ); + const claimed = await db + .selectFrom("interactive_session_credential_policies") + .select([ + "lookup_id", + "state", + "registration_generation", + "registration_claim", + "registration_claim_expires_at", + ]) + .where("session_id", "=", sessionId) + .where("sandbox_id", "=", sandboxId) + .where("lookup_id", "in", lookupIds) + .execute(); + if ( + claimed.length !== lookupIds.length || + claimed.some( + (row) => + row.state !== "registering" || + row.registration_generation !== registration.generation || + row.registration_claim !== registration.claim || + row.registration_claim_expires_at !== registrationExpiresAt, + ) + ) { + throw new Error("legacy sandbox credential policy repair claim was not acquired"); + } + return registration; +} + +async function repairLegacySandboxCredentialPolicy( + env: RuntimeEnv, + sessionId: string, + sandboxId: string, +): Promise { + const stub = sandboxControlStub(env); + if (!stub || !env.SANDBOX) { + throw new Error("legacy sandbox credential policy repair is unavailable"); + } + const session = await database(env) + .selectFrom("interactive_sessions") + .selectAll() + .where("id", "=", sessionId) + .executeTakeFirst(); + if (!session?.lease_id) { + throw new Error("legacy sandbox credential policy owner is unavailable"); + } + const lease = sandboxLeaseInfo({ + id: session.id, + adapter: session.adapter, + leaseId: session.lease_id, + }); + if (lease.sandboxId !== sandboxId) { + throw new Error("legacy sandbox credential policy lease does not match"); + } + const ownership: SandboxCurrentLeaseFence = { leaseId: session.lease_id, sandboxId }; + const registration = await beginLegacySandboxCredentialPolicyRepair( + env, + sessionId, + sandboxId, + ownership, + ); + try { + const registrationExpiresAt = await renewSandboxCredentialPolicyRegistration( + env, + sessionId, + sandboxId, + registration, + ownership, + ); + if (!registrationExpiresAt) { + throw new Error("legacy sandbox credential policy repair claim was revoked"); + } + const response = await stub.fetch( + "https://crabfleet.internal/api/session-control/migrate-legacy", + { + method: "POST", + body: JSON.stringify({ + generation: registration.generation, + registrationClaim: registration.claim, + registrationExpiresAt, + sandboxIds: registration.lookupIds, + sessionId, + } satisfies SandboxCredentialPolicyLegacyMigration), + headers: { "content-type": "application/json" }, + }, + ); + if (!response.ok) { + throw new Error("legacy sandbox credential policy repair failed"); + } + if ( + !(await finishSandboxCredentialPolicyRegistration( + env, + sessionId, + sandboxId, + registration, + ownership, + )) + ) { + throw new Error("legacy sandbox credential policy repair lost ownership"); + } + } catch (error) { + await database(env) + .updateTable("interactive_session_credential_policies") + .set({ + last_error: clean(error instanceof Error ? error.message : String(error), 500), + updated_at: Date.now(), + }) + .where("session_id", "=", sessionId) + .where("sandbox_id", "=", sandboxId) + .where("registration_generation", "=", registration.generation) + .where("registration_claim", "=", registration.claim) + .execute(); + throw error; + } +} + +async function repairLegacySandboxCredentialPolicyBatch( + env: RuntimeEnv, + now: number, + sessionId?: string, +): Promise { + if (!env.SANDBOX || !env.SESSION_CONTROL) return; + let query = database(env) + .selectFrom("interactive_session_credential_policies") + .select(["session_id", "sandbox_id"]) + .select(({ fn }) => fn.min("updated_at").as("repair_updated_at")) + .where((expression) => + expression.or([ + expression.and([ + expression("state", "=", "active"), + expression( + "registration_generation", + "like", + `${credentialPolicyLegacyGenerationPrefix}%`, + ), + ]), + expression.and([ + expression("state", "=", "registering"), + expression("registration_claim", "like", `${credentialPolicyLegacyRepairClaimPrefix}%`), + expression("registration_claim_expires_at", "<=", now), + ]), + ]), + ) + .groupBy(["session_id", "sandbox_id"]) + .orderBy("repair_updated_at", "asc") + .orderBy("session_id", "asc") + .orderBy("sandbox_id", "asc") + .limit(credentialPolicyCleanupLimit); + if (sessionId) query = query.where("session_id", "=", sessionId); + const candidates = await query.execute(); + await mapWithConcurrency(candidates, 3, async (candidate) => { + await repairLegacySandboxCredentialPolicy( + env, + candidate.session_id, + candidate.sandbox_id, + ).catch((error) => { + console.error("legacy sandbox credential policy repair failed", error); + }); + }); +} + +async function renewSandboxCredentialPolicyRegistration( + env: RuntimeEnv, + sessionId: string, + sandboxId: string, + registration: SandboxCredentialPolicyRegistration, + ownershipFence: SandboxCredentialPolicyOwnershipFence, +): Promise { + const now = Date.now(); + const registrationExpiresAt = now + credentialPolicyRegistrationClaimMs; + const renewed = await database(env) + .updateTable("interactive_session_credential_policies") + .set({ + registration_claim_expires_at: registrationExpiresAt, + updated_at: now, + }) + .where("session_id", "=", sessionId) + .where("sandbox_id", "=", sandboxId) + .where("lookup_id", "in", registration.lookupIds) + .where("state", "=", "registering") + .where("registration_generation", "=", registration.generation) + .where("registration_claim", "=", registration.claim) + .where(sandboxCredentialPolicyOwnerCondition(sessionId, sandboxId, ownershipFence, now)) + .executeTakeFirst(); + return Number(renewed.numUpdatedRows ?? 0n) === registration.lookupIds.length + ? registrationExpiresAt + : null; +} + +async function finishSandboxCredentialPolicyRegistration( + env: RuntimeEnv, + sessionId: string, + sandboxId: string, + registration: SandboxCredentialPolicyRegistration, + ownershipFence: SandboxCredentialPolicyOwnershipFence, +): Promise { + const now = Date.now(); + const db = database(env); + await db + .updateTable("interactive_session_credential_policies") + .set({ + state: "active", + registration_claim: null, + registration_claim_expires_at: null, + updated_at: now, + }) + .where("session_id", "=", sessionId) + .where("sandbox_id", "=", sandboxId) + .where("lookup_id", "in", registration.lookupIds) + .where("state", "=", "registering") + .where("registration_generation", "=", registration.generation) + .where("registration_claim", "=", registration.claim) + .where(sandboxCredentialPolicyOwnerCondition(sessionId, sandboxId, ownershipFence, now)) + .execute(); + const active = await db + .selectFrom("interactive_session_credential_policies") + .select(["lookup_id", "state", "registration_generation", "registration_claim"]) + .where("session_id", "=", sessionId) + .where("sandbox_id", "=", sandboxId) + .where("lookup_id", "in", registration.lookupIds) + .execute(); + return ( + active.length === registration.lookupIds.length && + active.every( + (row) => + row.state === "active" && + row.registration_generation === registration.generation && + row.registration_claim === null, + ) + ); +} + +async function abandonSandboxCredentialPolicyRegistration( + env: RuntimeEnv, + sessionId: string, + sandboxId: string, + registration: SandboxCredentialPolicyRegistration, + reason: string, +): Promise { + const now = Date.now(); + await database(env) + .updateTable("interactive_session_credential_policies") + .set({ + state: sql<"registering" | "cleanup_pending">`CASE + WHEN ${sandboxCredentialPolicyCleanupAuthorizedCondition(sessionId, sandboxId, now)} + THEN 'cleanup_pending' + ELSE 'registering' + END`, + registration_claim: null, + registration_claim_expires_at: null, + last_error: reason, + updated_at: now, + }) + .where("session_id", "=", sessionId) + .where("sandbox_id", "=", sandboxId) + .where("registration_generation", "=", registration.generation) + .where("registration_claim", "=", registration.claim) + .execute(); +} + +async function registerSandboxCredentialPolicy( + env: RuntimeEnv, + session: SandboxRuntimeSession, + sandboxId: string, + ownershipFence: SandboxCredentialPolicyOwnershipFence, +): Promise { + const stub = sandboxControlStub(env); + if (!stub) throw new Error("SESSION_CONTROL Durable Object is not configured"); + const policyExpiresAt = + "provisionId" in ownershipFence + ? await standaloneSandboxPolicyExpiresAt(env, session.id, sandboxId, ownershipFence) + : null; + if ("provisionId" in ownershipFence && !policyExpiresAt) { + throw new Error("standalone Sandbox credential expiry is unavailable"); + } + const registration = await beginSandboxCredentialPolicyRegistration( + env, + session.id, + sandboxId, + ownershipFence, + ); + try { + const githubToken = "githubToken" in session ? session.githubToken : undefined; + const githubTokenCiphertext = githubToken ? await sealSecret(env, githubToken) : null; + if (githubToken && !githubTokenCiphertext) { + throw new Error( + "CRABBOX_TOKEN_ENCRYPTION_KEY or GITHUB_CLIENT_SECRET is required for user GitHub tokens", + ); + } + const effectiveGithubToken = githubToken ?? env.GITHUB_TOKEN; + const githubCredentialSource = githubTokenCiphertext + ? "session" + : env.GITHUB_TOKEN + ? "worker" + : "none"; + const githubRepoNodeId = effectiveGithubToken + ? await fetchGithubRepoNodeId(session.repo, effectiveGithubToken) + : null; + const policy: SandboxCredentialPolicy = { + allowedHosts: sandboxBackupAllowedHosts(env), + ...(policyExpiresAt ? { expiresAt: policyExpiresAt } : {}), + githubCredentialSource, + githubRepo: session.repo, + owner: session.owner, + sandboxId, + sessionId: session.id, + ...(githubRepoNodeId ? { githubRepoNodeId } : {}), + ...(githubTokenCiphertext ? { githubTokenCiphertext } : {}), + ...(env.OPENAI_BASE_URL ? { openAIBaseUrl: env.OPENAI_BASE_URL } : {}), + ...(env.OPENAI_ORG_ID ? { openAIOrgId: env.OPENAI_ORG_ID } : {}), + }; + for (const lookupId of registration.lookupIds) { + const registrationExpiresAt = await renewSandboxCredentialPolicyRegistration( + env, + session.id, + sandboxId, + registration, + ownershipFence, + ); + if (!registrationExpiresAt) { + throw new Error("sandbox credential policy registration claim was revoked"); + } + const response = await stub.fetch("https://crabfleet.internal/api/session-control/register", { + method: "POST", + body: JSON.stringify({ + generation: registration.generation, + registrationClaim: registration.claim, + registrationExpiresAt, + policy: { ...policy, sandboxId: lookupId }, + } satisfies StoredSandboxCredentialPolicy), + headers: { "content-type": "application/json" }, + }); + if (!response.ok) { + throw new Error("sandbox credential policy registration failed"); + } + } + if ( + !(await finishSandboxCredentialPolicyRegistration( + env, + session.id, + sandboxId, + registration, + ownershipFence, + )) + ) { + throw new Error("sandbox credential policy cleanup became pending during registration"); + } + } catch (error) { + await abandonSandboxCredentialPolicyRegistration( + env, + session.id, + sandboxId, + registration, + clean(error instanceof Error ? error.message : String(error), 500), + ).catch(() => undefined); + throw error; + } +} + +async function standaloneSandboxPolicyExpiresAt( + env: RuntimeEnv, + sessionId: string, + sandboxId: string, + fence: StandaloneSandboxProvisionFence, +): Promise { + const now = Date.now(); + const owner = await database(env) + .selectFrom("standalone_sandbox_provisions") + .select("expires_at") + .where("id", "=", sessionId) + .where("id", "=", fence.provisionId) + .where("sandbox_id", "=", sandboxId) + .where("sandbox_id", "=", fence.sandboxId) + .where("state", "=", "provisioning") + .where("ownership_claim", "=", fence.claim) + .where("ownership_claim_expires_at", ">", now) + .where("expires_at", ">", now) + .executeTakeFirst(); + return owner?.expires_at ?? null; +} + +async function ensureSandboxCredentialPolicy( + env: RuntimeEnv, + session: InteractiveSession & { githubToken?: string }, + sandboxId: string, +): Promise { + const leaseId = session.leaseId; + if (!leaseId || !leaseId.startsWith(sandboxLeasePrefix)) { + throw new Error("sandbox credential policy requires a current durable lease"); + } + const lease = sandboxLeaseInfo(session); + if (lease.sandboxId !== sandboxId) { + throw new Error("sandbox credential policy lease ownership does not match"); + } + const ownership: SandboxCurrentLeaseFence = { leaseId, sandboxId }; + const hasFreshUserToken = Boolean("githubToken" in session && session.githubToken); + let generation = await existingSandboxCredentialPolicyGeneration(env, session.id, sandboxId); + if (generation?.startsWith(credentialPolicyLegacyGenerationPrefix)) { + await repairLegacySandboxCredentialPolicy(env, session.id, sandboxId); + if (!hasFreshUserToken) return; + generation = await existingSandboxCredentialPolicyGeneration(env, session.id, sandboxId); + } + if ( + !hasFreshUserToken && + generation && + (await sandboxCredentialPolicyExists(env, sandboxId, generation)) + ) { + if ( + !(await recordSandboxCredentialPolicyRefs( + env, + session.id, + sandboxId, + "active", + generation, + ownership, + )) + ) { + throw new Error("sandbox credential policy lifecycle is unavailable"); + } + return; + } + await registerSandboxCredentialPolicy(env, session, sandboxId, ownership); +} + +async function recordSandboxCredentialPolicyRefs( + env: RuntimeEnv, + sessionId: string, + sandboxId: string, + state: "registering" | "active" | "cleanup_pending", + generation: string, + ownershipFence: SandboxCredentialPolicyOwnershipFence, + now = Date.now(), +): Promise { + const lookupIds = sandboxLookupIds(env, sandboxId); + if (state === "active") { + await promoteSandboxCredentialPolicyRegistration( + env, + sessionId, + sandboxId, + generation, + ownershipFence, + now, + ); + } + await executeBatch( + env, + sandboxCredentialPolicyRefQueries( + env, + sessionId, + sandboxId, + state, + generation, + now, + sandboxCredentialPolicyOwnerCondition(sessionId, sandboxId, ownershipFence, now), + ), + ); + const refs = await database(env) + .selectFrom("interactive_session_credential_policies") + .select(["lookup_id", "state", "registration_generation", "registration_claim"]) + .where("session_id", "=", sessionId) + .where("sandbox_id", "=", sandboxId) + .where("lookup_id", "in", lookupIds) + .execute(); + return ( + refs.length === lookupIds.length && + refs.every( + (ref) => + ref.state === state && + ref.registration_generation === generation && + ref.registration_claim === null, + ) + ); +} + +async function promoteSandboxCredentialPolicyRegistration( + env: RuntimeEnv, + sessionId: string, + sandboxId: string, + generation: string, + ownershipFence: SandboxCredentialPolicyOwnershipFence, + now: number, +): Promise { + await database(env) + .updateTable("interactive_session_credential_policies") + .set({ + state: "active", + registration_claim: null, + registration_claim_expires_at: null, + last_error: null, + updated_at: now, + }) + .where("session_id", "=", sessionId) + .where("sandbox_id", "=", sandboxId) + .where("lookup_id", "in", sandboxLookupIds(env, sandboxId)) + .where("state", "=", "registering") + .where("registration_generation", "=", generation) + .where((expression) => + expression.or([ + expression("registration_claim", "is", null), + expression("registration_claim_expires_at", "<=", now), + ]), + ) + .where(sandboxCredentialPolicyOwnerCondition(sessionId, sandboxId, ownershipFence, now)) + .execute(); +} + +async function sandboxCredentialPolicyExists( + env: RuntimeEnv, + sandboxId: string, + generation: string, +): Promise { + const stub = sandboxControlStub(env); + if (!stub) return false; + const responses = await Promise.all( + sandboxLookupIds(env, sandboxId).map((lookupId) => + stub.fetch( + `https://crabfleet.internal/api/session-control/egress/${encodeURIComponent(lookupId)}`, + ), + ), + ); + if (responses.some((response) => !response.ok && response.status !== 404)) { + throw new Error("sandbox credential policy lookup failed"); + } + return responses.every( + (response) => + response.ok && response.headers.get("x-crabfleet-policy-generation") === generation, + ); +} + +async function fetchGithubRepoNodeId(repo: string, token: string): Promise { + const response = await fetch(`https://api.github.com/repos/${repo}`, { + headers: { + accept: "application/vnd.github+json", + authorization: `Bearer ${token}`, + "user-agent": "crabfleet", + "x-github-api-version": "2022-11-28", + }, + }); + if (!response.ok) { + throw new Error(`GitHub repository metadata lookup failed for ${repo}`); + } + const body = (await response.json()) as { node_id?: unknown }; + if (typeof body.node_id !== "string" || !body.node_id) { + throw new Error(`GitHub repository metadata lookup did not include node_id for ${repo}`); + } + return body.node_id; +} + +async function githubNodeBelongsToRepo( + nodeId: string, + repo: string, + token: string, +): Promise { + const [owner, name] = repo.toLowerCase().split("/"); + if (!owner || !name) return false; + const response = await fetch("https://api.github.com/graphql", { + method: "POST", + body: JSON.stringify({ + query: `query($id: ID!) { + node(id: $id) { + __typename + ... on Repository { + owner { login } + name + } + ... on RepositoryNode { + repository { owner { login } name } + } + } + }`, + variables: { id: nodeId }, + }), + headers: { + accept: "application/vnd.github+json", + authorization: `Bearer ${token}`, + "content-type": "application/json", + "user-agent": "crabfleet", + "x-github-api-version": "2022-11-28", + }, + }); + if (!response.ok) return false; + const body = (await response.json().catch(() => null)) as { + data?: { + node?: { + name?: unknown; + owner?: { login?: unknown }; + repository?: { name?: unknown; owner?: { login?: unknown } }; + }; + }; + errors?: unknown; + } | null; + if (!body || body.errors) return false; + const node = body.data?.node; + const repository = node?.repository ?? node; + return ( + typeof repository?.owner?.login === "string" && + typeof repository.name === "string" && + repository.owner.login.toLowerCase() === owner && + repository.name.toLowerCase() === name + ); +} + +function sandboxLookupIds(env: RuntimeEnv, sandboxId: string): string[] { + const ids = new Set([sandboxId]); + if (env.SANDBOX) ids.add(env.SANDBOX.idFromName(sandboxId).toString()); + return [...ids]; +} + +async function ensureCurrentSandboxLease( + request: Request, + env: RuntimeEnv, + user: User | null, + session: InteractiveSession & { githubToken?: string }, +): Promise { + if (!env.SANDBOX) return session; + if (!isSandboxInteractiveSession(session)) { + throw serviceUnavailable("session is not backed by a Cloudflare Sandbox lease"); + } + if (isCurrentSandboxLease(session.leaseId)) { + await ensureSandboxCredentialPolicy(env, session, sandboxLeaseInfo(session).sandboxId); + return session; + } + const originalLeaseId = session.leaseId; + if (!originalLeaseId) { + throw serviceUnavailable("Cloudflare Sandbox lease refresh is already in progress"); + } + const refreshStartedAt = sandboxLeaseRefreshStartedAt(originalLeaseId); + const now = Date.now(); + if (refreshStartedAt && now - refreshStartedAt < 2 * 60_000) { + throw serviceUnavailable("Cloudflare Sandbox lease refresh is already in progress"); + } + if (!user || actor(user) !== session.owner) { + throw serviceUnavailable("session owner must reconnect to refresh Cloudflare Sandbox lease"); + } + const githubToken = user?.subject.startsWith("github:") + ? (session.githubToken ?? (await sessionGitHubToken(request, env))) + : undefined; + if (user.subject.startsWith("github:") && !githubToken) { + throw forbidden("GitHub PR credentials are not connected; sign in with GitHub again"); + } + const refreshPayload: InteractiveProvisionRequest = { + id: session.id, + parentSessionId: session.parentSessionId, + rootSessionId: session.rootSessionId ?? session.id, + repo: session.repo, + branch: session.branch, + runtime: session.runtime, + profile: session.profile, + command: session.command, + prompt: session.prompt, + purpose: session.purpose, + summary: session.summary, + owner: session.owner, + createdBy: session.createdBy, + ...(githubToken ? { githubToken } : {}), + }; + const preflightError = sandboxProvisionPreflightError(env, refreshPayload); + if (preflightError) throw serviceUnavailable(preflightError); + const fallbackLeaseId = sandboxLeaseWithoutRefresh(originalLeaseId); + const oldSandboxId = originalLeaseId.startsWith(sandboxLeasePrefix) + ? sandboxLeaseInfo({ id: session.id, leaseId: fallbackLeaseId }).sandboxId + : null; + const refreshLeaseId = `${fallbackLeaseId}:refreshing-${now}-${crypto.randomUUID().slice(0, 8)}`; + const refreshLease = newSandboxLease(session.id); + const agentToken = newAgentToken(); + const agentTokenHash = await sha256(agentToken); + const refreshFence: SandboxLeaseRefreshFence = { + claim: `refresh:${crypto.randomUUID()}`, + expiresAt: now + credentialPolicyProvisioningStaleMs, + refreshLeaseId, + sandboxId: refreshLease.sandboxId, + }; + const claim = await database(env) + .updateTable("interactive_sessions") + .set({ + lease_id: refreshLeaseId, + sandbox_refresh_sandbox_id: refreshFence.sandboxId, + sandbox_refresh_claim: refreshFence.claim, + sandbox_refresh_claim_expires_at: refreshFence.expiresAt, + agent_token_hash: agentTokenHash, + last_event: "Cloudflare Sandbox lease refresh started", + updated_at: sql`MAX(updated_at + 1, ${now})`, + }) + .where("id", "=", session.id) + .where("lease_id", "=", originalLeaseId) + .where("status", "in", ["ready", "attached", "detached"]) + .executeTakeFirst(); + if ((claim.numUpdatedRows ?? 0n) === 0n) { + const current = await readInteractiveSession(env, session.id); + if (current && isSandboxInteractiveSession(current) && isCurrentSandboxLease(current.leaseId)) { + return current; + } + throw serviceUnavailable("Cloudflare Sandbox lease refresh is already in progress"); + } + let provisioned: InteractiveProvisionResult; + try { + provisioned = await provisionWithSandbox( + env, + refreshPayload, + agentToken, + refreshLease, + refreshFence, + ); + } catch (error) { + const message = `Cloudflare Sandbox lease refresh failed: ${safeProviderError(error)}`; + await stageFailedManagedSandboxProvision(env, session.id, refreshFence, message, Date.now()); + throw serviceUnavailable(message); + } + if (provisioned.status !== "ready") { + await stageFailedManagedSandboxProvision( + env, + session.id, + refreshFence, + provisioned.message, + Date.now(), + ); + throw serviceUnavailable(provisioned.message); + } + const refreshedAt = Date.now(); + const expectedLeaseId = sandboxLeaseId(refreshLease); + if (provisioned.leaseId !== expectedLeaseId) { + await stageFailedManagedSandboxProvision( + env, + session.id, + refreshFence, + "Cloudflare Sandbox lease refresh returned an unexpected lease", + refreshedAt, + ); + throw serviceUnavailable("Cloudflare Sandbox lease refresh returned an unexpected lease"); + } + const db = database(env); + const commitQueries: CompilableQuery[] = [ + db + .updateTable("interactive_sessions") + .set({ + status: provisioned.status, + lease_id: provisioned.leaseId, + attach_url: provisioned.attachUrl, + vnc_url: provisioned.vncUrl, + sandbox_refresh_sandbox_id: null, + sandbox_refresh_claim: null, + sandbox_refresh_claim_expires_at: null, + last_event: "Cloudflare Sandbox lease refreshed", + updated_at: sql`MAX(updated_at + 1, ${refreshedAt})`, + }) + .where("id", "=", session.id) + .where(sql`lease_id IS ${refreshFence.refreshLeaseId}`) + .where("sandbox_refresh_sandbox_id", "=", refreshFence.sandboxId) + .where("sandbox_refresh_claim", "=", refreshFence.claim) + .where("sandbox_refresh_claim_expires_at", "=", refreshFence.expiresAt) + .where("sandbox_refresh_claim_expires_at", ">", refreshedAt) + .where("agent_token_hash", "=", agentTokenHash) + .where("status", "in", ["ready", "attached", "detached"]), + ]; + if (oldSandboxId && oldSandboxId !== refreshLease.sandboxId) { + commitQueries.push( + db + .updateTable("interactive_session_credential_policies") + .set({ + state: "cleanup_pending", + cleanup_claim: null, + cleanup_claim_expires_at: null, + updated_at: refreshedAt, + }) + .where("session_id", "=", session.id) + .where("sandbox_id", "=", oldSandboxId).where(sql` + EXISTS ( + SELECT 1 + FROM interactive_sessions AS session + WHERE session.id = ${session.id} + AND session.lease_id = ${provisioned.leaseId} + AND session.sandbox_refresh_claim IS NULL + ) + `), + ); + } + await executeBatch(env, commitQueries); + const committed = await db + .selectFrom("interactive_sessions") + .select(["lease_id", "status", "credential_cleanup_terminal_status", "agent_token_hash"]) + .where("id", "=", session.id) + .executeTakeFirst(); + if ( + committed?.lease_id !== provisioned.leaseId || + committed.agent_token_hash !== agentTokenHash || + committed.credential_cleanup_terminal_status !== null || + !["ready", "attached", "detached"].includes(committed.status) + ) { + await stageFailedManagedSandboxProvision( + env, + session.id, + refreshFence, + "Cloudflare Sandbox lease refresh ownership changed", + refreshedAt, + ); + throw serviceUnavailable("Cloudflare Sandbox lease refresh is already in progress"); + } + if (oldSandboxId && oldSandboxId !== refreshLease.sandboxId) { + await reconcileCredentialPolicyCleanupBatch(env, refreshedAt, session.id); + } + const current = await readInteractiveSession(env, session.id); + if ( + !current || + current.leaseId !== provisioned.leaseId || + !["ready", "attached", "detached"].includes(current.status) + ) { + throw serviceUnavailable("previous Cloudflare Sandbox credential cleanup stopped the session"); + } + await appendInteractiveSessionLog( + env, + session.id, + user, + "Cloudflare Sandbox lease refreshed", + refreshedAt, + ); + const latest = await readInteractiveSession(env, session.id); + if ( + !latest || + latest.leaseId !== provisioned.leaseId || + !["ready", "attached", "detached"].includes(latest.status) + ) { + throw serviceUnavailable("previous Cloudflare Sandbox credential cleanup stopped the session"); + } + return { ...latest, ...(githubToken ? { githubToken } : {}) }; +} + +async function prepareSandboxWorkspace( + sandbox: SandboxSessionTarget, + env: RuntimeEnv, + session: SandboxRuntimeSession, + workdir: string, +): Promise { + const repoUrl = `https://github.com/${session.repo}.git`; + const quotedRepoUrl = shellQuote(repoUrl); + const quotedBranch = shellQuote(session.branch); + const quotedWorkdir = shellQuote(workdir); + const quotedPrompt = shellQuote(session.prompt); + const checkoutErrorPath = sandboxCheckoutErrorPath(session.id); + const quotedCheckoutErrorPath = shellQuote(checkoutErrorPath); + const resetResult = await sandbox.exec( + [ + `if [ ! -d ${quotedWorkdir}/.git ]; then`, + ` rm -rf ${quotedWorkdir}`, + ` mkdir -p ${quotedWorkdir}`, + `fi`, + `rm -f ${quotedCheckoutErrorPath}`, + ].join("\n"), + { timeout: 30_000 }, + ); + if (!resetResult.success) { + throw new Error( + clean(resetResult.stderr || resetResult.stdout || "workspace reset failed", 500), + ); + } + + const result = await sandbox.exec( + [ + "checkout_status=0", + "cat > /tmp/crabbox-git-askpass-placeholder.sh <<'EOF'", + "#!/bin/sh", + 'case "$1" in', + " *Username*) printf '%s\\n' x-access-token ;;", + ` *Password*) printf '%s\\n' ${shellQuote(sandboxPlaceholderGitHubToken)} ;;`, + " *) exit 1 ;;", + "esac", + "EOF", + "chmod 700 /tmp/crabbox-git-askpass-placeholder.sh", + "git_with_github_auth() {", + ` GIT_TERMINAL_PROMPT=0 GIT_USERNAME=x-access-token GIT_PASSWORD=${shellQuote( + sandboxPlaceholderGitHubToken, + )} GIT_ASKPASS=${shellQuote("/tmp/crabbox-git-askpass-placeholder.sh")} git -c credential.helper= "$@"`, + "}", + `if [ ! -d ${quotedWorkdir}/.git ]; then`, + ` tmp="${workdir}.clone.$$"`, + ` rm -rf "$tmp"`, + ` rm -f ${quotedCheckoutErrorPath}`, + ` if git_with_github_auth clone --depth 1 --branch ${quotedBranch} ${quotedRepoUrl} "$tmp" 2>/tmp/crabbox-git-clone.log || git_with_github_auth clone --depth 1 ${quotedRepoUrl} "$tmp" 2>>/tmp/crabbox-git-clone.log; then`, + ` if rm -rf ${quotedWorkdir} && mkdir -p ${quotedWorkdir} && cp -a "$tmp"/. ${quotedWorkdir}/; then`, + ` :`, + ` else`, + ` checkout_status=$?`, + ` printf 'Repository checkout copy failed for %s branch %s.\\n' ${quotedRepoUrl} ${quotedBranch} > ${quotedCheckoutErrorPath}`, + ` fi`, + ` else`, + ` printf 'Repository checkout failed for %s branch %s. See /tmp/crabbox-git-clone.log.\\n' ${quotedRepoUrl} ${quotedBranch} > ${quotedCheckoutErrorPath}`, + ` cat /tmp/crabbox-git-clone.log >> ${quotedCheckoutErrorPath} || true`, + ` checkout_status=70`, + ` fi`, + ` rm -rf "$tmp"`, + "fi", + `if [ "$checkout_status" -eq 0 ] && [ ! -d ${quotedWorkdir}/.git ]; then`, + ` if [ ! -s ${quotedCheckoutErrorPath} ]; then`, + ` printf 'Repository checkout failed for %s branch %s.\\n' ${quotedRepoUrl} ${quotedBranch} > ${quotedCheckoutErrorPath}`, + ` fi`, + ` checkout_status=70`, + `fi`, + `if [ "$checkout_status" -eq 0 ]; then`, + ` rm -f ${quotedCheckoutErrorPath}`, + ` cd ${quotedWorkdir} || checkout_status=$?`, + `fi`, + `if [ "$checkout_status" -eq 0 ]; then git config --global --add safe.directory ${quotedWorkdir} || true; fi`, + `if [ "$checkout_status" -eq 0 ]; then git remote set-url origin ${quotedRepoUrl} || true; fi`, + `if [ "$checkout_status" -eq 0 ]; then git_with_github_auth fetch --depth 1 origin ${quotedBranch} || checkout_status=$?; fi`, + `if [ "$checkout_status" -eq 0 ]; then git checkout -B ${quotedBranch} FETCH_HEAD || checkout_status=$?; fi`, + `if [ "$checkout_status" -eq 0 ]; then git rev-parse --verify HEAD >/dev/null || checkout_status=$?; fi`, + `if [ "$checkout_status" -eq 0 ]; then test "$(git rev-parse --abbrev-ref HEAD)" = ${quotedBranch} || checkout_status=$?; fi`, + `if [ "$checkout_status" -eq 0 ]; then test "$(git config --get remote.origin.url)" = ${quotedRepoUrl} || checkout_status=$?; fi`, + quotedPrompt + ? `if [ "$checkout_status" -eq 0 ]; then printf '%s\n' ${quotedPrompt} > .crabbox-initial-prompt.txt || checkout_status=$?; fi` + : `if [ "$checkout_status" -eq 0 ]; then rm -f .crabbox-initial-prompt.txt || checkout_status=$?; fi`, + `if [ "$checkout_status" -eq 0 ]; then`, + ` printf '\\nCRABBOX_CHECKOUT_OK\\n'`, + `else`, + ` if [ -s ${quotedCheckoutErrorPath} ]; then cat ${quotedCheckoutErrorPath}; fi`, + ` printf '\\nCRABBOX_CHECKOUT_FAILED %s\\n' "$checkout_status"`, + `fi`, + ].join("\n"), + { timeout: 120_000 }, + ); + const checkoutMarker = result.stdout.trim().split(/\r?\n/).at(-1); + if (!result.success || checkoutMarker !== "CRABBOX_CHECKOUT_OK") { + throw new Error( + clean( + [result.stdout, result.stderr].filter(Boolean).join("\n") || "repository checkout failed", + 700, + ), + ); + } +} + +async function prepareSandboxCodexAuth( + sandbox: SandboxSessionTarget, + env: RuntimeEnv, + workdir: string, +): Promise { + const projectKey = JSON.stringify(workdir); + const workspaceKey = JSON.stringify("/workspace"); + const result = await sandbox.exec( + ` +set -eu +export CODEX_HOME="$HOME/.codex" +mkdir -p "$CODEX_HOME" +cat > "$CODEX_HOME/config.toml" <<'EOF' +cli_auth_credentials_store = "file" +forced_login_method = "api" +preferred_auth_method = "apikey" +approval_policy = "never" +sandbox_mode = "danger-full-access" + +[shell_environment_policy] +inherit = "all" +ignore_default_excludes = true + +[features] +goals = true + +[projects.${projectKey}] +trust_level = "trusted" + +[projects.${workspaceKey}] +trust_level = "trusted" +EOF +if command -v node >/dev/null 2>&1; then + node - <<'NODE' +const fs = require("fs"); +const path = require("path"); +const home = process.env.CODEX_HOME; +const apiKey = process.env.OPENAI_API_KEY || ""; +if (!apiKey) process.exit(0); +fs.writeFileSync( + path.join(home, "auth.json"), + JSON.stringify({ OPENAI_API_KEY: apiKey, auth_mode: "apikey" }), + { mode: 0o600 } +); +NODE +elif command -v codex >/dev/null 2>&1 && [ -n "\${OPENAI_API_KEY:-}" ]; then + printf '%s' "$OPENAI_API_KEY" | codex -c 'forced_login_method="api"' login --with-api-key >/dev/null 2>&1 || true +fi +`, + { + timeout: 60_000, + env: { + OPENAI_API_KEY: env.OPENAI_API_KEY ? sandboxPlaceholderOpenAIKey : undefined, + OPENAI_BASE_URL: env.OPENAI_BASE_URL, + OPENAI_ORG_ID: env.OPENAI_ORG_ID, + }, + }, + ); + if (!result.success) { + throw new Error(clean(result.stderr || result.stdout || "Codex auth setup failed", 700)); + } +} + +async function prepareSandboxRuntimeTools( + sandbox: SandboxSessionTarget, + env: RuntimeEnv, + session: SandboxRuntimeSession, + workdir: string, + commandEnv: Record = {}, + agentToken?: string, +): Promise { + const autostartScript = sandboxAutostartScriptPath(session.id); + const terminalShell = sandboxTerminalShellPath(session.id); + const result = await sandbox.exec( + ` +set -eu +export CODEX_HOME="$HOME/.codex" +missing_tools="" +for tool in git node npm pnpm codex gh rg fd jq python3 make gcc time ssh rsync crabbox; do + if ! command -v "$tool" >/dev/null 2>&1; then + missing_tools="$missing_tools $tool" + fi +done +if [ -n "$missing_tools" ]; then + printf 'Crabfleet sandbox image is missing required tools:%s\\n' "$missing_tools" >/tmp/crabbox-runtime-tools.log + if command -v crabbox-diagnostics >/dev/null 2>&1; then + crabbox-diagnostics >>/tmp/crabbox-runtime-tools.log 2>&1 || true + fi + cat /tmp/crabbox-runtime-tools.log + exit 72 +fi +installed_codex="$(npm list -g @openai/codex --depth=0 --json 2>/dev/null | node -e 'let s="";process.stdin.on("data",d=>s+=d);process.stdin.on("end",()=>{try{const v=JSON.parse(s).dependencies?.["@openai/codex"]?.version||""; if (v) console.log(v);}catch{}})' || true)" +latest_codex="$(npm view @openai/codex version 2>/dev/null || true)" +if [ -z "$installed_codex" ] || { [ -n "$latest_codex" ] && [ "$installed_codex" != "$latest_codex" ]; }; then + if command -v timeout >/dev/null 2>&1; then + timeout 120s npm install -g @openai/codex@latest >/tmp/crabbox-codex-install.log 2>&1 + else + npm install -g @openai/codex@latest >/tmp/crabbox-codex-install.log 2>&1 + fi +fi +rm -f "$HOME/.config/crabbox/github-credential" 2>/dev/null || true +rm -rf "$HOME/.config/gh" "$HOME/.local/share/gh" 2>/dev/null || true +git config --global --unset-all credential.helper 2>/dev/null || true +git config --global credential.helper "!f() { test \\"\\$1\\" = get || exit 0; printf 'username=x-access-token\\n'; printf 'password=%s\\n' ${shellQuote(sandboxPlaceholderGitHubToken)}; }; f" +git config --global user.name ${shellQuote(session.owner)} +git config --global user.email ${shellQuote(`${session.owner}@users.noreply.github.com`)} +mkdir -p "$(dirname ${shellQuote(autostartScript)})" +cat > ${shellQuote(autostartScript)} <<'EOF' +export CODEX_HOME="$HOME/.codex" +export GITHUB_TOKEN=${shellQuote(sandboxPlaceholderGitHubToken)} +export GH_TOKEN=${shellQuote(sandboxPlaceholderGitHubToken)} +export CRABBOX_SESSION_ID=${shellQuote(session.id)} +export CRABFLEET_SESSION_ID=${shellQuote(session.id)} +export CRABFLEET_PARENT_SESSION_ID=${shellQuote(session.parentSessionId ?? "")} +export CRABFLEET_ROOT_SESSION_ID=${shellQuote(session.rootSessionId ?? session.id)} +export CRABFLEET_AGENT_TOKEN=${shellQuote(agentToken ?? "")} +export CRABFLEET_API_URL=${shellQuote(deploymentConfig(env).canonicalUrl)} +export CRABBOX_REPO=${shellQuote(session.repo)} +export CRABBOX_BRANCH=${shellQuote(session.branch)} +export CRABBOX_RUNTIME=${shellQuote(session.runtime)} +export CRABBOX_COMMAND=${shellQuote(session.command)} +export CRABBOX_CHECKOUT_ERROR=${shellQuote(sandboxCheckoutErrorPath(session.id))} +export CRABBOX_WORKDIR=${shellQuote(workdir)} +if [ -z "\${CRABBOX_SHELL_BOOTSTRAPPED:-}" ]; then + export CRABBOX_SHELL_BOOTSTRAPPED=1 + cd "$CRABBOX_WORKDIR" 2>/dev/null || true +fi +if [ -z "\${CRABBOX_CODEX_AUTOSTART_CHECKED:-}" ]; then + export CRABBOX_CODEX_AUTOSTART_CHECKED=1 + crabbox_autostart_marker="$HOME/.cache/crabbox/\${CRABBOX_SESSION_ID:-session}.codex-autostarted" + mkdir -p "$HOME/.cache/crabbox" 2>/dev/null || true + if [ ! -e "$crabbox_autostart_marker" ]; then + if [ -s "\${CRABBOX_CHECKOUT_ERROR:-}" ]; then + printf '\\nCrabfleet repository checkout failed:\\n' + cat "$CRABBOX_CHECKOUT_ERROR" + printf '\\n' + elif [ -n "\${CRABBOX_COMMAND:-}" ]; then + touch "$crabbox_autostart_marker" 2>/dev/null || true + ( + cd "$CRABBOX_WORKDIR" 2>/dev/null || { + printf 'Crabfleet workdir is unavailable: %s\\n' "$CRABBOX_WORKDIR" + exit 127 + } + env -u BASH_ENV -u PROMPT_COMMAND /bin/bash -c "$CRABBOX_COMMAND" + ) + fi + fi +fi +EOF +marker=${shellQuote(sandboxBashrcMarker(session))} +bashrc_tmp="$HOME/.bashrc.crabbox.$$" +{ + printf '%s\\n' "$marker" + printf '%s\\n' 'source ${shellQuote(autostartScript)} 2>/dev/null || true' + if [ -f "$HOME/.bashrc" ]; then + awk -v marker="$marker" '$0 == marker { getline; next } { print }' "$HOME/.bashrc" + fi +} > "$bashrc_tmp" +mv "$bashrc_tmp" "$HOME/.bashrc" +cat > ${shellQuote(terminalShell)} <<'EOF' +#!/bin/bash +cd ${shellQuote(workdir)} 2>/dev/null || true +source ${shellQuote(autostartScript)} 2>/dev/null || true +exec /bin/bash -i +EOF +chmod +x ${shellQuote(terminalShell)} +`, + { + timeout: 300_000, + env: commandEnv, + }, + ); + if (!result.success) { + throw new Error(clean(result.stderr || result.stdout || "runtime tool setup failed", 700)); + } } -async function provisionInteractiveEndpoint( +async function openSandboxTerminalResponse( request: Request, env: RuntimeEnv, -): Promise { - authorizeProvisionEndpoint(request, env); - const session = await readJson>(request); - const id = clean(session.id, 120); - const repo = normalizeRepo(session.repo); - const branch = clean(session.branch, 120) || "main"; - const runtime = oneOf(session.runtime, ["crabbox", "container"], "container") as - | "crabbox" - | "container"; - const command = interactiveCommand(session.command); - const prompt = clean(session.prompt, 4000); - const purpose = interactiveSessionPurpose(session.purpose, prompt, repo, branch, command); - const summary = interactiveSessionSummary(session.summary, purpose, prompt); - const owner = clean(session.owner, 240); - const githubToken = clean(session.githubToken, 4000) || undefined; - if (!id || !repo || !owner) { - return failedProvision("interactive provision failed: invalid session request"); - } - - const payload: InteractiveProvisionRequest = { - id, - repo, - branch, - runtime, - command, - prompt, - purpose, - summary, - owner, - createdBy: clean(session.createdBy, 240) || owner, - parentSessionId: clean(session.parentSessionId, 120) || null, - rootSessionId: clean(session.rootSessionId, 120) || id, - ...(githubToken ? { githubToken } : {}), + sandbox: ReturnType, + session: InteractiveSession & { githubToken?: string }, + size: { cols: number; rows: number }, +): Promise { + const lease = sandboxLeaseInfo(session); + const options = { + cols: size.cols, + rows: size.rows, + shell: sandboxTerminalShellPath(session.id), + }; + await ensureSandboxTerminalPrepared(sandbox, env, session, lease.terminalSessionId); + const open = async () => { + const terminalSession = await sandbox.getSession(lease.terminalSessionId); + return terminalSession.terminal(request, options); }; - return provisionInteractivePayload(env, payload); -} -function isBuiltInInteractiveProvisionUrl(value: string): boolean { - if (value === "/api/provision/interactive") return true; try { - const url = new URL(value); - return ( - url.pathname === "/api/provision/interactive" && - (url.hostname === appCanonicalHost || appRedirectHosts.has(url.hostname)) - ); + const response = await open(); + if (response.webSocket && response.status === 101) return response; } catch { - return false; - } -} - -async function provisionInteractivePayload( - env: RuntimeEnv, - payload: InteractiveProvisionRequest, -): Promise { - if (payload.runtime === "container" && env.SANDBOX) { - return provisionWithSandbox(env, payload); - } - if (env.CRABBOX_RUNTIME_PROVISION_URL) { - return forwardRuntimeProvision(env, payload); - } - if (payload.runtime === "container" && env.CRABBOX_CLOUDFLARE_RUNNER_URL) { - return provisionWithCloudflareRunner(env, payload); - } - if (payload.runtime === "crabbox" && env.CRABBOX_CLAWFLEET_URL) { - return provisionWithClawFleet(env, payload); + // A previous PTY disconnect can leave the SDK execution session terminated. } - return { - status: "pending_adapter", - leaseId: null, - attachUrl: null, - vncUrl: null, - message: "provision route live; runtime backend not configured", - }; -} -function authorizeProvisionEndpoint(request: Request, env: RuntimeEnv): void { - const hasBackend = Boolean( - env.SANDBOX || - env.CRABBOX_RUNTIME_PROVISION_URL || - env.CRABBOX_CLOUDFLARE_RUNNER_URL || - env.CRABBOX_CLAWFLEET_URL, - ); - if (!env.CRABBOX_INTERACTIVE_PROVISION_TOKEN) { - if (hasBackend) { - throw serviceUnavailable("interactive provision token is not configured"); - } - return; - } - const expected = `Bearer ${env.CRABBOX_INTERACTIVE_PROVISION_TOKEN}`; - if (request.headers.get("authorization") !== expected) throw unauthorized(); + await recreateSandboxTerminalSession(sandbox, env, session, lease.terminalSessionId); + return open(); } -async function provisionWithSandbox( +async function ensureSandboxTerminalPrepared( + sandbox: ReturnType, env: RuntimeEnv, - session: InteractiveProvisionRequest, - agentToken?: string, -): Promise { - if (!env.SANDBOX) { - return failedProvision("Cloudflare Sandbox binding is not configured"); - } - if (!env.SESSION_CONTROL) { - return failedProvision("SESSION_CONTROL Durable Object is not configured"); - } - if (!env.OPENAI_API_KEY) { - return failedProvision("OPENAI_API_KEY is not configured for Cloudflare Sandbox Codex"); - } - - const lease = newSandboxLease(session.id); + session: InteractiveSession & { githubToken?: string }, + terminalSessionId: string, +): Promise { const workdir = sandboxWorkdir(session.id); - const sandbox = getSandbox(env.SANDBOX, lease.sandboxId); try { - await registerSandboxCredentialPolicy(env, session, lease.sandboxId); - await setupSandboxTerminalSession( - sandbox, - env, - session, - workdir, - lease.terminalSessionId, - agentToken, - ); - } catch (error) { - await unregisterSandboxCredentialPolicy(env, lease.sandboxId); - const message = clean(error instanceof Error ? error.message : String(error), 240); - return failedProvision(`Cloudflare Sandbox provision failed: ${message}`); + if (await sandboxTerminalProfileExists(sandbox, env, session, workdir)) return; + await setupSandboxTerminalSession(sandbox, env, session, workdir, terminalSessionId); + return; + } catch { + // Missing or terminated default shell; recreate the sandbox below. } + await recreateSandboxTerminalSession(sandbox, env, session, terminalSessionId); +} - return { - status: "ready", - leaseId: sandboxLeaseId(lease), - attachUrl: `/api/interactive-sessions/${encodeURIComponent(session.id)}/pty`, - vncUrl: null, - message: `Cloudflare Sandbox ready for ${session.repo}`, - }; +async function sandboxTerminalProfileExists( + sandbox: CloudflareSandbox, + env: RuntimeEnv, + session: InteractiveSession & { githubToken?: string }, + workdir: string, +): Promise { + const setup = await createSandboxSession( + sandbox, + sandboxSetupSessionId(session.id), + "/workspace", + { + CRABBOX_SESSION_ID: session.id, + }, + ); + const marker = shellQuote(sandboxBashrcMarker(session)); + const autostartScript = sandboxAutostartScriptPath(session.id); + const terminalShell = sandboxTerminalShellPath(session.id); + const repoUrl = `https://github.com/${session.repo}.git`; + const checks = [ + `test -d ${shellQuote(workdir)}`, + `test -d ${shellQuote(workdir)}/.git`, + `test ! -s ${shellQuote(sandboxCheckoutErrorPath(session.id))}`, + `git -C ${shellQuote(workdir)} rev-parse --verify HEAD >/dev/null`, + `test "$(git -C ${shellQuote(workdir)} rev-parse --abbrev-ref HEAD)" = ${shellQuote(session.branch)}`, + `test "$(git -C ${shellQuote(workdir)} config --get remote.origin.url)" = ${shellQuote(repoUrl)}`, + `test -s ${shellQuote(autostartScript)}`, + `test -x ${shellQuote(terminalShell)}`, + `grep -Fqx '[shell_environment_policy]' "$HOME/.codex/config.toml"`, + `grep -Fqx '[projects."/workspace"]' "$HOME/.codex/config.toml"`, + `node -e 'const fs=require("fs"); const p=process.env.HOME+"/.codex/auth.json"; const auth=JSON.parse(fs.readFileSync(p,"utf8")); process.exit(auth.OPENAI_API_KEY==="crabfleet-worker-injected"?0:1)'`, + `grep -Fqx ' cd "$CRABBOX_WORKDIR" 2>/dev/null || {' ${shellQuote(autostartScript)}`, + `grep -Fqx ${marker} "$HOME/.bashrc"`, + `test ! -e "$HOME/.config/crabbox/github-credential"`, + ]; + const result = await setup.exec(checks.join(" && "), { timeout: 10_000 }); + return result.success; } -async function registerSandboxCredentialPolicy( +async function setupSandboxTerminalSession( + sandbox: CloudflareSandbox, env: RuntimeEnv, session: SandboxRuntimeSession, - sandboxId: string, + workdir: string, + terminalSessionId: string, + agentToken?: string, ): Promise { - const stub = sandboxControlStub(env); - if (!stub) throw new Error("SESSION_CONTROL Durable Object is not configured"); - const githubToken = "githubToken" in session ? session.githubToken : undefined; - const githubTokenCiphertext = githubToken ? await sealSecret(env, githubToken) : null; - if (githubToken && !githubTokenCiphertext) { - throw new Error( - "CRABBOX_TOKEN_ENCRYPTION_KEY or GITHUB_CLIENT_SECRET is required for user GitHub tokens", - ); - } - const effectiveGithubToken = githubToken ?? env.GITHUB_TOKEN; - const githubCredentialSource = githubTokenCiphertext - ? "session" - : env.GITHUB_TOKEN - ? "worker" - : "none"; - const githubRepoNodeId = effectiveGithubToken - ? await fetchGithubRepoNodeId(session.repo, effectiveGithubToken) - : null; - const policy: SandboxCredentialPolicy = { - allowedHosts: sandboxBackupAllowedHosts(env), - githubCredentialSource, - githubRepo: session.repo, - owner: session.owner, - sandboxId, - sessionId: session.id, - ...(githubRepoNodeId ? { githubRepoNodeId } : {}), - ...(githubTokenCiphertext ? { githubTokenCiphertext } : {}), - ...(env.OPENAI_BASE_URL ? { openAIBaseUrl: env.OPENAI_BASE_URL } : {}), - ...(env.OPENAI_ORG_ID ? { openAIOrgId: env.OPENAI_ORG_ID } : {}), - }; - for (const lookupId of sandboxLookupIds(env, sandboxId)) { - const response = await stub.fetch("https://crabfleet.internal/api/session-control/register", { - method: "POST", - body: JSON.stringify({ ...policy, sandboxId: lookupId }), - headers: { "content-type": "application/json" }, - }); - if (!response.ok) { - throw new Error("sandbox credential policy registration failed"); - } - } + const sessionEnv = sandboxSessionEnv(env, session, agentToken); + const setup = await createSandboxSession( + sandbox, + sandboxSetupSessionId(session.id), + "/workspace", + sessionEnv, + ); + await runSandboxSetupStep("workspace mkdir", () => setup.mkdir(workdir, { recursive: true })); + await runSandboxSetupStep("repository checkout", () => + prepareSandboxWorkspace(setup, env, session, workdir), + ); + await runSandboxSetupStep("Codex auth", () => prepareSandboxCodexAuth(setup, env, workdir)); + await runSandboxSetupStep("runtime tools", () => + prepareSandboxRuntimeTools(setup, env, session, workdir, {}, agentToken), + ); + await runSandboxSetupStep("terminal session", () => + createFreshSandboxSession(sandbox, terminalSessionId, workdir, sessionEnv), + ); } -async function ensureSandboxCredentialPolicy( +async function recreateSandboxTerminalSession( + sandbox: ReturnType, env: RuntimeEnv, - session: SandboxRuntimeSession, - sandboxId: string, + session: InteractiveSession & { githubToken?: string }, + terminalSessionId: string, ): Promise { - const hasFreshUserToken = Boolean("githubToken" in session && session.githubToken); - if (!hasFreshUserToken && (await sandboxCredentialPolicyExists(env, sandboxId))) return; - await registerSandboxCredentialPolicy(env, session, sandboxId); -} - -async function sandboxCredentialPolicyExists(env: RuntimeEnv, sandboxId: string): Promise { - const stub = sandboxControlStub(env); - if (!stub) return false; - const responses = await Promise.all( - sandboxLookupIds(env, sandboxId).map((lookupId) => - stub.fetch( - `https://crabfleet.internal/api/session-control/egress/${encodeURIComponent(lookupId)}`, - ), - ), + await setupSandboxTerminalSession( + sandbox, + env, + session, + sandboxWorkdir(session.id), + terminalSessionId, ); - return responses.some((response) => response.ok); } -async function fetchGithubRepoNodeId(repo: string, token: string): Promise { - const response = await fetch(`https://api.github.com/repos/${repo}`, { - headers: { - accept: "application/vnd.github+json", - authorization: `Bearer ${token}`, - "user-agent": "crabfleet", - "x-github-api-version": "2022-11-28", - }, - }); - if (!response.ok) { - throw new Error(`GitHub repository metadata lookup failed for ${repo}`); - } - const body = (await response.json()) as { node_id?: unknown }; - if (typeof body.node_id !== "string" || !body.node_id) { - throw new Error(`GitHub repository metadata lookup did not include node_id for ${repo}`); - } - return body.node_id; +function sandboxSessionEnv( + env: RuntimeEnv, + session: SandboxRuntimeSession, + agentToken?: string, +): Record { + return { + CRABBOX_SESSION_ID: session.id, + CRABFLEET_SESSION_ID: session.id, + CRABFLEET_PARENT_SESSION_ID: session.parentSessionId ?? undefined, + CRABFLEET_ROOT_SESSION_ID: session.rootSessionId ?? session.id, + CRABFLEET_AGENT_TOKEN: agentToken, + CRABFLEET_API_URL: deploymentConfig(env).canonicalUrl, + CRABBOX_REPO: session.repo, + CRABBOX_BRANCH: session.branch, + CRABBOX_RUNTIME: session.runtime, + TERM: "xterm-256color", + COLORTERM: "truecolor", + GH_TOKEN: sandboxHasGitHubCredential(env, session) ? sandboxPlaceholderGitHubToken : undefined, + GITHUB_TOKEN: sandboxHasGitHubCredential(env, session) + ? sandboxPlaceholderGitHubToken + : undefined, + TERM_PROGRAM: "ghostty", + TERM_PROGRAM_VERSION: "web", + OPENAI_API_KEY: env.OPENAI_API_KEY ? sandboxPlaceholderOpenAIKey : undefined, + OPENAI_BASE_URL: env.OPENAI_BASE_URL, + OPENAI_ORG_ID: env.OPENAI_ORG_ID, + }; } -async function githubNodeBelongsToRepo( - nodeId: string, - repo: string, - token: string, -): Promise { - const [owner, name] = repo.toLowerCase().split("/"); - if (!owner || !name) return false; - const response = await fetch("https://api.github.com/graphql", { - method: "POST", - body: JSON.stringify({ - query: `query($id: ID!) { - node(id: $id) { - __typename - ... on Repository { - owner { login } - name - } - ... on RepositoryNode { - repository { owner { login } name } - } - } - }`, - variables: { id: nodeId }, - }), - headers: { - accept: "application/vnd.github+json", - authorization: `Bearer ${token}`, - "content-type": "application/json", - "user-agent": "crabfleet", - "x-github-api-version": "2022-11-28", - }, - }); - if (!response.ok) return false; - const body = (await response.json().catch(() => null)) as { - data?: { - node?: { - name?: unknown; - owner?: { login?: unknown }; - repository?: { name?: unknown; owner?: { login?: unknown } }; - }; - }; - errors?: unknown; - } | null; - if (!body || body.errors) return false; - const node = body.data?.node; - const repository = node?.repository ?? node; - return ( - typeof repository?.owner?.login === "string" && - typeof repository.name === "string" && - repository.owner.login.toLowerCase() === owner && - repository.name.toLowerCase() === name - ); +function sandboxHasGitHubCredential(env: RuntimeEnv, session: SandboxRuntimeSession): boolean { + return Boolean(("githubToken" in session && session.githubToken) || env.GITHUB_TOKEN); } -function sandboxLookupIds(env: RuntimeEnv, sandboxId: string): string[] { - const ids = new Set([sandboxId]); - if (env.SANDBOX) ids.add(env.SANDBOX.idFromName(sandboxId).toString()); - return [...ids]; +function githubTokenEnv(session: Pick): { + GITHUB_TOKEN?: string; + GH_TOKEN?: string; +} { + return session.githubToken + ? { GITHUB_TOKEN: session.githubToken, GH_TOKEN: session.githubToken } + : {}; } -async function ensureCurrentSandboxLease( - request: Request, +async function provisionWithRuntimeAdapter( env: RuntimeEnv, - user: User | null, - session: InteractiveSession & { githubToken?: string }, -): Promise { - if (!env.SANDBOX) return session; - if (isCurrentSandboxLease(session.leaseId)) { - await ensureSandboxCredentialPolicy(env, session, sandboxLeaseInfo(session).sandboxId); - return session; + session: InteractiveProvisionRequest, + _agentToken?: string, +): Promise { + const namespace = normalizeAdapterNamespace(env.CRABBOX_RUNTIME_ADAPTER_NAMESPACE ?? ""); + const adapterWorkspaceId = session.adapterWorkspaceId + ? normalizeAdapterWorkspaceId(session.adapterWorkspaceId) === session.adapterWorkspaceId + ? session.adapterWorkspaceId + : null + : namespace + ? namespacedAdapterWorkspaceId(namespace, session.id) + : null; + if (!adapterWorkspaceId) { + return failedProvision( + "runtime adapter provision failed: persisted workspace id or valid namespace is required", + ); } - const originalLeaseId = session.leaseId; - if (!originalLeaseId) { - throw serviceUnavailable("Cloudflare Sandbox lease refresh is already in progress"); + const fallbackCapabilities = + session.runtime === "crabbox" ? crabboxCapabilities : containerCapabilities; + let baseUrl: string; + try { + baseUrl = requireRegisteredRuntimeAdapterControlPlane(env, session.adapterControlPlane); + } catch (error) { + return unresolvedRuntimeAdapterProvision( + session, + adapterWorkspaceId, + fallbackCapabilities, + clean(error instanceof Error ? error.message : String(error), 240), + ); } - const refreshStartedAt = sandboxLeaseRefreshStartedAt(originalLeaseId); - const now = Date.now(); - if (refreshStartedAt && now - refreshStartedAt < 2 * 60_000) { - throw serviceUnavailable("Cloudflare Sandbox lease refresh is already in progress"); + const requestedCapabilities = session.adapterRequestedCapabilities; + const ttlSeconds = persistedRuntimeAdapterSeconds(session.adapterTtlSeconds); + const idleTimeoutSeconds = persistedRuntimeAdapterSeconds(session.adapterIdleTimeoutSeconds); + if (!requestedCapabilities || !ttlSeconds || !idleTimeoutSeconds) { + return releaseFailedRuntimeAdapterProvision( + env, + session.id, + runtimeAdapterFailureProvision( + session, + adapterWorkspaceId, + requestedCapabilities ?? fallbackCapabilities, + "runtime adapter provision failed: persisted create settings are incomplete", + ), + ); } - if (!user || actor(user) !== session.owner) { - throw serviceUnavailable("session owner must reconnect to refresh Cloudflare Sandbox lease"); + const generatedPayload = session.adapterCreatePayloadJson + ? null + : runtimeAdapterCreatePayload( + { + namespace: namespace ?? "", + id: session.id, + parentSessionId: session.parentSessionId, + rootSessionId: session.rootSessionId, + repo: session.repo, + branch: session.branch, + runtime: session.runtime, + profile: session.profile, + command: session.command, + prompt: session.prompt, + purpose: session.purpose, + summary: session.summary, + owner: session.owner, + createdBy: session.createdBy, + ttlSeconds, + idleTimeoutSeconds, + desktop: requestedCapabilities.desktop, + }, + adapterWorkspaceId, + ); + const createPayloadJson = validatedRuntimeAdapterCreatePayloadJson( + session.adapterCreatePayloadJson ?? (generatedPayload ? JSON.stringify(generatedPayload) : ""), + { + workspaceId: adapterWorkspaceId, + ttlSeconds, + idleTimeoutSeconds, + desktop: requestedCapabilities.desktop, + }, + ); + if (!createPayloadJson) { + return releaseFailedRuntimeAdapterProvision( + env, + session.id, + runtimeAdapterFailureProvision( + session, + adapterWorkspaceId, + requestedCapabilities, + "runtime adapter provision failed: persisted create payload is invalid", + ), + ); } - const githubToken = user?.subject.startsWith("github:") - ? (session.githubToken ?? (await sessionGitHubToken(request, env))) - : undefined; - if (user.subject.startsWith("github:") && !githubToken) { - throw forbidden("GitHub PR credentials are not connected; sign in with GitHub again"); + if ( + !(await stageRuntimeAdapterProvision( + env, + session, + baseUrl, + adapterWorkspaceId, + requestedCapabilities, + ttlSeconds, + idleTimeoutSeconds, + createPayloadJson, + )) + ) { + return unresolvedRuntimeAdapterProvision( + session, + adapterWorkspaceId, + requestedCapabilities, + "runtime adapter control-plane registration changed before create", + ); + } + let response: Response; + try { + response = await runtimeAdapterFetch(env, runtimeAdapterCollectionUrl(baseUrl), { + method: "POST", + headers: { "idempotency-key": adapterWorkspaceId }, + body: createPayloadJson, + }); + } catch (error) { + return ambiguousRuntimeAdapterProvision( + session, + adapterWorkspaceId, + requestedCapabilities, + `runtime adapter create outcome unknown: ${safeProviderError(error, [adapterWorkspaceId])}`, + ); } - const fallbackLeaseId = sandboxLeaseWithoutRefresh(originalLeaseId); - const oldSandboxId = originalLeaseId.startsWith(sandboxLeasePrefix) - ? sandboxLeaseInfo({ id: session.id, leaseId: fallbackLeaseId }).sandboxId - : null; - const refreshLeaseId = `${fallbackLeaseId}:refreshing-${now}-${crypto.randomUUID().slice(0, 8)}`; - const claim = await database(env) - .updateTable("interactive_sessions") - .set({ - lease_id: refreshLeaseId, - last_event: "Cloudflare Sandbox lease refresh started", - updated_at: now, - }) - .where("id", "=", session.id) - .where("lease_id", "=", originalLeaseId) - .where("status", "in", ["ready", "attached", "detached"]) - .executeTakeFirst(); - if ((claim.numUpdatedRows ?? 0n) === 0n) { - const current = await readInteractiveSession(env, session.id); - if (current && isCurrentSandboxLease(current.leaseId)) return current; - throw serviceUnavailable("Cloudflare Sandbox lease refresh is already in progress"); + let responseBody: unknown; + try { + responseBody = await readRuntimeAdapterResponseBody(response); + } catch (error) { + return ambiguousRuntimeAdapterProvision( + session, + adapterWorkspaceId, + requestedCapabilities, + `runtime adapter create outcome unknown: ${safeProviderError(error, [adapterWorkspaceId])}`, + ); } - const provisioned = await provisionWithSandbox(env, { - id: session.id, - parentSessionId: session.parentSessionId, - rootSessionId: session.rootSessionId ?? session.id, - repo: session.repo, - branch: session.branch, - runtime: session.runtime, - command: session.command, - prompt: session.prompt, - purpose: session.purpose, - summary: session.summary, - owner: session.owner, - createdBy: session.createdBy, - ...(githubToken ? { githubToken } : {}), + if (!response.ok) { + const responseMessage = redactedAdapterResponseMessage( + responseBody, + `HTTP ${response.status}`, + [adapterWorkspaceId], + ); + if (definitiveRuntimeAdapterCreateFailure(response.status)) { + return releaseFailedRuntimeAdapterProvision( + env, + session.id, + runtimeAdapterFailureProvision( + session, + adapterWorkspaceId, + requestedCapabilities, + `runtime adapter provision failed: ${responseMessage}`, + ), + ); + } + return ambiguousRuntimeAdapterProvision( + session, + adapterWorkspaceId, + requestedCapabilities, + `runtime adapter create outcome unknown: ${responseMessage}`, + ); + } + const parsed = parseAdapterWorkspaceResult(responseBody, { + workspaceId: adapterWorkspaceId, + profile: session.profile, }); - if (provisioned.status === "failed") { - await database(env) - .updateTable("interactive_sessions") - .set({ - lease_id: fallbackLeaseId, - last_event: provisioned.message, - updated_at: Date.now(), - }) - .where("id", "=", session.id) - .where("lease_id", "=", refreshLeaseId) - .execute(); - throw serviceUnavailable(provisioned.message); + if (!parsed) { + return ambiguousRuntimeAdapterProvision( + session, + adapterWorkspaceId, + requestedCapabilities, + "runtime adapter create outcome unknown: invalid workspace response", + ); } - const refreshedAt = Date.now(); - const update = await database(env) + if (!adapterWorkspaceIdMatches(parsed, adapterWorkspaceId)) { + return ambiguousRuntimeAdapterProvision( + session, + adapterWorkspaceId, + requestedCapabilities, + "runtime adapter create outcome unknown: workspace identity mismatch", + ); + } + const result = runtimeAdapterProvisionResult( + parsed, + session, + Date.now(), + adapterWorkspaceId, + true, + ); + return result.status === "failed" + ? releaseFailedRuntimeAdapterProvision(env, session.id, result) + : result; +} + +function persistedRuntimeAdapterSeconds(value: number | null | undefined): number | null { + return typeof value === "number" && Number.isInteger(value) && value > 0 ? value : null; +} + +async function stageRuntimeAdapterProvision( + env: RuntimeEnv, + session: InteractiveProvisionRequest, + adapterControlPlane: string, + adapterWorkspaceId: string, + capabilities: RuntimeCapabilities, + ttlSeconds: number, + idleTimeoutSeconds: number, + createPayloadJson: string, +): Promise { + const staged = await database(env) .updateTable("interactive_sessions") .set({ - status: provisioned.status, - lease_id: provisioned.leaseId, - attach_url: provisioned.attachUrl, - vnc_url: provisioned.vncUrl, - last_event: "Cloudflare Sandbox lease refreshed", - updated_at: refreshedAt, + adapter: runtimeAdapterName, + profile: session.profile, + adapter_workspace_id: adapterWorkspaceId, + capabilities_json: JSON.stringify(capabilities), + adapter_ttl_seconds: ttlSeconds, + adapter_idle_timeout_seconds: idleTimeoutSeconds, + adapter_requested_capabilities_json: JSON.stringify(capabilities), + adapter_create_payload_json: createPayloadJson, + adapter_create_pending: 1, + reconcile_error: "runtime adapter create pending", }) .where("id", "=", session.id) - .where("lease_id", "=", refreshLeaseId) - .where("status", "in", ["ready", "attached", "detached"]) + .where("adapter_control_plane", "=", adapterControlPlane) + .where("status", "in", ["provisioning", "pending_adapter"]) .executeTakeFirst(); - if ((update.numUpdatedRows ?? 0n) === 0n) { - if (provisioned.leaseId?.startsWith(sandboxLeasePrefix)) { - await unregisterSandboxCredentialPolicy( + return (staged.numUpdatedRows ?? 0n) > 0n; +} + +function ambiguousRuntimeAdapterProvision( + session: Pick, + adapterWorkspaceId: string, + capabilities: RuntimeCapabilities, + message: string, +): InteractiveProvisionResult { + return { + status: "provisioning", + leaseId: null, + attachUrl: null, + attachUrlPresent: true, + vncUrl: null, + message, + adapter: runtimeAdapterName, + profile: session.profile, + adapterWorkspaceId, + providerResourceId: null, + capabilities, + capabilitiesPresent: true, + expiresAt: null, + expiresAtPresent: false, + reconciledAt: Date.now(), + reconcileError: message, + terminalStatus: null, + createPending: true, + }; +} + +function runtimeAdapterFailureProvision( + session: Pick, + adapterWorkspaceId: string, + capabilities: RuntimeCapabilities, + message: string, +): InteractiveProvisionResult { + return { + ...ambiguousRuntimeAdapterProvision(session, adapterWorkspaceId, capabilities, message), + status: "failed", + terminalStatus: null, + createPending: false, + }; +} + +function unresolvedRuntimeAdapterProvision( + session: Pick, + adapterWorkspaceId: string, + capabilities: RuntimeCapabilities, + message: string, +): InteractiveProvisionResult { + return { + ...runtimeAdapterFailureProvision(session, adapterWorkspaceId, capabilities, message), + status: "stopping", + message: `${message}; runtime workspace outcome unresolved`, + reconcileError: message, + terminalStatus: "failed", + createPending: true, + }; +} + +async function releaseFailedRuntimeAdapterProvision( + env: RuntimeEnv, + sessionId: string, + result: InteractiveProvisionResult, +): Promise { + const adapterWorkspaceId = result.adapterWorkspaceId; + if (!adapterWorkspaceId) return result; + await stageFailedRuntimeAdapterRelease(env, sessionId, adapterWorkspaceId, result.message); + try { + const release = await stopRuntimeAdapterWorkspaceForSession(env, sessionId, adapterWorkspaceId); + const releaseState = adapterFailureReleaseState(release.status); + if (release.status === "stopped") { + await recordConfirmedRuntimeAdapterRelease( env, - sandboxLeaseInfo({ id: session.id, leaseId: provisioned.leaseId }).sandboxId, + sessionId, + adapterWorkspaceId, + Date.now(), + release.message, ); } - const current = await readInteractiveSession(env, session.id); - if (current && isCurrentSandboxLease(current.leaseId)) return current; - throw serviceUnavailable("Cloudflare Sandbox lease refresh is already in progress"); - } - const newSandboxId = provisioned.leaseId?.startsWith(sandboxLeasePrefix) - ? sandboxLeaseInfo({ id: session.id, leaseId: provisioned.leaseId }).sandboxId - : null; - if (oldSandboxId && oldSandboxId !== newSandboxId) { - await unregisterSandboxCredentialPolicy(env, oldSandboxId); + const releaseMessage = `${result.message}; ${release.message}`; + if (release.status === "stopping") { + await persistRuntimeAdapterStopEvidence( + env, + sessionId, + adapterWorkspaceId, + releaseMessage, + Date.now(), + ); + } + return { + ...result, + status: releaseState.status, + attachUrl: null, + vncUrl: null, + message: releaseMessage, + reconcileError: release.status === "stopping" ? releaseMessage : result.message, + terminalStatus: releaseState.terminalStatus, + }; + } catch (error) { + const releaseError = safeProviderError( + error, + [adapterWorkspaceId, result.providerResourceId ?? null], + [result.attachUrl], + ); + const releaseState = adapterFailureReleaseState("stopping"); + const pendingMessage = `${result.message}; ${releaseState.message}: ${releaseError}`; + await persistRuntimeAdapterStopEvidence( + env, + sessionId, + adapterWorkspaceId, + pendingMessage, + Date.now(), + ); + return { + ...result, + status: releaseState.status, + attachUrl: null, + vncUrl: null, + message: pendingMessage, + reconcileError: pendingMessage, + terminalStatus: releaseState.terminalStatus, + }; } - await appendInteractiveSessionLog( - env, - session.id, - user, - "Cloudflare Sandbox lease refreshed", - refreshedAt, - ); - return { - ...session, - status: provisioned.status, - leaseId: provisioned.leaseId, - attachUrl: provisioned.attachUrl, - vncUrl: provisioned.vncUrl, - lastEvent: "Cloudflare Sandbox lease refreshed", - ...(githubToken ? { githubToken } : {}), - }; } -async function prepareSandboxWorkspace( - sandbox: SandboxSessionTarget, +async function persistRuntimeAdapterStopEvidence( env: RuntimeEnv, - session: SandboxRuntimeSession, - workdir: string, + sessionId: string, + adapterWorkspaceId: string, + message: string, + now: number, + reconcileError: string | null = message, + eventActor = "system", ): Promise { - const repoUrl = `https://github.com/${session.repo}.git`; - const quotedRepoUrl = shellQuote(repoUrl); - const quotedBranch = shellQuote(session.branch); - const quotedWorkdir = shellQuote(workdir); - const quotedPrompt = shellQuote(session.prompt); - const checkoutErrorPath = sandboxCheckoutErrorPath(session.id); - const quotedCheckoutErrorPath = shellQuote(checkoutErrorPath); - const resetResult = await sandbox.exec( - [ - `if [ ! -d ${quotedWorkdir}/.git ]; then`, - ` rm -rf ${quotedWorkdir}`, - ` mkdir -p ${quotedWorkdir}`, - `fi`, - `rm -f ${quotedCheckoutErrorPath}`, - ].join("\n"), - { timeout: 30_000 }, - ); - if (!resetResult.success) { - throw new Error( - clean(resetResult.stderr || resetResult.stdout || "workspace reset failed", 500), - ); - } + const evidence = clean(message, 500); + const errorEvidence = reconcileError ? clean(reconcileError, 500) : null; + const actorName = clean(eventActor, 120) || "system"; + const reconcileErrorOwner = errorEvidence + ? sql`reconcile_error = ${errorEvidence}` + : sql`reconcile_error IS NULL`; + const db = database(env); + await executeBatch(env, [ + db + .updateTable("interactive_sessions") + .set({ + last_reconciled_at: now, + reconcile_error: errorEvidence, + last_event: evidence, + updated_at: sql`MAX(updated_at + 1, ${now})`, + }) + .where("id", "=", sessionId) + .where("adapter", "=", runtimeAdapterName) + .where("adapter_workspace_id", "=", adapterWorkspaceId) + .where("status", "=", "stopping").where(sql` + COALESCE(last_event, '') != ${evidence} + OR COALESCE(reconcile_error, '') != ${errorEvidence ?? ""} + `), + sql` + INSERT INTO interactive_session_events (session_id, actor, message, created_at) + SELECT ${sessionId}, ${actorName}, ${evidence}, ${now} + WHERE EXISTS ( + SELECT 1 + FROM interactive_sessions + WHERE id = ${sessionId} + AND adapter = ${runtimeAdapterName} + AND adapter_workspace_id = ${adapterWorkspaceId} + AND status = 'stopping' + AND last_event = ${evidence} + AND ${reconcileErrorOwner} + ) + AND NOT EXISTS ( + SELECT 1 + FROM interactive_session_events + WHERE session_id = ${sessionId} + AND actor = ${actorName} + AND message = ${evidence} + ) + `, + ]); + await archiveInteractiveSessionLogs(env, sessionId, now).catch(() => undefined); +} - const result = await sandbox.exec( - [ - "checkout_status=0", - "cat > /tmp/crabbox-git-askpass-placeholder.sh <<'EOF'", - "#!/bin/sh", - 'case "$1" in', - " *Username*) printf '%s\\n' x-access-token ;;", - ` *Password*) printf '%s\\n' ${shellQuote(sandboxPlaceholderGitHubToken)} ;;`, - " *) exit 1 ;;", - "esac", - "EOF", - "chmod 700 /tmp/crabbox-git-askpass-placeholder.sh", - "git_with_github_auth() {", - ` GIT_TERMINAL_PROMPT=0 GIT_USERNAME=x-access-token GIT_PASSWORD=${shellQuote( - sandboxPlaceholderGitHubToken, - )} GIT_ASKPASS=${shellQuote("/tmp/crabbox-git-askpass-placeholder.sh")} git -c credential.helper= "$@"`, - "}", - `if [ ! -d ${quotedWorkdir}/.git ]; then`, - ` tmp="${workdir}.clone.$$"`, - ` rm -rf "$tmp"`, - ` rm -f ${quotedCheckoutErrorPath}`, - ` if git_with_github_auth clone --depth 1 --branch ${quotedBranch} ${quotedRepoUrl} "$tmp" 2>/tmp/crabbox-git-clone.log || git_with_github_auth clone --depth 1 ${quotedRepoUrl} "$tmp" 2>>/tmp/crabbox-git-clone.log; then`, - ` if rm -rf ${quotedWorkdir} && mkdir -p ${quotedWorkdir} && cp -a "$tmp"/. ${quotedWorkdir}/; then`, - ` :`, - ` else`, - ` checkout_status=$?`, - ` printf 'Repository checkout copy failed for %s branch %s.\\n' ${quotedRepoUrl} ${quotedBranch} > ${quotedCheckoutErrorPath}`, - ` fi`, - ` else`, - ` printf 'Repository checkout failed for %s branch %s. See /tmp/crabbox-git-clone.log.\\n' ${quotedRepoUrl} ${quotedBranch} > ${quotedCheckoutErrorPath}`, - ` cat /tmp/crabbox-git-clone.log >> ${quotedCheckoutErrorPath} || true`, - ` checkout_status=70`, - ` fi`, - ` rm -rf "$tmp"`, - "fi", - `if [ "$checkout_status" -eq 0 ] && [ ! -d ${quotedWorkdir}/.git ]; then`, - ` if [ ! -s ${quotedCheckoutErrorPath} ]; then`, - ` printf 'Repository checkout failed for %s branch %s.\\n' ${quotedRepoUrl} ${quotedBranch} > ${quotedCheckoutErrorPath}`, - ` fi`, - ` checkout_status=70`, - `fi`, - `if [ "$checkout_status" -eq 0 ]; then`, - ` rm -f ${quotedCheckoutErrorPath}`, - ` cd ${quotedWorkdir} || checkout_status=$?`, - `fi`, - `if [ "$checkout_status" -eq 0 ]; then git config --global --add safe.directory ${quotedWorkdir} || true; fi`, - `if [ "$checkout_status" -eq 0 ]; then git remote set-url origin ${quotedRepoUrl} || true; fi`, - `if [ "$checkout_status" -eq 0 ]; then git_with_github_auth fetch --depth 1 origin ${quotedBranch} || checkout_status=$?; fi`, - `if [ "$checkout_status" -eq 0 ]; then git checkout -B ${quotedBranch} FETCH_HEAD || checkout_status=$?; fi`, - `if [ "$checkout_status" -eq 0 ]; then git rev-parse --verify HEAD >/dev/null || checkout_status=$?; fi`, - `if [ "$checkout_status" -eq 0 ]; then test "$(git rev-parse --abbrev-ref HEAD)" = ${quotedBranch} || checkout_status=$?; fi`, - `if [ "$checkout_status" -eq 0 ]; then test "$(git config --get remote.origin.url)" = ${quotedRepoUrl} || checkout_status=$?; fi`, - quotedPrompt - ? `if [ "$checkout_status" -eq 0 ]; then printf '%s\n' ${quotedPrompt} > .crabbox-initial-prompt.txt || checkout_status=$?; fi` - : `if [ "$checkout_status" -eq 0 ]; then rm -f .crabbox-initial-prompt.txt || checkout_status=$?; fi`, - `if [ "$checkout_status" -eq 0 ]; then`, - ` printf '\\nCRABBOX_CHECKOUT_OK\\n'`, - `else`, - ` if [ -s ${quotedCheckoutErrorPath} ]; then cat ${quotedCheckoutErrorPath}; fi`, - ` printf '\\nCRABBOX_CHECKOUT_FAILED %s\\n' "$checkout_status"`, - `fi`, - ].join("\n"), - { timeout: 120_000 }, +async function stageFailedRuntimeAdapterRelease( + env: RuntimeEnv, + sessionId: string, + adapterWorkspaceId: string, + message: string, +): Promise { + const now = Date.now(); + await database(env) + .updateTable("interactive_sessions") + .set({ + status: "stopping", + lease_id: null, + attach_url: null, + vnc_url: null, + terminal_status: "failed", + terminal_failure_reason: message, + adapter_create_pending: 0, + last_reconciled_at: now, + reconcile_error: message, + agent_token_hash: null, + controller: null, + control_requested_by: null, + control_requested_at: null, + control_granted_at: null, + control_expires_at: null, + updated_at: sql`MAX(updated_at + 1, ${now})`, + last_event: `${message}; runtime workspace release pending`, + }) + .where("id", "=", sessionId) + .where("adapter", "=", runtimeAdapterName) + .where("adapter_workspace_id", "=", adapterWorkspaceId) + .where("status", "in", [ + "provisioning", + "pending_adapter", + "ready", + "attached", + "detached", + "stopping", + ]) + .execute(); +} + +function runtimeAdapterProvisionResult( + result: AdapterWorkspaceResult, + session: Pick & { + capabilities_json?: string; + }, + reconciledAt: number, + adapterWorkspaceId: string, + initialCreate: boolean, +): InteractiveProvisionResult { + const defaultCapabilities = session.capabilities_json + ? runtimeCapabilities(session.runtime, session.capabilities_json) + : session.runtime === "crabbox" + ? crabboxCapabilities + : containerCapabilities; + const capabilities = effectiveAdapterCapabilities(result, defaultCapabilities, initialCreate); + return { + status: result.status, + leaseId: null, + attachUrl: result.terminalUrl, + attachUrlPresent: initialCreate || result.terminalUrlPresent, + // Desktop access is minted only after Crabfleet authenticates the viewer. + vncUrl: null, + message: result.message, + adapter: runtimeAdapterName, + profile: result.profile ?? session.profile, + adapterWorkspaceId, + providerResourceId: result.providerResourceId, + ...(capabilities === undefined ? {} : { capabilities, capabilitiesPresent: true }), + ...(initialCreate || result.expiresAtPresent + ? { expiresAt: result.expiresAt, expiresAtPresent: true } + : {}), + reconciledAt, + reconcileError: null, + createPending: false, + }; +} + +async function inspectRuntimeAdapterWorkspace( + env: RuntimeEnv, + session: InteractiveSessionRow, +): Promise { + const adapterWorkspaceId = session.adapter_workspace_id; + const providerResourceId = session.provider_resource_id; + if (!adapterWorkspaceId) { + throw new Error("runtime adapter workspace reference is incomplete"); + } + const controlPlane = requireRegisteredRuntimeAdapterControlPlane( + env, + session.adapter_control_plane, ); - const checkoutMarker = result.stdout.trim().split(/\r?\n/).at(-1); - if (!result.success || checkoutMarker !== "CRABBOX_CHECKOUT_OK") { + if (session.status === "stopping") { + return reconcileStoppingRuntimeAdapterWorkspace(env, session); + } + const response = await runtimeAdapterFetch( + env, + runtimeAdapterWorkspaceUrl(controlPlane, adapterWorkspaceId), + { method: "GET" }, + ); + const responseBody = await readRuntimeAdapterResponseBody(response); + if (response.status === 404) { + if (shouldReplayRuntimeAdapterCreate(session.status, session.adapter_create_pending === 1)) { + return provisionWithRuntimeAdapter(env, runtimeAdapterReplayRequest(session)); + } + return { + status: "expired", + leaseId: null, + attachUrl: null, + attachUrlPresent: true, + vncUrl: null, + message: "runtime adapter workspace is gone", + adapter: runtimeAdapterName, + profile: session.profile, + adapterWorkspaceId, + providerResourceId, + reconciledAt: Date.now(), + reconcileError: null, + createPending: false, + }; + } + if (!response.ok) { throw new Error( - clean( - [result.stdout, result.stderr].filter(Boolean).join("\n") || "repository checkout failed", - 700, + redactedAdapterResponseMessage( + responseBody, + `runtime adapter inspect HTTP ${response.status}`, + [adapterWorkspaceId, providerResourceId], ), ); } + const parsed = parseAdapterWorkspaceResult(responseBody, { + workspaceId: adapterWorkspaceId, + providerResourceId, + profile: session.profile, + }); + if (!parsed) throw new Error("runtime adapter inspect returned an invalid workspace"); + if (!adapterWorkspaceIdMatches(parsed, adapterWorkspaceId)) { + throw new Error("runtime adapter inspect returned a different workspace id"); + } + const result = runtimeAdapterProvisionResult( + parsed, + session, + Date.now(), + adapterWorkspaceId, + false, + ); + return result.status === "failed" + ? releaseFailedRuntimeAdapterProvision(env, session.id, result) + : result; } -async function prepareSandboxCodexAuth( - sandbox: SandboxSessionTarget, +async function reconcileStoppingRuntimeAdapterWorkspace( env: RuntimeEnv, - workdir: string, -): Promise { - const projectKey = JSON.stringify(workdir); - const workspaceKey = JSON.stringify("/workspace"); - const result = await sandbox.exec( - ` -set -eu -export CODEX_HOME="$HOME/.codex" -mkdir -p "$CODEX_HOME" -cat > "$CODEX_HOME/config.toml" <<'EOF' -cli_auth_credentials_store = "file" -forced_login_method = "api" -preferred_auth_method = "apikey" -approval_policy = "never" -sandbox_mode = "danger-full-access" - -[shell_environment_policy] -inherit = "all" -ignore_default_excludes = true - -[features] -goals = true + session: InteractiveSessionRow, +): Promise { + const adapterWorkspaceId = session.adapter_workspace_id; + if (!adapterWorkspaceId) throw new Error("runtime adapter workspace reference is incomplete"); -[projects.${projectKey}] -trust_level = "trusted" + let replayMessage: string | null = null; + if (session.adapter_create_pending === 1) { + const replay = await replayStoppingRuntimeAdapterCreate(env, session); + replayMessage = replay.message; + } -[projects.${workspaceKey}] -trust_level = "trusted" -EOF -if command -v node >/dev/null 2>&1; then - node - <<'NODE' -const fs = require("fs"); -const path = require("path"); -const home = process.env.CODEX_HOME; -const apiKey = process.env.OPENAI_API_KEY || ""; -if (!apiKey) process.exit(0); -fs.writeFileSync( - path.join(home, "auth.json"), - JSON.stringify({ OPENAI_API_KEY: apiKey, auth_mode: "apikey" }), - { mode: 0o600 } -); -NODE -elif command -v codex >/dev/null 2>&1 && [ -n "\${OPENAI_API_KEY:-}" ]; then - printf '%s' "$OPENAI_API_KEY" | codex -c 'forced_login_method="api"' login --with-api-key >/dev/null 2>&1 || true -fi -`, - { - timeout: 60_000, - env: { - OPENAI_API_KEY: env.OPENAI_API_KEY ? sandboxPlaceholderOpenAIKey : undefined, - OPENAI_BASE_URL: env.OPENAI_BASE_URL, - OPENAI_ORG_ID: env.OPENAI_ORG_ID, - }, - }, - ); - if (!result.success) { - throw new Error(clean(result.stderr || result.stdout || "Codex auth setup failed", 700)); + let release: RuntimeAdapterStopResult; + try { + release = await stopRuntimeAdapterWorkspace( + env, + requireRegisteredRuntimeAdapterControlPlane(env, session.adapter_control_plane), + adapterWorkspaceId, + ); + } catch (error) { + const message = `runtime adapter stop pending: ${safeProviderError( + error, + [adapterWorkspaceId, session.provider_resource_id], + [session.attach_url], + )}`; + return { + status: "stopping", + leaseId: null, + attachUrl: null, + attachUrlPresent: true, + vncUrl: null, + message: replayMessage ? `${replayMessage}; ${message}` : message, + adapter: runtimeAdapterName, + profile: session.profile, + adapterWorkspaceId, + providerResourceId: session.provider_resource_id, + reconciledAt: Date.now(), + reconcileError: message, + terminalStatus: session.terminal_status, + createPending: session.adapter_create_pending === 1, + }; + } + if (release.status === "stopped") { + await recordConfirmedRuntimeAdapterRelease( + env, + session.id, + adapterWorkspaceId, + Date.now(), + release.message, + ); } + const lifecycle = await database(env) + .selectFrom("interactive_sessions") + .select(["status", "terminal_status", "adapter_create_pending"]) + .where("id", "=", session.id) + .where("adapter", "=", runtimeAdapterName) + .where("adapter_workspace_id", "=", adapterWorkspaceId) + .executeTakeFirst(); + const status = lifecycle?.status ?? (release.status === "stopped" ? "stopped" : "stopping"); + const createPending = lifecycle?.adapter_create_pending === 1; + const releaseMessage = createPending + ? `${release.message}; runtime adapter stop waiting for create resolution` + : release.message; + return { + status, + leaseId: null, + attachUrl: null, + attachUrlPresent: true, + vncUrl: null, + message: replayMessage ? `${replayMessage}; ${releaseMessage}` : releaseMessage, + adapter: runtimeAdapterName, + profile: session.profile, + adapterWorkspaceId, + providerResourceId: session.provider_resource_id, + reconciledAt: Date.now(), + reconcileError: null, + terminalStatus: lifecycle?.terminal_status ?? null, + createPending, + }; } -async function prepareSandboxRuntimeTools( - sandbox: SandboxSessionTarget, - session: SandboxRuntimeSession, - workdir: string, - commandEnv: Record = {}, - agentToken?: string, -): Promise { - const autostartScript = sandboxAutostartScriptPath(session.id); - const terminalShell = sandboxTerminalShellPath(session.id); - const result = await sandbox.exec( - ` -set -eu -export CODEX_HOME="$HOME/.codex" -missing_tools="" -for tool in git node npm pnpm codex gh rg fd jq python3 make gcc time ssh rsync crabbox; do - if ! command -v "$tool" >/dev/null 2>&1; then - missing_tools="$missing_tools $tool" - fi -done -if [ -n "$missing_tools" ]; then - printf 'Crabfleet sandbox image is missing required tools:%s\\n' "$missing_tools" >/tmp/crabbox-runtime-tools.log - if command -v crabbox-diagnostics >/dev/null 2>&1; then - crabbox-diagnostics >>/tmp/crabbox-runtime-tools.log 2>&1 || true - fi - cat /tmp/crabbox-runtime-tools.log - exit 72 -fi -installed_codex="$(npm list -g @openai/codex --depth=0 --json 2>/dev/null | node -e 'let s="";process.stdin.on("data",d=>s+=d);process.stdin.on("end",()=>{try{const v=JSON.parse(s).dependencies?.["@openai/codex"]?.version||""; if (v) console.log(v);}catch{}})' || true)" -latest_codex="$(npm view @openai/codex version 2>/dev/null || true)" -if [ -z "$installed_codex" ] || { [ -n "$latest_codex" ] && [ "$installed_codex" != "$latest_codex" ]; }; then - if command -v timeout >/dev/null 2>&1; then - timeout 120s npm install -g @openai/codex@latest >/tmp/crabbox-codex-install.log 2>&1 - else - npm install -g @openai/codex@latest >/tmp/crabbox-codex-install.log 2>&1 - fi -fi -rm -f "$HOME/.config/crabbox/github-credential" 2>/dev/null || true -rm -rf "$HOME/.config/gh" "$HOME/.local/share/gh" 2>/dev/null || true -git config --global --unset-all credential.helper 2>/dev/null || true -git config --global credential.helper "!f() { test \\"\\$1\\" = get || exit 0; printf 'username=x-access-token\\n'; printf 'password=%s\\n' ${shellQuote(sandboxPlaceholderGitHubToken)}; }; f" -git config --global user.name ${shellQuote(session.owner)} -git config --global user.email ${shellQuote(`${session.owner}@users.noreply.github.com`)} -mkdir -p "$(dirname ${shellQuote(autostartScript)})" -cat > ${shellQuote(autostartScript)} <<'EOF' -export CODEX_HOME="$HOME/.codex" -export GITHUB_TOKEN=${shellQuote(sandboxPlaceholderGitHubToken)} -export GH_TOKEN=${shellQuote(sandboxPlaceholderGitHubToken)} -export CRABBOX_SESSION_ID=${shellQuote(session.id)} -export CRABFLEET_SESSION_ID=${shellQuote(session.id)} -export CRABFLEET_PARENT_SESSION_ID=${shellQuote(session.parentSessionId ?? "")} -export CRABFLEET_ROOT_SESSION_ID=${shellQuote(session.rootSessionId ?? session.id)} -export CRABFLEET_AGENT_TOKEN=${shellQuote(agentToken ?? "")} -export CRABFLEET_API_URL=${shellQuote(appCanonicalOrigin)} -export CRABBOX_REPO=${shellQuote(session.repo)} -export CRABBOX_BRANCH=${shellQuote(session.branch)} -export CRABBOX_RUNTIME=${shellQuote(session.runtime)} -export CRABBOX_COMMAND=${shellQuote(session.command)} -export CRABBOX_CHECKOUT_ERROR=${shellQuote(sandboxCheckoutErrorPath(session.id))} -export CRABBOX_WORKDIR=${shellQuote(workdir)} -if [ -z "\${CRABBOX_SHELL_BOOTSTRAPPED:-}" ]; then - export CRABBOX_SHELL_BOOTSTRAPPED=1 - cd "$CRABBOX_WORKDIR" 2>/dev/null || true -fi -if [ -z "\${CRABBOX_CODEX_AUTOSTART_CHECKED:-}" ]; then - export CRABBOX_CODEX_AUTOSTART_CHECKED=1 - crabbox_autostart_marker="$HOME/.cache/crabbox/\${CRABBOX_SESSION_ID:-session}.codex-autostarted" - mkdir -p "$HOME/.cache/crabbox" 2>/dev/null || true - if [ ! -e "$crabbox_autostart_marker" ]; then - if [ -s "\${CRABBOX_CHECKOUT_ERROR:-}" ]; then - printf '\\nCrabfleet repository checkout failed:\\n' - cat "$CRABBOX_CHECKOUT_ERROR" - printf '\\n' - elif [ -n "\${CRABBOX_COMMAND:-}" ]; then - touch "$crabbox_autostart_marker" 2>/dev/null || true - ( - cd "$CRABBOX_WORKDIR" 2>/dev/null || { - printf 'Crabfleet workdir is unavailable: %s\\n' "$CRABBOX_WORKDIR" - exit 127 - } - env -u BASH_ENV -u PROMPT_COMMAND /bin/bash -c "$CRABBOX_COMMAND" - ) - fi - fi -fi -EOF -marker=${shellQuote(sandboxBashrcMarker(session))} -bashrc_tmp="$HOME/.bashrc.crabbox.$$" -{ - printf '%s\\n' "$marker" - printf '%s\\n' 'source ${shellQuote(autostartScript)} 2>/dev/null || true' - if [ -f "$HOME/.bashrc" ]; then - awk -v marker="$marker" '$0 == marker { getline; next } { print }' "$HOME/.bashrc" - fi -} > "$bashrc_tmp" -mv "$bashrc_tmp" "$HOME/.bashrc" -cat > ${shellQuote(terminalShell)} <<'EOF' -#!/bin/bash -cd ${shellQuote(workdir)} 2>/dev/null || true -source ${shellQuote(autostartScript)} 2>/dev/null || true -exec /bin/bash -i -EOF -chmod +x ${shellQuote(terminalShell)} -`, +type StoppingRuntimeAdapterReplay = { + message: string; + resolved: boolean; +}; + +async function replayStoppingRuntimeAdapterCreate( + env: RuntimeEnv, + session: InteractiveSessionRow, +): Promise { + const adapterWorkspaceId = session.adapter_workspace_id; + const replay = runtimeAdapterReplayRequest(session); + const requestedCapabilities = replay.adapterRequestedCapabilities; + const ttlSeconds = persistedRuntimeAdapterSeconds(replay.adapterTtlSeconds); + const idleTimeoutSeconds = persistedRuntimeAdapterSeconds(replay.adapterIdleTimeoutSeconds); + if ( + !adapterWorkspaceId || + !requestedCapabilities || + !ttlSeconds || + !idleTimeoutSeconds || + !session.adapter_requested_capabilities_json + ) { + return { + message: "runtime adapter create replay blocked: persisted lifecycle is incomplete", + resolved: false, + }; + } + let controlPlane: string; + try { + controlPlane = requireRegisteredRuntimeAdapterControlPlane(env, session.adapter_control_plane); + } catch (error) { + return { + message: safeProviderError(error, [adapterWorkspaceId]), + resolved: false, + }; + } + const createPayloadJson = validatedRuntimeAdapterCreatePayloadJson( + replay.adapterCreatePayloadJson ?? "", { - timeout: 300_000, - env: commandEnv, + workspaceId: adapterWorkspaceId, + ttlSeconds, + idleTimeoutSeconds, + desktop: requestedCapabilities.desktop, }, ); - if (!result.success) { - throw new Error(clean(result.stderr || result.stdout || "runtime tool setup failed", 700)); + if (!createPayloadJson) { + return { + message: "runtime adapter create replay blocked: persisted payload is invalid", + resolved: false, + }; + } + let ownership = database(env) + .selectFrom("interactive_sessions") + .select("id") + .where("id", "=", session.id) + .where("adapter", "=", runtimeAdapterName) + .where("adapter_workspace_id", "=", adapterWorkspaceId) + .where("adapter_control_plane", "=", controlPlane) + .where("adapter_create_payload_json", "=", createPayloadJson) + .where("adapter_requested_capabilities_json", "=", session.adapter_requested_capabilities_json) + .where("adapter_ttl_seconds", "=", ttlSeconds) + .where("adapter_idle_timeout_seconds", "=", idleTimeoutSeconds) + .where("adapter_create_pending", "=", 1) + .where("status", "=", "stopping") + .where("updated_at", "=", session.updated_at); + ownership = session.terminal_status + ? ownership.where("terminal_status", "=", session.terminal_status) + : ownership.where("terminal_status", "is", null); + if (!(await ownership.executeTakeFirst())) { + return { + message: "runtime adapter create replay deferred: lifecycle ownership changed", + resolved: false, + }; } -} - -async function openSandboxTerminalResponse( - request: Request, - env: RuntimeEnv, - sandbox: ReturnType, - session: InteractiveSession & { githubToken?: string }, - size: { cols: number; rows: number }, -): Promise { - const lease = sandboxLeaseInfo(session); - const options = { - cols: size.cols, - rows: size.rows, - shell: sandboxTerminalShellPath(session.id), - }; - await ensureSandboxTerminalPrepared(sandbox, env, session, lease.terminalSessionId); - const open = async () => { - const terminalSession = await sandbox.getSession(lease.terminalSessionId); - return terminalSession.terminal(request, options); - }; + let response: Response; try { - const response = await open(); - if (response.webSocket && response.status === 101) return response; - } catch { - // A previous PTY disconnect can leave the SDK execution session terminated. + response = await runtimeAdapterFetch(env, runtimeAdapterCollectionUrl(controlPlane), { + method: "POST", + headers: { "idempotency-key": adapterWorkspaceId }, + body: createPayloadJson, + }); + } catch (error) { + return { + message: `runtime adapter create replay pending: ${safeProviderError(error, [adapterWorkspaceId])}`, + resolved: false, + }; } - await recreateSandboxTerminalSession(sandbox, env, session, lease.terminalSessionId); - return open(); -} - -async function ensureSandboxTerminalPrepared( - sandbox: ReturnType, - env: RuntimeEnv, - session: InteractiveSession & { githubToken?: string }, - terminalSessionId: string, -): Promise { - const workdir = sandboxWorkdir(session.id); + let responseBody: unknown; try { - if (await sandboxTerminalProfileExists(sandbox, env, session, workdir)) return; - await setupSandboxTerminalSession(sandbox, env, session, workdir, terminalSessionId); - return; - } catch { - // Missing or terminated default shell; recreate the sandbox below. + responseBody = await readRuntimeAdapterResponseBody(response); + } catch (error) { + return { + message: `runtime adapter create replay pending: ${safeProviderError(error, [adapterWorkspaceId])}`, + resolved: false, + }; } - await recreateSandboxTerminalSession(sandbox, env, session, terminalSessionId); -} - -async function sandboxTerminalProfileExists( - sandbox: CloudflareSandbox, - env: RuntimeEnv, - session: InteractiveSession & { githubToken?: string }, - workdir: string, -): Promise { - const setup = await createSandboxSession( - sandbox, - sandboxSetupSessionId(session.id), - "/workspace", - { - CRABBOX_SESSION_ID: session.id, - }, + let message: string; + if (!response.ok) { + const responseMessage = redactedAdapterResponseMessage( + responseBody, + `HTTP ${response.status}`, + [adapterWorkspaceId], + ); + if (!definitiveRuntimeAdapterCreateFailure(response.status)) { + return { + message: `runtime adapter create replay pending: ${responseMessage}`, + resolved: false, + }; + } + message = `runtime adapter create replay resolved: ${responseMessage}`; + } else { + const parsed = parseAdapterWorkspaceResult(responseBody, { + workspaceId: adapterWorkspaceId, + profile: session.profile, + }); + if (!parsed || !adapterWorkspaceIdMatches(parsed, adapterWorkspaceId)) { + return { + message: "runtime adapter create replay pending: invalid workspace identity", + resolved: false, + }; + } + message = `runtime adapter create replay resolved: ${parsed.status}`; + } + + const resolvedAt = Date.now(); + const terminalStatusOwner = session.terminal_status + ? sql`terminal_status = ${session.terminal_status}` + : sql`terminal_status IS NULL`; + const expectedOwner = sql` + id = ${session.id} + AND adapter = ${runtimeAdapterName} + AND adapter_workspace_id = ${adapterWorkspaceId} + AND adapter_control_plane = ${controlPlane} + AND adapter_create_payload_json = ${createPayloadJson} + AND adapter_requested_capabilities_json = ${session.adapter_requested_capabilities_json} + AND adapter_ttl_seconds = ${ttlSeconds} + AND adapter_idle_timeout_seconds = ${idleTimeoutSeconds} + AND adapter_create_pending = 1 + AND status = 'stopping' + AND updated_at = ${session.updated_at} + AND ${terminalStatusOwner} + `; + const db = database(env); + const update = db + .updateTable("interactive_sessions") + .set({ + adapter_create_pending: 0, + last_reconciled_at: resolvedAt, + reconcile_error: message, + last_event: message, + updated_at: sql`MAX(updated_at + 1, ${resolvedAt})`, + }) + .where(expectedOwner) + .returning("updated_at"); + const event = sql` + INSERT INTO interactive_session_events (session_id, actor, message, created_at) + SELECT ${session.id}, 'system', ${clean(message, 1000)}, ${resolvedAt} + FROM interactive_sessions + WHERE ${expectedOwner} + `; + const results = await env.DB.batch<{ updated_at: number }>( + [event, update].map((query) => { + const compiled = query.compile(db); + return env.DB.prepare(compiled.sql).bind(...compiled.parameters); + }), ); - const marker = shellQuote(sandboxBashrcMarker(session)); - const autostartScript = sandboxAutostartScriptPath(session.id); - const terminalShell = sandboxTerminalShellPath(session.id); - const repoUrl = `https://github.com/${session.repo}.git`; - const checks = [ - `test -d ${shellQuote(workdir)}`, - `test -d ${shellQuote(workdir)}/.git`, - `test ! -s ${shellQuote(sandboxCheckoutErrorPath(session.id))}`, - `git -C ${shellQuote(workdir)} rev-parse --verify HEAD >/dev/null`, - `test "$(git -C ${shellQuote(workdir)} rev-parse --abbrev-ref HEAD)" = ${shellQuote(session.branch)}`, - `test "$(git -C ${shellQuote(workdir)} config --get remote.origin.url)" = ${shellQuote(repoUrl)}`, - `test -s ${shellQuote(autostartScript)}`, - `test -x ${shellQuote(terminalShell)}`, - `grep -Fqx '[shell_environment_policy]' "$HOME/.codex/config.toml"`, - `grep -Fqx '[projects."/workspace"]' "$HOME/.codex/config.toml"`, - `node -e 'const fs=require("fs"); const p=process.env.HOME+"/.codex/auth.json"; const auth=JSON.parse(fs.readFileSync(p,"utf8")); process.exit(auth.OPENAI_API_KEY==="crabfleet-worker-injected"?0:1)'`, - `grep -Fqx ' cd "$CRABBOX_WORKDIR" 2>/dev/null || {' ${shellQuote(autostartScript)}`, - `grep -Fqx ${marker} "$HOME/.bashrc"`, - `test ! -e "$HOME/.config/crabbox/github-credential"`, - ]; - const result = await setup.exec(checks.join(" && "), { timeout: 10_000 }); - return result.success; + const resolved = Boolean(results.at(-1)?.results.length); + if (resolved) { + await archiveInteractiveSessionLogs(env, session.id, resolvedAt).catch(() => undefined); + } + return { message, resolved }; } -async function setupSandboxTerminalSession( - sandbox: CloudflareSandbox, +async function stopRuntimeAdapterWorkspace( env: RuntimeEnv, - session: SandboxRuntimeSession, - workdir: string, - terminalSessionId: string, - agentToken?: string, -): Promise { - const sessionEnv = sandboxSessionEnv(env, session, agentToken); - const setup = await createSandboxSession( - sandbox, - sandboxSetupSessionId(session.id), - "/workspace", - sessionEnv, - ); - await runSandboxSetupStep("workspace mkdir", () => setup.mkdir(workdir, { recursive: true })); - await runSandboxSetupStep("repository checkout", () => - prepareSandboxWorkspace(setup, env, session, workdir), - ); - await runSandboxSetupStep("Codex auth", () => prepareSandboxCodexAuth(setup, env, workdir)); - await runSandboxSetupStep("runtime tools", () => - prepareSandboxRuntimeTools(setup, session, workdir, {}, agentToken), - ); - await runSandboxSetupStep("terminal session", () => - createFreshSandboxSession(sandbox, terminalSessionId, workdir, sessionEnv), + registeredControlPlane: string, + adapterWorkspaceId: string, +): Promise { + const controlPlane = requireRegisteredRuntimeAdapterControlPlane(env, registeredControlPlane); + const response = await runtimeAdapterFetch( + env, + runtimeAdapterWorkspaceUrl(controlPlane, adapterWorkspaceId), + { method: "DELETE" }, ); -} + const body = response.status === 204 ? null : await readRuntimeAdapterResponseBody(response); + const parsed = parseAdapterWorkspaceResult(body, { workspaceId: adapterWorkspaceId }); + if (parsed && !adapterWorkspaceIdMatches(parsed, adapterWorkspaceId)) { + throw new Error("runtime adapter stop returned a different workspace id"); + } + const fallbackMessage = + response.status === 404 || response.status === 204 + ? "runtime adapter workspace released" + : `runtime adapter stop HTTP ${response.status}`; + const message = + parsed?.message ?? redactedAdapterResponseMessage(body, fallbackMessage, [adapterWorkspaceId]); + if (response.status === 404 || response.status === 204) { + return { status: "stopped", message }; + } + if (!response.ok) throw new Error(message); + const outcome = runtimeAdapterStopOutcome(response.status, parsed, adapterWorkspaceId); + if (outcome === "identity_mismatch") { + throw new Error("runtime adapter stop returned a different workspace id"); + } + return { status: outcome, message }; +} + +type RuntimeAdapterStopResult = { + status: "stopping" | "stopped"; + message: string; +}; -async function recreateSandboxTerminalSession( - sandbox: ReturnType, +async function stopRuntimeAdapterWorkspaceForSession( env: RuntimeEnv, - session: InteractiveSession & { githubToken?: string }, - terminalSessionId: string, -): Promise { - await setupSandboxTerminalSession( - sandbox, + sessionId: string, + adapterWorkspaceId: string, +): Promise { + const controlPlane = await registeredRuntimeAdapterControlPlaneForSession( env, - session, - sandboxWorkdir(session.id), - terminalSessionId, + sessionId, + adapterWorkspaceId, ); + return stopRuntimeAdapterWorkspace(env, controlPlane, adapterWorkspaceId); } -function sandboxSessionEnv( +async function runtimeAdapterFetch( env: RuntimeEnv, - session: SandboxRuntimeSession, - agentToken?: string, -): Record { - return { - CRABBOX_SESSION_ID: session.id, - CRABFLEET_SESSION_ID: session.id, - CRABFLEET_PARENT_SESSION_ID: session.parentSessionId ?? undefined, - CRABFLEET_ROOT_SESSION_ID: session.rootSessionId ?? session.id, - CRABFLEET_AGENT_TOKEN: agentToken, - CRABFLEET_API_URL: appCanonicalOrigin, - CRABBOX_REPO: session.repo, - CRABBOX_BRANCH: session.branch, - CRABBOX_RUNTIME: session.runtime, - TERM: "xterm-256color", - COLORTERM: "truecolor", - GH_TOKEN: sandboxHasGitHubCredential(env, session) ? sandboxPlaceholderGitHubToken : undefined, - GITHUB_TOKEN: sandboxHasGitHubCredential(env, session) - ? sandboxPlaceholderGitHubToken - : undefined, - TERM_PROGRAM: "ghostty", - TERM_PROGRAM_VERSION: "web", - OPENAI_API_KEY: env.OPENAI_API_KEY ? sandboxPlaceholderOpenAIKey : undefined, - OPENAI_BASE_URL: env.OPENAI_BASE_URL, - OPENAI_ORG_ID: env.OPENAI_ORG_ID, - }; + url: string, + init: RequestInit, +): Promise { + const token = runtimeAdapterToken(env); + if (!token) throw new Error("runtime adapter token is not configured"); + const safeTarget = safeDesktopUrl(url); + if (!safeTarget) throw new Error("runtime adapter URL must use HTTPS or loopback HTTP"); + const target = new URL(safeTarget); + const headers = new Headers(init.headers); + headers.set("authorization", `Bearer ${token}`); + headers.set("accept", "application/json"); + if (init.body) headers.set("content-type", "application/json"); + return fetch(target, { + ...init, + headers, + redirect: "error", + signal: AbortSignal.timeout(10_000), + }); } -function sandboxHasGitHubCredential(env: RuntimeEnv, session: SandboxRuntimeSession): boolean { - return Boolean(("githubToken" in session && session.githubToken) || env.GITHUB_TOKEN); +async function readRuntimeAdapterResponseBody(response: Response): Promise { + const body = await readBoundedResponseText(response); + if (!body) return null; + try { + return JSON.parse(body); + } catch { + return { message: body }; + } } -function githubTokenEnv(session: Pick): { - GITHUB_TOKEN?: string; - GH_TOKEN?: string; -} { - return session.githubToken - ? { GITHUB_TOKEN: session.githubToken, GH_TOKEN: session.githubToken } - : {}; +function runtimeAdapterToken(env: RuntimeEnv): string { + return clean(env.CRABBOX_RUNTIME_ADAPTER_TOKEN, 4000); +} + +function runtimeAdapterProviderConfigured(env: RuntimeEnv): boolean { + return Boolean(configuredRuntimeAdapterControlPlane(env) && runtimeAdapterToken(env)); } async function forwardRuntimeProvision( @@ -5358,7 +11437,7 @@ async function forwardRuntimeProvision( body: JSON.stringify(session), }); } catch (error) { - return failedProvision(`interactive provision failed: ${clean(String(error), 240)}`); + return failedProvision(`interactive provision failed: ${safeProviderError(error)}`); } if (!response.ok) { return failedProvision(`interactive provision failed: runtime HTTP ${response.status}`); @@ -5411,7 +11490,7 @@ async function provisionWithCloudflareRunner( }), }); } catch (error) { - return failedProvision(`cloudflare runner provision failed: ${clean(String(error), 240)}`); + return failedProvision(`cloudflare runner provision failed: ${safeProviderError(error)}`); } if (!response.ok) { return failedProvision(`cloudflare runner provision failed: HTTP ${response.status}`); @@ -5457,7 +11536,7 @@ async function provisionWithClawFleet( body: JSON.stringify({ count: 1, runtime_type: "openclaw" }), }); } catch (error) { - return failedProvision(`clawfleet provision failed: ${clean(String(error), 240)}`); + return failedProvision(`clawfleet provision failed: ${safeProviderError(error)}`); } if (!response.ok) { return failedProvision(`clawfleet provision failed: HTTP ${response.status}`); @@ -5484,14 +11563,22 @@ function provisionResultFromBody( body: Record, invalidMessage: string, ): InteractiveProvisionResult { - const status = optionalOneOf(body.status, interactiveSessionStatuses); + const status = createOnlyAdapterStatus(body.status); if (!status) return failedProvision(invalidMessage); + const leaseId = clean(body.leaseId ?? body.lease_id, 240) || null; + const attachUrl = clean(body.attachUrl ?? body.attach_url, 1000) || null; + const vncUrl = clean(body.vncUrl ?? body.vnc_url, 1000) || null; return { status, - leaseId: clean(body.leaseId ?? body.lease_id, 240) || null, - attachUrl: clean(body.attachUrl ?? body.attach_url, 1000) || null, - vncUrl: clean(body.vncUrl ?? body.vnc_url, 1000) || null, - message: clean(body.message, 500) || `interactive workspace ${status}`, + leaseId, + attachUrl, + vncUrl, + message: redactedAdapterMessage( + clean(body.message, 500) || null, + status, + [leaseId], + [attachUrl, vncUrl], + ), }; } @@ -5505,6 +11592,19 @@ function failedProvision(message: string): InteractiveProvisionResult { }; } +function safeProviderError( + error: unknown, + identifiers: Array = [], + connectionValues: Array = [], +): string { + return redactedAdapterMessage( + clean(error instanceof Error ? error.message : String(error), 2000), + "failed", + identifiers, + connectionValues, + ); +} + function cloudflareRunnerWorkdir(env: RuntimeEnv, session: InteractiveProvisionRequest): string { const base = clean(env.CRABBOX_CLOUDFLARE_RUNNER_WORKDIR, 160) || "/workspace/crabbox"; const suffix = session.id.toLowerCase().replace(/[^a-z0-9_-]/g, "-"); @@ -5796,7 +11896,7 @@ async function evaluateWorkflow( user: User, ): Promise> { const body = await readJson<{ repo?: string }>(request); - const repo = normalizeRepo(body.repo) || preferredRepo; + const repo = normalizeRepo(body.repo) || deploymentConfig(env).preferredRepo; await requireRepo(env, repo); const workflow = await refreshWorkflowForRepo(env, repo, Date.now()); await audit(env, user, `workflow evaluated ${repo} status=${workflow.status}`, Date.now()); @@ -5942,7 +12042,9 @@ async function fetchGitHubReferences( const item = payload.data?.[`r${index}`]?.issueOrPullRequest; return item ? [githubReferenceFromGraphql(target.repo, item)] : []; }) - .sort((left, right) => sortRepoNames(left.repo, right.repo)); + .sort((left, right) => + sortRepoNames(left.repo, right.repo, deploymentConfig(env).preferredRepo), + ); } async function fetchPublicGitHubReferences( @@ -5950,7 +12052,8 @@ async function fetchPublicGitHubReferences( repos: string[], number: number, ): Promise { - const repo = repos.includes(preferredRepo) ? preferredRepo : repos[0]; + const preferred = deploymentConfig(env).preferredRepo; + const repo = repos.includes(preferred) ? preferred : repos[0]; if (!repo) return []; const match = await fetchGitHubReference(env, repo, number); return match ? [match] : []; @@ -6275,10 +12378,7 @@ async function readRunsForCard(env: RuntimeEnv, cardId: string): Promise { +async function readInteractiveSessions(env: RuntimeEnv, user: User): Promise { const rows = await database(env) .selectFrom("interactive_sessions") .selectAll() @@ -6318,6 +12418,16 @@ async function readInteractiveSession( return interactiveSession(row, logs.get(id) ?? [], archives.get(id) ?? null); } +async function readFreshInteractiveSession( + env: RuntimeEnv, + id: string, +): Promise { + await reconcileExternalInteractiveSessionById(env, id).catch((error) => { + console.error("targeted runtime adapter reconciliation failed", error); + }); + return readInteractiveSession(env, id); +} + async function readSharedInteractiveSession( env: RuntimeEnv, id: string, @@ -6338,9 +12448,16 @@ async function readSharedInteractiveSession( return { session: { ...session, + adapter: null, + profile: "", + adapterWorkspaceId: null, + providerResourceId: null, + lastReconciledAt: null, + reconcileError: null, leaseId: null, attachUrl: null, vncUrl: null, + ptyAvailable: false, controller: activeController, controlGrantedAt: activeController ? session.controlGrantedAt : null, controlExpiresAt: activeController ? session.controlExpiresAt : null, @@ -6416,20 +12533,15 @@ async function updateInteractiveSessionSummary( const summary = clean(body.summary, 500); if (!purpose && !summary) throw badRequest("summary or purpose is required"); const now = Date.now(); - await database(env) - .updateTable("interactive_sessions") - .set({ - ...(purpose ? { purpose } : {}), - ...(summary ? { summary } : {}), - updated_at: now, - }) - .where("id", "=", id) - .execute(); - await appendInteractiveSessionEvent( + await mutateInteractiveSessionMetadataAtomically( env, - id, + session, user, summary ? "session summary updated" : "session purpose updated", + { + ...(purpose ? { purpose } : {}), + ...(summary ? { summary } : {}), + }, now, ); return { @@ -6520,15 +12632,16 @@ async function appendInteractiveSessionEvent( message: string, now = Date.now(), ): Promise { - await database(env) - .insertInto("interactive_session_events") - .values({ + const db = database(env); + await executeBatch(env, [ + db.insertInto("interactive_session_events").values({ session_id: id, actor: actor(user), message: clean(message, 1000), created_at: now, - }) - .execute(); + }), + terminalFinalizationPendingQuery(db, id), + ]); await archiveInteractiveSessionLogs(env, id, now).catch(() => undefined); } @@ -6543,18 +12656,152 @@ async function appendInteractiveSessionLog( await appendInteractiveSessionEvent(env, id, user, message, now); return; } - await database(env) - .insertInto("interactive_session_events") - .values({ + const db = database(env); + await executeBatch(env, [ + db.insertInto("interactive_session_events").values({ session_id: id, actor: "system", message: clean(message, 1000), created_at: now, - }) - .execute(); + }), + terminalFinalizationPendingQuery(db, id), + ]); await archiveInteractiveSessionLogs(env, id, now).catch(() => undefined); } +function terminalFinalizationPendingQuery(db: Kysely, id: string): CompilableQuery { + return db + .updateTable("interactive_sessions") + .set({ terminal_finalize_pending: 1 }) + .where("id", "=", id) + .where("status", "in", deadInteractiveSessionStatuses); +} + +async function finalizeTerminalInteractiveSession( + env: RuntimeEnv, + id: string, + status: "stopped" | "expired" | "failed", + now: number, +): Promise { + const db = database(env); + const terminal = await db + .selectFrom("interactive_sessions") + .select(["terminal_failure_reason", "reconcile_error", "last_event"]) + .where("id", "=", id) + .where("status", "=", status) + .executeTakeFirst(); + const message = + status === "failed" + ? retainedRuntimeAdapterFailureMessage( + terminal?.terminal_failure_reason ?? null, + terminal?.reconcile_error ?? null, + terminal?.last_event ?? null, + ) + : status === "expired" + ? "interactive workspace expired" + : "interactive workspace stopped"; + await completeTerminalFinalization({ + ensureEvent: async () => { + await executeBatch(env, [ + sql` + INSERT INTO interactive_session_events (session_id, actor, message, created_at) + SELECT ${id}, 'system', ${message}, COALESCE(stopped_at, ${now}) + FROM interactive_sessions AS session + WHERE session.id = ${id} + AND session.status = ${status} + AND NOT EXISTS ( + SELECT 1 + FROM interactive_session_events AS event + WHERE event.session_id = session.id + AND event.actor = 'system' + AND event.message = ${message} + ) + `, + terminalFinalizationPendingQuery(db, id), + ]); + return true; + }, + readArchiveState: async () => { + const [currentArchive, eventCount, currentSession] = await Promise.all([ + db + .selectFrom("interactive_session_log_archives") + .selectAll() + .where("session_id", "=", id) + .executeTakeFirst(), + countInteractiveSessionEvents(env, id), + db + .selectFrom("interactive_sessions") + .select("updated_at") + .where("id", "=", id) + .executeTakeFirst(), + ]); + return { + eventCount, + archiveEventCount: currentArchive?.event_count ?? null, + archiveSessionVersionMatches: + currentArchive?.session_updated_at === currentSession?.updated_at, + archiveObjectsReady: Boolean( + !env.SESSION_LOGS || + (currentArchive?.events_key && + currentArchive.transcript_key && + currentArchive.summary_key), + ), + }; + }, + archive: () => archiveInteractiveSessionLogs(env, id, now, { force: true }), + clearPending: async () => { + const cleared = await sql` + UPDATE interactive_sessions + SET terminal_finalize_pending = 0 + WHERE id = ${id} + AND status = ${status} + AND terminal_finalize_pending > 0 + AND EXISTS ( + SELECT 1 + FROM interactive_session_log_archives AS archive + WHERE archive.session_id = interactive_sessions.id + AND archive.session_updated_at = interactive_sessions.updated_at + ) + AND NOT EXISTS ( + SELECT 1 + FROM interactive_session_credential_policies + WHERE session_id = ${id} + ) + AND COALESCE( + ( + SELECT event_count + FROM interactive_session_log_archives + WHERE session_id = ${id} + ), + -1 + ) >= ( + SELECT count(*) + FROM interactive_session_events + WHERE session_id = ${id} + ) + AND ( + ${env.SESSION_LOGS ? 1 : 0} = 0 + OR EXISTS ( + SELECT 1 + FROM interactive_session_log_archives + WHERE session_id = ${id} + AND events_key IS NOT NULL + AND transcript_key IS NOT NULL + AND summary_key IS NOT NULL + ) + ) + `.execute(db); + if ((cleared.numAffectedRows ?? 0n) > 0n) return true; + const current = await db + .selectFrom("interactive_sessions") + .select("terminal_finalize_pending") + .where("id", "=", id) + .executeTakeFirst(); + return !current || current.terminal_finalize_pending === 0; + }, + }); +} + async function archiveInteractiveSessionLogs( env: RuntimeEnv, id: string, @@ -6577,11 +12824,16 @@ async function archiveInteractiveSessionLogs( } const events = await readInteractiveSessionEventRows(env, id); const latestEventAt = events.at(-1)?.created_at ?? now; - const archiveVersion = `${String(events.length).padStart(8, "0")}-${String(latestEventAt).padStart(13, "0")}-${now}`; - const base = `${sessionLogArchiveBase(id)}/${archiveVersion}`; - const eventsKey = `${base}/events.ndjson`; - const transcriptKey = `${base}/transcript.md`; - const summaryKey = `${base}/summary.json`; + const attemptedArchive = sessionArchiveAttemptKeys( + sessionLogArchiveBase(id), + events.length, + latestEventAt, + now, + crypto.randomUUID(), + ); + const eventsKey = attemptedArchive.events_key; + const transcriptKey = attemptedArchive.transcript_key; + const summaryKey = attemptedArchive.summary_key; if (env.SESSION_LOGS) { await Promise.all([ env.SESSION_LOGS.put( @@ -6603,6 +12855,7 @@ async function archiveInteractiveSessionLogs( INSERT INTO interactive_session_log_archives ( session_id, event_count, + session_updated_at, events_key, transcript_key, summary_key, @@ -6612,6 +12865,7 @@ async function archiveInteractiveSessionLogs( VALUES ( ${id}, ${events.length}, + ${sessionRow.updated_at}, ${env.SESSION_LOGS ? eventsKey : null}, ${env.SESSION_LOGS ? transcriptKey : null}, ${env.SESSION_LOGS ? summaryKey : null}, @@ -6620,6 +12874,7 @@ async function archiveInteractiveSessionLogs( ) ON CONFLICT(session_id) DO UPDATE SET event_count = excluded.event_count, + session_updated_at = excluded.session_updated_at, events_key = excluded.events_key, transcript_key = excluded.transcript_key, summary_key = excluded.summary_key, @@ -6627,7 +12882,24 @@ async function archiveInteractiveSessionLogs( WHERE excluded.event_count > interactive_session_log_archives.event_count OR ( excluded.event_count = interactive_session_log_archives.event_count - AND excluded.updated_at >= interactive_session_log_archives.updated_at + AND ( + ( + excluded.session_updated_at IS NOT NULL + AND interactive_session_log_archives.session_updated_at IS NULL + ) + OR ( + excluded.session_updated_at > interactive_session_log_archives.session_updated_at + ) + OR ( + excluded.session_updated_at IS interactive_session_log_archives.session_updated_at + AND ( + interactive_session_log_archives.events_key IS NULL + OR interactive_session_log_archives.transcript_key IS NULL + OR interactive_session_log_archives.summary_key IS NULL + OR excluded.updated_at >= interactive_session_log_archives.updated_at + ) + ) + ) ) `.execute(db); if (!env.SESSION_LOGS) return; @@ -6636,12 +12908,9 @@ async function archiveInteractiveSessionLogs( .selectAll() .where("session_id", "=", id) .executeTakeFirst(); - const archiveWon = latestArchive?.events_key === eventsKey; await cleanupSessionLogArchiveObjects( env, - archiveWon - ? currentArchive - : { events_key: eventsKey, transcript_key: transcriptKey, summary_key: summaryKey }, + obsoleteSessionArchiveObjectKeys(latestArchive, currentArchive, attemptedArchive), ); } @@ -6840,12 +13109,19 @@ async function nextRunAttempt(env: RuntimeEnv, cardId: string): Promise } async function nextInteractiveSessionId(env: RuntimeEnv): Promise { - const row = await database(env) - .selectFrom("interactive_sessions") - .select(sql`max(CAST(substr(id, 4) AS INTEGER))`.as("max_id")) - .where("id", "like", "IS-%") - .executeTakeFirst(); - return `IS-${String((row?.max_id ?? 100) + 1)}`; + const db = database(env); + for (let attempt = 0; attempt < 100; attempt += 1) { + const result = await sql.raw<{ next_id: number }>(allocateInteractiveSessionIdSql).execute(db); + const id = formatInteractiveSessionId(Number(result.rows[0]?.next_id)); + if (!id) throw new Error("failed to allocate interactive session id"); + const standalone = await db + .selectFrom("standalone_sandbox_provisions") + .select("id") + .where(sql`id = ${id} COLLATE NOCASE`) + .executeTakeFirst(); + if (!standalone) return id; + } + throw new Error("failed to allocate an unreserved interactive session id"); } async function requireRepo(env: RuntimeEnv, repo: string): Promise { @@ -7116,23 +13392,16 @@ function authMethods(env: RuntimeEnv, request?: Request): Record): string { - return user.login ?? user.email ?? user.subject; +function devIdentityEnabled(env: RuntimeEnv, request: Request): boolean { + return developmentIdentityEnabled(env.CRABFLEET_DEV_LOGIN_ENABLED, request.url); } -function isLocalDevRequest(request: Request): boolean { - const hostname = new URL(request.url).hostname.toLowerCase(); - return ( - hostname === "localhost" || - hostname.endsWith(".localhost") || - hostname === "127.0.0.1" || - hostname === "::1" || - hostname === "[::1]" - ); +function actor(user: Pick): string { + return user.login ?? user.email ?? user.subject; } function devIdentityId(value: unknown): string { @@ -7187,6 +13456,7 @@ function isChangedFile(value: unknown): value is ChangedFile { } function runAttempt(row: RunAttemptTable): RunAttempt { + const capabilities = runtimeCapabilities(row.runtime, row.capabilities_json); return { id: row.id, cardId: row.card_id, @@ -7195,10 +13465,11 @@ function runAttempt(row: RunAttemptTable): RunAttempt { status: row.status, controlIntent: row.control_intent, leaseId: row.lease_id, - attachUrl: row.attach_url, + attachUrl: capabilities.terminal ? row.attach_url : null, vncUrl: row.vnc_url, + ptyAvailable: false, selectionReason: row.selection_reason, - capabilities: runtimeCapabilities(row.runtime, row.capabilities_json), + capabilities, operator: row.operator, lastHeartbeatAt: row.last_heartbeat_at, startedAt: row.started_at, @@ -7210,10 +13481,11 @@ function runAttempt(row: RunAttemptTable): RunAttempt { } function interactiveSession( - row: InteractiveSessionTable, + row: InteractiveSessionRow, logs: string[], logArchive: InteractiveSessionLogArchive | null = null, ): InteractiveSession { + const capabilities = runtimeCapabilities(row.runtime, row.capabilities_json); return { id: row.id, parentSessionId: row.parent_session_id, @@ -7221,6 +13493,14 @@ function interactiveSession( repo: row.repo, branch: row.branch, runtime: row.runtime, + adapter: row.adapter, + profile: row.profile, + adapterWorkspaceId: row.adapter_workspace_id, + providerResourceId: row.provider_resource_id, + capabilities, + expiresAt: row.expires_at, + lastReconciledAt: row.last_reconciled_at, + reconcileError: row.reconcile_error, command: row.command, prompt: row.prompt, purpose: row.purpose, @@ -7229,7 +13509,7 @@ function interactiveSession( createdBy: row.created_by, status: row.status, leaseId: row.lease_id, - attachUrl: row.attach_url, + attachUrl: capabilities.terminal ? row.attach_url : null, vncUrl: row.vnc_url, lastEvent: row.last_event, createdAt: row.created_at, @@ -7276,7 +13556,7 @@ function sessionLogArchiveBase(id: string): string { } function sessionLogTranscript( - session: InteractiveSession | InteractiveSessionTable, + session: InteractiveSession | InteractiveSessionRow, events: InteractiveSessionEventRow[], ): string { const parentSessionId = @@ -7308,7 +13588,7 @@ function sessionLogTranscript( } function sessionLogSummary( - session: InteractiveSessionTable, + session: InteractiveSessionRow, events: InteractiveSessionEventRow[], ): Record { return { @@ -7333,21 +13613,49 @@ function sessionLogSummary( function decorateInteractiveSession( session: InteractiveSession, - user?: User, - env?: RuntimeEnv, + user: User, + env: RuntimeEnv, ): InteractiveSession { - if (!user) return session; const now = Date.now(); - const delegatedControl = env ? canGrantDelegatedControl(env, session) : true; + const delegatedControl = canGrantDelegatedControl(env, session); const canManage = canManageInteractiveSession(user, session); const canChangeMultiplayer = canChangeInteractiveSessionMultiplayer(user, session); const canControl = canControlInteractiveSession(user, session, now, delegatedControl); const activeController = activeDelegatedController(session, now); + const desktopActive = !["stopping", "stopped", "expired", "failed"].includes(session.status); + const versionedDesktopReady = ["ready", "attached", "detached"].includes(session.status); + const versionedDesktopAvailable = + versionedDesktopReady && + session.adapter === runtimeAdapterName && + (session.capabilities.vnc || session.capabilities.desktop); + const legacyDesktopUrl = desktopActive ? safeDesktopUrl(session.vncUrl) : null; + const ptyAvailable = + canControl && + session.capabilities.terminal && + ["ready", "attached", "detached"].includes(session.status) && + Boolean(interactivePtyRouteKind(env, session)); + const attachUrl = + ptyAvailable && session.adapter === runtimeAdapterName + ? `/api/interactive-sessions/${encodeURIComponent(session.id)}/pty` + : canControl && session.capabilities.terminal + ? session.attachUrl + : null; return { ...session, - leaseId: canControl ? session.leaseId : null, - attachUrl: canControl ? session.attachUrl : null, - vncUrl: canControl ? session.vncUrl : null, + adapter: canControl ? session.adapter : null, + profile: canControl ? session.profile : "", + adapterWorkspaceId: canControl ? session.adapterWorkspaceId : null, + providerResourceId: canControl ? session.providerResourceId : null, + lastReconciledAt: canControl ? session.lastReconciledAt : null, + reconcileError: canControl ? session.reconcileError : null, + leaseId: canControl ? legacyInteractiveSessionLeaseId(session) : null, + attachUrl, + ptyAvailable, + vncUrl: canControl + ? versionedDesktopAvailable + ? runtimeAdapterBrowserVncUrl(deploymentConfig(env).canonicalUrl, session.id) + : legacyDesktopUrl + : null, controller: activeController, controlGrantedAt: activeController ? session.controlGrantedAt : null, controlExpiresAt: activeController ? session.controlExpiresAt : null, @@ -7412,7 +13720,8 @@ async function canControlInteractiveSessionById( .executeTakeFirst(); if (!row) return false; const session = interactiveSession(row, []); - if (["expired", "failed", "stopped"].includes(session.status)) return false; + if (!session.capabilities.terminal) return false; + if (["stopping", "expired", "failed", "stopped"].includes(session.status)) return false; return canControlInteractiveSession( user, session, @@ -7422,7 +13731,7 @@ async function canControlInteractiveSessionById( } function canGrantDelegatedControl(env: RuntimeEnv, session: InteractiveSession): boolean { - if (!env.SANDBOX && session.leaseId?.startsWith(sandboxLeasePrefix)) return false; + if (!env.SANDBOX && isSandboxInteractiveSession(session)) return false; return true; } @@ -7820,9 +14129,10 @@ function sandboxLeaseWithoutRefresh(leaseId: string): string { function sandboxLeaseInfo( session: Pick & { leaseId?: string | null; + adapter?: string | null; }, ): { sandboxId: string; terminalSessionId: string } { - const rawLease = "leaseId" in session ? session.leaseId : null; + const rawLease = legacyLeaseIdForAdapter(session.adapter ?? null, session.leaseId ?? null); const raw = rawLease?.startsWith(sandboxLeasePrefix) ? rawLease.slice(sandboxLeasePrefix.length) : ""; @@ -7879,7 +14189,7 @@ function terminalDimension(value: number | null, fallback: number): number { } function terminalCloseMessage(code: number, reason: string): string { - const suffix = reason ? `: ${clean(reason, 120)}` : ""; + const suffix = reason ? `: ${clean(redactedAdapterMessage(reason, "detached"), 120)}` : ""; return `PTY detached ${code || 1000}${suffix}`; } @@ -8051,13 +14361,13 @@ function numberSetting(value: string | undefined, fallback: number): number { return Number.isFinite(parsed) ? parsed : fallback; } -function sortRepos(repos: string[]): string[] { - return [...repos].sort(sortRepoNames); +function sortRepos(repos: string[], preferred = preferredRepo): string[] { + return [...repos].sort((left, right) => sortRepoNames(left, right, preferred)); } -function sortRepoNames(left: string, right: string): number { - if (left === preferredRepo) return -1; - if (right === preferredRepo) return 1; +function sortRepoNames(left: string, right: string, preferred = preferredRepo): number { + if (left === preferred) return -1; + if (right === preferred) return 1; return left.localeCompare(right); } @@ -8138,6 +14448,10 @@ function forbidden(message: string): Error { return Object.assign(new Error(message), { status: 403 }); } +function conflict(message: string): Error { + return Object.assign(new Error(message), { status: 409 }); +} + function serviceUnavailable(message: string): Error { return Object.assign(new Error(message), { status: 503 }); } diff --git a/src/oauth.ts b/src/oauth.ts index a220260..abd1dd6 100644 --- a/src/oauth.ts +++ b/src/oauth.ts @@ -1,8 +1,119 @@ export const githubOAuthCallbackPath = "/auth/github/callback"; +export const githubOAuthLoginPath = "/login/github"; export function githubOAuthRedirectUri(requestUrl: string | URL, configured?: string): string { - const trimmed = configured?.trim(); - if (trimmed) return new URL(trimmed).toString(); + if (configured !== undefined) return configuredGitHubOAuthRedirectUri(configured); const url = new URL(requestUrl); + if (url.username || url.password) + throw new Error("GitHub OAuth request URL cannot use credentials"); + if ( + url.protocol !== "https:" && + !(url.protocol === "http:" && isLiteralLoopbackHostname(url.hostname)) + ) { + throw new Error("GitHub OAuth request URL must use HTTPS or loopback HTTP"); + } return `${url.origin}${githubOAuthCallbackPath}`; } + +export function githubOAuthCanonicalLoginUrl( + requestUrl: string | URL, + configured?: string, +): string | null { + if (configured === undefined) return null; + const callback = new URL(configuredGitHubOAuthRedirectUri(configured)); + const request = new URL(requestUrl); + if ( + request.protocol === "https:" && + !request.username && + !request.password && + request.origin === callback.origin + ) { + return null; + } + return `${callback.origin}${githubOAuthLoginPath}`; +} + +export function githubOAuthCanonicalSshLinkUrl( + requestUrl: string | URL, + code: string, + configured?: string, +): string | null { + if (configured === undefined) return null; + const callback = new URL(configuredGitHubOAuthRedirectUri(configured)); + const request = new URL(requestUrl); + if ( + request.protocol === "https:" && + !request.username && + !request.password && + request.origin === callback.origin + ) { + return null; + } + return `${callback.origin}/ssh/link/${encodeURIComponent(code)}`; +} + +export function githubOAuthCallbackRequestMatches( + requestUrl: string | URL, + configured?: string, +): boolean { + if (configured === undefined) return true; + const callback = new URL(configuredGitHubOAuthRedirectUri(configured)); + const request = new URL(requestUrl); + return ( + request.protocol === "https:" && + !request.username && + !request.password && + request.origin === callback.origin && + request.pathname === githubOAuthCallbackPath + ); +} + +function configuredGitHubOAuthRedirectUri(configured: string): string { + if (!configured || configured !== configured.trim()) { + throw new Error("GITHUB_REDIRECT_URI must be an exact HTTPS URL"); + } + if (/\s/u.test(configured)) { + throw new Error("GITHUB_REDIRECT_URI cannot contain whitespace or controls"); + } + if (!/^https:\/\/[^/]/iu.test(configured)) { + throw new Error("GITHUB_REDIRECT_URI must be an absolute HTTPS URL"); + } + for (let index = 0; index < configured.length; index += 1) { + const code = configured.charCodeAt(index); + if (code <= 0x20 || (code >= 0x7f && code <= 0x9f)) { + throw new Error("GITHUB_REDIRECT_URI cannot contain whitespace or controls"); + } + } + let url: URL; + try { + url = new URL(configured); + } catch { + throw new Error("GITHUB_REDIRECT_URI must be an absolute HTTPS URL"); + } + if (url.protocol !== "https:" || !url.hostname) { + throw new Error("GITHUB_REDIRECT_URI must be an absolute HTTPS URL"); + } + const authority = configured.slice(configured.indexOf("//") + 2).split("/", 1)[0] ?? ""; + if (url.username || url.password || authority.includes("@")) { + throw new Error("GITHUB_REDIRECT_URI cannot use URL credentials"); + } + if (url.pathname !== githubOAuthCallbackPath) { + throw new Error(`GITHUB_REDIRECT_URI path must be ${githubOAuthCallbackPath}`); + } + if (url.search || configured.includes("?")) { + throw new Error("GITHUB_REDIRECT_URI cannot include a query string"); + } + if (url.hash || configured.includes("#")) { + throw new Error("GITHUB_REDIRECT_URI cannot include a fragment"); + } + return `${url.origin}${githubOAuthCallbackPath}`; +} + +function isLiteralLoopbackHostname(hostname: string): boolean { + return ( + hostname === "localhost" || + hostname === "127.0.0.1" || + hostname === "::1" || + hostname === "[::1]" + ); +} diff --git a/src/repo-selection.ts b/src/repo-selection.ts new file mode 100644 index 0000000..355dc03 --- /dev/null +++ b/src/repo-selection.ts @@ -0,0 +1,3 @@ +export function preferredEnabledRepo(repos: string[], preferred: string): string | undefined { + return repos.includes(preferred) ? preferred : repos[0]; +} diff --git a/src/runtime-adapter.ts b/src/runtime-adapter.ts new file mode 100644 index 0000000..8c8cc1e --- /dev/null +++ b/src/runtime-adapter.ts @@ -0,0 +1,890 @@ +export const runtimeAdapterVersion = "crabfleet/v1"; +export const runtimeAdapterName = "runtime-v1"; +export const runtimeAdapterDesktopMaxTtlMs = 15 * 60 * 1000; + +export type AdapterSessionStatus = + | "provisioning" + | "pending_adapter" + | "ready" + | "attached" + | "detached" + | "stopping" + | "stopped" + | "expired" + | "failed"; + +export type AdapterCapabilities = { + terminal: boolean; + takeover: boolean; + vnc: boolean; + desktop: boolean; + logs: boolean; + artifacts: boolean; +}; + +export const clearedAdapterCapabilities: AdapterCapabilities = { + terminal: false, + takeover: false, + vnc: false, + desktop: false, + logs: false, + artifacts: false, +}; + +export type AdapterWorkspaceResult = { + status: AdapterSessionStatus; + workspaceId: string | null; + reportedWorkspaceId: string | null; + providerResourceId: string | null; + terminalUrl: string | null; + terminalUrlPresent: boolean; + capabilities: AdapterCapabilities | null; + capabilitiesPresent: boolean; + capabilitiesOverlay?: Partial | null; + terminalCapabilityInferred?: boolean; + expiresAt: number | null; + expiresAtPresent: boolean; + profile: string | null; + message: string; +}; + +export type AdapterDesktopConnection = { + url: string; + expiresAt: number | null; + expiresAtPresent: boolean; +}; + +export type AdapterCreateInput = { + namespace: string; + id: string; + parentSessionId: string | null; + rootSessionId: string | null; + repo: string; + branch: string; + runtime: "crabbox" | "container"; + profile: string; + command: string; + prompt: string; + purpose: string; + summary: string; + owner: string; + createdBy: string; + ttlSeconds: number; + idleTimeoutSeconds: number; + desktop: boolean; +}; + +export type AdapterProvisionRecord = { + id: string; + adapter_workspace_id: string | null; + adapter_control_plane: string | null; + parent_session_id: string | null; + root_session_id: string | null; + repo: string; + branch: string; + runtime: "crabbox" | "container"; + profile: string; + command: string; + prompt: string; + purpose: string; + summary: string; + owner: string; + created_by: string; + adapter_ttl_seconds: number | null; + adapter_idle_timeout_seconds: number | null; + adapter_requested_capabilities_json: string | null; + adapter_create_payload_json: string | null; +}; + +export type AdapterProvisionRequest = { + id: string; + adapterWorkspaceId: string | null; + adapterControlPlane: string | null; + parentSessionId: string | null; + rootSessionId: string; + repo: string; + branch: string; + runtime: "crabbox" | "container"; + profile: string; + command: string; + prompt: string; + purpose: string; + summary: string; + owner: string; + createdBy: string; + adapterTtlSeconds: number | null; + adapterIdleTimeoutSeconds: number | null; + adapterRequestedCapabilities: AdapterCapabilities | null; + adapterCreatePayloadJson: string | null; +}; + +export type AdapterFailureReleaseState = { + status: "failed" | "stopping"; + terminalStatus: "failed" | null; + message: "runtime workspace released" | "runtime workspace release pending"; +}; + +const statusMap: Record = { + pending: "provisioning", + provisioning: "provisioning", + creating: "provisioning", + starting: "provisioning", + ready: "ready", + running: "ready", + healthy: "ready", + attached: "attached", + detached: "detached", + stopping: "stopping", + deleting: "stopping", + stopped: "stopped", + deleted: "stopped", + released: "stopped", + expired: "expired", + failed: "failed", + error: "failed", +}; + +const createOnlyAdapterStatuses = new Set([ + "provisioning", + "pending_adapter", + "ready", + "attached", + "detached", + "stopped", + "expired", + "failed", +]); + +export function runtimeAdapterCollectionUrl(base: string): string { + return joinAdapterUrl(base, "/v1/workspaces"); +} + +export function runtimeAdapterControlPlaneIdentity(value: unknown): string | null { + if (typeof value !== "string" || value.includes("?") || value.includes("#")) return null; + const safe = safeDesktopUrl(value); + if (!safe) return null; + const url = new URL(safe); + if (url.search || url.hash) return null; + url.pathname = url.pathname.replace(/\/+$/, "") || "/"; + return url.toString(); +} + +export function runtimeAdapterWorkspaceUrl(base: string, adapterWorkspaceId: string): string { + return joinAdapterUrl(base, `/v1/workspaces/${encodeURIComponent(adapterWorkspaceId)}`); +} + +export function runtimeAdapterDesktopUrl(base: string, adapterWorkspaceId: string): string { + return `${runtimeAdapterWorkspaceUrl(base, adapterWorkspaceId)}/connections/desktop`; +} + +export function runtimeAdapterBrowserVncUrl(canonicalUrl: string, sessionId: string): string { + return new URL( + `/api/interactive-sessions/${encodeURIComponent(sessionId)}/vnc`, + canonicalUrl, + ).toString(); +} + +export function normalizeAdapterWorkspaceId(value: string): string | null { + const normalized = value + .toLowerCase() + .replace(/[^a-z0-9-]+/g, "-") + .replace(/^-+|-+$/g, "") + .slice(0, 63) + .replace(/-+$/g, ""); + return normalized || null; +} + +export function normalizeAdapterNamespace(value: string): string | null { + const normalized = value.trim().toLowerCase(); + if (normalized.length > 32) return null; + return /^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/.test(normalized) ? normalized : null; +} + +export function namespacedAdapterWorkspaceId(namespace: string, value: string): string | null { + const normalizedNamespace = normalizeAdapterNamespace(namespace); + const normalizedId = normalizeAdapterWorkspaceId(value); + if (!normalizedNamespace || !normalizedId) return null; + const id = `${normalizedNamespace}-${normalizedId}`; + return id.length <= 63 ? id : null; +} + +export function runtimeAdapterCreatePayload( + input: AdapterCreateInput, + persistedWorkspaceId?: string, +): Record | null { + const id = persistedWorkspaceId + ? normalizeAdapterWorkspaceId(persistedWorkspaceId) === persistedWorkspaceId + ? persistedWorkspaceId + : null + : namespacedAdapterWorkspaceId(input.namespace, input.id); + if (!id) return null; + return { + id, + parentSessionId: input.parentSessionId, + rootSessionId: input.rootSessionId, + repo: input.repo, + branch: input.branch, + runtime: input.runtime, + profile: input.profile, + command: input.command, + prompt: input.prompt, + purpose: input.purpose, + summary: input.summary, + owner: input.owner, + createdBy: input.createdBy, + ttlSeconds: input.ttlSeconds, + idleTimeoutSeconds: input.idleTimeoutSeconds, + capabilities: { + desktop: input.desktop, + }, + }; +} + +export function shouldReplayRuntimeAdapterCreate( + status: AdapterSessionStatus, + createPending: boolean, +): boolean { + return createPending && (status === "provisioning" || status === "pending_adapter"); +} + +export function createOnlyAdapterStatus(value: unknown): AdapterSessionStatus | null { + return typeof value === "string" && createOnlyAdapterStatuses.has(value as AdapterSessionStatus) + ? (value as AdapterSessionStatus) + : null; +} + +export function runtimeAdapterTerminalFailureStatus( + adapter: string | null, +): "detached" | "expired" { + return adapter === runtimeAdapterName ? "detached" : "expired"; +} + +export function legacyLeaseIdForAdapter( + adapter: string | null, + leaseId: string | null, +): string | null { + return adapter === runtimeAdapterName ? null : leaseId; +} + +export function resolveCreateAfterStopRace( + createPending: boolean, + requestedTerminalStatus: "failed" | null, +): { + status: "stopping" | "stopped" | "failed"; + terminalStatus: "failed" | null; +} { + return createPending + ? { status: "stopping", terminalStatus: requestedTerminalStatus } + : { status: requestedTerminalStatus ?? "stopped", terminalStatus: null }; +} + +export function runtimeAdapterReplayRequest( + session: AdapterProvisionRecord, +): AdapterProvisionRequest { + return { + id: session.id, + adapterWorkspaceId: session.adapter_workspace_id, + adapterControlPlane: session.adapter_control_plane, + parentSessionId: session.parent_session_id, + rootSessionId: session.root_session_id ?? session.id, + repo: session.repo, + branch: session.branch, + runtime: session.runtime, + profile: session.profile, + command: session.command, + prompt: session.prompt, + purpose: session.purpose, + summary: session.summary, + owner: session.owner, + createdBy: session.created_by, + adapterTtlSeconds: session.adapter_ttl_seconds, + adapterIdleTimeoutSeconds: session.adapter_idle_timeout_seconds, + adapterRequestedCapabilities: storedAdapterCapabilities( + session.adapter_requested_capabilities_json, + ), + adapterCreatePayloadJson: session.adapter_create_payload_json, + }; +} + +export function validatedRuntimeAdapterCreatePayloadJson( + value: string, + expected: { + workspaceId: string; + ttlSeconds: number; + idleTimeoutSeconds: number; + desktop: boolean; + }, +): string | null { + if (!value || value.length > 100_000) return null; + try { + const payload = objectValue(JSON.parse(value)); + const capabilities = objectValue(payload.capabilities); + return payload.id === expected.workspaceId && + payload.ttlSeconds === expected.ttlSeconds && + payload.idleTimeoutSeconds === expected.idleTimeoutSeconds && + capabilities.desktop === expected.desktop + ? value + : null; + } catch { + return null; + } +} + +export function adapterFailureReleaseState( + release: "stopped" | "stopping", +): AdapterFailureReleaseState { + return release === "stopped" + ? { + status: "failed", + terminalStatus: null, + message: "runtime workspace released", + } + : { + status: "stopping", + terminalStatus: "failed", + message: "runtime workspace release pending", + }; +} + +export function retainedRuntimeAdapterFailureMessage( + failureReason: string | null, + reconcileError: string | null, + lastEvent: string | null, +): string { + return ( + firstString(failureReason, reconcileError, lastEvent) ?? + "interactive workspace failed after release" + ); +} + +export function definitiveRuntimeAdapterCreateFailure(status: number): boolean { + return status >= 400 && status < 500 && ![408, 409, 423, 425, 429].includes(status); +} + +export function effectiveAdapterCapabilities( + result: Pick< + AdapterWorkspaceResult, + "capabilities" | "capabilitiesPresent" | "capabilitiesOverlay" | "terminalCapabilityInferred" + >, + defaults: AdapterCapabilities, + initialCreate: boolean, +): AdapterCapabilities | null | undefined { + if (!initialCreate && !result.capabilitiesPresent && !result.terminalCapabilityInferred) { + return undefined; + } + const capabilities = result.capabilitiesPresent + ? result.capabilities === null + ? clearedAdapterCapabilities + : result.capabilitiesOverlay + ? { ...defaults, ...result.capabilitiesOverlay } + : result.capabilities + : initialCreate || result.terminalCapabilityInferred + ? defaults + : null; + return capabilities && result.terminalCapabilityInferred + ? { ...capabilities, terminal: true } + : capabilities; +} + +export function parseAdapterWorkspaceResult( + value: unknown, + fallback: { + workspaceId?: string | null; + providerResourceId?: string | null; + profile?: string | null; + } = {}, +): AdapterWorkspaceResult | null { + const body = objectValue(value); + const workspace = objectValue(body.workspace ?? body.data); + const status = adapterStatus(body.status ?? body.state ?? workspace.status ?? workspace.state); + if (!status) return null; + + const connections = objectValue(body.connections ?? workspace.connections); + const terminal = objectValue(connections.terminal ?? body.terminal ?? workspace.terminal); + const terminalUrlField = firstPresentField([ + [terminal, "url"], + [connections, "terminalUrl"], + [connections, "terminal_url"], + [body, "terminalUrl"], + [body, "terminal_url"], + [body, "attachUrl"], + [body, "attach_url"], + [workspace, "attachUrl"], + [workspace, "attach_url"], + ]); + const terminalUrl = safeWebSocketUrl(terminalUrlField.value); + const capabilityField = firstPresentField([ + [body, "capabilities"], + [workspace, "capabilities"], + [body, "features"], + [workspace, "features"], + ]); + let capabilities = adapterCapabilities(capabilityField.value); + const capabilitiesOverlay = adapterCapabilityOverlay(capabilityField.value); + const terminalCapabilitySpecified = + capabilityField.present && + (capabilitiesOverlay === null || Object.hasOwn(capabilitiesOverlay, "terminal")); + const terminalCapabilityInferred = Boolean(terminalUrl && !terminalCapabilitySpecified); + if (terminalCapabilityInferred && capabilities) { + capabilities = { ...capabilities, terminal: true }; + } + const workspaceIdentity = exactReportedAdapterWorkspaceId([ + [body, "workspaceId"], + [body, "workspace_id"], + [body, "id"], + [workspace, "workspaceId"], + [workspace, "workspace_id"], + [workspace, "id"], + ]); + if (!workspaceIdentity) return null; + const reportedWorkspaceId = workspaceIdentity.value; + const fallbackWorkspaceId = exactAdapterWorkspaceId(fallback.workspaceId); + if (fallback.workspaceId !== undefined && fallback.workspaceId !== null && !fallbackWorkspaceId) { + return null; + } + const workspaceId = reportedWorkspaceId ?? fallbackWorkspaceId; + const providerResourceId = firstString( + body.providerResourceId, + body.provider_resource_id, + body.leaseId, + body.lease_id, + workspace.providerResourceId, + workspace.provider_resource_id, + workspace.leaseId, + workspace.lease_id, + fallback.providerResourceId, + ); + const expiryField = firstPresentField([ + [body, "expiresAt"], + [body, "expires_at"], + [workspace, "expiresAt"], + [workspace, "expires_at"], + ]); + const expiresAt = adapterTimestamp(expiryField.value); + if (expiryField.present && expiryField.value !== null && expiresAt === null) return null; + + return { + status, + workspaceId, + reportedWorkspaceId, + providerResourceId, + terminalUrl, + terminalUrlPresent: terminalUrlField.present, + capabilities, + capabilitiesPresent: capabilityField.present, + capabilitiesOverlay, + terminalCapabilityInferred, + expiresAt, + expiresAtPresent: expiryField.present, + profile: firstString(body.profile, workspace.profile, fallback.profile), + message: redactedAdapterMessage( + firstString(body.message, workspace.message), + status, + [reportedWorkspaceId, providerResourceId], + adapterConnectionValues(body, workspace), + ), + }; +} + +export function redactedAdapterResponseMessage( + value: unknown, + fallback: string, + identifiers: Array = [], +): string { + const body = objectValue(value); + const workspace = objectValue(body.workspace ?? body.data); + const error = objectValue(body.error); + const errorWorkspace = objectValue(error.workspace ?? error.data); + return redactedAdapterMessage( + firstString( + body.message, + error.message, + error.detail, + error.reason, + body.error, + body.detail, + body.reason, + workspace.message, + workspace.error, + workspace.detail, + workspace.reason, + ) ?? fallback, + "failed", + [...identifiers, ...adapterProviderIdentifiers(body, workspace, error, errorWorkspace)], + adapterConnectionValues(body, workspace), + ); +} + +function adapterProviderIdentifiers( + ...objects: Array> +): Array { + return objects.flatMap((object) => [ + firstRawString(object.providerResourceId), + firstRawString(object.provider_resource_id), + firstRawString(object.leaseId), + firstRawString(object.lease_id), + ]); +} + +function storedAdapterCapabilities(value: string | null): AdapterCapabilities | null { + if (!value) return null; + try { + return adapterCapabilities(JSON.parse(value)); + } catch { + return null; + } +} + +export function adapterWorkspaceIdMatches( + result: AdapterWorkspaceResult, + expectedWorkspaceId: string, +): boolean { + return result.reportedWorkspaceId === expectedWorkspaceId; +} + +export function runtimeAdapterStopOutcome( + responseStatus: number, + result: AdapterWorkspaceResult | null, + expectedWorkspaceId: string, +): "stopped" | "stopping" | "identity_mismatch" { + if (responseStatus === 404 || responseStatus === 204) return "stopped"; + if (result && !adapterWorkspaceIdMatches(result, expectedWorkspaceId)) { + return "identity_mismatch"; + } + return result?.status === "stopped" || result?.status === "expired" ? "stopped" : "stopping"; +} + +export function parseAdapterDesktopConnection(value: unknown): AdapterDesktopConnection | null { + const body = objectValue(value); + const connection = objectValue(body.connection ?? body.desktop ?? body.data); + const url = safeDesktopUrl( + firstRawString( + body.url, + body.desktopUrl, + body.desktop_url, + body.vncUrl, + body.vnc_url, + connection.url, + connection.vncUrl, + connection.vnc_url, + ), + ); + if (!url) return null; + const expiryField = firstPresentField([ + [body, "expiresAt"], + [body, "expires_at"], + [connection, "expiresAt"], + [connection, "expires_at"], + ]); + const expiresAt = adapterTimestamp(expiryField.value); + if (expiryField.present && expiryField.value !== null && expiresAt === null) return null; + return { + url, + expiresAt, + expiresAtPresent: expiryField.present && expiryField.value !== null, + }; +} + +export function currentAdapterDesktopConnection( + value: unknown, + now = Date.now(), + maxTtlMs = runtimeAdapterDesktopMaxTtlMs, +): AdapterDesktopConnection | null { + const connection = parseAdapterDesktopConnection(value); + if ( + !connection || + (connection.expiresAtPresent && + (connection.expiresAt === null || + connection.expiresAt <= now || + connection.expiresAt > now + maxTtlMs)) + ) { + return null; + } + return connection; +} + +export function safeDesktopUrl(value: unknown): string | null { + const raw = exactUrlString(value); + if (!raw) return null; + try { + const url = new URL(raw); + if (url.username || url.password) return null; + if (url.protocol === "https:") return raw; + if (url.protocol !== "http:" || !isExactLiteralLoopbackUrl(raw, url)) return null; + return raw; + } catch { + return null; + } +} + +export function safeWebSocketUrl(value: unknown): string | null { + const raw = exactUrlString(value); + if (!raw) return null; + try { + const url = new URL(raw); + if (url.username || url.password) return null; + if (url.protocol === "wss:") return raw; + if (url.protocol !== "ws:" || !isExactLiteralLoopbackUrl(raw, url)) return null; + return raw; + } catch { + return null; + } +} + +function exactUrlString(value: unknown): string | null { + if (typeof value !== "string" || !value) return null; + for (let index = 0; index < value.length; index += 1) { + const code = value.charCodeAt(index); + if (code <= 0x20 || (code >= 0x7f && code <= 0x9f)) return null; + } + return value; +} + +function isExactLiteralLoopbackUrl(raw: string, parsed: URL): boolean { + if (!isLiteralLoopbackHostname(parsed.hostname)) return false; + return /^[a-z][a-z0-9+.-]*:\/\/(?:localhost|127\.0\.0\.1|\[::1\])(?::[0-9]+)?(?:[/?#]|$)/iu.test( + raw, + ); +} + +function adapterStatus(value: unknown): AdapterSessionStatus | null { + const normalized = typeof value === "string" ? value.trim().toLowerCase() : ""; + return statusMap[normalized] ?? null; +} + +function adapterCapabilities(value: unknown): AdapterCapabilities | null { + const defaults: AdapterCapabilities = { + terminal: false, + takeover: false, + vnc: false, + desktop: false, + logs: true, + artifacts: false, + }; + if (Array.isArray(value)) { + const features = new Set(value.map((item) => String(item).toLowerCase())); + return { + ...defaults, + terminal: features.has("terminal") || features.has("pty") || features.has("ssh"), + takeover: features.has("takeover"), + vnc: features.has("vnc") || features.has("webvnc") || features.has("desktop"), + desktop: features.has("desktop") || features.has("vnc") || features.has("webvnc"), + artifacts: features.has("artifacts"), + }; + } + const overlay = adapterCapabilityOverlay(value); + return overlay ? { ...defaults, ...overlay } : null; +} + +function adapterCapabilityOverlay(value: unknown): Partial | null { + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + const capabilities = value as Record; + const overlay: Partial = {}; + const terminal = firstPresentField([ + [capabilities, "terminal"], + [capabilities, "pty"], + [capabilities, "ssh"], + ]); + if (terminal.present) overlay.terminal = booleanValue(terminal.value); + const takeover = firstPresentField([[capabilities, "takeover"]]); + if (takeover.present) overlay.takeover = booleanValue(takeover.value); + const desktop = firstPresentField([[capabilities, "desktop"]]); + const vnc = firstPresentField([ + [capabilities, "vnc"], + [capabilities, "webvnc"], + ]); + if (desktop.present || vnc.present) { + const desktopAvailable = booleanValue(desktop.value) || booleanValue(vnc.value); + overlay.desktop = desktopAvailable; + overlay.vnc = desktopAvailable; + } + const logs = firstPresentField([[capabilities, "logs"]]); + if (logs.present) overlay.logs = booleanValue(logs.value); + const artifacts = firstPresentField([[capabilities, "artifacts"]]); + if (artifacts.present) overlay.artifacts = booleanValue(artifacts.value); + return overlay; +} + +function adapterTimestamp(value: unknown): number | null { + if (typeof value === "number" && Number.isFinite(value)) { + return value < 1_000_000_000_000 ? Math.trunc(value * 1000) : Math.trunc(value); + } + if (typeof value !== "string" || !value.trim()) return null; + const numeric = Number(value); + if (Number.isFinite(numeric)) { + return numeric < 1_000_000_000_000 ? Math.trunc(numeric * 1000) : Math.trunc(numeric); + } + const parsed = Date.parse(value); + return Number.isFinite(parsed) ? parsed : null; +} + +function joinAdapterUrl(base: string, path: string): string { + const normalized = base.endsWith("/") ? base : `${base}/`; + return new URL(path.replace(/^\/+/, ""), normalized).toString(); +} + +function objectValue(value: unknown): Record { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? (value as Record) + : {}; +} + +function firstPresentField(fields: Array<[Record, string]>): { + present: boolean; + value: unknown; +} { + for (const [object, key] of fields) { + if (Object.hasOwn(object, key)) return { present: true, value: object[key] }; + } + return { present: false, value: undefined }; +} + +function exactReportedAdapterWorkspaceId( + fields: Array<[Record, string]>, +): { value: string | null } | null { + let value: string | null = null; + for (const [object, key] of fields) { + if (!Object.hasOwn(object, key)) continue; + const candidate = exactAdapterWorkspaceId(object[key]); + if (!candidate || (value !== null && value !== candidate)) return null; + value = candidate; + } + return { value }; +} + +function exactAdapterWorkspaceId(value: unknown): string | null { + return typeof value === "string" && normalizeAdapterWorkspaceId(value) === value ? value : null; +} + +function firstString(...values: unknown[]): string | null { + for (const value of values) { + if (typeof value === "string" && value.trim()) return value.trim(); + } + return null; +} + +function firstRawString(...values: unknown[]): string | null { + for (const value of values) { + if (typeof value === "string" && value) return value; + } + return null; +} + +function booleanValue(value: unknown): boolean { + return value === true || value === 1 || value === "true"; +} + +export function redactedAdapterMessage( + value: string | null, + status: AdapterSessionStatus, + identifiers: Array = [], + connectionValues: Array = [], +): string { + let message = value || `runtime adapter workspace ${status}`; + const sensitiveName = + "access[_-]?token|api[_-]?key|apikey|auth|authorization|client[_-]?secret|code|cookie|credential|id[_-]?token|key|password|passwd|refresh[_-]?token|secret|session[_-]?token|sig|signature|ticket|token|x-amz-signature"; + message = message + .replace(/\b(?:https?|wss?):(?:\\+\/){2}[^\s<>{}"'`]+/giu, "[connection]") + .replace(/\b(?:https?|wss?):\/\/[^\s<>{}"'`]+/giu, "[connection]") + .replace( + /\b(?:authorization|proxy-authorization|x-api-key|api-key)\s*:\s*[^\r\n]+/giu, + "[credential]", + ) + .replace(/\b(?:bearer|basic)\s+[^\s,;}\x5d]+/giu, "[credential]") + .replace(/\b(?:cookie|set-cookie)\s*:\s*[^\r\n]+/giu, "[credential]") + .replace( + new RegExp( + `(?:\\\\?["']?)(?:${sensitiveName})(?:\\\\?["']?)\\s*[:=]\\s*(?!\\[credential\\])(?:\\\\?["'](?:\\\\.|[^"'\\\\])*\\\\?["']|[^\\s,;}&}\\x5d]+)`, + "giu", + ), + "[credential]", + ); + const orderedConnections = [ + ...new Set(connectionValues.filter((connection): connection is string => Boolean(connection))), + ].sort((left, right) => right.length - left.length); + for (const connection of orderedConnections) { + for (const representation of escapedConnectionRepresentations(connection)) { + message = message.replaceAll(representation, "[connection]"); + } + } + const orderedIdentifiers = [ + ...new Set(identifiers.filter((identifier): identifier is string => Boolean(identifier))), + ].sort((left, right) => right.length - left.length); + for (const identifier of orderedIdentifiers) { + message = message.replaceAll(identifier, "[workspace]"); + } + let withoutControls = ""; + for (let index = 0; index < message.length; index += 1) { + const code = message.charCodeAt(index); + withoutControls += code < 0x20 || (code >= 0x7f && code <= 0x9f) ? " " : message[index]; + } + return withoutControls.replace(/\s+/gu, " ").trim().slice(0, 500); +} + +function adapterConnectionValues( + body: Record, + workspace: Record, +): Array { + const connections = objectValue(body.connections ?? workspace.connections); + const terminal = objectValue(connections.terminal ?? body.terminal ?? workspace.terminal); + const desktop = objectValue(connections.desktop ?? body.desktop ?? workspace.desktop); + const values: Array = [ + terminal.url, + connections.terminalUrl, + connections.terminal_url, + body.terminalUrl, + body.terminal_url, + body.attachUrl, + body.attach_url, + workspace.attachUrl, + workspace.attach_url, + desktop.url, + connections.desktopUrl, + connections.desktop_url, + body.desktopUrl, + body.desktop_url, + body.vncUrl, + body.vnc_url, + workspace.desktopUrl, + workspace.desktop_url, + workspace.vncUrl, + workspace.vnc_url, + ].map((value) => (typeof value === "string" && value ? value : null)); + const connectionField = /^(?:attach|desktop|terminal|vnc)?_?url$/iu; + const visited = new WeakSet(); + const visit = (value: unknown, depth: number) => { + if (!value || typeof value !== "object" || depth > 5) return; + if (visited.has(value)) return; + visited.add(value); + for (const [key, nested] of Object.entries(value)) { + if (connectionField.test(key) && typeof nested === "string" && nested) { + values.push(nested); + } else if (nested && typeof nested === "object") { + visit(nested, depth + 1); + } + } + }; + visit(body, 0); + return values; +} + +function escapedConnectionRepresentations(connection: string): string[] { + const representations = new Set(); + for (const initial of [connection, connection.replaceAll("/", "\\/")]) { + let escaped = initial; + for (let depth = 0; depth < 3; depth += 1) { + representations.add(escaped); + escaped = JSON.stringify(escaped).slice(1, -1); + } + } + return [...representations].sort((left, right) => right.length - left.length); +} + +function isLiteralLoopbackHostname(hostname: string): boolean { + return ( + hostname === "localhost" || + hostname === "127.0.0.1" || + hostname === "::1" || + hostname === "[::1]" + ); +} diff --git a/src/session-archive.ts b/src/session-archive.ts new file mode 100644 index 0000000..d9f8be0 --- /dev/null +++ b/src/session-archive.ts @@ -0,0 +1,49 @@ +export type SessionArchiveObjectKeys = { + events_key: string | null; + transcript_key: string | null; + summary_key: string | null; +}; + +export type PopulatedSessionArchiveObjectKeys = { + events_key: string; + transcript_key: string; + summary_key: string; +}; + +export function sessionArchiveAttemptKeys( + archiveBase: string, + eventCount: number, + latestEventAt: number, + now: number, + attemptId: string, +): PopulatedSessionArchiveObjectKeys { + const version = `${String(eventCount).padStart(8, "0")}-${String(latestEventAt).padStart(13, "0")}-${now}-${attemptId}`; + const base = `${archiveBase}/${version}`; + return { + events_key: `${base}/events.ndjson`, + transcript_key: `${base}/transcript.md`, + summary_key: `${base}/summary.json`, + }; +} + +export function sameSessionArchiveObjectKeys( + left: SessionArchiveObjectKeys | undefined, + right: SessionArchiveObjectKeys | undefined, +): boolean { + return Boolean( + left && + right && + left.events_key === right.events_key && + left.transcript_key === right.transcript_key && + left.summary_key === right.summary_key, + ); +} + +export function obsoleteSessionArchiveObjectKeys( + latest: SessionArchiveObjectKeys | undefined, + previous: SessionArchiveObjectKeys | undefined, + attempted: PopulatedSessionArchiveObjectKeys, +): SessionArchiveObjectKeys | undefined { + const candidate = sameSessionArchiveObjectKeys(latest, attempted) ? previous : attempted; + return sameSessionArchiveObjectKeys(candidate, latest) ? undefined : candidate; +} diff --git a/src/session-id.ts b/src/session-id.ts new file mode 100644 index 0000000..0f6947d --- /dev/null +++ b/src/session-id.ts @@ -0,0 +1,10 @@ +export const allocateInteractiveSessionIdSql = ` +INSERT INTO id_sequences (name, last_id) +VALUES ('interactive_sessions', 101) +ON CONFLICT(name) DO UPDATE SET last_id = id_sequences.last_id + 1 +RETURNING last_id AS next_id +`; + +export function formatInteractiveSessionId(value: number): string | null { + return Number.isSafeInteger(value) && value > 100 ? `IS-${value}` : null; +} diff --git a/src/terminal-authorization.ts b/src/terminal-authorization.ts new file mode 100644 index 0000000..87c4f70 --- /dev/null +++ b/src/terminal-authorization.ts @@ -0,0 +1,27 @@ +export function cachedBooleanGrant( + read: () => Promise, + ttlMs = 1000, + now: () => number = Date.now, +): () => Promise { + let cached = false; + let validUntil = 0; + let inFlight: Promise | null = null; + return async () => { + const checkedAt = now(); + if (checkedAt < validUntil) return cached; + if (!inFlight) { + inFlight = Promise.resolve() + .then(read) + .catch(() => false) + .then((allowed) => { + cached = allowed; + validUntil = now() + ttlMs; + return allowed; + }) + .finally(() => { + inFlight = null; + }); + } + return inFlight; + }; +} diff --git a/src/terminal-finalization.ts b/src/terminal-finalization.ts new file mode 100644 index 0000000..5c13dcb --- /dev/null +++ b/src/terminal-finalization.ts @@ -0,0 +1,42 @@ +export type TerminalArchiveState = { + eventCount: number; + archiveEventCount: number | null; + archiveObjectsReady: boolean; + archiveSessionVersionMatches: boolean; +}; + +export type TerminalFinalizationOperations = { + ensureEvent: () => Promise; + readArchiveState: () => Promise; + archive: () => Promise; + clearPending: () => Promise; +}; + +export function terminalArchiveNeedsRefresh( + eventInserted: boolean, + state: TerminalArchiveState, +): boolean { + return ( + eventInserted || + state.archiveEventCount === null || + state.archiveEventCount < state.eventCount || + !state.archiveObjectsReady || + !state.archiveSessionVersionMatches + ); +} + +export async function completeTerminalFinalization( + operations: TerminalFinalizationOperations, +): Promise { + const eventInserted = await operations.ensureEvent(); + for (let attempt = 0; attempt < 3; attempt += 1) { + const state = await operations.readArchiveState(); + if (terminalArchiveNeedsRefresh(eventInserted && attempt === 0, state)) { + await operations.archive(); + } + const verified = await operations.readArchiveState(); + if (terminalArchiveNeedsRefresh(false, verified)) continue; + if (await operations.clearPending()) return; + } + throw new Error("terminal archive coverage changed during finalization"); +} diff --git a/src/terminal-target.ts b/src/terminal-target.ts new file mode 100644 index 0000000..573fd30 --- /dev/null +++ b/src/terminal-target.ts @@ -0,0 +1,18 @@ +export type TerminalRouteKind = "sandbox" | "bridge" | "attach" | "cloudflare"; + +export function sizedTerminalTargetUrl( + rawUrl: string, + routeKind: TerminalRouteKind | null, + cols: number, + rows: number, +): string { + if (routeKind !== "bridge" && routeKind !== "cloudflare") return rawUrl; + try { + const url = new URL(rawUrl); + url.searchParams.set("cols", String(cols)); + url.searchParams.set("rows", String(rows)); + return url.toString(); + } catch { + return ""; + } +} diff --git a/src/url-security.ts b/src/url-security.ts new file mode 100644 index 0000000..9fffcdc --- /dev/null +++ b/src/url-security.ts @@ -0,0 +1,33 @@ +export function isLiteralLoopbackHostname(hostname: string): boolean { + return ( + hostname === "localhost" || + hostname === "127.0.0.1" || + hostname === "::1" || + hostname === "[::1]" + ); +} + +export function developmentIdentityEnabled(value: string | undefined, requestUrl: string): boolean { + if (value !== "true") return false; + try { + return isLiteralLoopbackHostname(new URL(requestUrl).hostname.toLowerCase()); + } catch { + return false; + } +} + +export function configuredHttpOrigin(value: string | undefined, fallback: string): string { + const candidate = String(value ?? "") + .trim() + .slice(0, 1000); + if (!candidate) return fallback; + try { + const url = new URL(candidate); + if (url.username || url.password) return fallback; + if (url.protocol === "https:") return url.origin; + if (url.protocol === "http:" && isLiteralLoopbackHostname(url.hostname)) return url.origin; + return fallback; + } catch { + return fallback; + } +} diff --git a/tests/app-utils.test.ts b/tests/app-utils.test.ts index 6d038e0..3d398a0 100644 --- a/tests/app-utils.test.ts +++ b/tests/app-utils.test.ts @@ -4,11 +4,13 @@ import { humanStatus, isActiveRun, isDeadInteractiveSession, + isFleetSessionAttachable, isTerminalReadyInteractiveSession, interactiveSessionStatus, interactiveCommand, linkedInteractiveSessionPlaceholder, optimisticInteractiveSession, + runCapabilities, sessionItems, terminalText, } from "../src/app/utils.js"; @@ -111,17 +113,66 @@ test("linked session placeholders render a best-effort Codex card", () => { test("interactive lifecycle helpers keep UI and terminal state aligned", () => { const live = { kind: "interactive", status: "attached" }; const rawLive = { status: "ready" }; + const terminalWithdrawn = { + kind: "interactive", + status: "ready", + capabilities: { terminal: false }, + }; + const rawTerminalWithdrawn = { status: "ready", capabilities: { terminal: false } }; + const controlledWithoutPty = { + kind: "interactive", + status: "ready", + capabilities: { terminal: true }, + canControl: true, + ptyAvailable: false, + }; + const controlledWithPty = { ...controlledWithoutPty, ptyAvailable: true }; + const sharedReadOnly = { + ...controlledWithoutPty, + canControl: false, + sharedReadOnly: true, + }; const provisioning = { kind: "interactive", status: "pending_adapter" }; + const stopping = { kind: "interactive", status: "stopping" }; const failed = { kind: "interactive", status: "failed" }; assert.equal(isActiveRun(live), true); assert.equal(isTerminalReadyInteractiveSession(live), true); assert.equal(isTerminalReadyInteractiveSession(rawLive), true); + assert.equal(isTerminalReadyInteractiveSession(terminalWithdrawn), false); + assert.equal(isTerminalReadyInteractiveSession(rawTerminalWithdrawn), false); + assert.equal(isTerminalReadyInteractiveSession(controlledWithoutPty), false); + assert.equal(isTerminalReadyInteractiveSession(controlledWithPty), true); + assert.equal(isTerminalReadyInteractiveSession(sharedReadOnly), true); + assert.equal(runCapabilities(rawTerminalWithdrawn).terminal, false); + assert.equal( + isFleetSessionAttachable({ + ...rawLive, + attachUrl: "wss://terminal.example/session", + fleet: { attachable: true }, + capabilities: { terminal: false }, + }), + false, + ); + assert.equal( + isFleetSessionAttachable({ + ...rawLive, + attachUrl: "wss://terminal.example/session", + fleet: { attachable: false }, + capabilities: { terminal: true }, + }), + false, + ); assert.deepEqual(interactiveSessionStatus(live), { label: "Live", tone: "live" }); assert.deepEqual(interactiveSessionStatus(provisioning), { label: "Provisioning", tone: "provisioning", }); + assert.equal(isActiveRun(stopping), false); + assert.deepEqual(interactiveSessionStatus(stopping), { + label: "Stopping", + tone: "provisioning", + }); assert.equal(isDeadInteractiveSession(failed), true); assert.deepEqual(interactiveSessionStatus(failed), { label: "Failed", tone: "failed" }); assert.equal(humanStatus("pending_adapter"), "Pending Adapter"); diff --git a/tests/bounded-response.test.ts b/tests/bounded-response.test.ts new file mode 100644 index 0000000..087ee6e --- /dev/null +++ b/tests/bounded-response.test.ts @@ -0,0 +1,32 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { readBoundedResponseText, ResponseBodyLimitError } from "../src/bounded-response.ts"; + +test("bounded response reader accepts a body exactly at the limit", async () => { + const response = new Response("abcd", { headers: { "content-length": "4" } }); + assert.equal(await readBoundedResponseText(response, 4), "abcd"); +}); + +test("bounded response reader rejects an oversized declared body before parsing", async () => { + const response = new Response("small", { headers: { "content-length": "100" } }); + await assert.rejects(readBoundedResponseText(response, 8), ResponseBodyLimitError); +}); + +test("bounded response reader cancels an oversized chunked body", async () => { + let canceled = false; + const response = new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode("1234")); + controller.enqueue(new TextEncoder().encode("56789")); + }, + cancel() { + canceled = true; + }, + }), + ); + + await assert.rejects(readBoundedResponseText(response, 8), ResponseBodyLimitError); + assert.equal(canceled, true); +}); diff --git a/tests/credential-policy-fence.test.ts b/tests/credential-policy-fence.test.ts new file mode 100644 index 0000000..17911aa --- /dev/null +++ b/tests/credential-policy-fence.test.ts @@ -0,0 +1,290 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + credentialPolicyCleanupMatches, + credentialPolicyMigrationCleanupMatches, + credentialPolicyRegistrationAccepted, + credentialPolicySandboxIsExpected, + migratedCredentialPolicyRecord, + type CredentialPolicyGenerationRecord, + type CredentialPolicyGenerationTombstone, +} from "../src/credential-policy-fence.ts"; + +type Policy = { sessionId: string; value: string }; + +const registration = ( + generation: string, + claim: string, + registrationExpiresAt = 200, +): CredentialPolicyGenerationRecord => ({ + generation, + registrationClaim: claim, + registrationExpiresAt, + policy: { sessionId: "IS-101", value: claim }, +}); + +test("generation tombstones reject late registration regardless of operation order", () => { + const incoming = registration("generation-1", "claim-1"); + const tombstone: CredentialPolicyGenerationTombstone = { + generation: "generation-1", + sessionId: "IS-101", + tombstonedAt: 100, + }; + + assert.equal(credentialPolicyRegistrationAccepted(undefined, undefined, incoming, 100), true); + assert.equal(credentialPolicyCleanupMatches(incoming, "generation-1", "IS-101"), true); + assert.equal(credentialPolicyRegistrationAccepted(undefined, tombstone, incoming, 100), false); + assert.equal( + credentialPolicyRegistrationAccepted( + undefined, + { ...tombstone, sessionId: "IS-102" }, + incoming, + 100, + ), + false, + ); +}); + +test("generation fences isolate new policies from stale cleanup", () => { + const current = registration("generation-2", "claim-2"); + assert.equal( + credentialPolicyRegistrationAccepted( + current, + undefined, + registration("generation-1", "claim-1"), + 100, + ), + false, + ); + assert.equal(credentialPolicyCleanupMatches(current, "generation-1", "IS-101"), false); + assert.equal( + credentialPolicyRegistrationAccepted( + undefined, + undefined, + registration("generation-1", "claim-1"), + 200, + ), + false, + ); +}); + +test("same-generation registration claims advance monotonically", () => { + const current = registration("generation-1", "claim-current", 300); + + assert.equal( + credentialPolicyRegistrationAccepted( + current, + undefined, + registration("generation-1", "claim-current", 299), + 100, + ), + false, + ); + assert.equal( + credentialPolicyRegistrationAccepted( + current, + undefined, + registration("generation-1", "claim-current", 300), + 100, + ), + true, + ); + assert.equal( + credentialPolicyRegistrationAccepted( + current, + undefined, + registration("generation-1", "claim-current", 301), + 100, + ), + true, + ); + assert.equal( + credentialPolicyRegistrationAccepted( + current, + undefined, + registration("generation-1", "claim-replacement", 300), + 100, + ), + false, + ); + assert.equal( + credentialPolicyRegistrationAccepted( + current, + undefined, + registration("generation-1", "claim-replacement", 301), + 100, + ), + true, + ); +}); + +test("delayed abandoned registration cannot replace a newer claim", () => { + const newer = registration("generation-1", "claim-new", 400); + const abandoned = registration("generation-1", "claim-old", 300); + + assert.equal(credentialPolicyRegistrationAccepted(newer, undefined, abandoned, 200), false); +}); + +test("legacy policy migration resumes idempotently after a crash", () => { + const legacy: Policy = { sessionId: "IS-101", value: "legacy-secret" }; + const promoted = migratedCredentialPolicyRecord( + registration("legacy:IS-101:sandbox", "legacy-claim", 200), + undefined, + undefined, + { + generation: "generation-repaired", + registrationClaim: "legacy-repair:first", + registrationExpiresAt: 300, + sessionId: "IS-101", + }, + 100, + ); + assert.equal(promoted?.generation, "generation-repaired"); + assert.equal( + migratedCredentialPolicyRecord( + registration("legacy:IS-101:sandbox", "legacy-newer", 400), + undefined, + undefined, + { + generation: "generation-repaired", + registrationClaim: "legacy-repair:stale", + registrationExpiresAt: 300, + sessionId: "IS-101", + }, + 100, + ), + undefined, + ); + const first = migratedCredentialPolicyRecord( + undefined, + legacy, + undefined, + { + generation: "generation-repaired", + registrationClaim: "legacy-repair:first", + registrationExpiresAt: 300, + sessionId: "IS-101", + }, + 100, + ); + assert.deepEqual(first, { + generation: "generation-repaired", + registrationClaim: "legacy-repair:first", + registrationExpiresAt: 300, + policy: legacy, + }); + + const retry = migratedCredentialPolicyRecord( + first, + undefined, + undefined, + { + generation: "generation-repaired", + registrationClaim: "legacy-repair:retry", + registrationExpiresAt: 400, + sessionId: "IS-101", + }, + 301, + ); + assert.equal(retry?.registrationClaim, "legacy-repair:retry"); + assert.equal(retry?.policy, legacy); + assert.equal( + migratedCredentialPolicyRecord( + retry, + undefined, + undefined, + { + generation: "generation-repaired", + registrationClaim: "legacy-repair:first", + registrationExpiresAt: 300, + sessionId: "IS-101", + }, + 200, + ), + undefined, + ); +}); + +test("legacy migration honors identity, tombstones, and raced cleanup", () => { + const legacy: Policy = { sessionId: "IS-101", value: "legacy-secret" }; + const migration = { + generation: "generation-repaired", + registrationClaim: "legacy-repair:claim", + registrationExpiresAt: 300, + sessionId: "IS-101", + }; + assert.equal( + migratedCredentialPolicyRecord( + undefined, + { ...legacy, sessionId: "IS-102" }, + undefined, + migration, + 100, + ), + undefined, + ); + assert.equal( + migratedCredentialPolicyRecord( + undefined, + legacy, + { + generation: migration.generation, + sessionId: migration.sessionId, + tombstonedAt: 99, + }, + migration, + 100, + ), + undefined, + ); + assert.equal( + credentialPolicyMigrationCleanupMatches( + registration("legacy:IS-101:sandbox", "old-claim", 200), + migration.generation, + migration.sessionId, + ), + true, + ); + assert.equal( + credentialPolicyMigrationCleanupMatches( + registration("generation-other", "other-claim", 200), + migration.generation, + migration.sessionId, + ), + false, + ); +}); + +test("live lease refresh fences both current and expected sandbox policies", () => { + assert.equal( + credentialPolicySandboxIsExpected("sandbox-old", "sandbox-old", null, null, null, 100), + true, + ); + assert.equal( + credentialPolicySandboxIsExpected( + "sandbox-old", + "sandbox-new", + "sandbox-new", + "refresh-claim", + 200, + 100, + ), + true, + ); + assert.equal( + credentialPolicySandboxIsExpected( + "sandbox-old", + "sandbox-new", + "sandbox-new", + "refresh-claim", + 100, + 100, + ), + false, + ); + assert.equal( + credentialPolicySandboxIsExpected("sandbox-old", "sandbox-new", "sandbox-new", null, 200, 100), + false, + ); +}); diff --git a/tests/d1-execution.test.ts b/tests/d1-execution.test.ts new file mode 100644 index 0000000..e2cd4e6 --- /dev/null +++ b/tests/d1-execution.test.ts @@ -0,0 +1,70 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { D1Connection, executeD1Statement } from "../src/d1-execution.ts"; + +test("D1 connection executes INSERT RETURNING through all and preserves rows", async () => { + let allCalls = 0; + let runCalls = 0; + let boundParameters: readonly unknown[] = []; + const sql = "INSERT INTO id_sequences(name, last_id) VALUES (?, ?) RETURNING last_id AS next_id"; + const connection = new D1Connection({ + prepare(preparedSql: string) { + assert.equal(preparedSql, sql); + return { + bind(...parameters: unknown[]) { + boundParameters = parameters; + return { + async all() { + allCalls += 1; + return { + results: [{ next_id: 117 }], + meta: { changes: 1, last_row_id: 117 }, + }; + }, + async run() { + runCalls += 1; + return { meta: { changes: 1 } }; + }, + }; + }, + }; + }, + } as unknown as D1Database); + const result = await connection.executeQuery<{ next_id: number }>({ + sql, + parameters: ["interactive_sessions", 117], + query: {} as never, + queryId: {} as never, + }); + + assert.deepEqual(boundParameters, ["interactive_sessions", 117]); + assert.equal(allCalls, 1); + assert.equal(runCalls, 0); + assert.deepEqual(result.rows, [{ next_id: 117 }]); + assert.equal(result.numAffectedRows, 1n); + assert.equal(result.insertId, 117n); +}); + +test("D1 executes non-returning mutations through run", async () => { + let allCalls = 0; + let runCalls = 0; + const result = await executeD1Statement( + { + async all() { + allCalls += 1; + return { results: [], meta: {} }; + }, + async run() { + runCalls += 1; + return { meta: { changes: 2 } }; + }, + }, + "UPDATE sessions SET status = 'stopped'", + ); + + assert.equal(allCalls, 0); + assert.equal(runCalls, 1); + assert.deepEqual(result.rows, []); + assert.equal(result.changes, 2); +}); diff --git a/tests/fleet-state.test.ts b/tests/fleet-state.test.ts index 1513647..962f959 100644 --- a/tests/fleet-state.test.ts +++ b/tests/fleet-state.test.ts @@ -61,6 +61,7 @@ test("fleet state aggregates sessions and redacted sandbox policies", () => { defaultEgressHosts: ["github.com", "api.github.com"], generatedAt: 100, productUrl: "https://crabfleet.ai", + sandboxAvailable: true, }, ); @@ -95,3 +96,160 @@ test("sandbox lease parser ignores non-sandbox leases", () => { assert.equal(sandboxIdFromLeaseId("crabbox:external"), null); assert.equal(sandboxIdFromLeaseId(null), null); }); + +test("fleet VNC availability follows adapter capabilities without persisting a URL", () => { + const fleet = buildFleetState( + [ + { + ...baseSession, + runtime: "crabbox", + adapter: "runtime-v1", + leaseId: "provider/resource", + vncUrl: null, + capabilities: { terminal: true, desktop: true, vnc: true }, + }, + ], + [], + { + canonicalUrl: "https://fleet.example", + defaultEgressHosts: [], + generatedAt: 100, + productUrl: "https://product.example", + }, + ); + + assert.equal(fleet.totals.vnc, 1); + assert.equal(fleet.sessions[0]?.vnc, true); +}); + +test("stopping sessions are inactive and not attachable", () => { + const fleet = buildFleetState( + [ + { + ...baseSession, + status: "stopping", + attachUrl: "wss://terminal.example/session", + capabilities: { desktop: true, vnc: true }, + }, + ], + [], + { + canonicalUrl: "https://fleet.example", + defaultEgressHosts: [], + generatedAt: 100, + productUrl: "https://product.example", + }, + ); + + assert.equal(fleet.totals.active, 0); + assert.equal(fleet.totals.attachable, 0); + assert.equal(fleet.totals.vnc, 0); + assert.equal(fleet.sessions[0]?.active, false); +}); + +test("withdrawn terminal capability suppresses fleet attachability", () => { + const fleet = buildFleetState( + [ + { + ...baseSession, + adapter: "runtime-v1", + attachUrl: "wss://terminal.example/session", + capabilities: { terminal: false }, + }, + ], + [], + { + canonicalUrl: "https://fleet.example", + defaultEgressHosts: [], + generatedAt: 100, + productUrl: "https://product.example", + }, + ); + + assert.equal(fleet.totals.attachable, 0); + assert.equal(fleet.sessions[0]?.attachable, false); +}); + +test("fleet attachability follows resolvable PTY routes", () => { + const options = { + canonicalUrl: "https://fleet.example", + defaultEgressHosts: [], + generatedAt: 100, + productUrl: "https://product.example", + }; + const summary = ( + session: Parameters[0][number], + routing: Partial[2]> = {}, + ) => buildFleetState([session], [], { ...options, ...routing }).sessions[0]?.attachable; + + assert.equal( + summary({ ...baseSession, leaseId: null, attachUrl: "wss://terminal.example/session" }), + true, + ); + assert.equal( + summary({ + ...baseSession, + status: "provisioning", + leaseId: null, + attachUrl: "wss://terminal.example/session", + }), + false, + ); + assert.equal( + summary({ ...baseSession, leaseId: null, attachUrl: "https://terminal.example/console" }), + false, + ); + assert.equal( + summary({ ...baseSession, leaseId: null, attachUrl: "ws://terminal.example/session" }), + false, + ); + assert.equal( + summary({ ...baseSession, leaseId: null, attachUrl: "ws://127.0.0.1:9000/session" }), + true, + ); + assert.equal( + summary( + { ...baseSession, leaseId: "cloudflare:workspace-1", attachUrl: null }, + { cloudflareRunnerUrl: "https://runner.example" }, + ), + true, + ); + assert.equal( + summary( + { ...baseSession, leaseId: null, attachUrl: null }, + { ptyBridgeUrl: "https://bridge.example/pty/{id}" }, + ), + true, + ); + assert.equal( + summary( + { ...baseSession, leaseId: null, attachUrl: null, canControl: false }, + { ptyBridgeUrl: "https://bridge.example/pty/{id}" }, + ), + false, + ); +}); + +test("legacy sessions require an actual VNC URL", () => { + const fleet = buildFleetState( + [ + { + ...baseSession, + runtime: "crabbox", + adapter: null, + vncUrl: null, + capabilities: { desktop: true, vnc: true }, + }, + ], + [], + { + canonicalUrl: "https://fleet.example", + defaultEgressHosts: [], + generatedAt: 100, + productUrl: "https://product.example", + }, + ); + + assert.equal(fleet.totals.vnc, 0); + assert.equal(fleet.sessions[0]?.vnc, false); +}); diff --git a/tests/html-dialogs.test.ts b/tests/html-dialogs.test.ts index 2ecc871..5b6da43 100644 --- a/tests/html-dialogs.test.ts +++ b/tests/html-dialogs.test.ts @@ -8,4 +8,14 @@ test("app actions use styled HTML dialogs instead of browser prompts", async () assert.doesNotMatch(source, /\bwindow\.(?:alert|confirm|prompt)\s*\(/); assert.match(source, / setRepo\(preferred\), \[preferred\]\)/); +}); + +test("fleet terminal affordances require attachable session state", async () => { + const source = await readFile(new URL("../src/app/fleet.jsx", import.meta.url), "utf8"); + + assert.match(source, /const attachable = isFleetSessionAttachable\(session\)/); + assert.match(source, /\{attachable \? \(/); + assert.match(source, /cli=\{totals\.attachable \?\? props\.cli\}/); }); diff --git a/tests/oauth.test.ts b/tests/oauth.test.ts index 51c9ff3..e5c4018 100644 --- a/tests/oauth.test.ts +++ b/tests/oauth.test.ts @@ -1,14 +1,20 @@ import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; import { test } from "node:test"; -import { githubOAuthRedirectUri } from "../src/oauth.ts"; +import { + githubOAuthCallbackRequestMatches, + githubOAuthCanonicalLoginUrl, + githubOAuthCanonicalSshLinkUrl, + githubOAuthRedirectUri, +} from "../src/oauth.ts"; test("githubOAuthRedirectUri uses configured callback when present", () => { assert.equal( githubOAuthRedirectUri( "https://crabfleet.openclaw.ai/login/github", - " https://crabfleet.openclaw.ai/auth/github/callback ", + "https://fleet.example/auth/github/callback", ), - "https://crabfleet.openclaw.ai/auth/github/callback", + "https://fleet.example/auth/github/callback", ); }); @@ -17,4 +23,120 @@ test("githubOAuthRedirectUri defaults to request origin callback", () => { githubOAuthRedirectUri("https://crabfleet.openclaw.ai/login/github"), "https://crabfleet.openclaw.ai/auth/github/callback", ); + assert.equal( + githubOAuthRedirectUri("http://localhost:8787/login/github"), + "http://localhost:8787/auth/github/callback", + ); + assert.throws( + () => githubOAuthRedirectUri("http://attacker.example/login/github"), + /HTTPS or loopback HTTP/, + ); +}); + +test("configured GitHub callback validation fails closed", () => { + for (const configured of [ + "", + " https://fleet.example/auth/github/callback", + "http://fleet.example/auth/github/callback", + "https:fleet.example/auth/github/callback", + "https://user:secret@fleet.example/auth/github/callback", + "https://@fleet.example/auth/github/callback", + "https://fleet.example/auth/github/call back", + "https://fleet.example/auth/github/callback?tenant=a", + "https://fleet.example/auth/github/callback#fragment", + "https://fleet.example/auth/github/callback/", + "https://fleet.example/other/callback", + "/auth/github/callback", + ]) { + assert.throws( + () => githubOAuthRedirectUri("https://request.example/login/github", configured), + /GITHUB_REDIRECT_URI/, + configured, + ); + } +}); + +test("configured GitHub origin is authoritative across host mismatches", () => { + const configured = "https://fleet.example/auth/github/callback"; + assert.equal( + githubOAuthCanonicalLoginUrl("https://attacker.example/login/github", configured), + "https://fleet.example/login/github", + ); + assert.equal( + githubOAuthCanonicalLoginUrl("https://fleet.example/login/github", configured), + null, + ); + assert.equal( + githubOAuthCallbackRequestMatches( + "https://fleet.example/auth/github/callback?code=a&state=b", + configured, + ), + true, + ); + assert.equal( + githubOAuthCallbackRequestMatches( + "https://attacker.example/auth/github/callback?code=a&state=b", + configured, + ), + false, + ); + assert.equal( + githubOAuthCallbackRequestMatches( + "http://fleet.example/auth/github/callback?code=a&state=b", + configured, + ), + false, + ); + assert.equal( + githubOAuthCallbackRequestMatches( + "https://fleet.example/login/github?code=a&state=b", + configured, + ), + false, + ); +}); + +test("SSH link state canonicalizes before host-only OAuth cookies", async () => { + const configured = "https://fleet.example/auth/github/callback"; + assert.equal( + githubOAuthCanonicalSshLinkUrl( + "https://alias.example/ssh/link/code%2Fwith%2Fslashes", + "code/with/slashes", + configured, + ), + "https://fleet.example/ssh/link/code%2Fwith%2Fslashes", + ); + assert.equal( + githubOAuthCanonicalSshLinkUrl( + "https://fleet.example/ssh/link/code%2Fwith%2Fslashes", + "code/with/slashes", + configured, + ), + null, + ); + const source = await readFile(new URL("../src/index.ts", import.meta.url), "utf8"); + const linkStart = source.indexOf("async function sshLink("); + const linkEnd = source.indexOf("async function consumeSshLink", linkStart); + const linkSource = source.slice(linkStart, linkEnd); + assert.match(linkSource, /githubOAuthCanonicalSshLinkUrl/); + assert.ok(linkSource.indexOf("canonicalLinkUrl") < linkSource.indexOf("sshLinkCookie")); +}); + +test("OAuth initiation and token exchange share the authoritative callback", async () => { + const source = await readFile(new URL("../src/index.ts", import.meta.url), "utf8"); + const loginStart = source.indexOf("async function githubLogin"); + const callbackStart = source.indexOf("async function githubCallback", loginStart); + const loginSource = source.slice(loginStart, callbackStart); + const callbackEnd = source.indexOf("async function sshLink", callbackStart); + const callbackSource = source.slice(callbackStart, callbackEnd); + + assert.match(loginSource, /githubOAuthRedirectUri\(url, env\.GITHUB_REDIRECT_URI\)/); + assert.match(loginSource, /githubOAuthCanonicalLoginUrl\(url, env\.GITHUB_REDIRECT_URI\)/); + assert.ok(loginSource.indexOf("canonicalLoginUrl") < loginSource.indexOf("crypto.randomUUID")); + assert.match(callbackSource, /githubOAuthCallbackRequestMatches/); + assert.ok( + callbackSource.indexOf("githubOAuthCallbackRequestMatches") < + callbackSource.indexOf('fetch("https://github.com/login/oauth/access_token"'), + ); + assert.match(callbackSource, /redirect_uri: redirectUri/); }); diff --git a/tests/repo-selection.test.ts b/tests/repo-selection.test.ts new file mode 100644 index 0000000..8ff3e31 --- /dev/null +++ b/tests/repo-selection.test.ts @@ -0,0 +1,16 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { preferredEnabledRepo } from "../src/repo-selection.ts"; + +test("repo-less SSH create prefers the configured repo only when enabled", () => { + assert.equal( + preferredEnabledRepo(["alpha/project", "tenant/preferred"], "tenant/preferred"), + "tenant/preferred", + ); + assert.equal( + preferredEnabledRepo(["alpha/project", "zeta/project"], "tenant/preferred"), + "alpha/project", + ); + assert.equal(preferredEnabledRepo([], "tenant/preferred"), undefined); +}); diff --git a/tests/runtime-adapter.test.ts b/tests/runtime-adapter.test.ts new file mode 100644 index 0000000..21ba505 --- /dev/null +++ b/tests/runtime-adapter.test.ts @@ -0,0 +1,1922 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; + +import { + adapterFailureReleaseState, + adapterWorkspaceIdMatches, + clearedAdapterCapabilities, + createOnlyAdapterStatus, + definitiveRuntimeAdapterCreateFailure, + effectiveAdapterCapabilities, + currentAdapterDesktopConnection, + legacyLeaseIdForAdapter, + namespacedAdapterWorkspaceId, + normalizeAdapterNamespace, + normalizeAdapterWorkspaceId, + parseAdapterDesktopConnection, + parseAdapterWorkspaceResult, + redactedAdapterMessage, + redactedAdapterResponseMessage, + runtimeAdapterControlPlaneIdentity, + runtimeAdapterCreatePayload, + runtimeAdapterBrowserVncUrl, + runtimeAdapterDesktopUrl, + runtimeAdapterReplayRequest, + retainedRuntimeAdapterFailureMessage, + runtimeAdapterStopOutcome, + runtimeAdapterTerminalFailureStatus, + runtimeAdapterWorkspaceUrl, + resolveCreateAfterStopRace, + safeDesktopUrl, + safeWebSocketUrl, + shouldReplayRuntimeAdapterCreate, + validatedRuntimeAdapterCreatePayloadJson, +} from "../src/runtime-adapter.ts"; + +test("adapter create payload matches the strict controller contract", () => { + const payload = runtimeAdapterCreatePayload({ + namespace: "fleet-a", + id: "IS-101", + parentSessionId: null, + rootSessionId: "IS-101", + repo: "example/project", + branch: "main", + runtime: "crabbox", + profile: "default", + command: "codex --yolo", + prompt: "investigate the failure", + purpose: "investigate", + summary: "starting", + owner: "operator", + createdBy: "operator", + ttlSeconds: 14_400, + idleTimeoutSeconds: 1_800, + desktop: true, + }); + + assert.deepEqual(payload, { + id: "fleet-a-is-101", + parentSessionId: null, + rootSessionId: "IS-101", + repo: "example/project", + branch: "main", + runtime: "crabbox", + profile: "default", + command: "codex --yolo", + prompt: "investigate the failure", + purpose: "investigate", + summary: "starting", + owner: "operator", + createdBy: "operator", + ttlSeconds: 14_400, + idleTimeoutSeconds: 1_800, + capabilities: { desktop: true }, + }); + assert.equal("apiVersion" in (payload ?? {}), false); + assert.equal("sessionId" in (payload ?? {}), false); + assert.equal("idempotencyKey" in (payload ?? {}), false); + assert.equal( + runtimeAdapterCreatePayload( + { + namespace: "different-config", + id: "IS-101", + parentSessionId: null, + rootSessionId: "IS-101", + repo: "example/project", + branch: "main", + runtime: "crabbox", + profile: "default", + command: "codex --yolo", + prompt: "investigate the failure", + purpose: "investigate", + summary: "starting", + owner: "operator", + createdBy: "operator", + ttlSeconds: 14_400, + idleTimeoutSeconds: 1_800, + desktop: true, + }, + "fleet-a-is-101", + )?.id, + "fleet-a-is-101", + ); +}); + +test("adapter workspace id stays distinct from provider resource id", () => { + const result = parseAdapterWorkspaceResult({ + id: "fleet-a-is-101", + providerResourceId: "cloud/project/box-42", + status: "ready", + attachUrl: "wss://controller.example/terminal/is-101", + capabilities: { desktop: true, browser: true, code: true }, + expiresAt: "2026-06-12T12:00:00Z", + message: "cloud/project/box-42 is ready as fleet-a-is-101", + }); + + assert.equal(result?.workspaceId, "fleet-a-is-101"); + assert.equal(result?.providerResourceId, "cloud/project/box-42"); + assert.equal(result?.terminalUrl, "wss://controller.example/terminal/is-101"); + assert.equal(result?.terminalUrlPresent, true); + assert.equal(result?.capabilities?.terminal, true); + assert.equal(result?.capabilities?.desktop, true); + assert.equal(result?.capabilities?.vnc, true); + assert.equal(result?.terminalCapabilityInferred, true); + assert.equal(result?.message, "[workspace] is ready as [workspace]"); + assert.deepEqual( + effectiveAdapterCapabilities( + result!, + { + ...clearedAdapterCapabilities, + takeover: true, + logs: true, + artifacts: true, + }, + false, + ), + { + terminal: true, + takeover: true, + vnc: true, + desktop: true, + logs: true, + artifacts: true, + }, + ); + + const explicitlyDisabled = parseAdapterWorkspaceResult({ + id: "fleet-a-is-102", + status: "ready", + attachUrl: "wss://controller.example/terminal/is-102", + capabilities: { terminal: false, desktop: true }, + }); + assert.equal(explicitlyDisabled?.capabilities?.terminal, false); + assert.equal(explicitlyDisabled?.terminalCapabilityInferred, false); + + const explicitlyNull = parseAdapterWorkspaceResult({ + id: "fleet-a-is-102-null", + status: "ready", + attachUrl: "wss://controller.example/terminal/is-102-null", + capabilities: { terminal: null, desktop: true }, + }); + assert.equal(explicitlyNull?.capabilities?.terminal, false); + assert.equal(explicitlyNull?.terminalCapabilityInferred, false); + assert.equal( + effectiveAdapterCapabilities( + explicitlyNull!, + { ...clearedAdapterCapabilities, terminal: true, artifacts: true }, + false, + )?.terminal, + false, + ); + + const authoritativeList = parseAdapterWorkspaceResult({ + id: "fleet-a-is-103", + status: "ready", + attachUrl: "wss://controller.example/terminal/is-103", + capabilities: ["desktop"], + }); + assert.equal(authoritativeList?.capabilities?.terminal, false); + assert.equal(authoritativeList?.terminalCapabilityInferred, false); + + const omittedCapabilities = parseAdapterWorkspaceResult({ + id: "fleet-a-is-104", + status: "ready", + attachUrl: "wss://controller.example/terminal/is-104", + }); + assert.equal(omittedCapabilities?.capabilitiesPresent, false); + assert.equal(omittedCapabilities?.terminalCapabilityInferred, true); + assert.equal(omittedCapabilities?.capabilities, null); + assert.deepEqual( + effectiveAdapterCapabilities( + omittedCapabilities!, + { ...clearedAdapterCapabilities, desktop: true, vnc: true, logs: true, artifacts: true }, + false, + ), + { + ...clearedAdapterCapabilities, + terminal: true, + desktop: true, + vnc: true, + logs: true, + artifacts: true, + }, + ); + + const overlapping = parseAdapterWorkspaceResult({ + id: "fleet-a-is-101", + providerResourceId: "fleet-a-is-101-provider-suffix", + status: "ready", + message: "fleet-a-is-101-provider-suffix is ready for fleet-a-is-101", + }); + assert.equal(overlapping?.message, "[workspace] is ready for [workspace]"); +}); + +test("adapter workspace identity is namespaced, bounded, and exact", () => { + assert.equal(normalizeAdapterNamespace("Fleet-A"), "fleet-a"); + assert.equal(normalizeAdapterNamespace("fleet_a"), null); + assert.equal(namespacedAdapterWorkspaceId("Fleet-A", "IS-101"), "fleet-a-is-101"); + assert.equal(namespacedAdapterWorkspaceId("a".repeat(32), "b".repeat(31)), null); + + const exact = parseAdapterWorkspaceResult({ id: "fleet-a-is-101", status: "ready" }); + const wrong = parseAdapterWorkspaceResult({ id: "fleet-b-is-101", status: "ready" }); + const missing = parseAdapterWorkspaceResult( + { status: "ready" }, + { workspaceId: "fleet-a-is-101" }, + ); + assert.equal(adapterWorkspaceIdMatches(exact!, "fleet-a-is-101"), true); + assert.equal(adapterWorkspaceIdMatches(wrong!, "fleet-a-is-101"), false); + assert.equal(adapterWorkspaceIdMatches(missing!, "fleet-a-is-101"), false); + assert.equal(exact?.providerResourceId, null); + assert.equal(parseAdapterWorkspaceResult({ id: " fleet-a-is-101", status: "ready" }), null); + assert.equal(parseAdapterWorkspaceResult({ id: "Fleet-A-Is-101", status: "ready" }), null); + assert.equal( + parseAdapterWorkspaceResult({ + id: "fleet-a-is-101", + workspaceId: "fleet-a-is-102", + status: "ready", + }), + null, + ); +}); + +test("create-only adapters cannot return an unowned stopping lifecycle", () => { + assert.equal(createOnlyAdapterStatus("ready"), "ready"); + assert.equal(createOnlyAdapterStatus("failed"), "failed"); + assert.equal(createOnlyAdapterStatus("stopping"), null); + assert.equal(createOnlyAdapterStatus(" ready "), null); +}); + +test("status-only inspect preserves omitted capability and expiry fields", () => { + const omitted = parseAdapterWorkspaceResult({ id: "fleet-a-is-101", status: "ready" }); + assert.equal(omitted?.capabilitiesPresent, false); + assert.equal(omitted?.capabilities, null); + assert.equal(omitted?.expiresAtPresent, false); + assert.equal(omitted?.expiresAt, null); + assert.equal(omitted?.terminalUrlPresent, false); + + const cleared = parseAdapterWorkspaceResult({ + id: "fleet-a-is-101", + status: "ready", + capabilities: null, + expiresAt: null, + attachUrl: null, + }); + assert.equal(cleared?.capabilitiesPresent, true); + assert.equal(cleared?.capabilities, null); + assert.equal(cleared?.expiresAtPresent, true); + assert.equal(cleared?.expiresAt, null); + assert.equal(cleared?.terminalUrlPresent, true); + assert.equal(cleared?.terminalUrl, null); + assert.equal( + parseAdapterWorkspaceResult({ + id: "fleet-a-is-101", + status: "ready", + expiresAt: "not-a-date", + }), + null, + ); + assert.equal( + parseAdapterWorkspaceResult({ + id: "fleet-a-is-101", + status: "ready", + expiresAt: "", + }), + null, + ); + assert.deepEqual(clearedAdapterCapabilities, { + terminal: false, + takeover: false, + vnc: false, + desktop: false, + logs: false, + artifacts: false, + }); + assert.deepEqual( + effectiveAdapterCapabilities( + cleared!, + { ...clearedAdapterCapabilities, vnc: true, desktop: true }, + true, + ), + clearedAdapterCapabilities, + ); + assert.equal( + effectiveAdapterCapabilities(omitted!, clearedAdapterCapabilities, false), + undefined, + ); +}); + +test("missing ambiguous workspaces replay the complete persisted request", () => { + const createPayloadJson = JSON.stringify({ + id: "fleet-a-is-101", + purpose: "immutable original purpose", + summary: "immutable original summary", + profile: "desktop-large", + ttlSeconds: 14_400, + idleTimeoutSeconds: 1_800, + capabilities: { desktop: true }, + }); + const request = runtimeAdapterReplayRequest({ + id: "IS-101", + adapter_workspace_id: "fleet-a-is-101", + adapter_control_plane: "https://controller.example/api", + parent_session_id: "IS-100", + root_session_id: "IS-99", + repo: "example/project", + branch: "feature/retry", + runtime: "crabbox", + profile: "desktop-large", + command: "codex --yolo", + prompt: "continue after timeout", + purpose: "repair create", + summary: "create outcome unknown", + owner: "operator", + created_by: "service", + adapter_ttl_seconds: 14_400, + adapter_idle_timeout_seconds: 1_800, + adapter_requested_capabilities_json: JSON.stringify({ + terminal: true, + takeover: true, + vnc: true, + desktop: true, + logs: true, + artifacts: true, + }), + adapter_create_payload_json: createPayloadJson, + }); + + assert.deepEqual(request, { + id: "IS-101", + adapterWorkspaceId: "fleet-a-is-101", + adapterControlPlane: "https://controller.example/api", + parentSessionId: "IS-100", + rootSessionId: "IS-99", + repo: "example/project", + branch: "feature/retry", + runtime: "crabbox", + profile: "desktop-large", + command: "codex --yolo", + prompt: "continue after timeout", + purpose: "repair create", + summary: "create outcome unknown", + owner: "operator", + createdBy: "service", + adapterTtlSeconds: 14_400, + adapterIdleTimeoutSeconds: 1_800, + adapterRequestedCapabilities: { + terminal: true, + takeover: true, + vnc: true, + desktop: true, + logs: true, + artifacts: true, + }, + adapterCreatePayloadJson: createPayloadJson, + }); + assert.equal(shouldReplayRuntimeAdapterCreate("provisioning", true), true); + assert.equal(shouldReplayRuntimeAdapterCreate("pending_adapter", true), true); + assert.equal(shouldReplayRuntimeAdapterCreate("provisioning", false), false); + assert.equal(shouldReplayRuntimeAdapterCreate("ready", true), false); + assert.equal(shouldReplayRuntimeAdapterCreate("stopping", true), false); + assert.equal( + validatedRuntimeAdapterCreatePayloadJson(createPayloadJson, { + workspaceId: "fleet-a-is-101", + ttlSeconds: 14_400, + idleTimeoutSeconds: 1_800, + desktop: true, + }), + createPayloadJson, + ); +}); + +test("failed adapter workspaces become terminal only after release", () => { + assert.deepEqual(adapterFailureReleaseState("stopping"), { + status: "stopping", + terminalStatus: "failed", + message: "runtime workspace release pending", + }); + assert.deepEqual(adapterFailureReleaseState("stopped"), { + status: "failed", + terminalStatus: null, + message: "runtime workspace released", + }); +}); + +test("adapter failure release retains the actual failure reason", () => { + assert.equal( + retainedRuntimeAdapterFailureMessage( + "runtime adapter provision failed: HTTP 422", + "release transport failed", + "generic release text", + ), + "runtime adapter provision failed: HTTP 422", + ); +}); + +test("runtime adapter terminal failures stay retryable until lifecycle release", () => { + assert.equal(runtimeAdapterTerminalFailureStatus("runtime-v1"), "detached"); + assert.equal(runtimeAdapterTerminalFailureStatus("legacy"), "expired"); + assert.equal(runtimeAdapterTerminalFailureStatus(null), "expired"); +}); + +test("runtime adapter provider identities never become legacy lease ids", () => { + assert.equal(legacyLeaseIdForAdapter("runtime-v1", "sandbox:provider-owned"), null); + assert.equal(legacyLeaseIdForAdapter("runtime-v1", "cloudflare:provider-owned"), null); + assert.equal(legacyLeaseIdForAdapter("legacy", "sandbox:legacy-owned"), "sandbox:legacy-owned"); +}); + +test("confirmed stop races terminalize only after create ambiguity clears", () => { + assert.deepEqual(resolveCreateAfterStopRace(true, "failed"), { + status: "stopping", + terminalStatus: "failed", + }); + assert.deepEqual(resolveCreateAfterStopRace(false, "failed"), { + status: "failed", + terminalStatus: null, + }); + assert.deepEqual(resolveCreateAfterStopRace(false, null), { + status: "stopped", + terminalStatus: null, + }); +}); + +test("runtime adapter lifecycle cannot escape durable session ownership", async () => { + const source = await readFile(new URL("../src/index.ts", import.meta.url), "utf8"); + const stopStart = source.indexOf("async function stopSupersededRuntimeAdapterProvision"); + const stopEnd = source.indexOf("async function resolveInteractiveSessionLineage", stopStart); + const stopSource = source.slice(stopStart, stopEnd); + const reconcileStart = source.indexOf("async function reconcileExternalInteractiveSession("); + const reconcileEnd = source.indexOf("function reconciledInteractiveStatus", reconcileStart); + const reconcileSource = source.slice(reconcileStart, reconcileEnd); + const releaseStart = source.indexOf("async function releaseFailedRuntimeAdapterProvision"); + const releaseEnd = source.indexOf("function runtimeAdapterProvisionResult", releaseStart); + const releaseSource = source.slice(releaseStart, releaseEnd); + + assert.match( + source, + /versioned runtime adapter requires a durable interactive session lifecycle/, + ); + assert.match(stopSource, /recordConfirmedRuntimeAdapterRelease/); + assert.match(stopSource, /select\(\[[\s\S]*"adapter_create_pending"[\s\S]*"terminal_status"/); + assert.match(stopSource, /AND adapter_create_pending = \$\{lifecycle\.adapter_create_pending\}/); + assert.match(stopSource, /terminal_status IS NULL/); + assert.match(stopSource, /AND updated_at = \$\{lifecycle\.updated_at\}/); + assert.match(stopSource, /MAX\(updated_at \+ 1, \$\{now\}\)/); + assert.match(stopSource, /env\.DB\.batch/); + assert.match(stopSource, /INSERT INTO interactive_session_events/); + assert.match(stopSource, /finalizeTerminalInteractiveSession/); + assert.match(stopSource, /terminal_finalize_pending: 1/); + assert.ok( + stopSource.indexOf("clearRuntimeAdapterCreatePending") < + stopSource.indexOf("const release = await stopRuntimeAdapterWorkspaceForSession"), + ); + assert.ok( + releaseSource.indexOf("stageFailedRuntimeAdapterRelease") < + releaseSource.indexOf("stopRuntimeAdapterWorkspace"), + ); + assert.match(releaseSource, /status: "stopping"/); + assert.match(releaseSource, /release\.message/); + assert.match(releaseSource, /pendingMessage/); + assert.match(releaseSource, /terminal_status: "failed"/); + assert.match(releaseSource, /adapter_create_pending: 0/); + assert.match(reconcileSource, /current\.stoppedAt \?\? now/); + assert.match(reconcileSource, /finalizeTerminalInteractiveSession/); + assert.match(source, /AND NOT EXISTS \(/); + assert.match(source, /archiveInteractiveSessionLogs\(env, id, now, \{ force: true \}\)/); +}); + +test("confirmed adapter failure release keeps the original failure evidence", async () => { + const source = await readFile(new URL("../src/index.ts", import.meta.url), "utf8"); + const migration = await readFile( + new URL("../migrations/0021_runtime_adapter_hardening.sql", import.meta.url), + "utf8", + ); + const releaseStart = source.indexOf("async function recordConfirmedRuntimeAdapterRelease"); + const releaseEnd = source.indexOf( + "async function clearRuntimeAdapterCreatePending", + releaseStart, + ); + const releaseSource = source.slice(releaseStart, releaseEnd); + const finalizeStart = source.indexOf("async function finalizeTerminalInteractiveSession"); + const finalizeEnd = source.indexOf("async function archiveInteractiveSessionLogs", finalizeStart); + const finalizeSource = source.slice(finalizeStart, finalizeEnd); + + assert.match(releaseSource, /"terminal_failure_reason"/); + assert.match(releaseSource, /retainedRuntimeAdapterFailureMessage/); + assert.match( + releaseSource, + /terminal_failure_reason: resolved\.status === "failed" \? failureMessage/, + ); + assert.match(releaseSource, /reconcile_error: resolved\.status === "failed" \? failureMessage/); + assert.match(releaseSource, /\? failureMessage/); + assert.match(finalizeSource, /retainedRuntimeAdapterFailureMessage/); + assert.match(finalizeSource, /INSERT INTO interactive_session_events/); + assert.match(finalizeSource, /SELECT \$\{id\}, 'system', \$\{message\}/); + assert.match(migration, /ADD COLUMN terminal_failure_reason TEXT/); +}); + +test("terminal archive finalization remains durably retryable", async () => { + const source = await readFile(new URL("../src/index.ts", import.meta.url), "utf8"); + const migration = await readFile( + new URL("../migrations/0021_runtime_adapter_hardening.sql", import.meta.url), + "utf8", + ); + const appendStart = source.indexOf("async function appendInteractiveSessionEvent"); + const appendEnd = source.indexOf( + "async function finalizeTerminalInteractiveSession", + appendStart, + ); + const appendSource = source.slice(appendStart, appendEnd); + const finalizeStart = source.indexOf("async function finalizeTerminalInteractiveSession"); + const finalizeEnd = source.indexOf("async function archiveInteractiveSessionLogs", finalizeStart); + const finalizeSource = source.slice(finalizeStart, finalizeEnd); + + assert.match(source, /expression\("terminal_finalize_pending", "=", 1\)/); + assert.match(source, /row\.terminal_finalize_pending === 1/); + assert.match(source, /const terminalCleanupDeletePending = 2/); + assert.match(source, /completeTerminalFinalization/); + assert.match(source, /SET terminal_finalize_pending = 0/); + assert.match(source, /interactive_session_log_archives\.events_key IS NULL/); + assert.match(source, /interactive_session_log_archives\.transcript_key IS NULL/); + assert.match(source, /interactive_session_log_archives\.summary_key IS NULL/); + assert.match(source, /archive\.session_updated_at = interactive_sessions\.updated_at/); + assert.match( + source, + /excluded\.session_updated_at > interactive_session_log_archives\.session_updated_at/, + ); + assert.match( + source, + /excluded\.session_updated_at IS interactive_session_log_archives\.session_updated_at/, + ); + assert.doesNotMatch( + source, + /session_updated_at IS NOT excluded\.session_updated_at[\s\S]*excluded\.updated_at >=/, + ); + assert.match(appendSource, /executeBatch\(env, \[/); + assert.match(appendSource, /insertInto\("interactive_session_events"\)/); + assert.match(appendSource, /terminalFinalizationPendingQuery\(db, id\)/); + assert.match(finalizeSource, /executeBatch\(env, \[/); + assert.match(finalizeSource, /INSERT INTO interactive_session_events/); + assert.match(finalizeSource, /terminalFinalizationPendingQuery\(db, id\)/); + assert.match(migration, /ADD COLUMN terminal_finalize_pending INTEGER NOT NULL DEFAULT 0/); + assert.match(migration, /ADD COLUMN session_updated_at INTEGER/); + assert.match(migration, /status IN \('stopped', 'expired', 'failed'\)/); +}); + +test("enabling R2 requeues D1-only terminal archives for object backfill", async () => { + const source = await readFile(new URL("../src/index.ts", import.meta.url), "utf8"); + const backfillStart = source.indexOf("async function requeueTerminalArchiveObjectBackfill"); + const batchStart = source.indexOf( + "async function reconcileExternalInteractiveSessionBatch", + backfillStart, + ); + const targetedStart = source.indexOf( + "async function reconcileExternalInteractiveSessionById", + batchStart, + ); + const reconcileStart = source.indexOf( + "async function reconcileExternalInteractiveSession(", + targetedStart, + ); + const backfillSource = source.slice(backfillStart, batchStart); + const batchSource = source.slice(batchStart, targetedStart); + const targetedSource = source.slice(targetedStart, reconcileStart); + + assert.match(backfillSource, /if \(!env\.SESSION_LOGS\) return/); + assert.match(backfillSource, /session\.terminal_finalize_pending = 0/); + assert.match(backfillSource, /archive\.events_key IS NULL/); + assert.match(backfillSource, /archive\.transcript_key IS NULL/); + assert.match(backfillSource, /archive\.summary_key IS NULL/); + assert.match(backfillSource, /SET terminal_finalize_pending = 1/); + assert.match(backfillSource, /last_reconciled_at = NULL/); + assert.match(batchSource, /await requeueTerminalArchiveObjectBackfill\(env\)/); + assert.match(targetedSource, /await requeueTerminalArchiveObjectBackfill\(env, id\)/); +}); + +test("runtime reconciliation has scheduled and targeted lifecycle clocks", async () => { + const source = await readFile(new URL("../src/index.ts", import.meta.url), "utf8"); + const config = await readFile(new URL("../wrangler.jsonc", import.meta.url), "utf8"); + const targetedStart = source.indexOf("async function reconcileExternalInteractiveSessionById"); + const targetedEnd = source.indexOf( + "async function reconcileExternalInteractiveSession(", + targetedStart, + ); + const targetedSource = source.slice(targetedStart, targetedEnd); + const batchStart = source.indexOf("async function reconcileExternalInteractiveSessionBatch"); + const batchEnd = source.indexOf("async function reconcileExternalInteractiveSessionById"); + const batchSource = source.slice(batchStart, batchEnd); + const reconcileStart = source.indexOf("async function reconcileExternalInteractiveSession("); + const reconcileEnd = source.indexOf("function reconciledInteractiveStatus", reconcileStart); + const reconcileSource = source.slice(reconcileStart, reconcileEnd); + + assert.match(source, /async scheduled\(/); + assert.match(source, /context\.waitUntil\(\s*reconcileInteractiveSessionLifecycleBatch/); + assert.match(config, /"crons": \["\* \* \* \* \*"\]/); + assert.match(batchSource, /expression\("terminal_finalize_pending", "=", 1\)/); + assert.match(batchSource, /expression\("adapter", "=", runtimeAdapterName\)/); + assert.match(targetedSource, /reconcileCredentialPolicyCleanupBatch\(env, now, id\)/); + assert.match(targetedSource, /row\.terminal_finalize_pending === 1/); + assert.match(targetedSource, /row\.adapter !== runtimeAdapterName/); + assert.match(targetedSource, /runtimeAdapterReconcileIntervalMs/); + assert.match(targetedSource, /reconcileExternalInteractiveSession\(env, row, now\)/); + assert.ok( + reconcileSource.indexOf("if (terminalFinalizationStatus)") < + reconcileSource.indexOf("inspectRuntimeAdapterWorkspace"), + ); + assert.match(source, /async function readFreshInteractiveSession/); + assert.match(source, /async function interactiveSessionPty[\s\S]*readFreshInteractiveSession/); + assert.match(source, /async function interactiveSessionVnc[\s\S]*readFreshInteractiveSession/); + assert.match(source, /scheduled interactive session reconciliation failed/); + assert.match( + source, + /async function reconcileInteractiveSessionLifecycleBatch[\s\S]*reconcileCredentialPolicyCleanupBatch[\s\S]*reconcileExternalInteractiveSessionBatch/, + ); + assert.match(reconcileSource, /const claimAt = Math\.max/); + assert.match(reconcileSource, /where\("updated_at", "=", row\.updated_at\)/); + assert.match(reconcileSource, /const completedAt = Math\.max\(Date\.now\(\), claimAt\)/); + assert.match( + reconcileSource, + /const completionVersion = Math\.max\(completedAt, row\.updated_at \+ 1\)/, + ); + assert.match(reconcileSource, /last_reconciled_at: completedAt/); + assert.match(reconcileSource, /updated_at: completionVersion/); + assert.match(reconcileSource, /INSERT INTO interactive_session_events/); + assert.match(reconcileSource, /env\.DB\.batch/); + assert.match(reconcileSource, /reconcile_error: safeProviderError/); + assert.doesNotMatch(reconcileSource, /updated_at: now/); +}); + +test("recurring terminal authorization never awaits provider reconciliation", async () => { + const source = await readFile(new URL("../src/index.ts", import.meta.url), "utf8"); + const grantStart = source.indexOf("function terminalInputGrant"); + const grantEnd = source.indexOf("function sendTerminalFrame", grantStart); + const grantSource = source.slice(grantStart, grantEnd); + const controlStart = source.indexOf("async function canControlInteractiveSessionById"); + const controlEnd = source.indexOf("function canGrantDelegatedControl", controlStart); + const controlSource = source.slice(controlStart, controlEnd); + const shareStart = source.indexOf("async function isSharedSessionToken"); + const shareEnd = source.indexOf("function sendTerminalFrame", shareStart); + const shareSource = source.slice(shareStart, shareEnd); + const bridgeStart = source.indexOf("function bridgeWebSockets"); + const bridgeEnd = source.indexOf("async function webSocketMessageData", bridgeStart); + const bridgeSource = source.slice(bridgeStart, bridgeEnd); + + assert.match(grantSource, /cachedBooleanGrant/); + assert.match(grantSource, /terminalSubscriptionReconciler/); + assert.match(grantSource, /void reconcileExternalInteractiveSessionById/); + assert.doesNotMatch(controlSource, /reconcileExternalInteractiveSessionById/); + assert.doesNotMatch(controlSource, /reconcileCredentialPolicyCleanupBatch|runtimeAdapterFetch/); + assert.doesNotMatch(shareSource, /reconcileExternalInteractiveSessionById/); + assert.doesNotMatch(shareSource, /reconcileCredentialPolicyCleanupBatch|runtimeAdapterFetch/); + assert.match(bridgeSource, /reconcileSubscription\?\.\(\)/); + assert.doesNotMatch(bridgeSource, /await reconcileSubscription/); +}); + +test("public auth deployment metadata excludes runtime routing", async () => { + const source = await readFile(new URL("../src/index.ts", import.meta.url), "utf8"); + const publicStart = source.indexOf("function publicDeploymentConfig"); + const publicEnd = source.indexOf("class D1Dialect", publicStart); + const publicSource = source.slice(publicStart, publicEnd); + + assert.match(source, /deployment: publicDeploymentConfig\(env\)/); + assert.match(publicSource, /label, canonicalUrl, productUrl, sshHost/); + assert.doesNotMatch(publicSource, /preferredRepo|defaultRuntime|defaultProfile|RUNTIME_ADAPTER/); +}); + +test("strict session rows and cleanup preserve terminal finalization anchors", async () => { + const source = await readFile(new URL("../src/index.ts", import.meta.url), "utf8"); + const cleanupStart = source.indexOf("async function cleanupInteractiveSessions"); + const cleanupEnd = source.indexOf("async function mutateInteractiveSession", cleanupStart); + const cleanupSource = source.slice(cleanupStart, cleanupEnd); + + assert.match(source, /type InteractiveSessionRow = Selectable/); + assert.match(cleanupSource, /where\("terminal_finalize_pending", "=", 0\)/); + assert.match(cleanupSource, /deleteFinalizedInteractiveSession\(env, row, archive\)/); + assert.match(cleanupSource, /terminalCleanupDeletePending/); + assert.match(cleanupSource, /updated_at", "=", row\.updated_at/); + assert.match(cleanupSource, /executeBatch\(env, \[/); + const archiveIndex = cleanupSource.indexOf('.selectFrom("interactive_session_log_archives")'); + const deleteIndex = cleanupSource.indexOf("deleteFinalizedInteractiveSession(env, row, archive)"); + const objectCleanupIndex = cleanupSource.indexOf("cleanupSessionLogArchiveObjects(env, archive)"); + assert.ok(archiveIndex >= 0 && deleteIndex > archiveIndex && objectCleanupIndex > deleteIndex); + assert.match(cleanupSource, /session archive object cleanup leaked/); + assert.match(cleanupSource, /events_key IS \$\{archive\?\.events_key/); + assert.match(cleanupSource, /transcript_key IS \$\{archive\?\.transcript_key/); + assert.match(cleanupSource, /summary_key IS \$\{archive\?\.summary_key/); + assert.match(cleanupSource, /deleteFrom\("interactive_session_events"\)/); + assert.match(cleanupSource, /deleteFrom\("interactive_session_log_archives"\)/); + assert.match(cleanupSource, /deleteFrom\("interactive_sessions"\)/); + assert.match(cleanupSource, /FROM interactive_session_credential_policies/); + assert.match(source, /terminalFinalizationPendingQuery/); + assert.match(source, /executeBatch\(env, \[[\s\S]*interactive_session_events/); + assert.match(source, /COALESCE\([\s\S]*event_count[\s\S]*count\(\*\)/); + assert.match(source, /events_key IS NOT NULL/); +}); + +test("summary and sharing events invalidate terminal cleanup snapshots", async () => { + const source = await readFile(new URL("../src/index.ts", import.meta.url), "utf8"); + const cleanupStart = source.indexOf("async function deleteFinalizedInteractiveSession"); + const cleanupEnd = source.indexOf("async function mutateInteractiveSession", cleanupStart); + const cleanupSource = source.slice(cleanupStart, cleanupEnd); + const shareStart = source.indexOf('if (action === "share_link")'); + const shareEnd = source.indexOf('if (action === "enable_multiplayer")', shareStart); + const shareSource = source.slice(shareStart, shareEnd); + const summaryStart = source.indexOf("async function updateInteractiveSessionSummary"); + const summaryEnd = source.indexOf("async function readInteractiveSessionLogs", summaryStart); + const summarySource = source.slice(summaryStart, summaryEnd); + const metadataStart = source.indexOf("async function mutateInteractiveSessionMetadataAtomically"); + const metadataEnd = source.indexOf("async function mutateInteractiveSession(", metadataStart); + const metadataSource = source.slice(metadataStart, metadataEnd); + + assert.match(cleanupSource, /where\("updated_at", "=", row\.updated_at\)/); + assert.match(cleanupSource, /terminal_finalize_pending: terminalCleanupDeletePending/); + assert.match(cleanupSource, /event_count = \$\{archive\?\.event_count/); + assert.match(cleanupSource, /archived_at = \$\{archive\?\.archived_at/); + assert.match(cleanupSource, /count\(\*\)/); + assert.match(shareSource, /mutateInteractiveSessionMetadataAtomically/); + assert.match(summarySource, /mutateInteractiveSessionMetadataAtomically/); + assert.match(metadataSource, /INSERT INTO interactive_session_events/); + assert.match(metadataSource, /terminal_finalize_pending: sql`CASE/); + assert.match(metadataSource, /WHEN status IN \('stopped', 'expired', 'failed'\) THEN 1/); + assert.match(metadataSource, /updated_at = \$\{session\.updatedAt\}/); + assert.match(metadataSource, /\.returning\("updated_at"\)/); + assert.match(metadataSource, /env\.DB\.batch/); + assert.ok(metadataSource.indexOf("eventQuery") < metadataSource.indexOf("updateQuery")); +}); + +test("development identity login requires an explicit local gate", async () => { + const source = await readFile(new URL("../src/index.ts", import.meta.url), "utf8"); + const config = await readFile(new URL("../wrangler.jsonc", import.meta.url), "utf8"); + const loginSource = source.slice( + source.indexOf("async function devIdentityLogin"), + source.indexOf("async function githubLogin"), + ); + const requireSource = source.slice( + source.indexOf("async function requireUser"), + source.indexOf("async function optionalUser"), + ); + const authStart = source.indexOf("function authMethods"); + const authSource = source.slice(authStart, source.indexOf("function actor", authStart)); + + assert.match(source, /function devIdentityEnabled\(env: RuntimeEnv, request: Request\)/); + assert.match(loginSource, /devIdentityEnabled\(env, request\)/); + assert.match(requireSource, /devIdentityEnabled\(env, request\)/); + assert.match(requireSource, /deleteFrom\("sessions"\)/); + assert.match(authSource, /devIdentityEnabled\(env, request\)/); + assert.match( + authSource, + /developmentIdentityEnabled\(env\.CRABFLEET_DEV_LOGIN_ENABLED, request\.url\)/, + ); + assert.match(config, /"CRABFLEET_DEV_LOGIN_ENABLED": "false"/); +}); + +test("runtime adapter credentials are preflighted before session allocation", async () => { + const source = await readFile(new URL("../src/index.ts", import.meta.url), "utf8"); + const createStart = source.indexOf("async function createInteractiveSessionFromInput"); + const createEnd = source.indexOf("function initialRuntimeAdapterWorkspaceId", createStart); + const createSource = source.slice(createStart, createEnd); + const preflightStart = source.indexOf("function requireRuntimeAdapterCreatePreflight"); + const preflightEnd = source.indexOf( + "async function stopSupersededRuntimeAdapterProvision", + preflightStart, + ); + const preflightSource = source.slice(preflightStart, preflightEnd); + + assert.ok( + createSource.indexOf("requireRuntimeAdapterCreatePreflight(env, runtime)") < + createSource.indexOf("nextInteractiveSessionId(env)"), + ); + assert.ok( + createSource.indexOf("requireRuntimeAdapterCreatePreflight(env, runtime)") < + createSource.indexOf('.insertInto("interactive_sessions")'), + ); + assert.match(preflightSource, /runtime === "container" && env\.SANDBOX/); + assert.match(preflightSource, /runtimeAdapterToken\(env\)/); + assert.match(preflightSource, /configuredRuntimeAdapterControlPlane\(env\)/); + assert.match(preflightSource, /runtime adapter token is not configured/); +}); + +test("runtime adapter operations stay bound to the registered control plane", async () => { + const source = await readFile(new URL("../src/index.ts", import.meta.url), "utf8"); + const migration = await readFile( + new URL("../migrations/0020_runtime_adapter_lifecycle.sql", import.meta.url), + "utf8", + ); + const bindingStart = source.indexOf("function configuredRuntimeAdapterControlPlane"); + const bindingEnd = source.indexOf( + "async function stopSupersededRuntimeAdapterProvision", + bindingStart, + ); + const bindingSource = source.slice(bindingStart, bindingEnd); + const provisionStart = source.indexOf("async function provisionWithRuntimeAdapter"); + const provisionEnd = source.indexOf("function persistedRuntimeAdapterSeconds", provisionStart); + const provisionSource = source.slice(provisionStart, provisionEnd); + const inspectStart = source.indexOf("async function inspectRuntimeAdapterWorkspace"); + const inspectEnd = source.indexOf( + "async function reconcileStoppingRuntimeAdapterWorkspace", + inspectStart, + ); + const inspectSource = source.slice(inspectStart, inspectEnd); + const stopStart = source.indexOf("async function stopRuntimeAdapterWorkspace("); + const stopEnd = source.indexOf("async function runtimeAdapterFetch", stopStart); + const stopSource = source.slice(stopStart, stopEnd); + + assert.match(migration, /ADD COLUMN adapter_control_plane TEXT/); + assert.match(source, /adapter_control_plane: adapterControlPlane/); + assert.match(bindingSource, /configuredControlPlane !== registeredControlPlane/); + assert.match(bindingSource, /control plane differs from workspace registration/); + assert.match(provisionSource, /requireRegisteredRuntimeAdapterControlPlane/); + assert.match(provisionSource, /runtimeAdapterCollectionUrl\(baseUrl\)/); + assert.match(inspectSource, /session\.adapter_control_plane/); + assert.match(inspectSource, /runtimeAdapterWorkspaceUrl\(controlPlane, adapterWorkspaceId\)/); + assert.match(stopSource, /requireRegisteredRuntimeAdapterControlPlane/); + assert.match(stopSource, /runtimeAdapterWorkspaceUrl\(controlPlane, adapterWorkspaceId\)/); + assert.doesNotMatch( + inspectSource, + /runtimeAdapterWorkspaceUrl\(env\.CRABBOX_RUNTIME_ADAPTER_URL/, + ); + assert.doesNotMatch(stopSource, /response\.status === 404[\s\S]*CRABBOX_RUNTIME_ADAPTER_URL/); +}); + +test("stopping create replay owns the exact persisted lifecycle", async () => { + const source = await readFile(new URL("../src/index.ts", import.meta.url), "utf8"); + const reconcileStart = source.indexOf("async function reconcileStoppingRuntimeAdapterWorkspace"); + const replayStart = source.indexOf("async function replayStoppingRuntimeAdapterCreate"); + const replayEnd = source.indexOf("async function stopRuntimeAdapterWorkspace(", replayStart); + const reconcileSource = source.slice(reconcileStart, replayStart); + const replaySource = source.slice(replayStart, replayEnd); + + assert.match(reconcileSource, /replayStoppingRuntimeAdapterCreate\(env, session\)/); + assert.doesNotMatch(reconcileSource, /provisionWithRuntimeAdapter/); + assert.match(replaySource, /AND adapter_control_plane = \$\{controlPlane\}/); + assert.match(replaySource, /AND adapter_create_payload_json = \$\{createPayloadJson\}/); + assert.match(replaySource, /AND adapter_create_pending = 1/); + assert.match(replaySource, /AND status = 'stopping'/); + assert.match(replaySource, /AND updated_at = \$\{session\.updated_at\}/); + assert.match(replaySource, /runtimeAdapterCollectionUrl\(controlPlane\)/); + assert.match(replaySource, /idempotency-key": adapterWorkspaceId/); + assert.match(replaySource, /adapter_create_pending: 0/); + assert.match(replaySource, /reconcile_error: message/); + assert.match(replaySource, /INSERT INTO interactive_session_events/); + assert.match(replaySource, /env\.DB\.batch/); +}); + +test("stateless Sandbox provision hook acquires durable standalone ownership", async () => { + const source = await readFile(new URL("../src/index.ts", import.meta.url), "utf8"); + const migration = await readFile( + new URL("../migrations/0022_credential_policy_cleanup.sql", import.meta.url), + "utf8", + ); + const expiryMigration = await readFile( + new URL("../migrations/0023_standalone_sandbox_expiry.sql", import.meta.url), + "utf8", + ); + const endpointStart = source.indexOf("async function provisionInteractiveEndpoint"); + const endpointEnd = source.indexOf("function isBuiltInInteractiveProvisionUrl", endpointStart); + const endpointSource = source.slice(endpointStart, endpointEnd); + const ownershipStart = source.indexOf("function sandboxCredentialPolicyOwnerCondition"); + const ownershipEnd = source.indexOf( + "function sandboxCredentialPolicyRegistrationQueries", + ownershipStart, + ); + const ownershipSource = source.slice(ownershipStart, ownershipEnd); + const managedStart = source.indexOf("async function provisionManagedSandboxEndpoint"); + const managedEnd = source.indexOf("async function provisionStandaloneSandbox", managedStart); + const managedSource = source.slice(managedStart, managedEnd); + const managedCommitSource = managedSource.slice(managedSource.indexOf("const commitRevision")); + const ptyStart = source.indexOf("async function standaloneSandboxPty"); + const ptyEnd = source.indexOf("function isBuiltInInteractiveProvisionUrl", ptyStart); + const ptySource = source.slice(ptyStart, ptyEnd); + const sandboxStart = source.indexOf("async function provisionWithSandbox"); + const sandboxEnd = source.indexOf("function sandboxManagedOwnershipCondition", sandboxStart); + const sandboxSource = source.slice(sandboxStart, sandboxEnd); + const activationSource = endpointSource.slice(endpointSource.indexOf("const activationVersion")); + const stopStart = source.indexOf("async function stopStandaloneSandboxProvision"); + const stopEnd = source.indexOf("function standaloneSandboxAttachUrl", stopStart); + const stopSource = source.slice(stopStart, stopEnd); + const expiryStart = source.indexOf("async function expireStandaloneSandboxProvisions"); + const expirySource = source.slice(expiryStart, stopStart); + const standaloneCleanupStart = source.indexOf( + "async function completeStandaloneSandboxProvisionCleanup", + ); + const standaloneCleanupEnd = source.indexOf( + "async function completeCredentialPolicyCleanupSession", + standaloneCleanupStart, + ); + const standaloneCleanupSource = source.slice(standaloneCleanupStart, standaloneCleanupEnd); + const strictAuthStart = source.indexOf("function authorizeProvisionBearerToken"); + const strictAuthEnd = source.indexOf("function sandboxProvisionPreflightError", strictAuthStart); + const strictAuthSource = source.slice(strictAuthStart, strictAuthEnd); + + assert.match(endpointSource, /selectFrom\("interactive_sessions"\)/); + assert.match(endpointSource, /if \(managed\)/); + assert.ok( + endpointSource.indexOf("if (managed)") < + endpointSource.indexOf("return provisionStandaloneSandbox(env, payload)"), + ); + assert.match(endpointSource, /provisionManagedSandboxEndpoint\(env, payload, managed\)/); + assert.match(endpointSource, /managedInteractiveSessionId\(payload\.id\)/); + assert.match(endpointSource, /managed session namespace/); + assert.match(endpointSource, /INSERT INTO standalone_sandbox_provisions/); + assert.match(endpointSource, /ownership_claim_expires_at/); + assert.match(endpointSource, /\$\{sandboxLeaseId\(lease\)\}/); + assert.match(endpointSource, /lease_id = excluded\.lease_id/); + assert.match(endpointSource, /provisionWithSandbox\([\s\S]*fence/); + assert.match(endpointSource, /state: "active"/); + assert.match(ownershipSource, /FROM standalone_sandbox_provisions AS owner/); + assert.match(ownershipSource, /owner\.ownership_claim = \$\{ownershipFence\.claim\}/); + assert.match(managedSource, /managedSandboxProvisionPayloadMatches/); + assert.ok( + managedSource.indexOf("managedSandboxProvisionPayloadMatches") < + managedSource.indexOf("newSandboxLease(payload.id)"), + ); + assert.match(managedSource, /where\("updated_at", "=", session\.updated_at\)/); + assert.match(managedSource, /sandbox_refresh_claim: fence\.claim/); + assert.match(managedSource, /const agentToken = newAgentToken\(\)/); + assert.match(managedSource, /const agentTokenHash = await sha256\(agentToken\)/); + assert.match(managedSource, /agent_token_hash: agentTokenHash/); + assert.match(managedSource, /agent_token_hash IS \$\{session\.agent_token_hash\}/); + assert.match(managedSource, /provisionWithSandbox\(env, payload, agentToken, lease, fence\)/); + assert.ok( + managedSource.indexOf("sandboxProvisionPreflightError(env, payload)") < + managedSource.indexOf("const agentToken = newAgentToken()"), + ); + assert.match(managedSource, /stageFailedManagedSandboxProvision/); + assert.match(managedSource, /where\("agent_token_hash", "=", agentTokenHash\)/); + assert.doesNotMatch(managedSource, /provisionWithSandbox\(env, payload, undefined/); + assert.match(managedSource, /executeBatch\(env, commitQueries\)/); + assert.match(managedCommitSource, /MAX\(updated_at \+ 1, \$\{commitRevision\}\)/); + assert.doesNotMatch(managedCommitSource, /where\("updated_at", "=", now\)/); + assert.match(managedCommitSource, /lease_id IS \$\{fence\.refreshLeaseId\}/); + assert.match(managedCommitSource, /sandbox_refresh_claim", "=", fence\.claim/); + assert.match(managedCommitSource, /sandbox_refresh_claim_expires_at", "=", fence\.expiresAt/); + assert.match(managedSource, /previousSandboxId/); + assert.match(managedSource, /state: "cleanup_pending"/); + assert.match(managedSource, /claimed\.numUpdatedRows/); + assert.match(ptySource, /authorizeProvisionEndpoint\(request, env\)/); + assert.match(ptySource, /standalone_sandbox_provisions/); + assert.match(ptySource, /where\("state", "=", "active"\)/); + assert.match(ptySource, /owner\.expires_at <= Date\.now\(\)/); + assert.match(ptySource, /stageStandaloneSandboxProvisionCleanup/); + assert.match(ptySource, /activeSandboxCredentialPolicyGeneration/); + assert.match(ptySource, /terminalHeaders\.delete\("authorization"\)/); + assert.match(ptySource, /const pair = new WebSocketPair\(\)/); + assert.match(ptySource, /response\.webSocket\.accept\(\)/); + assert.match(ptySource, /bridgeWebSockets\(/); + assert.match(ptySource, /standaloneSandboxTerminalGrant/); + assert.match(ptySource, /cachedBooleanGrant/); + assert.match(ptySource, /where\("request_hash", "=", ownership\.requestHash\)/); + assert.match(ptySource, /where\("expires_at", ">", now\)/); + assert.match(ptySource, /where\("updated_at", "=", ownership\.updatedAt\)/); + assert.match(ptySource, /activeSandboxCredentialPolicyCondition/); + assert.match(ptySource, /return new Response\(null, \{ status: 101, webSocket: client \}\)/); + assert.doesNotMatch(ptySource, /return response;/); + assert.match(standaloneCleanupSource, /deleteSession\(lease\.terminalSessionId\)/); + assert.match(standaloneCleanupSource, /isSandboxSessionAlreadyGone/); + assert.ok( + standaloneCleanupSource.indexOf("deleteSession(lease.terminalSessionId)") < + standaloneCleanupSource.indexOf('.deleteFrom("standalone_sandbox_provisions")'), + ); + assert.match(standaloneCleanupSource, /where\("updated_at", "=", owner\.updated_at\)/); + assert.match(sandboxSource, /standaloneSandboxAttachUrl\(env, session\.id\)/); + assert.match( + endpointSource, + /const activationVersion = Math\.max\(Date\.now\(\), finishedAt \+ 1\)/, + ); + assert.match(activationSource, /executeBatch\(env, \[/); + assert.ok( + activationSource.indexOf('.updateTable("interactive_session_credential_policies")') < + activationSource.indexOf('.updateTable("standalone_sandbox_provisions")'), + ); + assert.match(activationSource, /set\(\{ updated_at: activationVersion \}\)/); + assert.match( + endpointSource, + /activeSandboxCredentialPolicyCondition\([\s\S]*policyGeneration,[\s\S]*activationVersion/, + ); + assert.match(migration, /CREATE TABLE IF NOT EXISTS standalone_sandbox_provisions/); + assert.match(migration, /request_hash TEXT NOT NULL/); + assert.match(migration, /sandbox_id TEXT NOT NULL UNIQUE/); + assert.match(expiryMigration, /ADD COLUMN expires_at INTEGER/); + assert.match(expiryMigration, /substr\(lower\(id\), 4\) NOT GLOB '\*\[\^0-9\]\*'/); + assert.match(expiryMigration, /idx_standalone_sandbox_provision_expiry/); + assert.match(expirySource, /state = 'active'/); + assert.match(expirySource, /expires_at <= \$\{now\}/); + assert.match(expirySource, /substr\(lower\(id\), 4\) NOT GLOB '\*\[\^0-9\]\*'/); + assert.match(stopSource, /authorizeProvisionBearerToken\(request, env\)/); + assert.match(strictAuthSource, /if \(!env\.CRABBOX_INTERACTIVE_PROVISION_TOKEN\)/); + assert.match(strictAuthSource, /throw serviceUnavailable/); + assert.doesNotMatch(strictAuthSource, /hasBackend/); + assert.match(stopSource, /stageStandaloneSandboxProvisionCleanup/); + assert.match(stopSource, /reconcileCredentialPolicyCleanupBatch/); + assert.match(stopSource, /status: remaining \? "stopping" : "stopped"/); + assert.match(source, /CRABBOX_STANDALONE_SANDBOX_TTL_SECONDS/); + assert.match(source, /function managedInteractiveSessionId/); + assert.match(source, /standaloneProvisionStopMatch/); + assert.match(source, /stopStandaloneSandboxProvision/); + assert.match(source, /policy\?\.expiresAt !== undefined && policy\.expiresAt <= Date\.now\(\)/); + assert.match(source, /standaloneSandboxPolicyExpiresAt/); + assert.match( + source, + /selectFrom\("standalone_sandbox_provisions"\)[\s\S]*failed to allocate an unreserved interactive session id/, + ); + assert.match(source, /id = \$\{id\} COLLATE NOCASE/); + assert.match(expiryMigration, /UPDATE id_sequences/); + assert.match(expiryMigration, /MAX\(CAST\(substr\(lower\(id\), 4\) AS INTEGER\)\)/); +}); + +test("Sandbox credential egress proves the durable generation and owner", async () => { + const source = await readFile(new URL("../src/index.ts", import.meta.url), "utf8"); + const readStart = source.indexOf("async function sandboxCredentialPolicy("); + const readEnd = source.indexOf("async function sandboxOutbound", readStart); + const readSource = source.slice(readStart, readEnd); + const controlStart = source.indexOf("export class SessionControlDO"); + const controlEnd = source.indexOf( + "function validSandboxCredentialPolicyRegistration", + controlStart, + ); + const controlSource = source.slice(controlStart, controlEnd); + const egressStart = controlSource.indexOf("const egressMatch"); + const egressEnd = controlSource.indexOf("const sandboxMatch", egressStart); + const egressSource = controlSource.slice(egressStart, egressEnd); + + assert.match(readSource, /x-crabfleet-policy-generation/); + assert.match(readSource, /response\.status === 409/); + assert.match(readSource, /repairLegacySandboxCredentialPolicyBatch/); + assert.match(readSource, /response = await stub\.fetch\(policyUrl\)/); + assert.match(readSource, /sandboxCredentialPolicyHasDurableOwner/); + assert.match(readSource, /interactive_session_credential_policies/); + assert.match(readSource, /activeSandboxCredentialPolicyGeneration/); + assert.match(readSource, /sandboxCredentialPolicyCleanupAuthorizedCondition/); + assert.match(readSource, /policy\.expiresAt === standalone\.expires_at/); + assert.match(controlSource, /const current = storedSandboxCredentialPolicy\(stored\)/); + assert.match(controlSource, /if \(!current \|\| !policy\)/); + assert.doesNotMatch(egressSource, /storage\.delete/); + assert.match(egressSource, /legacy credential policy migration required/); + assert.match(egressSource, /status: 409/); + assert.match(controlSource, /return current\.policy/); + assert.doesNotMatch( + controlSource, + /storedSandboxCredentialPolicy\(value\)\?\.policy \?\? legacySandboxCredentialPolicy/, + ); +}); + +test("cron generation-wraps migrated legacy policies under exact live ownership", async () => { + const source = await readFile(new URL("../src/index.ts", import.meta.url), "utf8"); + const batchStart = source.indexOf("async function reconcileCredentialPolicyCleanupBatch"); + const batchEnd = source.indexOf("async function reconcileCredentialPolicyCleanup(", batchStart); + const batchSource = source.slice(batchStart, batchEnd); + const beginStart = source.indexOf("async function beginLegacySandboxCredentialPolicyRepair"); + const renewStart = source.indexOf( + "async function renewSandboxCredentialPolicyRegistration", + beginStart, + ); + const repairSource = source.slice(beginStart, renewStart); + const controlStart = source.indexOf("export class SessionControlDO"); + const controlEnd = source.indexOf("function storedSandboxCredentialPolicy", controlStart); + const controlSource = source.slice(controlStart, controlEnd); + + assert.ok( + batchSource.indexOf("repairLegacySandboxCredentialPolicyBatch") < + batchSource.indexOf("scanCredentialPolicyCleanupPage"), + ); + assert.match(repairSource, /credentialPolicyLegacyGenerationPrefix/); + assert.match(repairSource, /credentialPolicyLegacyRepairClaimPrefix/); + assert.match(repairSource, /sandboxCredentialPolicyRegistrationQueries/); + assert.match(repairSource, /const ownership: SandboxCurrentLeaseFence/); + assert.match(repairSource, /renewSandboxCredentialPolicyRegistration/); + assert.match(repairSource, /\/api\/session-control\/migrate-legacy/); + assert.match(repairSource, /sandboxIds: registration\.lookupIds/); + assert.match(repairSource, /finishSandboxCredentialPolicyRegistration/); + assert.match(repairSource, /registration_claim_expires_at", "<=", now/); + assert.match(controlSource, /migratedCredentialPolicyRecord/); + assert.match(controlSource, /const sourcePolicy = records/); + assert.match(controlSource, /migratedRecords/); + assert.match(controlSource, /credentialPolicyMigrationCleanupMatches/); + assert.match(controlSource, /this\.ctx\.storage\.transaction/); +}); + +test("active credential-policy generations recover after a post-DO crash", async () => { + const source = await readFile(new URL("../src/index.ts", import.meta.url), "utf8"); + const scanStart = source.indexOf("async function scanCredentialPolicyCleanupPage"); + const scanEnd = source.indexOf("async function readCredentialPolicyScanPage", scanStart); + const scanSource = source.slice(scanStart, scanEnd); + const repairStart = source.indexOf( + "async function repairActiveSandboxCredentialPolicyRegistration", + ); + const repairEnd = source.indexOf("function credentialPolicyScanRequiresCleanup", repairStart); + const repairSource = source.slice(repairStart, repairEnd); + const promoteStart = source.indexOf("async function promoteSandboxCredentialPolicyRegistration"); + const promoteEnd = source.indexOf("async function sandboxCredentialPolicyExists", promoteStart); + const promoteSource = source.slice(promoteStart, promoteEnd); + + assert.ok( + scanSource.indexOf("repairActiveSandboxCredentialPolicyRegistration") < + scanSource.indexOf("credentialPolicyScanRequiresCleanup"), + ); + assert.match(scanSource, /repairedRegistrations/); + assert.match(scanSource, /deferredRegistrations/); + assert.match(repairSource, /row\.policy_state !== "registering"/); + assert.match(repairSource, /row\.registration_claim_expires_at/); + assert.match(repairSource, /credentialPolicyScanOwnershipFence/); + assert.match(repairSource, /sandboxCredentialPolicyExists/); + assert.match(repairSource, /recordSandboxCredentialPolicyRefs\([\s\S]*"active"/); + assert.match(repairSource, /repair lost durable ownership/); + assert.match(promoteSource, /state: "active"/); + assert.match(promoteSource, /registration_claim: null/); + assert.match(promoteSource, /registration_claim_expires_at: null/); + assert.match(promoteSource, /where\("registration_generation", "=", generation\)/); + assert.match(promoteSource, /expression\("registration_claim_expires_at", "<=", now\)/); + assert.match(promoteSource, /sandboxCredentialPolicyOwnerCondition/); +}); + +test("Sandbox credential registration always proves exact durable ownership", async () => { + const source = await readFile(new URL("../src/index.ts", import.meta.url), "utf8"); + const ownerStart = source.indexOf("function sandboxManagedOwnershipCondition"); + const ownerEnd = source.indexOf("async function abandonSandboxCredentialPolicyRegistration"); + const ownerSource = source.slice(ownerStart, ownerEnd); + const createStart = source.indexOf("async function createInteractiveSessionFromInput"); + const createEnd = source.indexOf("function initialRuntimeAdapterWorkspaceId", createStart); + const createSource = source.slice(createStart, createEnd); + const ensureStart = source.indexOf("async function ensureSandboxCredentialPolicy"); + const ensureEnd = source.indexOf("async function recordSandboxCredentialPolicyRefs", ensureStart); + const ensureSource = source.slice(ensureStart, ensureEnd); + + assert.doesNotMatch(ownerSource, /ownershipFence\?:/); + assert.doesNotMatch( + ownerSource, + /ownershipFence: SandboxCredentialPolicyOwnershipFence \| undefined/, + ); + assert.doesNotMatch(ownerSource, /1 = 1/); + assert.match(ownerSource, /lease_id = \$\{ownershipFence\.leaseId\}/); + assert.match(ownerSource, /sandbox_refresh_claim = \$\{ownershipFence\.claim\}/); + assert.match(ownerSource, /AND \$\{sandboxId\} = \$\{ownershipFence\.sandboxId\}/); + assert.match(createSource, /const initialSandboxLease/); + assert.match(createSource, /const initialAgentTokenHash = await sha256\(agentToken\)/); + assert.match(createSource, /lease_id: initialSandboxOwnership\?\.leaseId/); + assert.match(createSource, /ownership: initialSandboxOwnership/); + assert.match(ensureSource, /sandboxCurrentLeaseFence|SandboxCurrentLeaseFence/); + assert.match(ensureSource, /credentialPolicyLegacyGenerationPrefix/); + assert.match(ensureSource, /repairLegacySandboxCredentialPolicy/); + assert.match( + ensureSource, + /registerSandboxCredentialPolicy\(env, session, sandboxId, ownership\)/, + ); +}); + +test("initial terminal adapter responses enter durable finalization", async () => { + const source = await readFile(new URL("../src/index.ts", import.meta.url), "utf8"); + const createStart = source.indexOf("async function createInteractiveSessionFromInput"); + const createEnd = source.indexOf("function initialRuntimeAdapterWorkspaceId", createStart); + const createSource = source.slice(createStart, createEnd); + const completionStart = createSource.indexOf("const provisionUpdate"); + const completionEnd = createSource.indexOf( + "if ((provisionUpdate.numUpdatedRows", + completionStart, + ); + const completionSource = createSource.slice(completionStart, completionEnd); + + assert.ok(createStart >= 0 && createEnd > createStart); + assert.match(createSource, /const initialTerminalStatus: "stopped" \| "expired" \| "failed"/); + assert.match(createSource, /terminal_finalize_pending: initialTerminalStatus \? 1 : 0/); + assert.match(createSource, /stopped_at: terminalAt/); + assert.match(createSource, /agent_token_hash: null/); + assert.match(createSource, /attach_url: initialTerminalStatus \? null/); + assert.match(createSource, /adapter_create_pending: initialTerminalStatus/); + assert.match(completionSource, /MAX\(updated_at \+ 1, \$\{completionVersionFloor\}\)/); + assert.doesNotMatch(completionSource, /where\("updated_at"/); + assert.match(createSource, /lease_id IS \$\{initialSandboxOwnership\?\.leaseId \?\? null\}/); + assert.match(createSource, /where\("agent_token_hash", "=", initialAgentTokenHash\)/); + assert.match(createSource, /where\("sandbox_refresh_sandbox_id", "is", null\)/); + assert.match(createSource, /where\("sandbox_refresh_claim", "is", null\)/); + assert.match(createSource, /where\("sandbox_refresh_claim_expires_at", "is", null\)/); + assert.match(createSource, /finalizeTerminalInteractiveSession/); +}); + +test("create-only adapters reject stopping responses before persistence", async () => { + const source = await readFile(new URL("../src/index.ts", import.meta.url), "utf8"); + const externalStart = source.indexOf("async function provisionInteractiveSession"); + const externalEnd = source.indexOf("async function provisionInteractiveEndpoint", externalStart); + const externalSource = source.slice(externalStart, externalEnd); + const forwardedStart = source.indexOf("function provisionResultFromBody"); + const forwardedEnd = source.indexOf("function failedProvision", forwardedStart); + const forwardedSource = source.slice(forwardedStart, forwardedEnd); + + assert.match(externalSource, /createOnlyAdapterStatus\(body\.status\)/); + assert.match(forwardedSource, /createOnlyAdapterStatus\(body\.status\)/); + assert.match(forwardedSource, /if \(!status\) return failedProvision/); +}); + +test("Sandbox cleanup and legacy stops use durable terminal transitions", async () => { + const source = await readFile(new URL("../src/index.ts", import.meta.url), "utf8"); + const actionStart = source.indexOf('if (action === "stop")'); + const actionEnd = source.indexOf('throw badRequest("unknown action")', actionStart); + const stopSource = source.slice(actionStart, actionEnd); + const legacyCompleteIndex = stopSource.indexOf("completeLegacyInteractiveSessionStop"); + const cleanupIndex = stopSource.lastIndexOf( + "stageTerminalCredentialPolicyCleanupById", + legacyCompleteIndex, + ); + const reconcileIndex = stopSource.indexOf("reconcileCredentialPolicyCleanupBatch", cleanupIndex); + const completeStart = source.indexOf("async function completeLegacyInteractiveSessionStop"); + const completeEnd = source.indexOf("async function mutateInteractiveSession(", completeStart); + const completeSource = source.slice(completeStart, completeEnd); + const scheduledStart = source.indexOf( + "async function reconcileLegacyStoppingInteractiveSessionBatch", + ); + const scheduledEnd = source.indexOf( + "async function requeueTerminalArchiveObjectBackfill", + scheduledStart, + ); + const scheduledSource = source.slice(scheduledStart, scheduledEnd); + + assert.ok(actionStart >= 0 && actionEnd > actionStart); + assert.ok( + cleanupIndex >= 0 && reconcileIndex > cleanupIndex && legacyCompleteIndex > reconcileIndex, + ); + assert.match(stopSource, /const staged = await stageTerminalCredentialPolicyCleanupById/); + assert.match(stopSource, /if \(!staged\)/); + assert.match(stopSource, /credential_cleanup_terminal_status/); + assert.match( + stopSource, + /completeLegacyInteractiveSessionStop\(env, session, actor\(user\), now\)/, + ); + assert.match(completeSource, /env\.DB\.batch/); + assert.match(completeSource, /interactive workspace stop requested/); + assert.match(completeSource, /interactive workspace stopped/); + assert.match(completeSource, /status: "stopped"/); + assert.match(completeSource, /terminal_finalize_pending: 1/); + assert.match(completeSource, /AND status = \$\{owner\.status\}/); + assert.match(completeSource, /AND updated_at = \$\{owner\.updatedAt\}/); + assert.match(completeSource, /finalizeTerminalInteractiveSession/); + assert.doesNotMatch(completeSource, /status: "stopping"/); + assert.match(scheduledSource, /where\("status", "=", "stopping"\)/); + assert.match(scheduledSource, /completeLegacyInteractiveSessionStop/); + assert.match(stopSource, /interactive session lifecycle changed; retry stop/); + assert.match(stopSource, /const current = await readInteractiveSession\(env, id\)/); + assert.match(stopSource, /current\.adapter !== runtimeAdapterName/); + assert.match(stopSource, /current\.adapterWorkspaceId !== session\.adapterWorkspaceId/); + assert.match(stopSource, /\["stopping", "stopped", "expired", "failed"\]\.includes/); +}); + +test("legacy expiry enters the shared retryable terminal finalizer", async () => { + const source = await readFile(new URL("../src/index.ts", import.meta.url), "utf8"); + const expiryStart = source.indexOf("async function markInteractiveTerminalUnavailable"); + const expiryEnd = source.indexOf("async function uploadInteractiveSessionClipboard", expiryStart); + const expirySource = source.slice(expiryStart, expiryEnd); + + assert.match(expirySource, /status: "expired"/); + assert.match(expirySource, /MAX\(updated_at \+ 1, \$\{now\}\)/); + assert.match(expirySource, /where\("updated_at", "=", existing\.updated_at\)/); + assert.match(expirySource, /terminal_finalize_pending: 1/); + assert.match(expirySource, /finalizeTerminalInteractiveSession\(env, id, "expired", now\)/); + assert.match(expirySource, /stageTerminalCredentialPolicyCleanupById/); + assert.match( + expirySource, + /stageTerminalCredentialPolicyCleanupById\([\s\S]*?"failed",\s*message,[\s\S]*?now,\s*message/, + ); + assert.match(expirySource, /reconcileCredentialPolicyCleanupBatch\(env, now, id\)/); + assert.ok( + expirySource.indexOf("stageTerminalCredentialPolicyCleanup") < + expirySource.indexOf("reconcileCredentialPolicyCleanupBatch"), + ); +}); + +test("idempotent legacy terminal stop verifies credential cleanup", async () => { + const source = await readFile(new URL("../src/index.ts", import.meta.url), "utf8"); + const actionStart = source.indexOf('if (action === "stop")'); + const legacyCompleteIndex = source.indexOf( + "completeLegacyInteractiveSessionStop(env, session", + actionStart, + ); + const fastPathSource = source.slice(actionStart, legacyCompleteIndex); + const unregisterStart = source.indexOf("async function unregisterSandboxCredentialPolicyLookup"); + const unregisterEnd = source.indexOf( + "function sandboxCredentialPolicyRefQueries", + unregisterStart, + ); + const unregisterSource = source.slice(unregisterStart, unregisterEnd); + + assert.match(fastPathSource, /isSandboxInteractiveSession\(session\)/); + assert.match(fastPathSource, /stageTerminalCredentialPolicyCleanup/); + assert.match(fastPathSource, /reconcileCredentialPolicyCleanupBatch/); + assert.match(unregisterSource, /if \(!stub\) throw serviceUnavailable/); + assert.match(unregisterSource, /if \(!response\.ok\)/); + assert.doesNotMatch(unregisterSource, /response\.status !== 404/); + assert.match(unregisterSource, /sandbox credential policy cleanup failed/); +}); + +test("sandbox credential cleanup is durably staged and retried", async () => { + const source = await readFile(new URL("../src/index.ts", import.meta.url), "utf8"); + const migration = await readFile( + new URL("../migrations/0022_credential_policy_cleanup.sql", import.meta.url), + "utf8", + ); + const stageStart = source.indexOf("async function stageTerminalCredentialPolicyCleanup"); + const stageEnd = source.indexOf("type CredentialPolicyScanRow", stageStart); + const stageSource = source.slice(stageStart, stageEnd); + const scanStart = stageEnd; + const batchStart = source.indexOf( + "async function reconcileCredentialPolicyCleanupBatch", + scanStart, + ); + const scanSource = source.slice(scanStart, batchStart); + const batchEnd = source.indexOf("async function reconcileCredentialPolicyCleanup(", batchStart); + const batchSource = source.slice(batchStart, batchEnd); + const cleanupStart = batchEnd; + const cleanupEnd = source.indexOf( + "async function completeCredentialPolicyCleanupSession", + cleanupStart, + ); + const cleanupSource = source.slice(cleanupStart, cleanupEnd); + const completionEnd = source.indexOf("function legacyInteractiveSessionLeaseId", cleanupEnd); + const completionSource = source.slice(cleanupEnd, completionEnd); + const provisionStart = source.indexOf("async function provisionWithSandbox"); + const provisionEnd = source.indexOf( + "async function registerSandboxCredentialPolicy", + provisionStart, + ); + const provisionSource = source.slice(provisionStart, provisionEnd); + const registerStart = provisionEnd; + const registerEnd = source.indexOf("async function ensureSandboxCredentialPolicy", registerStart); + const registerSource = source.slice(registerStart, registerEnd); + const registrationLifecycleStart = source.indexOf( + "function sandboxCredentialPolicyRegistrationQueries", + ); + const registrationLifecycleSource = source.slice(registrationLifecycleStart, registerEnd); + const finishStart = source.indexOf("async function finishSandboxCredentialPolicyRegistration"); + const finishEnd = source.indexOf( + "async function abandonSandboxCredentialPolicyRegistration", + finishStart, + ); + const finishSource = source.slice(finishStart, finishEnd); + const abandonEnd = source.indexOf("async function registerSandboxCredentialPolicy", finishEnd); + const abandonSource = source.slice(finishEnd, abandonEnd); + const scanDecisionStart = source.indexOf("function credentialPolicyScanRequiresCleanup"); + const scanDecisionEnd = source.indexOf( + "async function normalizeCredentialPolicyCleanupGroups", + scanDecisionStart, + ); + const scanDecisionSource = source.slice(scanDecisionStart, scanDecisionEnd); + const controlStart = source.indexOf("export class SessionControlDO"); + const controlEnd = source.indexOf("function dedupeSandboxPolicies", controlStart); + const controlSource = source.slice(controlStart, controlEnd); + const refreshStart = source.indexOf("async function ensureCurrentSandboxLease"); + const refreshEnd = source.indexOf("async function prepareSandboxWorkspace", refreshStart); + const refreshSource = source.slice(refreshStart, refreshEnd); + const ownershipStart = source.indexOf("function sandboxTerminalCleanupOwnership"); + const ownershipSource = source.slice(ownershipStart, stageStart); + + assert.match(stageSource, /executeBatch\(env, \[sessionTransition, \.\.\.policyTransitions\]\)/); + assert.match(stageSource, /status: "stopping"/); + assert.match(stageSource, /credential_cleanup_terminal_status: cleanupIntent/); + assert.match(stageSource, /credential_cleanup_terminal_status = 'failed'/); + assert.match(stageSource, /credential_cleanup_terminal_status = 'expired'/); + assert.match(stageSource, /terminalCleanupIntentRank/); + assert.match(stageSource, /sandbox_refresh_claim: null/); + assert.match(stageSource, /sandboxManagedStoredOwnershipCondition\(ownership\.fence\)/); + assert.match(stageSource, /where\("updated_at", "=", session\.updated_at\)/); + assert.match(stageSource, /sandboxCredentialPolicyCleanupAuthorizedCondition/); + assert.match(stageSource, /\.where\("sandbox_id", "=", sandboxId\)/); + assert.match(stageSource, /agent_token_hash: null/); + assert.match(stageSource, /controller: null/); + assert.match(stageSource, /terminal_failure_reason:/); + assert.match(stageSource, /failureReason/); + assert.match(stageSource, /NULLIF\(terminal_failure_reason, ''\)/); + assert.match(stageSource, /terminal_failure_reason: failureEvidence/); + assert.match(stageSource, /state: "cleanup_pending"/); + assert.match(stageSource, /for \(let attempt = 0; attempt < 3; attempt \+= 1\)/); + assert.match(ownershipSource, /sandboxLeaseWithoutRefresh/); + assert.match(ownershipSource, /sandbox_refresh_claim_expires_at/); + assert.match(ownershipSource, /sandboxIds: \[\.\.\.new Set/); + assert.match(scanSource, /credentialPolicyScanLimit/); + assert.match(scanSource, /scan_max_rowid/); + assert.match(scanSource, /maximumCredentialPolicyRowid/); + assert.match(scanSource, /policy\.rowid > \$\{cursor\}/); + assert.match(scanSource, /policy\.rowid <= \$\{maxRowid\}/); + assert.doesNotMatch(scanSource, /affectedSessions/); + assert.doesNotMatch(scanSource, /const transitioned = await update\.executeTakeFirst\(\)/); + assert.match(scanSource, /const sessionTransition = sql/); + assert.match(scanSource, /executeBatch\(env, \[sessionTransition, policyTransition\]\)/); + assert.match(scanSource, /executeBatch\(env, \[ownerTransition, policyTransition\]\)/); + assert.match(scanSource, /sandboxCredentialPolicyCleanupAuthorizedCondition/); + assert.match(scanSource, /AND status IS \$\{row\.session_status\}/); + assert.match(scanSource, /AND lease_id IS \$\{row\.session_lease_id\}/); + assert.match(scanSource, /AND agent_token_hash IS \$\{row\.session_agent_token_hash\}/); + assert.match(scanSource, /standalone\.ownership_claim AS standalone_claim/); + assert.match(scanSource, /AND updated_at IS \$\{row\.session_updated_at\}/); + assert.match(scanSource, /\.where\("sandbox_id", "=", row\.sandbox_id\)/); + assert.match(scanSource, /\.where\("lookup_id", "=", row\.lookup_id\)/); + assert.match( + scanSource, + /\.where\("registration_generation", "=", row\.registration_generation\)/, + ); + assert.match(scanSource, /terminal_failure_reason = CASE/); + assert.match(scanSource, /credential_cleanup_terminal_status = 'failed'/); + assert.match(scanSource, /credential_cleanup_terminal_status = 'expired'/); + assert.match(scanSource, /const transitionRevision = Math\.max/); + assert.match(scanSource, /agent_token_hash = NULL/); + assert.match(scanSource, /credentialPolicySandboxIsExpected/); + assert.match(scanSource, /sandbox_refresh_claim = NULL/); + assert.match(scanSource, /normalizeCredentialPolicyCleanupGroups/); + assert.match(scanSource, /group_max_session_id/); + assert.match(batchSource, /scanCredentialPolicyCleanupPage/); + assert.match(batchSource, /normalizeCredentialPolicyCleanupGroups/); + assert.match(batchSource, /COALESCE\(last_attempt_at, created_at\)/); + assert.match(batchSource, /\.limit\(credentialPolicyCleanupLimit\)/); + assert.match(batchSource, /completeStandaloneSandboxProvisionCleanupSafely/); + assert.match(cleanupSource, /cleanup_claim_expires_at/); + assert.match(cleanupSource, /registration_claim_expires_at > \$\{now\}/); + assert.match(cleanupSource, /sandboxCredentialPolicyCleanupAuthorizedCondition/); + assert.ok( + cleanupSource.indexOf("cleanup_claim = ${claim}") < + cleanupSource.indexOf("unregisterSandboxCredentialPolicyLookup"), + ); + assert.match(cleanupSource, /last_error:/); + assert.match(cleanupSource, /cleanup_claim: null/); + assert.match(cleanupSource, /async function completeStandaloneSandboxProvisionCleanupSafely/); + assert.match(cleanupSource, /standalone Sandbox cleanup pending/); + assert.match(cleanupSource, /standalone_sandbox_provisions/); + assert.match(cleanupSource, /MAX\(updated_at \+ 1, \$\{now\}\)/); + assert.match(cleanupSource, /cleanup failure persistence failed/); + assert.match(completionSource, /NOT EXISTS \(/); + assert.match(completionSource, /status: terminalStatus/); + assert.match(completionSource, /terminal_finalize_pending: 1/); + assert.match(completionSource, /MAX\(updated_at \+ 1, \$\{now\}\)/); + assert.match(completionSource, /\.where\("status", "=", session\.status\)/); + assert.match(completionSource, /\.where\("updated_at", "=", session\.updated_at\)/); + assert.match(completionSource, /retainedRuntimeAdapterFailureMessage/); + assert.match(completionSource, /terminal_failure_reason:/); + assert.match(completionSource, /\? failureMessage/); + assert.match(batchSource, /completeCredentialPolicyCleanupSession\(env, session\.id/); + assert.ok( + provisionSource.indexOf("stageTerminalCredentialPolicyCleanupById") < + provisionSource.indexOf("reconcileCredentialPolicyCleanupBatch"), + ); + assert.match(provisionSource, /failureAt,\s*cleanupMessage,\s*ownershipFence/); + assert.match(provisionSource, /try \{[\s\S]*sandboxProvisionPreflightError\(env, session\)/); + assert.match(provisionSource, /!agentToken/); + assert.match(provisionSource, /managed Sandbox agent token is unavailable/); + assert.match(registrationLifecycleSource, /registration_generation/); + assert.match(registrationLifecycleSource, /registration_claim/); + assert.match(registrationLifecycleSource, /registration_claim_expires_at/); + assert.doesNotMatch(registrationLifecycleSource, /ownershipFence\?:/); + assert.doesNotMatch(registrationLifecycleSource, /1 = 1/); + assert.match(registrationLifecycleSource, /sandboxCredentialPolicyOwnerCondition/); + assert.match(registrationLifecycleSource, /sandboxCredentialPolicyOwnerCondition/); + assert.ok( + registerSource.indexOf("beginSandboxCredentialPolicyRegistration") < + registerSource.indexOf( + 'stub.fetch("https://crabfleet.internal/api/session-control/register"', + ), + ); + assert.ok( + registerSource.indexOf("renewSandboxCredentialPolicyRegistration") < + registerSource.indexOf( + 'stub.fetch("https://crabfleet.internal/api/session-control/register"', + ), + ); + assert.ok( + registerSource.indexOf('stub.fetch("https://crabfleet.internal/api/session-control/register"') < + registerSource.indexOf("finishSandboxCredentialPolicyRegistration"), + ); + assert.doesNotMatch(finishSource, /INSERT INTO|insertInto/); + assert.match(finishSource, /state: "active"/); + assert.match(finishSource, /where\(sandboxCredentialPolicyOwnerCondition/); + assert.doesNotMatch(finishSource, /cleanup_pending/); + assert.match(registerSource, /abandonSandboxCredentialPolicyRegistration/); + assert.match(abandonSource, /sandboxCredentialPolicyCleanupAuthorizedCondition/); + assert.match(abandonSource, /THEN 'cleanup_pending'/); + assert.match(abandonSource, /ELSE 'registering'/); + assert.ok( + scanDecisionSource.indexOf("sandboxExpected") < + scanDecisionSource.indexOf("if (registrationAbandoned) return true"), + ); + assert.match(scanDecisionSource, /row\.session_agent_token_hash !== null/); + assert.match(controlSource, /sandboxPolicyTombstoneKey/); + assert.match(controlSource, /credentialPolicyRegistrationAccepted/); + assert.match(controlSource, /credentialPolicyCleanupMatches/); + assert.match(controlSource, /this\.ctx\.storage\.transaction/); + assert.doesNotMatch(source, /async function unregisterSandboxCredentialPolicy\(/); + assert.match(migration, /CREATE TABLE IF NOT EXISTS interactive_session_credential_policies/); + assert.match(migration, /state IN \('registering', 'active', 'cleanup_pending'\)/); + assert.match(migration, /registration_generation TEXT NOT NULL/); + assert.match(migration, /registration_claim_expires_at INTEGER/); + assert.match(migration, /CREATE TABLE IF NOT EXISTS credential_policy_reconcile_state/); + assert.match(migration, /scan_max_rowid INTEGER NOT NULL/); + assert.match(migration, /group_max_session_id TEXT NOT NULL/); + assert.match(migration, /ADD COLUMN sandbox_refresh_sandbox_id TEXT/); + assert.match(migration, /ADD COLUMN sandbox_refresh_claim TEXT/); + assert.match(migration, /ADD COLUMN sandbox_refresh_claim_expires_at INTEGER/); + assert.match(migration, /CREATE TABLE IF NOT EXISTS standalone_sandbox_provisions/); + assert.match(migration, /idx_interactive_policy_fair_cleanup/); + assert.match(migration, /COALESCE\(last_attempt_at, created_at\)/); + assert.match(migration, /terminal_failure_reason = CASE/); + assert.match(migration, /terminal_finalize_pending = 0/); + assert.match(migration, /agent_token_hash = NULL/); + assert.match(migration, /SET\s+status = 'stopping'/); + assert.match(refreshSource, /sandbox_refresh_sandbox_id: refreshFence\.sandboxId/); + assert.match(refreshSource, /sandbox_refresh_claim: refreshFence\.claim/); + assert.match(refreshSource, /sandbox_refresh_claim_expires_at: refreshFence\.expiresAt/); + assert.match(refreshSource, /const agentToken = newAgentToken\(\)/); + assert.ok( + refreshSource.indexOf("sandboxProvisionPreflightError(env, refreshPayload)") < + refreshSource.indexOf("const agentToken = newAgentToken()"), + ); + assert.match(refreshSource, /agent_token_hash: agentTokenHash/); + assert.match(refreshSource, /provisionWithSandbox\([\s\S]*?agentToken,[\s\S]*?refreshLease/); + assert.match(refreshSource, /where\("agent_token_hash", "=", agentTokenHash\)/); + assert.match(refreshSource, /sandbox_refresh_claim_expires_at", "=", refreshFence\.expiresAt/); + assert.match(refreshSource, /executeBatch\(env, commitQueries\)/); + assert.match(refreshSource, /state: "cleanup_pending"/); + assert.match(refreshSource, /stageFailedManagedSandboxProvision/); + assert.ok( + refreshSource.indexOf("sandbox_refresh_claim: null") < + refreshSource.lastIndexOf("reconcileCredentialPolicyCleanupBatch"), + ); + assert.doesNotMatch( + refreshSource, + /queueSandboxCredentialPolicyCleanup\(env, session\.id, oldSandboxId, refreshedAt\)/, + ); + assert.match(refreshSource, /const current = await readInteractiveSession/); +}); + +test("terminal endpoints enforce current runtime capabilities", async () => { + const source = await readFile(new URL("../src/index.ts", import.meta.url), "utf8"); + const decorateStart = source.indexOf("function decorateInteractiveSession"); + const decorateEnd = source.indexOf( + "function canChangeInteractiveSessionMultiplayer", + decorateStart, + ); + const decorateSource = source.slice(decorateStart, decorateEnd); + + assert.match(source, /type InteractiveSession = \{[\s\S]*ptyAvailable\?: boolean;/); + assert.match(source, /if \(!session\.capabilities\.terminal\)/); + assert.match(source, /runtimeCapabilities\(row\.runtime, row\.capabilities_json\)\.terminal/); + assert.match(source, /runtimeAdapterTerminalFailureStatus\(existing\.adapter\) === "detached"/); + assert.match(source, /attachUrl: capabilities\.terminal \? row\.attach_url : null/); + assert.match(decorateSource, /ptyAvailable && session\.adapter === runtimeAdapterName/); + assert.match( + decorateSource, + /`\/api\/interactive-sessions\/\$\{encodeURIComponent\(session\.id\)\}\/pty`/, + ); + assert.match(decorateSource, /attachUrl,/); + assert.doesNotMatch( + decorateSource, + /attachUrl: canControl && session\.capabilities\.terminal \? session\.attachUrl : null/, + ); +}); + +test("non-retryable adapter client errors do not enter ambiguous replay", () => { + assert.equal(definitiveRuntimeAdapterCreateFailure(413), true); + assert.equal(definitiveRuntimeAdapterCreateFailure(415), true); + assert.equal(definitiveRuntimeAdapterCreateFailure(422), true); + assert.equal(definitiveRuntimeAdapterCreateFailure(408), false); + assert.equal(definitiveRuntimeAdapterCreateFailure(409), false); + assert.equal(definitiveRuntimeAdapterCreateFailure(423), false); + assert.equal(definitiveRuntimeAdapterCreateFailure(425), false); + assert.equal(definitiveRuntimeAdapterCreateFailure(429), false); + assert.equal(definitiveRuntimeAdapterCreateFailure(503), false); +}); + +test("definitive adapter create errors retain a redacted provider reason before release", async () => { + const source = await readFile(new URL("../src/index.ts", import.meta.url), "utf8"); + const createStart = source.indexOf("async function provisionWithRuntimeAdapter"); + const createEnd = source.indexOf("function persistedRuntimeAdapterSeconds", createStart); + const createSource = source.slice(createStart, createEnd); + const replayStart = source.indexOf("async function replayStoppingRuntimeAdapterCreate"); + const replayEnd = source.indexOf("async function stopRuntimeAdapterWorkspace", replayStart); + const replaySource = source.slice(replayStart, replayEnd); + const releaseStart = source.indexOf("async function releaseFailedRuntimeAdapterProvision"); + const releaseEnd = source.indexOf( + "async function persistRuntimeAdapterStopEvidence", + releaseStart, + ); + const releaseSource = source.slice(releaseStart, releaseEnd); + const bodyStart = source.indexOf("async function readRuntimeAdapterResponseBody"); + const bodyEnd = source.indexOf("function runtimeAdapterToken", bodyStart); + const bodySource = source.slice(bodyStart, bodyEnd); + + const bodyReadIndex = createSource.indexOf( + "responseBody = await readRuntimeAdapterResponseBody(response)", + ); + assert.ok(bodyReadIndex >= 0 && bodyReadIndex < createSource.indexOf("if (!response.ok)")); + assert.match(createSource, /redactedAdapterResponseMessage/); + assert.match(createSource, /runtime adapter provision failed: \$\{responseMessage\}/); + assert.match(createSource, /releaseFailedRuntimeAdapterProvision/); + assert.doesNotMatch(createSource, /response\.json/); + assert.match(replaySource, /readRuntimeAdapterResponseBody\(response\)/); + assert.match(replaySource, /redactedAdapterResponseMessage/); + assert.match(replaySource, /reconcile_error: message/); + assert.match(replaySource, /INSERT INTO interactive_session_events/); + assert.doesNotMatch(replaySource, /response\.json/); + assert.ok( + releaseSource.indexOf("stageFailedRuntimeAdapterRelease") < + releaseSource.indexOf("stopRuntimeAdapterWorkspaceForSession"), + ); + assert.match(bodySource, /await readBoundedResponseText\(response\)/); + assert.doesNotMatch(bodySource, /response\.(?:json|text)\(/); + assert.match(bodySource, /JSON\.parse\(body\)/); + assert.equal( + redactedAdapterResponseMessage( + { detail: "capacity unavailable; token=private-value" }, + "HTTP 422", + ), + "capacity unavailable; [credential]", + ); + const opaqueErrorIds = ["provider-body", "provider-workspace", "provider-error"]; + const opaqueErrorMessage = redactedAdapterResponseMessage( + { + providerResourceId: opaqueErrorIds[0], + workspace: { provider_resource_id: opaqueErrorIds[1] }, + error: { + leaseId: opaqueErrorIds[2], + message: `failed ${opaqueErrorIds.join(" ")}`, + }, + }, + "HTTP 422", + ); + assert.equal(opaqueErrorMessage, "failed [workspace] [workspace] [workspace]"); + for (const identifier of opaqueErrorIds) { + assert.doesNotMatch(opaqueErrorMessage, new RegExp(identifier)); + } +}); + +test("successful DELETE requires an implicit or parsed release confirmation", () => { + const ready = parseAdapterWorkspaceResult({ id: "fleet-a-is-101", status: "ready" }); + const stopped = parseAdapterWorkspaceResult({ id: "fleet-a-is-101", status: "stopped" }); + const wrong = parseAdapterWorkspaceResult({ id: "fleet-b-is-101", status: "stopped" }); + assert.equal(runtimeAdapterStopOutcome(200, null, "fleet-a-is-101"), "stopping"); + assert.equal(runtimeAdapterStopOutcome(202, null, "fleet-a-is-101"), "stopping"); + assert.equal(runtimeAdapterStopOutcome(200, ready, "fleet-a-is-101"), "stopping"); + assert.equal(runtimeAdapterStopOutcome(200, stopped, "fleet-a-is-101"), "stopped"); + assert.equal(runtimeAdapterStopOutcome(204, null, "fleet-a-is-101"), "stopped"); + assert.equal(runtimeAdapterStopOutcome(200, wrong, "fleet-a-is-101"), "identity_mismatch"); +}); + +test("adapter DELETE evidence survives pending and confirmed release", async () => { + const source = await readFile(new URL("../src/index.ts", import.meta.url), "utf8"); + const deleteStart = source.indexOf("async function stopRuntimeAdapterWorkspace("); + const deleteEnd = source.indexOf("async function runtimeAdapterFetch", deleteStart); + const deleteSource = source.slice(deleteStart, deleteEnd); + const reconcileStart = source.indexOf("async function reconcileStoppingRuntimeAdapterWorkspace"); + const reconcileEnd = source.indexOf("type StoppingRuntimeAdapterReplay", reconcileStart); + const reconcileSource = source.slice(reconcileStart, reconcileEnd); + const releaseStart = source.indexOf("async function recordConfirmedRuntimeAdapterRelease"); + const releaseEnd = source.indexOf( + "async function clearRuntimeAdapterCreatePending", + releaseStart, + ); + const releaseSource = source.slice(releaseStart, releaseEnd); + const failedReleaseStart = source.indexOf("async function releaseFailedRuntimeAdapterProvision"); + const failedReleaseEnd = source.indexOf( + "async function stageFailedRuntimeAdapterRelease", + failedReleaseStart, + ); + const failedReleaseSource = source.slice(failedReleaseStart, failedReleaseEnd); + + assert.match(deleteSource, /await readRuntimeAdapterResponseBody\(response\)/); + assert.doesNotMatch(deleteSource, /response\.json/); + assert.match(deleteSource, /redactedAdapterResponseMessage/); + assert.match(deleteSource, /return \{ status: "stopped", message \}/); + assert.match(deleteSource, /return \{ status: outcome, message \}/); + assert.match(reconcileSource, /runtime adapter stop pending: \$\{safeProviderError/); + assert.match(reconcileSource, /reconcileError: message/); + assert.match(reconcileSource, /release\.message/); + assert.match(releaseSource, /retainedReleaseMessage/); + assert.match(releaseSource, /env\.DB\.batch/); + assert.match(releaseSource, /INSERT INTO interactive_session_events/); + assert.match(releaseSource, /terminal_finalize_pending: 1/); + assert.match(failedReleaseSource, /persistRuntimeAdapterStopEvidence/); + assert.match(failedReleaseSource, /await executeBatch\(env, \[/); + assert.match(failedReleaseSource, /INSERT INTO interactive_session_events/); + assert.match(failedReleaseSource, /AND NOT EXISTS/); +}); + +test("adapter workspace paths use the controller id and encode it", () => { + assert.equal(normalizeAdapterWorkspaceId("IS-101"), "is-101"); + assert.equal( + runtimeAdapterWorkspaceUrl("https://controller.example/base", "session/one"), + "https://controller.example/base/v1/workspaces/session%2Fone", + ); + assert.equal( + runtimeAdapterDesktopUrl("https://controller.example/base", "is-101"), + "https://controller.example/base/v1/workspaces/is-101/connections/desktop", + ); + assert.equal( + runtimeAdapterBrowserVncUrl("https://fleet.example", "IS/101"), + "https://fleet.example/api/interactive-sessions/IS%2F101/vnc", + ); +}); + +test("runtime adapter control-plane identity is canonical and origin-bound", () => { + assert.equal( + runtimeAdapterControlPlaneIdentity("https://controller.example/api/"), + "https://controller.example/api", + ); + assert.equal( + runtimeAdapterControlPlaneIdentity("https://controller.example"), + "https://controller.example/", + ); + assert.equal(runtimeAdapterControlPlaneIdentity("https://controller.example/api?tenant=a"), null); + assert.equal(runtimeAdapterControlPlaneIdentity("https://controller.example/api?"), null); + assert.equal(runtimeAdapterControlPlaneIdentity("https://controller.example/api#fragment"), null); + assert.equal(runtimeAdapterControlPlaneIdentity("https://controller.example/api#"), null); + assert.equal(runtimeAdapterControlPlaneIdentity("http://controller.example/api"), null); + assert.equal( + runtimeAdapterControlPlaneIdentity("http://127.0.0.1:8788/adapter/"), + "http://127.0.0.1:8788/adapter", + ); + assert.equal( + runtimeAdapterWorkspaceUrl("https://controller.example/root/adapter", "fleet-is-1"), + "https://controller.example/root/adapter/v1/workspaces/fleet-is-1", + ); +}); + +test("adapter bodies share the bounded stream reader", async () => { + const source = await readFile(new URL("../src/index.ts", import.meta.url), "utf8"); + const ranges = [ + ["async function provisionWithRuntimeAdapter", "function persistedRuntimeAdapterSeconds"], + [ + "async function inspectRuntimeAdapterWorkspace", + "async function reconcileStoppingRuntimeAdapterWorkspace", + ], + [ + "async function replayStoppingRuntimeAdapterCreate", + "async function stopRuntimeAdapterWorkspace(", + ], + ["async function stopRuntimeAdapterWorkspace(", "type RuntimeAdapterStopResult"], + ["async function interactiveSessionVnc", "function interactiveTerminalTarget"], + ] as const; + for (const [startMarker, endMarker] of ranges) { + const start = source.indexOf(startMarker); + const end = source.indexOf(endMarker, start + startMarker.length); + const operation = source.slice(start, end); + assert.match(operation, /readRuntimeAdapterResponseBody\(response\)/); + assert.doesNotMatch(operation, /response\.(?:json|text)\(/); + } + const readerStart = source.indexOf("async function readRuntimeAdapterResponseBody"); + const readerEnd = source.indexOf("function runtimeAdapterToken", readerStart); + const readerSource = source.slice(readerStart, readerEnd); + assert.match(readerSource, /readBoundedResponseText\(response\)/); + assert.doesNotMatch(readerSource, /response\.(?:json|text)\(/); +}); + +test("desktop mint revalidates current ownership before redirect", async () => { + const source = await readFile(new URL("../src/index.ts", import.meta.url), "utf8"); + const vncStart = source.indexOf("async function interactiveSessionVnc"); + const vncEnd = source.indexOf("function interactiveTerminalTarget", vncStart); + const vncSource = source.slice(vncStart, vncEnd); + + assert.ok( + vncSource.indexOf("currentAdapterDesktopConnection") < + vncSource.indexOf("currentRuntimeAdapterDesktopAccess"), + ); + assert.ok( + vncSource.indexOf("currentRuntimeAdapterDesktopAccess") < + vncSource.indexOf("target = connection.url"), + ); + assert.match(vncSource, /selectFrom\("interactive_sessions"\)/); + assert.match(vncSource, /adapter_workspace_id/); + assert.match(vncSource, /adapter_control_plane/); + assert.match(vncSource, /provider_resource_id/); + assert.match(vncSource, /adapter_create_pending/); + assert.match(vncSource, /\["ready", "attached", "detached"\]/); + assert.match(vncSource, /current\.capabilities\.vnc/); + assert.match(vncSource, /current\.capabilities\.desktop/); + assert.match(vncSource, /canControlInteractiveSession/); + assert.match(vncSource, /desktop authorization changed; retry/); +}); + +test("desktop redirects require HTTPS or literal loopback HTTP", () => { + assert.equal( + safeDesktopUrl("https://desktop.example/session"), + "https://desktop.example/session", + ); + assert.equal(safeDesktopUrl("http://127.0.0.1:6080/vnc"), "http://127.0.0.1:6080/vnc"); + assert.equal(safeDesktopUrl("http://localhost:6080/vnc"), "http://localhost:6080/vnc"); + assert.equal(safeDesktopUrl("http://desktop.example/vnc"), null); + assert.equal(safeDesktopUrl("http://2130706433:6080/vnc"), null); + assert.equal(safeDesktopUrl("https://user:secret@desktop.example/vnc"), null); + assert.equal(safeDesktopUrl("javascript:alert(1)"), null); + assert.equal(safeDesktopUrl(" https://desktop.example/vnc"), null); + const signed = "https://Desktop.Example:443/%7Edesktop?signature=a%2Bb%2Fc%3D&dup=1&dup=2"; + assert.equal(safeDesktopUrl(signed), signed); + assert.equal(parseAdapterDesktopConnection({ url: signed })?.url, signed); +}); + +test("terminal URLs require WSS except literal loopback WS", () => { + assert.equal( + safeWebSocketUrl("wss://terminal.example/session"), + "wss://terminal.example/session", + ); + assert.equal(safeWebSocketUrl("ws://localhost:8787/session"), "ws://localhost:8787/session"); + assert.equal(safeWebSocketUrl("ws://127.0.0.1:8787/session"), "ws://127.0.0.1:8787/session"); + assert.equal(safeWebSocketUrl("ws://terminal.example/session"), null); + assert.equal(safeWebSocketUrl("ws://127.1:8787/session"), null); + assert.equal(safeWebSocketUrl("wss://terminal.example/session\n"), null); + const signed = "wss://Terminal.Example:443/%7Epty?signature=a%2Bb%2Fc%3D&dup=1&dup=2"; + assert.equal(safeWebSocketUrl(signed), signed); + assert.equal( + parseAdapterWorkspaceResult({ status: "ready", attachUrl: signed })?.terminalUrl, + signed, + ); + const message = parseAdapterWorkspaceResult({ + id: "fleet-a-is-101", + status: "ready", + attachUrl: signed, + message: `attach ${signed}; Authorization: Bearer bearer-secret; token=query-secret`, + })?.message; + assert.equal(message, "attach [connection] [credential]"); + assert.doesNotMatch(message ?? "", /bearer-secret|query-secret|signature=/u); + const slashEscaped = signed.replaceAll("/", "\\/"); + const nestedEscaped = JSON.stringify(slashEscaped).slice(1, -1); + for (const escaped of [slashEscaped, nestedEscaped]) { + const redacted = redactedAdapterMessage(`provider terminal ${escaped}`, "failed", [], [signed]); + assert.equal(redacted, "provider terminal [connection]"); + assert.doesNotMatch(redacted, /signature|Terminal\.Example/iu); + } + for (const arbitrary of [ + String.raw`failed https:\/\/host.example\/pty?bearer=path-secret`, + String.raw`nested {\"detail\":\"wss:\\/\\/host.example\\/pty?token=nested-secret\"}`, + String.raw`mixed ws:\/\/host.example\/pty?authorization=Bearer-query-secret`, + ]) { + const redacted = redactedAdapterMessage(arbitrary, "failed"); + assert.match(redacted, /\[connection\]/u); + assert.doesNotMatch(redacted, /host\.example|path-secret|nested-secret|query-secret/iu); + } + assert.equal( + redactedAdapterResponseMessage( + { + message: `desktop ${slashEscaped}`, + desktopUrl: signed.replace("wss://", "https://"), + terminalUrl: signed, + }, + "fallback", + ), + "desktop [connection]", + ); + assert.equal( + redactedAdapterMessage( + "desktop https://desktop.example/vnc?ticket=secret and sig=second-secret", + "ready", + ), + "desktop [connection] and [credential]", + ); + const structured = redactedAdapterMessage( + `provider {"token":"json-secret","ticket":"ticket-secret","safe":"ok"}; Authorization: Basic dXNlcjpwYXNz`, + "failed", + ); + assert.doesNotMatch( + structured, + /json-secret|ticket-secret|dXNlcjpwYXNz|access_token|refresh_token/iu, + ); + assert.match(structured, /\[credential\]/u); + for (const providerMessage of [ + `X-Api-Key: colon-secret`, + `access_token: quoted-secret`, + String.raw`escaped {\"refresh_token\":\"escaped-secret\"}`, + `password=pass-secret; code: code-secret`, + ]) { + const redacted = redactedAdapterMessage(providerMessage, "failed"); + assert.doesNotMatch( + redacted, + /colon-secret|quoted-secret|escaped-secret|pass-secret|code-secret/iu, + ); + } + const opaqueIdentifierCollision = redactedAdapterMessage( + "provider token=identifier-hidden-secret", + "failed", + ["token"], + ); + assert.equal(opaqueIdentifierCollision, "provider [credential]"); + assert.doesNotMatch(opaqueIdentifierCollision, /identifier-hidden-secret/u); +}); + +test("desktop connection parser accepts current controller response aliases", () => { + assert.deepEqual( + parseAdapterDesktopConnection({ + vncUrl: "https://desktop.example/session?ticket=short-lived", + expiresAt: 1_800_000_000, + }), + { + url: "https://desktop.example/session?ticket=short-lived", + expiresAt: 1_800_000_000_000, + expiresAtPresent: true, + }, + ); +}); + +test("desktop connection expiry is optional but bounded when present", () => { + const now = 1_800_000_000_000; + const url = "https://desktop.example/session?ticket=transient"; + assert.equal(currentAdapterDesktopConnection({ url }, now)?.url, url); + assert.equal( + currentAdapterDesktopConnection({ url, expiresAt: now + 60_000 }, now)?.expiresAt, + now + 60_000, + ); + assert.equal(currentAdapterDesktopConnection({ url, expiresAt: now }, now), null); + assert.equal(currentAdapterDesktopConnection({ url, expiresAt: now + 16 * 60_000 }, now), null); + assert.equal(currentAdapterDesktopConnection({ url, expiresAt: "not-a-date" }, now), null); + assert.equal(parseAdapterDesktopConnection({ url, expiresAt: "" }), null); + assert.equal(parseAdapterDesktopConnection({ url, expiresAt: null })?.expiresAtPresent, false); +}); diff --git a/tests/session-archive.test.ts b/tests/session-archive.test.ts new file mode 100644 index 0000000..1a1f2c2 --- /dev/null +++ b/tests/session-archive.test.ts @@ -0,0 +1,24 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + obsoleteSessionArchiveObjectKeys, + sameSessionArchiveObjectKeys, + sessionArchiveAttemptKeys, +} from "../src/session-archive.ts"; + +test("concurrent archive attempts always use distinct object keys", () => { + const first = sessionArchiveAttemptKeys("sessions/IS-101", 4, 123, 456, "attempt-a"); + const second = sessionArchiveAttemptKeys("sessions/IS-101", 4, 123, 456, "attempt-b"); + assert.equal(sameSessionArchiveObjectKeys(first, second), false); +}); + +test("archive cleanup never selects the committed object keys", () => { + const previous = sessionArchiveAttemptKeys("sessions/IS-101", 3, 100, 400, "previous"); + const attempted = sessionArchiveAttemptKeys("sessions/IS-101", 4, 123, 456, "attempted"); + const winner = sessionArchiveAttemptKeys("sessions/IS-101", 4, 123, 456, "winner"); + + assert.deepEqual(obsoleteSessionArchiveObjectKeys(attempted, previous, attempted), previous); + assert.deepEqual(obsoleteSessionArchiveObjectKeys(winner, previous, attempted), attempted); + assert.equal(obsoleteSessionArchiveObjectKeys(attempted, attempted, attempted), undefined); +}); diff --git a/tests/session-id.test.ts b/tests/session-id.test.ts new file mode 100644 index 0000000..2d2ce28 --- /dev/null +++ b/tests/session-id.test.ts @@ -0,0 +1,271 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { DatabaseSync } from "node:sqlite"; +import test from "node:test"; + +import { allocateInteractiveSessionIdSql, formatInteractiveSessionId } from "../src/session-id.ts"; + +test("interactive session ids remain monotonic after cleanup", () => { + const database = new DatabaseSync(":memory:"); + database.exec( + `CREATE TABLE interactive_sessions ( + id TEXT PRIMARY KEY, + parent_session_id TEXT, + root_session_id TEXT, + repo TEXT NOT NULL DEFAULT 'example/project', + branch TEXT NOT NULL DEFAULT 'main', + runtime TEXT NOT NULL DEFAULT 'container', + adapter TEXT, + profile TEXT NOT NULL DEFAULT 'default', + adapter_workspace_id TEXT, + adapter_control_plane TEXT, + provider_resource_id TEXT, + terminal_status TEXT, + lease_id TEXT, + attach_url TEXT, + vnc_url TEXT, + command TEXT NOT NULL DEFAULT 'codex --yolo', + prompt TEXT NOT NULL DEFAULT '', + purpose TEXT NOT NULL DEFAULT '', + summary TEXT NOT NULL DEFAULT '', + owner TEXT NOT NULL DEFAULT 'operator', + created_by TEXT NOT NULL DEFAULT 'operator', + status TEXT NOT NULL DEFAULT 'ready', + updated_at INTEGER NOT NULL DEFAULT 1, + stopped_at INTEGER, + agent_token_hash TEXT, + control_requested_by TEXT, + control_requested_at INTEGER, + controller TEXT, + control_granted_at INTEGER, + control_expires_at INTEGER, + reconcile_error TEXT, + last_event TEXT NOT NULL DEFAULT '' + ); + CREATE TABLE interactive_session_log_archives ( + session_id TEXT PRIMARY KEY, + event_count INTEGER NOT NULL DEFAULT 0, + events_key TEXT, + transcript_key TEXT, + summary_key TEXT, + archived_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + )`, + ); + database.exec("INSERT INTO interactive_sessions(id) VALUES ('IS-101'), ('IS-109')"); + database.exec( + "INSERT INTO interactive_sessions(id, runtime, adapter, status, adapter_workspace_id, lease_id, reconcile_error) VALUES ('IS-108', 'crabbox', 'runtime-v1', 'failed', 'fleet-is-108', 'sandbox:provider-owned', 'provider create failed: quota')", + ); + database.exec( + "INSERT INTO interactive_sessions(id, runtime, adapter, status, adapter_workspace_id) VALUES ('IS-107', 'crabbox', 'runtime-v1', 'stopped', 'fleet-is-107')", + ); + database.exec( + "INSERT INTO interactive_sessions(id, runtime, status, lease_id, attach_url, vnc_url, agent_token_hash, control_requested_by, control_requested_at, controller, control_granted_at, control_expires_at) VALUES ('IS-106', 'container', 'expired', 'sandbox:legacy', 'wss://terminal', 'https://desktop', 'agent-hash', 'requester', 2, 'controller', 3, 4)", + ); + database.exec( + "INSERT INTO interactive_sessions(id, runtime, status, lease_id, agent_token_hash) VALUES ('IS-105', 'container', 'ready', 'sandbox:active:terminal', 'active-agent-hash')", + ); + database.exec( + "INSERT INTO interactive_sessions(id, runtime, status) VALUES ('IS-104', 'container', 'failed')", + ); + database.exec( + "INSERT INTO interactive_sessions(id, runtime, status, lease_id, reconcile_error, last_event) VALUES ('IS-103', 'container', 'failed', 'sandbox:failed', 'sandbox terminal failed: shell exited', 'generic cleanup pending')", + ); + database.exec( + readFileSync( + new URL("../migrations/0021_runtime_adapter_hardening.sql", import.meta.url), + "utf8", + ), + ); + + const allocate = database.prepare(allocateInteractiveSessionIdSql); + assert.equal(formatInteractiveSessionId(Number(allocate.get()?.next_id)), "IS-110"); + database.exec("DELETE FROM interactive_sessions WHERE id = 'IS-109'"); + assert.equal(formatInteractiveSessionId(Number(allocate.get()?.next_id)), "IS-111"); + const migrated = database + .prepare( + "SELECT status, terminal_status, terminal_failure_reason, adapter_create_pending, terminal_finalize_pending, adapter_ttl_seconds, adapter_idle_timeout_seconds, adapter_requested_capabilities_json, adapter_create_payload_json, adapter_control_plane, provider_resource_id, lease_id FROM interactive_sessions WHERE id = 'IS-108'", + ) + .get(); + assert.equal(migrated?.status, "stopping"); + assert.equal(migrated?.terminal_status, "failed"); + assert.equal(migrated?.terminal_failure_reason, "provider create failed: quota"); + assert.equal(migrated?.adapter_create_pending, 0); + assert.equal(migrated?.terminal_finalize_pending, 0); + assert.equal(migrated?.adapter_ttl_seconds, 14_400); + assert.equal(migrated?.adapter_idle_timeout_seconds, 1_800); + assert.equal(migrated?.adapter_control_plane, null); + assert.equal(migrated?.provider_resource_id, "sandbox:provider-owned"); + assert.equal(migrated?.lease_id, null); + assert.equal(JSON.parse(String(migrated?.adapter_requested_capabilities_json)).desktop, true); + const createPayload = JSON.parse(String(migrated?.adapter_create_payload_json)); + assert.equal(createPayload.id, "fleet-is-108"); + assert.equal(createPayload.capabilities.desktop, true); + const terminal = database + .prepare("SELECT terminal_finalize_pending FROM interactive_sessions WHERE id = 'IS-107'") + .get(); + assert.equal(terminal?.terminal_finalize_pending, 1); + const legacyTerminal = database + .prepare("SELECT terminal_finalize_pending FROM interactive_sessions WHERE id = 'IS-106'") + .get(); + assert.equal(legacyTerminal?.terminal_finalize_pending, 1); + const legacyFailure = database + .prepare("SELECT terminal_finalize_pending FROM interactive_sessions WHERE id = 'IS-104'") + .get(); + assert.equal(legacyFailure?.terminal_finalize_pending, 1); + + database.exec( + readFileSync( + new URL("../migrations/0022_credential_policy_cleanup.sql", import.meta.url), + "utf8", + ), + ); + const terminalPolicy = database + .prepare( + "SELECT state, sandbox_id, lookup_id, registration_generation, registration_claim, registration_claim_expires_at FROM interactive_session_credential_policies WHERE session_id = 'IS-106'", + ) + .get(); + assert.deepEqual( + { ...terminalPolicy }, + { + state: "cleanup_pending", + sandbox_id: "legacy", + lookup_id: "legacy", + registration_generation: "legacy:IS-106:legacy", + registration_claim: null, + registration_claim_expires_at: null, + }, + ); + const failedSandbox = database + .prepare( + "SELECT terminal_failure_reason, credential_cleanup_terminal_status FROM interactive_sessions WHERE id = 'IS-103'", + ) + .get(); + assert.deepEqual( + { ...failedSandbox }, + { + terminal_failure_reason: "sandbox terminal failed: shell exited", + credential_cleanup_terminal_status: "failed", + }, + ); + assert.deepEqual( + { + ...database + .prepare( + "SELECT last_rowid, scan_max_rowid, group_session_id, group_sandbox_id, group_max_session_id, group_max_sandbox_id FROM credential_policy_reconcile_state WHERE id = 1", + ) + .get(), + }, + { + last_rowid: 0, + scan_max_rowid: 0, + group_session_id: "", + group_sandbox_id: "", + group_max_session_id: "", + group_max_sandbox_id: "", + }, + ); + const migratedTerminalPolicy = database + .prepare( + "SELECT status, credential_cleanup_terminal_status, terminal_finalize_pending, agent_token_hash, attach_url, vnc_url, control_requested_by, control_requested_at, controller, control_granted_at, control_expires_at FROM interactive_sessions WHERE id = 'IS-106'", + ) + .get(); + assert.deepEqual( + { ...migratedTerminalPolicy }, + { + status: "stopping", + credential_cleanup_terminal_status: "expired", + terminal_finalize_pending: 0, + agent_token_hash: null, + attach_url: null, + vnc_url: null, + control_requested_by: null, + control_requested_at: null, + controller: null, + control_granted_at: null, + control_expires_at: null, + }, + ); + const activePolicy = database + .prepare( + "SELECT state, sandbox_id, lookup_id, registration_generation, registration_claim, registration_claim_expires_at FROM interactive_session_credential_policies WHERE session_id = 'IS-105'", + ) + .get(); + assert.deepEqual( + { ...activePolicy }, + { + state: "active", + sandbox_id: "active", + lookup_id: "active", + registration_generation: "legacy:IS-105:active", + registration_claim: null, + registration_claim_expires_at: null, + }, + ); + const migratedActivePolicy = database + .prepare( + "SELECT credential_cleanup_terminal_status, agent_token_hash FROM interactive_sessions WHERE id = 'IS-105'", + ) + .get(); + assert.deepEqual( + { ...migratedActivePolicy }, + { + credential_cleanup_terminal_status: null, + agent_token_hash: "active-agent-hash", + }, + ); + assert.equal( + database + .prepare("SELECT terminal_finalize_pending FROM interactive_sessions WHERE id = 'IS-107'") + .get()?.terminal_finalize_pending, + 1, + ); + const refreshFence = database + .prepare( + "SELECT sandbox_refresh_sandbox_id, sandbox_refresh_claim, sandbox_refresh_claim_expires_at FROM interactive_sessions WHERE id = 'IS-105'", + ) + .get(); + assert.deepEqual( + { ...refreshFence }, + { + sandbox_refresh_sandbox_id: null, + sandbox_refresh_claim: null, + sandbox_refresh_claim_expires_at: null, + }, + ); + const standaloneColumns = database + .prepare("PRAGMA table_info(standalone_sandbox_provisions)") + .all() + .map((column) => column.name); + assert.ok(standaloneColumns.includes("request_hash")); + assert.ok(standaloneColumns.includes("ownership_claim")); + assert.ok(standaloneColumns.includes("lease_id")); + database.exec(` + INSERT INTO standalone_sandbox_provisions ( + id, request_hash, sandbox_id, state, message, created_at, updated_at + ) VALUES + ('IS-42', 'managed-hash', 'managed-sandbox', 'active', 'legacy reserved id', 1000, 1000), + ('iS-142', 'mixed-case-hash', 'mixed-case-sandbox', 'active', 'legacy mixed-case reserved id', 1500, 1500), + ('is-1worker', 'worker-hash', 'worker-sandbox', 'active', 'ordinary id', 2000, 2000) + `); + database.exec( + readFileSync( + new URL("../migrations/0023_standalone_sandbox_expiry.sql", import.meta.url), + "utf8", + ), + ); + const standaloneExpiries = database + .prepare( + "SELECT id, expires_at FROM standalone_sandbox_provisions ORDER BY id COLLATE NOCASE ASC", + ) + .all(); + assert.deepEqual( + standaloneExpiries.map((row) => ({ ...row })), + [ + { id: "iS-142", expires_at: 0 }, + { id: "is-1worker", expires_at: 14_402_000 }, + { id: "IS-42", expires_at: 0 }, + ], + ); + assert.equal(formatInteractiveSessionId(Number(allocate.get()?.next_id)), "IS-143"); +}); diff --git a/tests/terminal-authorization.test.ts b/tests/terminal-authorization.test.ts new file mode 100644 index 0000000..3afe454 --- /dev/null +++ b/tests/terminal-authorization.test.ts @@ -0,0 +1,42 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { cachedBooleanGrant } from "../src/terminal-authorization.ts"; + +test("terminal authorization coalesces concurrent reads within a short TTL", async () => { + let now = 100; + let reads = 0; + let resolveRead: ((allowed: boolean) => void) | undefined; + const grant = cachedBooleanGrant( + () => { + reads += 1; + return new Promise((resolve) => { + resolveRead = resolve; + }); + }, + 10, + () => now, + ); + + const checks = [grant(), grant(), grant()]; + await Promise.resolve(); + assert.equal(reads, 1); + resolveRead?.(true); + assert.deepEqual(await Promise.all(checks), [true, true, true]); + assert.equal(await grant(), true); + assert.equal(reads, 1); + + now = 111; + const refreshed = grant(); + await Promise.resolve(); + assert.equal(reads, 2); + resolveRead?.(false); + assert.equal(await refreshed, false); +}); + +test("terminal authorization fails closed when its state read fails", async () => { + const grant = cachedBooleanGrant(async () => { + throw new Error("D1 unavailable"); + }); + assert.equal(await grant(), false); +}); diff --git a/tests/terminal-finalization.test.ts b/tests/terminal-finalization.test.ts new file mode 100644 index 0000000..0098dd4 --- /dev/null +++ b/tests/terminal-finalization.test.ts @@ -0,0 +1,120 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { completeTerminalFinalization } from "../src/terminal-finalization.ts"; + +test("terminal finalization retries after an archive failure", async () => { + let eventInserted = true; + let archiveAttempts = 0; + let cleared = false; + const operations = { + ensureEvent: async () => { + const inserted = eventInserted; + eventInserted = false; + return inserted; + }, + readArchiveState: async () => ({ + eventCount: 4, + archiveEventCount: archiveAttempts > 1 ? 4 : 3, + archiveObjectsReady: archiveAttempts > 1, + archiveSessionVersionMatches: archiveAttempts > 1, + }), + archive: async () => { + archiveAttempts += 1; + if (archiveAttempts === 1) throw new Error("R2 unavailable"); + }, + clearPending: async () => { + cleared = true; + return true; + }, + }; + + await assert.rejects(completeTerminalFinalization(operations), /R2 unavailable/); + assert.equal(cleared, false); + await completeTerminalFinalization(operations); + assert.equal(archiveAttempts, 2); + assert.equal(cleared, true); +}); + +test("terminal finalization resumes after interruption between archive and marker clear", async () => { + let archiveAttempts = 0; + let clearAttempts = 0; + const operations = { + ensureEvent: async () => false, + readArchiveState: async () => ({ + eventCount: 4, + archiveEventCount: 4, + archiveObjectsReady: true, + archiveSessionVersionMatches: true, + }), + archive: async () => { + archiveAttempts += 1; + }, + clearPending: async () => { + clearAttempts += 1; + if (clearAttempts === 1) throw new Error("interrupted"); + return true; + }, + }; + + await assert.rejects(completeTerminalFinalization(operations), /interrupted/); + await completeTerminalFinalization(operations); + assert.equal(archiveAttempts, 0); + assert.equal(clearAttempts, 2); +}); + +test("terminal finalization re-archives events racing marker clear", async () => { + let eventCount = 4; + let archiveEventCount = 4; + let archiveAttempts = 0; + let clearAttempts = 0; + const operations = { + ensureEvent: async () => false, + readArchiveState: async () => ({ + eventCount, + archiveEventCount, + archiveObjectsReady: true, + archiveSessionVersionMatches: true, + }), + archive: async () => { + archiveAttempts += 1; + archiveEventCount = eventCount; + }, + clearPending: async () => { + clearAttempts += 1; + if (clearAttempts === 1) { + eventCount += 1; + return false; + } + return archiveEventCount >= eventCount; + }, + }; + + await completeTerminalFinalization(operations); + assert.equal(archiveAttempts, 1); + assert.equal(clearAttempts, 2); + assert.equal(archiveEventCount, 5); +}); + +test("terminal finalization re-archives mutable session metadata", async () => { + let archiveSessionVersion = 10; + const currentSessionVersion = 11; + let archiveAttempts = 0; + const operations = { + ensureEvent: async () => false, + readArchiveState: async () => ({ + eventCount: 4, + archiveEventCount: 4, + archiveObjectsReady: true, + archiveSessionVersionMatches: archiveSessionVersion === currentSessionVersion, + }), + archive: async () => { + archiveAttempts += 1; + archiveSessionVersion = currentSessionVersion; + }, + clearPending: async () => archiveSessionVersion === currentSessionVersion, + }; + + await completeTerminalFinalization(operations); + assert.equal(archiveAttempts, 1); +}); diff --git a/tests/terminal-target.test.ts b/tests/terminal-target.test.ts new file mode 100644 index 0000000..98973d0 --- /dev/null +++ b/tests/terminal-target.test.ts @@ -0,0 +1,43 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; + +import { sizedTerminalTargetUrl } from "../src/terminal-target.ts"; + +test("opaque direct terminal URLs pass through the multiplex hub unchanged", async () => { + const signed = + "wss://controller.example/v1/pty?signature=a%2Bb%2Fc%3D&cols=provider-owned&opaque=1"; + assert.equal(sizedTerminalTargetUrl(signed, "attach", 120, 34), signed); + const source = await readFile(new URL("../src/index.ts", import.meta.url), "utf8"); + const upstreamStart = source.indexOf("async function openInteractiveTerminalUpstream"); + const upstreamEnd = source.indexOf( + "async function markInteractiveTerminalConnected", + upstreamStart, + ); + const upstreamSource = source.slice(upstreamStart, upstreamEnd); + assert.match( + upstreamSource, + /fetch\(sizedTerminalTargetUrl\(target\.url, routeKind, cols, rows\)/, + ); + assert.doesNotMatch(upstreamSource, /addQuery\(target\.url/); + + const directStart = source.indexOf("async function interactiveSessionPty"); + const directEnd = source.indexOf("function sendTerminalJson", directStart); + const directSource = source.slice(directStart, directEnd); + assert.match(directSource, /const targetUrl = sizedTerminalTargetUrl\(/); + assert.match(directSource, /terminalSize\(request, "cols", 120\)/); + assert.match(directSource, /terminalSize\(request, "rows", 34\)/); + assert.match(directSource, /upstreamResponse = await fetch\(targetUrl/); + assert.doesNotMatch(directSource, /upstreamResponse = await fetch\(target\.url/); +}); + +test("known bridge and runner targets receive terminal dimensions", () => { + assert.equal( + sizedTerminalTargetUrl("wss://bridge.example/pty?token=opaque", "bridge", 120, 34), + "wss://bridge.example/pty?token=opaque&cols=120&rows=34", + ); + assert.equal( + sizedTerminalTargetUrl("wss://runner.example/pty?cols=80", "cloudflare", 132, 40), + "wss://runner.example/pty?cols=132&rows=40", + ); +}); diff --git a/tests/url-security.test.ts b/tests/url-security.test.ts new file mode 100644 index 0000000..f7b94f0 --- /dev/null +++ b/tests/url-security.test.ts @@ -0,0 +1,34 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { configuredHttpOrigin, developmentIdentityEnabled } from "../src/url-security.ts"; + +test("configured origins require HTTPS except literal loopback HTTP", () => { + const fallback = "https://fleet.example"; + assert.equal( + configuredHttpOrigin("https://internal.example/path", fallback), + "https://internal.example", + ); + assert.equal( + configuredHttpOrigin("http://localhost:8787/path", fallback), + "http://localhost:8787", + ); + assert.equal( + configuredHttpOrigin("http://127.0.0.1:8787/path", fallback), + "http://127.0.0.1:8787", + ); + assert.equal(configuredHttpOrigin("http://internal.example", fallback), fallback); + assert.equal(configuredHttpOrigin("https://user:secret@internal.example", fallback), fallback); +}); + +test("development identity requires an explicit true gate and literal loopback host", () => { + assert.equal(developmentIdentityEnabled(undefined, "http://localhost:8787"), false); + assert.equal(developmentIdentityEnabled("false", "http://localhost:8787"), false); + assert.equal(developmentIdentityEnabled("TRUE", "http://localhost:8787"), false); + assert.equal(developmentIdentityEnabled("true", "http://localhost:8787"), true); + assert.equal(developmentIdentityEnabled("true", "http://127.0.0.1:8787"), true); + assert.equal(developmentIdentityEnabled("true", "http://[::1]:8787"), true); + assert.equal(developmentIdentityEnabled("true", "http://tenant.localhost:8787"), false); + assert.equal(developmentIdentityEnabled("true", "https://fleet.example"), false); + assert.equal(developmentIdentityEnabled("true", "not a url"), false); +}); diff --git a/worker-configuration.d.ts b/worker-configuration.d.ts index 56d40b8..8fc0b65 100644 --- a/worker-configuration.d.ts +++ b/worker-configuration.d.ts @@ -7,13 +7,20 @@ interface Env { CRABBOX_BOOTSTRAP_TOKEN?: string; GITHUB_CLIENT_ID?: string; GITHUB_CLIENT_SECRET?: string; + /** Optional authoritative HTTPS callback ending in /auth/github/callback. */ GITHUB_REDIRECT_URI?: string; GITHUB_ORG?: string; GITHUB_TOKEN?: string; CRABBOX_INTERACTIVE_PROVISION_URL?: string; CRABBOX_INTERACTIVE_PROVISION_TOKEN?: string; + CRABBOX_STANDALONE_SANDBOX_TTL_SECONDS?: string; CRABBOX_RUNTIME_PROVISION_URL?: string; CRABBOX_RUNTIME_PROVISION_TOKEN?: string; + CRABBOX_RUNTIME_ADAPTER_URL?: string; + CRABBOX_RUNTIME_ADAPTER_TOKEN?: string; + CRABBOX_RUNTIME_ADAPTER_NAMESPACE?: string; + CRABBOX_RUNTIME_ADAPTER_TTL_SECONDS?: string; + CRABBOX_RUNTIME_ADAPTER_IDLE_SECONDS?: string; CRABBOX_CLOUDFLARE_RUNNER_URL?: string; CRABBOX_CLOUDFLARE_RUNNER_TOKEN?: string; CRABBOX_CLOUDFLARE_RUNNER_INSTANCE_TYPE?: string; @@ -30,6 +37,14 @@ interface Env { BACKUP_BUCKET_NAME?: string; CLOUDFLARE_ACCOUNT_ID?: string; CRABFLEET_LOCAL_SANDBOX_BACKUPS?: string; + CRABFLEET_LABEL?: string; + CRABFLEET_CANONICAL_URL?: string; + CRABFLEET_PRODUCT_URL?: string; + CRABFLEET_SSH_HOST?: string; + CRABFLEET_PREFERRED_REPO?: string; + CRABFLEET_DEFAULT_RUNTIME?: string; + CRABFLEET_DEFAULT_PROFILE?: string; + CRABFLEET_DEV_LOGIN_ENABLED?: string; OPENAI_API_KEY?: string; OPENAI_BASE_URL?: string; OPENAI_ORG_ID?: string; diff --git a/wrangler.jsonc b/wrangler.jsonc index 3934e26..4e7c9dc 100644 --- a/wrangler.jsonc +++ b/wrangler.jsonc @@ -22,6 +22,7 @@ "vars": { "BACKUP_BUCKET_NAME": "crabfleet-session-logs", "CLOUDFLARE_ACCOUNT_ID": "91b59577e757131d68d55a471fe32aca", + "CRABFLEET_DEV_LOGIN_ENABLED": "false", "CRABBOX_INTERACTIVE_PROVISION_URL": "https://crabfleet.openclaw.ai/api/provision/interactive", "GITHUB_REDIRECT_URI": "https://crabfleet.openclaw.ai/auth/github/callback", }, @@ -73,6 +74,9 @@ }, ], "workers_dev": true, + "triggers": { + "crons": ["* * * * *"], + }, // The canonical app/API host and legacy OpenClaw aliases are Worker Custom // Domains. The public crabfleet.ai product site uses wrangler.product.jsonc. "routes": [ From 658bcbe9cf7a138f037124a64c904df4d8293e55 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 13 Jun 2026 01:22:44 -0400 Subject: [PATCH 2/2] fix: preserve migrated sandbox sessions --- CHANGELOG.md | 2 +- src/index.ts | 2 +- tests/runtime-adapter.test.ts | 3 ++- tests/session-id.test.ts | 4 ++-- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 838c225..eac8b8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ - Make Sandbox terminal intent monotonic under stop/failure races, fence initial provision completion against managed retries and refresh ownership without rejecting concurrent metadata writes, atomically version terminal metadata with its event and finalization marker, keep live credential-registration failures retryable, clean both sides of interrupted refreshes, expire standalone Sandboxes with authenticated stop, and exactly reserve managed session IDs from standalone use. - Keep standalone Sandbox terminals behind a lifetime-authorized Worker WebSocket proxy and terminate their execution sessions during durable cleanup, fail closed legacy unfenced credential policies, isolate and persist per-owner cleanup failures, reserve managed IDs case-insensitively across upgrades, preserve SSH-link state across canonical OAuth redirects, reject lost runtime-stop claims, redact arbitrary escaped connection URLs, make terminal completion/release revisions monotonic, and retain single-read redacted adapter create/DELETE evidence through final archives. - Reject stale same-generation credential-policy registrations, preflight and atomically stage failed managed Sandbox claims, require the provision bearer for standalone stop after backend removal, and backfill D1-only terminal archives when R2 is enabled later. -- Proactively generation-wrap migrated legacy Sandbox credential policies under a live durable lease before cleanup, with crash-safe cron retries that preserve unattended session credentials. +- Proactively generation-wrap migrated legacy Sandbox credential policies under a live durable lease before cleanup, preserve live pre-token sessions, and use crash-safe cron retries that retain unattended session credentials. - Bound every runtime-adapter response stream, revalidate desktop authorization after minting, make legacy local stops atomic with scheduled crash recovery, and redact credentials before opaque provider identifiers. - Recover active credential policies after a post-registration crash, redact provider identities from structured adapter errors, and propagate terminal dimensions through direct bridge and runner PTY routes without rewriting opaque adapter URLs. - Support an optional authoritative `GITHUB_REDIRECT_URI` deployment binding with strict HTTPS callback validation, canonical-origin login handoff, and callback host/path enforcement while retaining safe request-origin defaults. diff --git a/src/index.ts b/src/index.ts index f77cdd3..8ecd1a4 100644 --- a/src/index.ts +++ b/src/index.ts @@ -5873,12 +5873,12 @@ function credentialPolicyScanRequiresCleanup(row: CredentialPolicyScanRow, now: now, ); if ( - row.session_agent_token_hash !== null && sandboxExpected && (row.session_status === "ready" || row.session_status === "attached" || row.session_status === "detached") ) { + // Migrated live sessions can predate agent tokens; the durable lease/refresh fence owns policy. return false; } if (registrationAbandoned) return true; diff --git a/tests/runtime-adapter.test.ts b/tests/runtime-adapter.test.ts index 21ba505..af03bd1 100644 --- a/tests/runtime-adapter.test.ts +++ b/tests/runtime-adapter.test.ts @@ -1491,7 +1491,8 @@ test("sandbox credential cleanup is durably staged and retried", async () => { scanDecisionSource.indexOf("sandboxExpected") < scanDecisionSource.indexOf("if (registrationAbandoned) return true"), ); - assert.match(scanDecisionSource, /row\.session_agent_token_hash !== null/); + assert.doesNotMatch(scanDecisionSource, /row\.session_agent_token_hash !== null/); + assert.match(scanDecisionSource, /sandboxExpected &&[\s\S]*row\.session_status === "ready"/); assert.match(controlSource, /sandboxPolicyTombstoneKey/); assert.match(controlSource, /credentialPolicyRegistrationAccepted/); assert.match(controlSource, /credentialPolicyCleanupMatches/); diff --git a/tests/session-id.test.ts b/tests/session-id.test.ts index 2d2ce28..d3ec489 100644 --- a/tests/session-id.test.ts +++ b/tests/session-id.test.ts @@ -63,7 +63,7 @@ test("interactive session ids remain monotonic after cleanup", () => { "INSERT INTO interactive_sessions(id, runtime, status, lease_id, attach_url, vnc_url, agent_token_hash, control_requested_by, control_requested_at, controller, control_granted_at, control_expires_at) VALUES ('IS-106', 'container', 'expired', 'sandbox:legacy', 'wss://terminal', 'https://desktop', 'agent-hash', 'requester', 2, 'controller', 3, 4)", ); database.exec( - "INSERT INTO interactive_sessions(id, runtime, status, lease_id, agent_token_hash) VALUES ('IS-105', 'container', 'ready', 'sandbox:active:terminal', 'active-agent-hash')", + "INSERT INTO interactive_sessions(id, runtime, status, lease_id) VALUES ('IS-105', 'container', 'ready', 'sandbox:active:terminal')", ); database.exec( "INSERT INTO interactive_sessions(id, runtime, status) VALUES ('IS-104', 'container', 'failed')", @@ -211,7 +211,7 @@ test("interactive session ids remain monotonic after cleanup", () => { { ...migratedActivePolicy }, { credential_cleanup_terminal_status: null, - agent_token_hash: "active-agent-hash", + agent_token_hash: null, }, ); assert.equal(