From f799ff9ff5ff6004f9a263ea3e4bf8ebf98dc470 Mon Sep 17 00:00:00 2001 From: Jannes Stubbemann Date: Sat, 11 Jul 2026 00:13:05 +0200 Subject: [PATCH 01/11] docs(plan): pre-claimed checkout spec and implementation plan The gateway serves eligible creates from a buffer of already-activated, org-less sandboxes; checkout is one resourceVersion-guarded label patch that is both the mutual exclusion between gateway replicas and the claim-time org attribution. Decision record, failure behavior, threat notes, and the per-org-namespace migration note are in the spec. Co-Authored-By: Claude Fable 5 Signed-off-by: Jannes Stubbemann --- .../plans/2026-07-11-preclaimed-checkout.md | 1255 +++++++++++++++++ .../2026-07-11-preclaimed-checkout-design.md | 164 +++ 2 files changed, 1419 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-11-preclaimed-checkout.md create mode 100644 docs/superpowers/specs/2026-07-11-preclaimed-checkout-design.md diff --git a/docs/superpowers/plans/2026-07-11-preclaimed-checkout.md b/docs/superpowers/plans/2026-07-11-preclaimed-checkout.md new file mode 100644 index 00000000..1f0e71bd --- /dev/null +++ b/docs/superpowers/plans/2026-07-11-preclaimed-checkout.md @@ -0,0 +1,1255 @@ +# Pre-claimed Checkout Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** The hosted gateway serves eligible creates from a buffer of already-activated sandboxes, so the hot path is one label patch instead of the whole claim round trip. + +**Architecture:** Buffered sandboxes are real Sandbox CRs in the single-tenant namespace carrying `mitos.run/buffered: "true"` and NO org labels. Checkout is one resourceVersion-guarded patch that removes the buffered label and stamps the org labels (exclusivity + attribution in one write). A per-replica reconcile loop refills to a floor, adopts after restarts, and reaps stale entries. The controller propagates the CR org label to the backing husk pod (the trusted billing label) on the reconcile the checkout patch triggers. + +**Tech Stack:** Go, controller-runtime (fake client + interceptor in tests, envtest for the controller task), existing gateway create machinery from #895 (establishSandboxWatch/watchWait). + +**Spec:** docs/superpowers/specs/2026-07-11-preclaimed-checkout-design.md + +## Global Constraints + +- No em or en dashes anywhere; conventional commits; every commit `git commit -s` (DCO). +- Secret VALUES (tokens) never logged, never in errors or conditions. +- No public latency number claimed anywhere until bench/tti-latency.py reproduces it. +- Feature is default OFF (`checkout.pools` empty) and requires single-tenant namespace mode; per-org tenancy note lives in the spec. +- Lint gate is BOTH `~/go/bin/golangci-lint run --timeout=5m` and `GOOS=linux ~/go/bin/golangci-lint run --timeout=5m`. +- All controlplane tests live in package `controlplane` (internal test package) and reuse helpers from controlplane_test.go (newScheme, poolIn, flipToReadyWhenCreated, orgA/orgB) and readywatch/poolcheck test idioms. + +--- + +### Task 1: Checkout config, buffer type, and the eligibility gate + +**Files:** +- Create: `internal/saas/controlplane/checkout.go` +- Create: `internal/saas/controlplane/checkout_test.go` +- Modify: `internal/saas/controlplane/controlplane.go` (add fields + WithCheckout option) + +**Interfaces:** +- Produces: `const BufferedLabelKey = "mitos.run/buffered"`, `const bufferedPoolLabelKey = "mitos.run/pool"`, `type CheckoutConfig struct { Pools []string; Floor, Cap int; MaxAge time.Duration }`, `type checkoutBuffer struct` with `func (b *checkoutBuffer) eligible(body createBody, pool string) bool`, `func newCheckoutBuffer(k *K8sControlPlane, cfg CheckoutConfig) *checkoutBuffer`, `type bufferedSandbox struct { name, pool, endpoint, token, resourceVersion, podName string; createdAt time.Time }`, K8sControlPlane field `checkout *checkoutBuffer`, `func WithCheckout(cfg CheckoutConfig) Option`. +- Consumes: `createBody` (forward.go), `K8sControlPlane.now`, `singleTenantNamespace`. + +- [ ] **Step 1: Write the failing tests** (checkout_test.go) + +```go +package controlplane + +import ( + "testing" + "time" +) + +func checkoutCP(t *testing.T, objs ...client.Object) *K8sControlPlane { + t.Helper() + c := newFakeClient(t, objs...) + cp := New(c, + WithPollInterval(5*time.Millisecond), + WithReadyTimeout(2*time.Second), + WithSingleTenantNamespace("mitos"), + WithCheckout(CheckoutConfig{Pools: []string{"python"}, Floor: 2, Cap: 4, MaxAge: 10 * time.Minute}), + ) + return cp +} + +// TestCheckoutEligibilityGate pins which creates may be served from the +// buffer: a checkout pool with NO env, secrets, workspace, fan-out, or +// lifetime knobs. Everything else must take the classic path. +func TestCheckoutEligibilityGate(t *testing.T) { + cp := checkoutCP(t) + b := cp.checkout + if b == nil { + t.Fatal("checkout buffer not constructed despite WithCheckout + single-tenant mode") + } + cases := []struct { + name string + body createBody + pool string + want bool + }{ + {"plain create on a checkout pool", createBody{}, "python", true}, + {"pool not enabled", createBody{}, "default", false}, + {"env set", createBody{Env: map[string]string{"A": "b"}}, "python", false}, + {"workspace set", createBody{Workspace: "ws"}, "python", false}, + {"replicas fan-out", createBody{Replicas: 2}, "python", false}, + {"ttl set", createBody{TTL: "5m"}, "python", false}, + {"timeout set", createBody{Timeout: "5m"}, "python", false}, + } + for _, tc := range cases { + if got := b.eligible(tc.body, tc.pool); got != tc.want { + t.Errorf("%s: eligible = %v, want %v", tc.name, got, tc.want) + } + } +} + +// TestCheckoutRequiresSingleTenantNamespace asserts the buffer is NOT +// constructed under per-org namespacing: the design leans on the shared +// namespace (see the spec's migration note) and must fail safe to the +// classic path everywhere else. +func TestCheckoutRequiresSingleTenantNamespace(t *testing.T) { + cp := New(newFakeClient(t), WithCheckout(CheckoutConfig{Pools: []string{"python"}})) + if cp.checkout != nil { + t.Fatal("checkout buffer constructed without single-tenant namespace mode") + } +} +``` + +Note: `createBody` fields must match forward.go exactly; if `Secrets` needs coverage add a case with `Secrets: []secretMountReq{{}}` after checking the struct. + +- [ ] **Step 2: Run to verify failure** + +Run: `go test ./internal/saas/controlplane/ -run TestCheckout -v` +Expected: FAIL (undefined: CheckoutConfig, checkout field, WithCheckout) + +- [ ] **Step 3: Implement** (checkout.go) + +```go +package controlplane + +import ( + "sync" + "time" +) + +// BufferedLabelKey marks a pre-created, org-less sandbox held by the gateway +// checkout buffer. Removing it, atomically with stamping the org labels, IS +// the checkout (see docs/superpowers/specs/2026-07-11-preclaimed-checkout-design.md). +const BufferedLabelKey = "mitos.run/buffered" + +// bufferedPoolLabelKey carries the pool identity on a buffered Sandbox so the +// reconcile loop can LIST per pool (spec.source.poolRef is not selectable). +const bufferedPoolLabelKey = "mitos.run/pool" + +// CheckoutConfig configures the pre-claimed checkout buffer. Pools empty +// means the feature is off. +type CheckoutConfig struct { + Pools []string + Floor int + Cap int + MaxAge time.Duration +} + +// bufferedSandbox is one cached, already-Ready, org-less sandbox. +type bufferedSandbox struct { + name string + pool string + endpoint string + token string + resourceVersion string + podName string + createdAt time.Time +} + +// checkoutBuffer serves eligible creates from pre-activated sandboxes. The +// cluster (label state on the CRs) is the source of truth; this struct is a +// cache plus the refill/janitor loop. +type checkoutBuffer struct { + k *K8sControlPlane + cfg CheckoutConfig + + mu sync.Mutex + entries map[string][]bufferedSandbox + // nextRefill backs off refill attempts per pool after failures (#894 + // shape: a refill that cannot succeed must not retry in a tight loop). + nextRefill map[string]time.Time + refillFails map[string]int +} + +func newCheckoutBuffer(k *K8sControlPlane, cfg CheckoutConfig) *checkoutBuffer { + return &checkoutBuffer{ + k: k, + cfg: cfg, + entries: make(map[string][]bufferedSandbox), + nextRefill: make(map[string]time.Time), + refillFails: make(map[string]int), + } +} + +func (c CheckoutConfig) enabledFor(pool string) bool { + for _, p := range c.Pools { + if p == pool { + return true + } + } + return false +} + +// eligible reports whether this create may be served from the buffer: a +// checkout-enabled pool and NOTHING tenant-specific that the activation +// handshake would have had to deliver (env, secrets, workspace), no fan-out, +// and no lifetime knobs (a buffered sandbox's TTL clock would predate the +// claim). +func (b *checkoutBuffer) eligible(body createBody, pool string) bool { + if !b.cfg.enabledFor(pool) { + return false + } + if len(body.Env) > 0 || len(body.Secrets) > 0 || body.Workspace != "" { + return false + } + if body.Replicas > 1 { + return false + } + if body.TTL != "" || body.Timeout != "" { + return false + } + return true +} +``` + +In controlplane.go add to the struct (below poolSeen): + +```go + // checkout serves eligible creates from a buffer of pre-activated + // sandboxes (nil = feature off). Constructed by New only when BOTH + // WithCheckout and single-tenant mode are set; see the spec's migration + // note for the per-org-namespace story. + checkoutCfg *CheckoutConfig + checkout *checkoutBuffer +``` + +```go +// WithCheckout enables the pre-claimed checkout buffer for the named pools. +// Ignored (feature off) when Pools is empty or the control plane is not in +// single-tenant namespace mode. Zero Floor/Cap/MaxAge take defaults 2/4/10m. +func WithCheckout(cfg CheckoutConfig) Option { + return func(k *K8sControlPlane) { + if len(cfg.Pools) == 0 { + return + } + if cfg.Floor <= 0 { + cfg.Floor = 2 + } + if cfg.Cap < cfg.Floor { + cfg.Cap = cfg.Floor * 2 + } + if cfg.MaxAge <= 0 { + cfg.MaxAge = 10 * time.Minute + } + k.checkoutCfg = &cfg + } +} +``` + +And in `New(...)` AFTER the options loop (find it at the bottom of controlplane.go; it applies opts then returns): construct the buffer only when both conditions hold: + +```go + if k.checkoutCfg != nil && k.singleTenantNamespace != "" { + k.checkout = newCheckoutBuffer(k, *k.checkoutCfg) + } +``` + +- [ ] **Step 4: Run to verify pass** + +Run: `go test ./internal/saas/controlplane/ -run TestCheckout -v` +Expected: PASS (both tests) + +- [ ] **Step 5: Commit** + +```bash +git add internal/saas/controlplane/checkout.go internal/saas/controlplane/checkout_test.go internal/saas/controlplane/controlplane.go +git commit -s -m "feat(saas): checkout buffer scaffold, config, and the eligibility gate" +``` + +--- + +### Task 2: Extract createSandboxAndAwait from create() (pure refactor) + +**Files:** +- Modify: `internal/saas/controlplane/forward.go` + +**Interfaces:** +- Produces: `func (k *K8sControlPlane) createSandboxAndAwait(ctx context.Context, sb *v1.Sandbox, startedAt time.Time) (saas.ForwardResponse, error)` covering: pre-create watch establish, `k.c.Create`, error envelope mapping, watchWait, pollReady/pollReadyTicker fallbacks. Behavior byte-identical to today's create() tail. +- Consumes: establishSandboxWatch, watchWait, pollReady, pollReadyTicker (all landed in #895). + +- [ ] **Step 1: Refactor.** In forward.go, move everything in `create()` from the `// Establish the ready watch BEFORE the Create` comment through the final `return k.pollReady(...)` into: + +```go +// createSandboxAndAwait writes sb and blocks for its terminal outcome using +// the pre-established watch (one serialized round trip fewer, #895), failing +// open to the post-create watch or poll exactly as before. It is shared by +// the tenant create path and the checkout buffer's refill, which is why it +// takes a fully built Sandbox rather than a request body. +func (k *K8sControlPlane) createSandboxAndAwait(ctx context.Context, sb *v1.Sandbox, startedAt time.Time) (saas.ForwardResponse, error) { + ns, name := sb.Namespace, sb.Name + var wi watch.Interface + if w, ok := k.c.(client.WithWatch); ok { + watchCtx, cancel := context.WithCancel(ctx) + defer cancel() + if established, werr := establishSandboxWatch(watchCtx, w, ns, name); werr == nil { + wi = established + defer wi.Stop() + } else { + slog.Warn("could not establish the sandbox ready watch before create; will wait post-create", + "namespace", ns, "sandbox", name, "error", werr.Error()) + } + } + + if err := k.c.Create(ctx, sb); err != nil { + switch { + case apierrors.IsAlreadyExists(err): + return errResp(withStatus(apierr.Get(apierr.CodeInternal). + WithCause("a sandbox with the generated name already exists; retry"), http.StatusConflict)), nil + case isNamespaceMissing(err, ns): + return errResp(namespaceMissingErr(sb.Labels[tenant.OrgLabelKey], ns)), nil + case apierrors.IsInvalid(err), apierrors.IsBadRequest(err): + return errResp(apierr.Get(apierr.CodeInvalidInput). + WithCause("the api server rejected the sandbox object as invalid: " + err.Error())), nil + default: + return errResp(apierr.Get(apierr.CodeInternal). + WithCause("could not create the sandbox object")), nil + } + } + + if wi != nil { + deadline := k.now().Add(k.readyTimeout) + if resp, done := k.watchWait(ctx, wi, ns, name, startedAt, deadline, "Pending"); done { + return resp, nil + } + slog.Warn("sandbox ready watch closed before a terminal outcome; falling back to status polling", + "namespace", ns, "sandbox", name) + return k.pollReadyTicker(ctx, ns, name, startedAt, deadline) + } + return k.pollReady(ctx, ns, name, startedAt) +} +``` + +CAREFUL: `namespaceMissingErr(req.OrgID, ns)` in the original takes the org id; inside the helper use `sb.Labels[tenant.OrgLabelKey]` as shown (same value: create() sets `Labels: tenant.OrgLabels(req.OrgID)`). Verify `namespaceMissingErr`'s message renders acceptably with an empty org (the refill path); if it embeds the org id verbatim an empty string is fine. + +create() then ends with: + +```go + return k.createSandboxAndAwait(ctx, sb, startedAt) +``` + +- [ ] **Step 2: Run the full package to verify the refactor is invisible** + +Run: `go test ./internal/saas/controlplane/ -race` +Expected: ok (all existing tests, including the #895 watch tests, pass unchanged) + +- [ ] **Step 3: Commit** + +```bash +git add internal/saas/controlplane/forward.go +git commit -s -m "refactor(saas): extract createSandboxAndAwait for reuse by the checkout refill" +``` + +--- + +### Task 3: The claim path (pop, stamp, respond, fall back) + +**Files:** +- Modify: `internal/saas/controlplane/checkout.go` +- Modify: `internal/saas/controlplane/forward.go` (create() consults the buffer) +- Modify: `internal/saas/controlplane/checkout_test.go` + +**Interfaces:** +- Produces: `func (b *checkoutBuffer) claim(ctx context.Context, ns, org, pool string, startedAt time.Time) (saas.ForwardResponse, bool)`; `func (b *checkoutBuffer) pop(pool string) (bufferedSandbox, bool)`; `func (b *checkoutBuffer) add(e bufferedSandbox)`; `func (b *checkoutBuffer) stampOrg(ctx context.Context, ns, org string, e bufferedSandbox) bool`. +- Consumes: tenant.OrgLabels, jsonResp, BufferedLabelKey. + +- [ ] **Step 1: Write the failing tests** + +```go +// seedBuffered creates a Ready buffered sandbox CR (+ token Secret) in ns +// "mitos" and returns the entry the gateway would cache for it. +func seedBuffered(t *testing.T, c client.Client, name string) bufferedSandbox { + t.Helper() + sb := &v1.Sandbox{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, Namespace: "mitos", + Labels: map[string]string{BufferedLabelKey: "true", bufferedPoolLabelKey: "python"}, + }, + Spec: v1.SandboxSpec{Source: v1.SandboxSource{PoolRef: &v1.LocalObjectReference{Name: "python"}}}, + } + if err := c.Create(context.Background(), sb); err != nil { + t.Fatalf("seed buffered sandbox: %v", err) + } + sb.Status.Phase = v1.SandboxReady + sb.Status.Endpoint = "10.0.0.9:9091" + sb.Status.Pod = name + if err := c.Status().Update(context.Background(), sb); err != nil { + t.Fatalf("seed status: %v", err) + } + var cur v1.Sandbox + if err := c.Get(context.Background(), client.ObjectKey{Namespace: "mitos", Name: name}, &cur); err != nil { + t.Fatalf("re-get: %v", err) + } + return bufferedSandbox{ + name: name, pool: "python", endpoint: "10.0.0.9:9091", token: "tok-" + name, + resourceVersion: cur.ResourceVersion, podName: name, createdAt: time.Now(), + } +} + +// TestCheckoutServesCreateFromBuffer asserts an eligible create is served +// from the buffer: 201 with the cached endpoint and token, NO Sandbox Create +// on the hot path, and the CR atomically loses the buffered label and gains +// the caller's org labels BEFORE the response returns. +func TestCheckoutServesCreateFromBuffer(t *testing.T) { + var creates atomic.Int64 + base := fakeclient.NewClientBuilder(). + WithScheme(newScheme(t)). + WithStatusSubresource(&v1.Sandbox{}). + Build() + c := interceptor.NewClient(base, interceptor.Funcs{ + Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + if _, ok := obj.(*v1.Sandbox); ok { + creates.Add(1) + } + return cl.Create(ctx, obj, opts...) + }, + }) + cp := New(c, + WithPollInterval(5*time.Millisecond), WithReadyTimeout(2*time.Second), + WithSingleTenantNamespace("mitos"), + WithCheckout(CheckoutConfig{Pools: []string{"python"}})) + cp.checkout.add(seedBuffered(t, c, "sb-buffered1")) + creates.Store(0) + + resp, err := cp.Forward(context.Background(), saas.ForwardRequest{ + OrgID: orgA, Op: "sandbox.create", Body: []byte(`{"pool":"python"}`), + }) + if err != nil { + t.Fatalf("Forward: %v", err) + } + if resp.Status != http.StatusCreated { + t.Fatalf("status = %d, body = %s", resp.Status, resp.Body) + } + m := decodeBody(t, resp.Body) + if m["id"] != "sb-buffered1" || m["endpoint"] != "10.0.0.9:9091" || m["token"] != "tok-sb-buffered1" { + t.Fatalf("payload = %v, want the buffered sandbox's identity", m) + } + if n := creates.Load(); n != 0 { + t.Fatalf("checkout performed %d Sandbox Create(s) on the hot path, want 0", n) + } + var sb v1.Sandbox + if err := c.Get(context.Background(), client.ObjectKey{Namespace: "mitos", Name: "sb-buffered1"}, &sb); err != nil { + t.Fatalf("get: %v", err) + } + if _, still := sb.Labels[BufferedLabelKey]; still { + t.Error("buffered label survived the checkout") + } + if sb.Labels[tenant.OrgLabelKey] != orgA { + t.Errorf("org label = %q, want %q (runtime authz needs it before the first exec)", sb.Labels[tenant.OrgLabelKey], orgA) + } +} + +// TestCheckoutIsExclusiveAcrossReplicas asserts two gateways holding the same +// cached entry cannot both hand it out: the RV-guarded patch lets exactly one +// win, and the loser falls back to the classic path. +func TestCheckoutIsExclusiveAcrossReplicas(t *testing.T) { + c := newFakeClient(t, poolIn2("mitos", "python")) + mk := func() *K8sControlPlane { + return New(c, + WithPollInterval(5*time.Millisecond), WithReadyTimeout(2*time.Second), + WithSingleTenantNamespace("mitos"), + WithCheckout(CheckoutConfig{Pools: []string{"python"}})) + } + cpA, cpB := mk(), mk() + e := seedBuffered(t, c, "sb-shared") + cpA.checkout.add(e) + cpB.checkout.add(e) + + stop := flipToReadyWhenCreatedInNs(t, c, "mitos", "10.9.9.9:9091", "tok-classic") + defer stop() + + respA, errA := cpA.Forward(context.Background(), saas.ForwardRequest{OrgID: orgA, Op: "sandbox.create", Body: []byte(`{"pool":"python"}`)}) + respB, errB := cpB.Forward(context.Background(), saas.ForwardRequest{OrgID: orgB, Op: "sandbox.create", Body: []byte(`{"pool":"python"}`)}) + if errA != nil || errB != nil { + t.Fatalf("Forward: %v / %v", errA, errB) + } + if respA.Status != http.StatusCreated || respB.Status != http.StatusCreated { + t.Fatalf("statuses = %d/%d, bodies %s / %s", respA.Status, respB.Status, respA.Body, respB.Body) + } + idA := decodeBody(t, respA.Body)["id"] + idB := decodeBody(t, respB.Body)["id"] + got := 0 + if idA == "sb-shared" { + got++ + } + if idB == "sb-shared" { + got++ + } + if got != 1 { + t.Fatalf("the shared buffered sandbox was handed out %d times (ids %v, %v), want exactly 1", got, idA, idB) + } +} + +// TestCheckoutEmptyBufferFallsBackClassic asserts an eligible create with no +// buffered entry takes the classic path and still succeeds. +func TestCheckoutEmptyBufferFallsBackClassic(t *testing.T) { + c := newFakeClient(t, poolIn2("mitos", "python")) + cp := New(c, + WithPollInterval(5*time.Millisecond), WithReadyTimeout(2*time.Second), + WithSingleTenantNamespace("mitos"), + WithCheckout(CheckoutConfig{Pools: []string{"python"}})) + stop := flipToReadyWhenCreatedInNs(t, c, "mitos", "10.9.9.9:9091", "tok-classic") + defer stop() + resp, err := cp.Forward(context.Background(), saas.ForwardRequest{OrgID: orgA, Op: "sandbox.create", Body: []byte(`{"pool":"python"}`)}) + if err != nil { + t.Fatalf("Forward: %v", err) + } + if resp.Status != http.StatusCreated { + t.Fatalf("status = %d, body = %s", resp.Status, resp.Body) + } +} +``` + +Add the helper `poolIn2(ns, name string) *v1.SandboxPool` (namespace-explicit twin of poolIn) if controlplane_test.go does not already have one: + +```go +func poolIn2(ns, name string) *v1.SandboxPool { + return &v1.SandboxPool{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}} +} +``` + +- [ ] **Step 2: Run to verify failure** + +Run: `go test ./internal/saas/controlplane/ -run 'TestCheckoutServes|TestCheckoutIsExclusive|TestCheckoutEmptyBuffer' -v` +Expected: FAIL (undefined: add, claim; then behavioral failures) + +- [ ] **Step 3: Implement** (checkout.go; imports grow to include context, log/slog, net/http, apierrors, metav1, client, v1, saas, tenant) + +```go +// add caches one buffered sandbox (refill, adopt, tests). +func (b *checkoutBuffer) add(e bufferedSandbox) { + b.mu.Lock() + defer b.mu.Unlock() + b.entries[e.pool] = append(b.entries[e.pool], e) +} + +// pop removes and returns the oldest cached entry for pool. +func (b *checkoutBuffer) pop(pool string) (bufferedSandbox, bool) { + b.mu.Lock() + defer b.mu.Unlock() + q := b.entries[pool] + if len(q) == 0 { + return bufferedSandbox{}, false + } + e := q[0] + b.entries[pool] = q[1:] + return e, true +} + +// claim serves one eligible create from the buffer. ok=false means the +// caller must take the classic path (empty buffer, or every candidate was +// lost to the other replica or reaped). On success the CR already carries +// the org labels: runtime authz (getOwned) is satisfied before the caller's +// first exec can arrive. +func (b *checkoutBuffer) claim(ctx context.Context, ns, org, pool string, startedAt time.Time) (saas.ForwardResponse, bool) { + for { + e, ok := b.pop(pool) + if !ok { + return saas.ForwardResponse{}, false + } + if !b.stampOrg(ctx, ns, org, e) { + continue + } + payload := map[string]any{ + "id": e.name, + "endpoint": e.endpoint, + "token": e.token, + "phase": string(v1.SandboxReady), + "template_id": e.pool, + "fork_time_ms": float64(b.k.now().Sub(startedAt).Milliseconds()), + } + resp := jsonResp(http.StatusCreated, payload) + resp.Header.Set("X-Mitos-Pool", e.pool) + return resp, true + } +} + +// stampOrg is THE checkout write: one resourceVersion-guarded patch that +// atomically drops the buffered label and stamps the org labels. A conflict +// or a vanished CR means this entry is not ours (the other replica won, or +// the janitor or idle reaper got it): report false so claim tries the next. +func (b *checkoutBuffer) stampOrg(ctx context.Context, ns, org string, e bufferedSandbox) bool { + old := &v1.Sandbox{ObjectMeta: metav1.ObjectMeta{ + Name: e.name, + Namespace: ns, + ResourceVersion: e.resourceVersion, + Labels: map[string]string{ + BufferedLabelKey: "true", + bufferedPoolLabelKey: e.pool, + }, + }} + claimed := old.DeepCopy() + claimed.Labels = tenant.OrgLabels(org) + err := b.k.c.Patch(ctx, claimed, client.MergeFromWithOptions(old, client.MergeFromWithOptimisticLock{})) + if err == nil { + return true + } + if !apierrors.IsConflict(err) && !apierrors.IsNotFound(err) { + slog.Warn("checkout stamp failed; entry dropped, create falls back", + "sandbox", e.name, "error", err.Error()) + } + return false +} +``` + +In forward.go create(), immediately AFTER the `pool == ""` rejection block and the replicas bound check, BEFORE the pool pre-check (a buffered entry proves the pool exists): + +```go + ns := k.namespaceForOrg(req.OrgID) + + // Pre-claimed checkout: an eligible create is served from the buffer of + // already-activated sandboxes, paying ONE label patch instead of the + // whole claim round trip. Any miss falls through to the classic path. + if k.checkout != nil && k.checkout.eligible(body, pool) { + if resp, ok := k.checkout.claim(ctx, ns, req.OrgID, pool, startedAt); ok { + return resp, nil + } + } +``` + +(The existing `ns := k.namespaceForOrg(req.OrgID)` line moves up to serve both; do not declare it twice.) + +- [ ] **Step 4: Run to verify pass, then the whole package** + +Run: `go test ./internal/saas/controlplane/ -race` +Expected: ok + +- [ ] **Step 5: Commit** + +```bash +git add internal/saas/controlplane/checkout.go internal/saas/controlplane/checkout_test.go internal/saas/controlplane/forward.go +git commit -s -m "feat(saas): serve eligible creates from the checkout buffer with one attribution patch" +``` + +--- + +### Task 4: Refill with floor, cap, and failure backoff + +**Files:** +- Modify: `internal/saas/controlplane/checkout.go` +- Modify: `internal/saas/controlplane/checkout_test.go` + +**Interfaces:** +- Produces: `func (b *checkoutBuffer) reconcilePool(ctx context.Context, pool string)` (list, prune, adopt, reap, refill: Task 5 extends it; this task lands list-count + refill + backoff), `func (b *checkoutBuffer) refillOne(ctx context.Context, pool string) error`. +- Consumes: createSandboxAndAwait (Task 2), generateName (forward.go), readToken. + +- [ ] **Step 1: Write the failing tests** + +```go +// TestRefillFillsToFloorAndStopsAtIt asserts reconcilePool creates buffered +// sandboxes (org-less, buffered+pool labels) until the cluster count reaches +// the floor, one per pass, and a pass at the floor creates none. +func TestRefillFillsToFloorAndStopsAtIt(t *testing.T) { + c := newFakeClient(t, poolIn2("mitos", "python")) + cp := New(c, + WithPollInterval(5*time.Millisecond), WithReadyTimeout(2*time.Second), + WithSingleTenantNamespace("mitos"), + WithCheckout(CheckoutConfig{Pools: []string{"python"}, Floor: 2, Cap: 4, MaxAge: 10 * time.Minute})) + + countBuffered := func() int { + var list v1.SandboxList + if err := c.List(context.Background(), &list, client.InNamespace("mitos"), + client.MatchingLabels{BufferedLabelKey: "true"}); err != nil { + t.Fatalf("list: %v", err) + } + return len(list.Items) + } + + for i := 0; i < 2; i++ { + stop := flipToReadyWhenCreatedInNs(t, c, "mitos", "10.0.0.5:9091", "tok-refill") + cp.checkout.reconcilePool(context.Background(), "python") + stop() + } + if n := countBuffered(); n != 2 { + t.Fatalf("buffered count after two passes = %d, want floor 2", n) + } + cp.checkout.reconcilePool(context.Background(), "python") + if n := countBuffered(); n != 2 { + t.Fatalf("buffered count after an at-floor pass = %d, want 2 (no over-refill)", n) + } + // The refilled entries are cached and claimable. + if _, ok := cp.checkout.pop("python"); !ok { + t.Fatal("refill did not cache a claimable entry") + } +} + +// TestRefillBacksOffAfterFailure asserts a failed refill (no controller ever +// flips the sandbox Ready, so the create times out) sets a backoff: the next +// pass makes NO create attempt until the backoff expires. +func TestRefillBacksOffAfterFailure(t *testing.T) { + var creates atomic.Int64 + base := fakeclient.NewClientBuilder(). + WithScheme(newScheme(t)). + WithStatusSubresource(&v1.Sandbox{}). + WithObjects(poolIn2("mitos", "python")). + Build() + c := interceptor.NewClient(base, interceptor.Funcs{ + Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + if _, ok := obj.(*v1.Sandbox); ok { + creates.Add(1) + } + return cl.Create(ctx, obj, opts...) + }, + }) + cp := New(c, + WithPollInterval(5*time.Millisecond), WithReadyTimeout(50*time.Millisecond), + WithSingleTenantNamespace("mitos"), + WithCheckout(CheckoutConfig{Pools: []string{"python"}, Floor: 1, Cap: 2, MaxAge: 10 * time.Minute})) + + cp.checkout.reconcilePool(context.Background(), "python") + if n := creates.Load(); n != 1 { + t.Fatalf("first pass made %d create attempts, want 1", n) + } + cp.checkout.reconcilePool(context.Background(), "python") + if n := creates.Load(); n != 1 { + t.Fatalf("pass during backoff made a create attempt (total %d), want none", n) + } +} +``` + +- [ ] **Step 2: Run to verify failure** + +Run: `go test ./internal/saas/controlplane/ -run TestRefill -v` +Expected: FAIL (undefined: reconcilePool) + +- [ ] **Step 3: Implement** (checkout.go; add "encoding/json" and "fmt" imports) + +```go +// refillBackoffBase and refillBackoffCap bound the retry cadence after +// consecutive refill failures (the #894 lesson: a refill that cannot succeed +// must never spin at full cadence). +const ( + refillBackoffBase = time.Second + refillBackoffCap = time.Minute +) + +// reconcilePool is one pass of the buffer loop for one pool: count the +// cluster's buffered sandboxes and refill toward the floor, at most one +// create per pass so two replicas converge without a thundering herd. +// (Task 5 extends this pass with adopt, prune, and reap.) +func (b *checkoutBuffer) reconcilePool(ctx context.Context, pool string) { + ns := b.k.singleTenantNamespace + var list v1.SandboxList + if err := b.k.c.List(ctx, &list, client.InNamespace(ns), + client.MatchingLabels{BufferedLabelKey: "true", bufferedPoolLabelKey: pool}); err != nil { + slog.Warn("checkout buffer list failed", "pool", pool, "error", err.Error()) + return + } + count := len(list.Items) + if count >= b.cfg.Floor || count >= b.cfg.Cap { + return + } + b.mu.Lock() + wait := b.k.now().Before(b.nextRefill[pool]) + b.mu.Unlock() + if wait { + return + } + if err := b.refillOne(ctx, pool); err != nil { + b.mu.Lock() + b.refillFails[pool]++ + backoff := refillBackoffBase << (b.refillFails[pool] - 1) + if backoff > refillBackoffCap { + backoff = refillBackoffCap + } + b.nextRefill[pool] = b.k.now().Add(backoff) + b.mu.Unlock() + slog.Warn("checkout refill failed; backing off", "pool", pool, "error", err.Error()) + return + } + b.mu.Lock() + b.refillFails[pool] = 0 + delete(b.nextRefill, pool) + b.mu.Unlock() +} + +// refillOne creates ONE buffered sandbox through the normal create machinery +// (watch-before-create and all): buffered + pool labels, NO org labels, so +// it bills nobody and matches no caller until checkout. +func (b *checkoutBuffer) refillOne(ctx context.Context, pool string) error { + ns := b.k.singleTenantNamespace + name := generateName() + sb := &v1.Sandbox{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + Labels: map[string]string{BufferedLabelKey: "true", bufferedPoolLabelKey: pool}, + }, + Spec: v1.SandboxSpec{Source: v1.SandboxSource{PoolRef: &v1.LocalObjectReference{Name: pool}}}, + } + resp, err := b.k.createSandboxAndAwait(ctx, sb, b.k.now()) + if err != nil { + return err + } + if resp.Status != http.StatusCreated { + return fmt.Errorf("buffered create for pool %q did not become ready (status %d)", pool, resp.Status) + } + var payload struct { + ID string `json:"id"` + Endpoint string `json:"endpoint"` + Token string `json:"token"` + } + if err := json.Unmarshal(resp.Body, &payload); err != nil { + return fmt.Errorf("parse buffered create payload: %w", err) + } + // One off-hot-path read to snapshot the resourceVersion (the checkout + // patch's optimistic lock) and the backing pod name. + var cur v1.Sandbox + if err := b.k.c.Get(ctx, client.ObjectKey{Namespace: ns, Name: payload.ID}, &cur); err != nil { + return fmt.Errorf("snapshot buffered sandbox: %w", err) + } + b.add(bufferedSandbox{ + name: payload.ID, + pool: pool, + endpoint: payload.Endpoint, + token: payload.Token, + resourceVersion: cur.ResourceVersion, + podName: cur.Status.Pod, + createdAt: cur.CreationTimestamp.Time, + }) + return nil +} +``` + +NOTE: token values pass through memory exactly as in create(); never log them. + +- [ ] **Step 4: Run to verify pass** + +Run: `go test ./internal/saas/controlplane/ -run TestRefill -v && go test ./internal/saas/controlplane/ -race` +Expected: PASS, then ok + +- [ ] **Step 5: Commit** + +```bash +git add internal/saas/controlplane/checkout.go internal/saas/controlplane/checkout_test.go +git commit -s -m "feat(saas): checkout refill to a floor with capped backoff on failure" +``` + +--- + +### Task 5: Adopt, prune, reap (the janitor half of the pass) and the run loop + +**Files:** +- Modify: `internal/saas/controlplane/checkout.go` +- Modify: `internal/saas/controlplane/checkout_test.go` + +**Interfaces:** +- Produces: `func (k *K8sControlPlane) StartCheckout(ctx context.Context)` (no-op when checkout is nil; otherwise starts the loop goroutine), `func (b *checkoutBuffer) run(ctx context.Context)`; reconcilePool grows adopt/prune/reap. +- Consumes: readToken (forward.go). + +- [ ] **Step 1: Write the failing tests** + +```go +// TestReconcileAdoptsPrunesAndReaps asserts one pass: a Ready buffered CR +// unknown to this replica is ADOPTED (token from its Secret); a buffered CR +// past MaxAge is DELETED; a Failed buffered CR is DELETED; the memory cache +// ends up holding exactly the adoptable entry. +func TestReconcileAdoptsPrunesAndReaps(t *testing.T) { + c := newFakeClient(t, poolIn2("mitos", "python")) + cp := New(c, + WithPollInterval(5*time.Millisecond), WithReadyTimeout(2*time.Second), + WithSingleTenantNamespace("mitos"), + WithCheckout(CheckoutConfig{Pools: []string{"python"}, Floor: 1, Cap: 4, MaxAge: 10 * time.Minute})) + + // Adoptable: Ready, in-age, with its token Secret. + e := seedBuffered(t, c, "sb-adopt") + sec := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "sb-adopt" + tokenSecretSuffix, Namespace: "mitos"}, + Data: map[string][]byte{"token": []byte("tok-sb-adopt"), "endpoint": []byte(e.endpoint)}, + } + if err := c.Create(context.Background(), sec); err != nil { + t.Fatalf("seed secret: %v", err) + } + + // Over-age: Ready but "created" long ago; age comes from CreationTimestamp, + // which the fake client stamps at Create, so step the control plane clock + // forward instead and exempt sb-adopt by re-seeding it after the step. + _ = seedBuffered(t, c, "sb-old") + cp.now = func() time.Time { return time.Now().Add(11 * time.Minute) } + // Failed: buffered but not Ready. + fail := &v1.Sandbox{ObjectMeta: metav1.ObjectMeta{ + Name: "sb-failed", Namespace: "mitos", + Labels: map[string]string{BufferedLabelKey: "true", bufferedPoolLabelKey: "python"}, + }, Spec: v1.SandboxSpec{Source: v1.SandboxSource{PoolRef: &v1.LocalObjectReference{Name: "python"}}}} + if err := c.Create(context.Background(), fail); err != nil { + t.Fatalf("seed failed: %v", err) + } + failCur := &v1.Sandbox{} + _ = c.Get(context.Background(), client.ObjectKey{Namespace: "mitos", Name: "sb-failed"}, failCur) + failCur.Status.Phase = v1.SandboxFailed + _ = c.Status().Update(context.Background(), failCur) + + // NOTE: with the clock stepped +11m, sb-adopt is over-age too. This test + // wants exactly one survivor, so give the adoptable entry a fresh clock: + // run the pass with MaxAge large enough to keep sb-adopt (20m) but a + // separate assertion pass for the reap: adjust cfg inline. + cp.checkout.cfg.MaxAge = 20 * time.Minute + stop := flipToReadyWhenCreatedInNs(t, c, "mitos", "10.0.0.5:9091", "tok-refill") + cp.checkout.reconcilePool(context.Background(), "python") + stop() + + // sb-failed pruned+deleted; sb-adopt adopted; sb-old still in-age at 20m. + var gone v1.Sandbox + if err := c.Get(context.Background(), client.ObjectKey{Namespace: "mitos", Name: "sb-failed"}, &gone); !apierrors.IsNotFound(err) { + t.Fatalf("failed buffered sandbox not deleted: err=%v", err) + } + popped, ok := cp.checkout.pop("python") + if !ok { + t.Fatal("nothing adopted") + } + if popped.name != "sb-adopt" && popped.name != "sb-old" { + t.Fatalf("adopted %q, want a seeded buffered sandbox", popped.name) + } + if popped.token == "" { + t.Fatal("adopted entry has no token (Secret read failed)") + } + + // Now the reap: shrink MaxAge back to 10m; the pass must delete over-age CRs. + cp.checkout.cfg.MaxAge = 10 * time.Minute + cp.checkout.reconcilePool(context.Background(), "python") // reap pass; refill may create anew, that is fine + var old v1.Sandbox + if err := c.Get(context.Background(), client.ObjectKey{Namespace: "mitos", Name: "sb-old"}, &old); !apierrors.IsNotFound(err) { + t.Fatalf("over-age buffered sandbox not reaped: err=%v", err) + } +} +``` + +If the interleaving above proves brittle when run, split it into three focused tests (adopt / prune-failed / reap-over-age), each seeding only its own CR; keep the same assertions. + +- [ ] **Step 2: Run to verify failure** + +Run: `go test ./internal/saas/controlplane/ -run TestReconcileAdopts -v` +Expected: FAIL (adopt/prune/reap not implemented; pop returns nothing or the failed CR survives) + +- [ ] **Step 3: Implement.** Extend reconcilePool: after the LIST, before the refill decision: + +```go + // Sync the cache against the cluster: the CRs are the truth. + known := make(map[string]bool) + b.mu.Lock() + for _, e := range b.entries[pool] { + known[e.name] = true + } + b.mu.Unlock() + + live := 0 + for i := range list.Items { + sb := &list.Items[i] + age := b.k.now().Sub(sb.CreationTimestamp.Time) + switch { + case age > b.cfg.MaxAge: + // Bounded staleness: a buffered VM runs live while it waits, so + // entries past MaxAge are recycled, not handed to a tenant. + b.dropAndDelete(ctx, pool, sb) + case sb.Status.Phase == v1.SandboxFailed, + sb.Status.Phase == v1.SandboxReady && sb.Status.Endpoint == "": + b.dropAndDelete(ctx, pool, sb) + case sb.Status.Phase != v1.SandboxReady: + // In flight (a refill mid-activation): neither adoptable nor + // reapable yet; it counts toward the floor so we do not over-refill. + live++ + default: + live++ + if !known[sb.Name] { + b.adopt(ctx, pool, sb) + } + } + } + count := live +``` + +(replace the earlier `count := len(list.Items)` from Task 4 with this `live` count) and add: + +```go +// dropAndDelete removes sb from the cache and deletes the CR (terminate). +func (b *checkoutBuffer) dropAndDelete(ctx context.Context, pool string, sb *v1.Sandbox) { + b.mu.Lock() + q := b.entries[pool][:0] + for _, e := range b.entries[pool] { + if e.name != sb.Name { + q = append(q, e) + } + } + b.entries[pool] = q + b.mu.Unlock() + if err := b.k.c.Delete(ctx, sb); err != nil && !apierrors.IsNotFound(err) { + slog.Warn("checkout reap failed", "sandbox", sb.Name, "error", err.Error()) + } +} + +// adopt rebuilds a cache entry from the cluster after a gateway restart (or +// for a CR the other replica refilled): endpoint and pod from status, token +// from the controller-owned Secret. +func (b *checkoutBuffer) adopt(ctx context.Context, pool string, sb *v1.Sandbox) { + token, err := b.k.readToken(ctx, sb.Namespace, sb.Name+tokenSecretSuffix) + if err != nil { + slog.Warn("checkout adopt could not read the token secret; leaving for the next pass", + "sandbox", sb.Name, "error", err.Error()) + return + } + b.add(bufferedSandbox{ + name: sb.Name, + pool: pool, + endpoint: sb.Status.Endpoint, + token: token, + resourceVersion: sb.ResourceVersion, + podName: sb.Status.Pod, + createdAt: sb.CreationTimestamp.Time, + }) +} +``` + +And the loop plus its public entry point: + +```go +// checkoutReconcileInterval paces the buffer loop; the hot path never waits +// on it (claim reads only the cache). +const checkoutReconcileInterval = 15 * time.Second + +// StartCheckout starts the buffer's reconcile loop for the server's +// lifetime. A no-op when the checkout feature is off. +func (k *K8sControlPlane) StartCheckout(ctx context.Context) { + if k.checkout == nil { + return + } + go k.checkout.run(ctx) +} + +func (b *checkoutBuffer) run(ctx context.Context) { + t := time.NewTicker(checkoutReconcileInterval) + defer t.Stop() + for { + for _, pool := range b.cfg.Pools { + b.reconcilePool(ctx, pool) + } + select { + case <-ctx.Done(): + return + case <-t.C: + } + } +} +``` + +- [ ] **Step 4: Run to verify pass, whole package with race** + +Run: `go test ./internal/saas/controlplane/ -race` +Expected: ok + +- [ ] **Step 5: Commit** + +```bash +git add internal/saas/controlplane/checkout.go internal/saas/controlplane/checkout_test.go +git commit -s -m "feat(saas): checkout adopt-on-restart, staleness reap, and the reconcile loop" +``` + +--- + +### Task 6: Controller propagates the CR org label to the husk pod (billing) + +**Files:** +- Modify: `internal/controller/sandboxclaim_controller.go` (the Ready-claim block at ~line 413, inside `if claim.Status.Phase == v1.SandboxReady { if r.EnableHuskPods { ... } }`, right after the `reflectHuskBackingReadiness` block, where `lostPod` is in scope) +- Test: the existing envtest suite file that covers husk claims (grep `reflectHuskBackingReadiness` or `checkHuskPodLost` under `internal/controller/*_test.go` and add alongside) + +**Interfaces:** +- Consumes: `checkHuskPodLost` already returns the backing pod (`lostPod`) even when healthy; `tenant.OrgLabelKey`. +- Produces: nothing new; behavior: a Ready husk claim whose CR org label differs from its pod's org label patches the pod label (idempotent). + +**Why the controller and not the gateway:** the usage scraper bills the TRUSTED pod org label, and the trust chain is that the CONTROLLER writes pod labels. The gateway's checkout patch stamps the CR; the watch event it fires brings the reconciler here, which propagates. The gateway needs no pod RBAC at all. + +- [ ] **Step 1: Write the failing envtest** (in the husk-claim envtest file; follow its existing setup helpers for creating a pool, a fake dormant pod, and driving a claim; the new test can also be written directly against a Ready claim + pod fixture) + +```go +// TestReadyClaimPropagatesOrgLabelToHuskPod asserts claim-time org +// attribution reaches the backing pod: when a Ready claim's org label (set +// by the gateway checkout patch) differs from its husk pod's, one reconcile +// patches the pod. The usage scraper bills the pod label, so without this a +// checked-out sandbox meters nobody. +func TestReadyClaimPropagatesOrgLabelToHuskPod(t *testing.T) { + // Fixture: a Ready claim with status.Pod set + the healthy backing pod + // WITHOUT an org label; stamp tenant.OrgLabels("acme") on the claim; + // run one Reconcile; assert pod.Labels[tenant.OrgLabelKey] == "acme". + // Reuse the suite's existing husk fixtures for pool/pod/claim shape. +} +``` + +Fill the fixture with the suite's existing helper shapes (the file already builds Ready husk claims for the pod-lost tests; copy that fixture and add the org label to the claim only). + +- [ ] **Step 2: Run to verify failure** + +Run: `eval $(~/go/bin/setup-envtest use 1.31 -p env) && go test ./internal/controller/ -run TestReadyClaimPropagatesOrg -v` +Expected: FAIL (pod label never set) + +- [ ] **Step 3: Implement.** In the Ready-claim block, after the `reflectHuskBackingReadiness` if-block and before the hydrate call: + +```go + // Claim-time org attribution (pre-claimed checkout): the gateway + // stamps the org on a formerly buffered sandbox's CR labels; this + // reconcile, triggered by that very patch, propagates the org to + // the backing husk pod, whose TRUSTED label is what the usage + // scraper bills. Idempotent, and a no-op for classic claims whose + // pod was labeled at claim time. + if lostPod != nil { + if org := claim.Labels[tenant.OrgLabelKey]; org != "" && lostPod.Labels[tenant.OrgLabelKey] != org { + podPatch := client.MergeFrom(lostPod.DeepCopy()) + if lostPod.Labels == nil { + lostPod.Labels = map[string]string{} + } + lostPod.Labels[tenant.OrgLabelKey] = org + if err := r.Patch(ctx, lostPod, podPatch); err != nil { + return ctrl.Result{}, err + } + } + } +``` + +Verify `checkHuskPodLost` indeed returns the pod when NOT lost (read its body first; if it returns nil for the healthy case, Get the pod by `claim.Status.Pod` instead, still inside the EnableHuskPods block). + +- [ ] **Step 4: Run to verify pass, then the controller suite** + +Run: `eval $(~/go/bin/setup-envtest use 1.31 -p env) && go test ./internal/controller/` +Expected: ok (whole suite) + +- [ ] **Step 5: Commit** + +```bash +git add internal/controller/sandboxclaim_controller.go internal/controller/ +git commit -s -m "feat(controller): propagate claim org label to the husk pod for checkout billing" +``` + +--- + +### Task 7: Gateway wiring (flags, startup, logging) + +**Files:** +- Modify: `cmd/gateway/main.go` (flags near line 107, newControlPlane near line 78, startup logging near line 163) + +**Interfaces:** +- Consumes: `controlplane.WithCheckout`, `controlplane.CheckoutConfig`, `StartCheckout` (real control plane only; the mock path never starts it). + +- [ ] **Step 1: Add flags** (following the exact style of the existing flag block: flag.String with os.Getenv defaults) + +```go + checkoutPools := flag.String("checkout-pools", os.Getenv("MITOS_GATEWAY_CHECKOUT_POOLS"), "comma-separated pool names served by the pre-claimed checkout buffer (empty disables the feature); requires --single-tenant-namespace") + checkoutFloor := flag.Int("checkout-floor", envInt("MITOS_GATEWAY_CHECKOUT_FLOOR", 2), "buffered sandboxes to keep ready per checkout pool") + checkoutCap := flag.Int("checkout-cap", envInt("MITOS_GATEWAY_CHECKOUT_CAP", 4), "hard ceiling of buffered sandboxes per checkout pool") + checkoutMaxAge := flag.Duration("checkout-max-age", envDuration("MITOS_GATEWAY_CHECKOUT_MAX_AGE", 10*time.Minute), "buffered sandboxes older than this are recycled") +``` + +If `envInt`/`envDuration` helpers do not exist in main.go, add them (strconv.Atoi / time.ParseDuration with the default on any error). Thread the values through `newControlPlane` (extend its signature) into: + +```go + if pools := splitNonEmpty(*checkoutPools); len(pools) > 0 { + opts = append(opts, controlplane.WithCheckout(controlplane.CheckoutConfig{ + Pools: pools, Floor: *checkoutFloor, Cap: *checkoutCap, MaxAge: *checkoutMaxAge, + })) + } +``` + +with `splitNonEmpty` = strings.Split on "," with TrimSpace, dropping empties. After the real control plane is constructed and the server context exists, start the loop and log the state honestly: + +```go + real.(*controlplane.K8sControlPlane).StartCheckout(ctx) +``` + +(newControlPlane returns saas.ControlPlane; either return the concrete type alongside, or add StartCheckout to nothing else and type-assert as shown, logging and skipping if the assertion fails.) Log at startup: enabled pools + floor/cap/maxAge when on; a WARNING when --checkout-pools is set without --single-tenant-namespace (the buffer silently stays off otherwise, which must not be silent). + +- [ ] **Step 2: Build and run the gateway's own tests** + +Run: `go build ./cmd/gateway/ && go test ./cmd/gateway/...` +Expected: builds; existing tests ok + +- [ ] **Step 3: Commit** + +```bash +git add cmd/gateway/main.go +git commit -s -m "feat(saas): gateway flags and startup wiring for the pre-claimed checkout" +``` + +--- + +### Task 8: Chart values, schema, RBAC patch verb, threat model, spec amendments + +**Files:** +- Modify: `deploy/charts/mitos/values.yaml` (gateway block, after `enforce`) +- Modify: `deploy/charts/mitos/values.schema.json` (gateway properties; the schema rejects unknown keys, #667, so this is REQUIRED for the release to install) +- Modify: `deploy/charts/mitos/templates/gateway.yaml` (args) +- Modify: `deploy/charts/mitos/templates/gateway-rbac.yaml` (add `patch` to the sandboxes verbs, with a comment naming the checkout attribution patch as the reason) +- Modify: `docs/threat-model.md` (one row: org-less buffered Ready sandboxes; unreachable through the gateway pre-checkout because getOwned matches no org; token custody window extended to maxAge; no tenant data in the VM by the eligibility gate) +- Modify: `docs/superpowers/specs/2026-07-11-preclaimed-checkout-design.md` (three amendments: idle-reap needs NO controller exemption because a reaped buffered CR is self-healing via NotFound-tolerant pop; pod org label propagation is CONTROLLER-side; e2e proof is unit + prod acceptance, no kind gateway job) + +values.yaml: + +```yaml + # Pre-claimed checkout: keep a small buffer of already-activated sandboxes + # per listed pool and serve eligible creates (no env, secrets, workspace, + # fan-out, or TTL) from it with a single attribution patch. Requires + # single-tenant namespace mode; buffered sandboxes carry no org and bill + # nobody until claimed (platform cost, bounded by cap and maxAge). + checkout: + pools: [] + floor: 2 + cap: 4 + maxAge: 10m +``` + +gateway.yaml args (inside the existing args list, following the conditional style used by other optional args): + +```yaml + {{- if .Values.gateway.checkout.pools }} + - --checkout-pools={{ join "," .Values.gateway.checkout.pools }} + - --checkout-floor={{ .Values.gateway.checkout.floor }} + - --checkout-cap={{ .Values.gateway.checkout.cap }} + - --checkout-max-age={{ .Values.gateway.checkout.maxAge }} + {{- end }} +``` + +values.schema.json, inside the gateway properties object: + +```json +"checkout": { + "type": "object", + "additionalProperties": false, + "properties": { + "pools": {"type": "array", "items": {"type": "string"}}, + "floor": {"type": "integer", "minimum": 1}, + "cap": {"type": "integer", "minimum": 1}, + "maxAge": {"type": "string"} + } +} +``` + +- [ ] **Step 1: Make all six edits.** +- [ ] **Step 2: Verify the chart renders both ways** + +Run: `helm template mitos deploy/charts/mitos --kube-version 1.31.0 > /dev/null && helm template mitos deploy/charts/mitos --kube-version 1.31.0 --set gateway.checkout.pools='{python}' | grep -A2 checkout-pools` +Expected: clean render; the second shows the four args + +- [ ] **Step 3: Commit** + +```bash +git add deploy/charts/mitos docs/threat-model.md docs/superpowers/specs/2026-07-11-preclaimed-checkout-design.md +git commit -s -m "feat(chart): gateway.checkout values, RBAC patch verb, and the threat-model row" +``` + +--- + +### Task 9: Full verification and the PR + +- [ ] **Step 1: Whole-repo gates** + +Run, each must pass: +```bash +go build ./... +go test ./internal/saas/... -race +make test-unit +eval $(~/go/bin/setup-envtest use 1.31 -p env) && go test ./internal/controller/ +~/go/bin/golangci-lint run --timeout=5m +GOOS=linux ~/go/bin/golangci-lint run --timeout=5m +``` + +- [ ] **Step 2: Commit the spec + plan** (if not yet on the branch), push, open the PR with the repo template (Thinking Path, Verification, Model Used; conventional title `feat(saas): pre-claimed checkout, the gateway hands out an already-activated sandbox`; no em/en dashes; Related: the TTI issue #871 and spec/plan paths). State explicitly: default OFF, no public number claimed, prod acceptance = bench/tti-latency.py after a roll with `gateway.checkout.pools={python}`. +- [ ] **Step 3: Drive CI green, resolve every CodeRabbit comment, auto-merge on green.** + +--- + +## Self-Review Notes + +- Spec coverage: eligibility (T1), one-patch checkout + exclusivity + fallback (T3), refill floor/cap/backoff (T4), adopt/prune/reap + loop (T5), controller pod-label propagation replacing the spec's async-gateway-patch (T6 + spec amendment in T8), config/Helm/RBAC/threat model (T7/T8), honesty + acceptance (T9). The spec's "janitor re-patches org-less pods" sweep is superseded by T6 (the reconciler IS the sweep, watch-triggered); amended in T8. +- The fake client's Watch + interceptor idioms match the #895 tests; the exclusivity test relies on the fake client honoring resourceVersion on Patch with MergeFromWithOptimisticLock (it does; it 409s on mismatch). +- Type consistency: bufferedSandbox fields, CheckoutConfig fields, and label constants are used identically across T1-T5; tenant.OrgLabelKey/OrgLabels shared with T6. diff --git a/docs/superpowers/specs/2026-07-11-preclaimed-checkout-design.md b/docs/superpowers/specs/2026-07-11-preclaimed-checkout-design.md new file mode 100644 index 00000000..6c8aa108 --- /dev/null +++ b/docs/superpowers/specs/2026-07-11-preclaimed-checkout-design.md @@ -0,0 +1,164 @@ +# Pre-claimed checkout: the gateway hands out an already-activated sandbox + +Status: approved direction (2026-07-10, jannes: gateway buffer, k8s-native labels, +remaining decisions delegated to the recommended options). Companion to the +prepare-time restore plan (docs/superpowers/plans/2026-07-10-prepare-time-restore.md), +which attacks the engine share of the same budget. + +## Why + +Hosted TTI is ~308 ms P50 (bench/results/2026-07-10-tti-hosted.md). The measured +create budget (prod controller logs, 2026-07-10, successful claims n=33) is: +controller reconcile 137 ms P50 (activate_rpc 98.2, mark_pod_claimed 14.8, +status_write_ready 8.7) plus ~35 ms of k8s plumbing around it plus ~20 ms gateway +front. Even with prepare-time restore slices 2 and 3 landed (activate ~30 ms, +first exec warm ~35 ms), the projected TTI is ~165 ms: above Daytona (136 ms) and +far above Northflank (95.9 ms) on the ComputeSDK peer table. + +The remaining cost is the claim round trip itself: CR write, watch wake-ups, +reconcile, activate RPC, status writes. The peers we trail do not pay any of this +per request; their sequential number is a warm-pool checkout (their burst ratios +prove it, 5.2x for Daytona). This design gives mitos the same checkout shape: +create() pops a sandbox that is ALREADY Ready, and the only hot-path write is the +attribution patch. + +## Decision record + +1. **Placement: gateway buffer of real Sandboxes** (chosen over husk-level + generic-activate + attach). Fastest to the target, no controller, husk, or + guest changes, no overlap with the prepare-time restore workstream. The trade + is a dependency on the single-tenant namespace (see Migration). +2. **Source of truth: k8s labels, not gateway memory or Postgres.** Buffered + sandboxes carry `mitos.run/buffered: "true"`. Checkout is one + resourceVersion-guarded label patch that atomically removes the buffered label + and stamps the org labels: mutual exclusion between the two gateway replicas + AND claim-time org attribution in the same ~10 ms write. Memory is only a + cache; a restarted gateway re-adopts by LIST. +3. **Eligibility: pool-only creates.** A create qualifies iff it names a + checkout-enabled pool and sets no env, no secrets, no workspace, no + replicas > 1, and no TTL or timeout. Everything else, and every buffer miss, + takes the classic path unchanged. Rationale: env and secret values are + delivered by the fork-correctness handshake at activation, which for a + buffered sandbox already happened with empty tenant inputs. +4. **Token: the pre-minted token is reused, not rotated.** The gateway already + transits every sandbox token; the token lives in the controller-owned Secret + (which the gateway already reads) and in gateway memory. Rotation would cost + a husk round trip and re-serialize the path this design removes. +5. **Buffer sizing: floor 2, cap 4, per-pool opt-in, maxAge 10 m.** Hosted prod + enables it for the `python` pool only. Refill is triggered by a pop and by a + 15 s reconcile; both replicas count before creating (LIST), overshoot is + bounded by the cap. Each buffered sandbox holds one warm-pool slot and ~72 MiB + resident (lazy-restore measurement), platform-paid; floor 2 keeps that cost + under one extra dormant pod equivalent. +6. **Billing and quota: attribution at claim.** A buffered sandbox has NO org + labels, so the usage scraper (trusted pod label) bills nobody and the console + shows it to nobody; that unbilled window is deliberate platform cost. At + checkout the CR patch stamps the org synchronously (runtime authz getOwned + needs it before the first exec arrives); the husk POD org label (billing) is + patched asynchronously with retry, and the janitor sweeps for org-labeled CRs + whose pod is still org-less so no claimed sandbox can dodge metering for more + than one sweep interval. Quota is checked before checkout, exactly where the + classic path checks it before the CR create. +7. **Reaping: the buffered label exempts an entry from idle reap; the janitor + terminates buffered sandboxes older than maxAge.** A buffered sandbox is idle + by definition, so without the exemption the idle reaper would drain the + buffer. maxAge bounds staleness (clock and CRNG were stepped at activation; + the VM runs live while buffered) and bounds leak size if refill misbehaves. +8. **Health: the reconcile prunes non-Ready buffer entries.** A husk that dies + while buffered is dropped from the buffer at the next 15 s pass; a pop that + races a just-died sandbox surfaces to the tenant like any post-create death + does today. No per-pop health probe (it would re-serialize a round trip). + +## Data flow + +Refill (off the hot path, per gateway replica): + +1. LIST sandboxes with `mitos.run/buffered=true` for the pool; if count is at or + above floor, stop. +2. Create a Sandbox through the EXISTING create path (watch-before-create, #895) + with `mitos.run/buffered: "true"`, no org labels, in the shared namespace. +3. On Ready, cache {name, endpoint, token} in memory (token from the create + response it already receives). + +Checkout (the hot path): + +1. Gateway front as today: auth, quota, parse. Eligibility check (decision 3). +2. Pop a cached entry; issue ONE patch (resourceVersion-guarded): remove + `mitos.run/buffered`, set `tenant.OrgLabels(org)`. A 409 means the other + replica won that entry: drop it, pop the next; empty buffer means classic + path. +3. Return 201 {id, endpoint, token, phase Ready} from cache. fork_time_ms is the + observed wall time of this request, honestly tiny. +4. Async: patch the husk pod org label (billing); kick refill. + +Adopt (gateway start): LIST buffered sandboxes, read each token Secret, rebuild +the memory cache. + +Janitor (each reconcile pass): prune non-Ready entries; terminate entries older +than maxAge; re-patch pods for claimed CRs still missing the pod org label. + +## Security + +- Before checkout a buffered sandbox has no org labels, so getOwned matches NO + caller: it is unreachable through the gateway runtime surface. Its endpoint is + cluster-internal and bearer-token gated exactly like every Ready sandbox. +- No tenant data exists in a buffered VM: eligibility excludes env, secrets, and + workspaces, and the activation handshake ran with empty tenant inputs. +- The token custody chain is unchanged in kind (controller Secret -> gateway -> + tenant); what changes is duration (up to maxAge in gateway memory). Never + logged, as today. +- threat-model.md gets a row for the buffered state in the same PR. + +## Failure behavior (operating principle 4) + +- Gateway crash: memory cache lost, CRs remain; the surviving replica and the + restarted one re-adopt by LIST. Nothing leaks past maxAge. +- Both replicas down: buffered sandboxes idle until the janitor of whichever + replica returns reaps the over-age ones. +- Slow etcd: the checkout patch inherits it (one write); classic path inherits + it worse (several). Fallback ordering never blocks on the buffer. +- Capacity exhaustion: refill creates fail like any create (NoHuskPod pending); + the buffer floor is best-effort, the hot path falls back to classic. +- The 1752-retry pathology (#894) applies to refill creates too; refill backs + off on consecutive failures and never retries in a tight loop. + +## Honesty and measurement + +- No README or website number changes from this design until + bench/tti-latency.py reproduces the improvement against prod, per the + no-unverified-claims rule. The bench doc must state that eligible creates are + buffer checkouts and report the measured fallback (buffer-miss) cost beside + the headline, the same honesty bar we apply to Daytona's burst ratio. +- Projection, stated as a target not a claim: create ~35-50 ms server-side + (front ~20 + patch ~10 + response), TTI ~95-110 ms with prepare-time restore + slice 3, near the Northflank line. The next visible chunks after this land + are the per-create quota LIST and the activate RPC overhead (#893 adjacent). + +## Migration note (per-org namespaces) + +This design leans on the single-tenant namespace: a CR cannot move namespaces, +so per-org tenancy invalidates a shared buffer. At that cutover the options are +per-org buffers (paid per org, only above an activity threshold) or the +husk-level generic-activate + attach design (pre-activation at the POD, claim +stays a per-org CR create). This spec explicitly does NOT choose; the checkout +surface (eligibility gate, fallback ordering, bench honesty) survives either. + +## Configuration + +Helm: `gateway.checkout.pools` (list, default empty = feature off), +`gateway.checkout.floor` (default 2), `gateway.checkout.cap` (default 4), +`gateway.checkout.maxAge` (default 10m). Self-host gets the same knobs; the +feature is capability-neutral (no console surface change). + +## Testing + +- Unit (fake client, in-package with the #895 tests): eligibility gate; + pop-patch exclusivity (two concurrent pops, one 409 loser re-pops); fallback + on empty buffer; adopt-on-start rebuilds cache from CRs + Secrets; janitor + prunes non-Ready, reaps over-age, re-patches org-less pods; refill respects + floor/cap and backs off on failures; org labels land on CR before the + response returns. +- kind e2e: a checkout-enabled pool serves a create from the buffer (observable + via the buffered label lifecycle) and a create with env falls back classic. +- Prod acceptance: bench/tti-latency.py before/after, plus one burst run to + measure the fallback cost honestly. From 44f65767719e33156d6d21309eb796a682d2df50 Mon Sep 17 00:00:00 2001 From: Jannes Stubbemann Date: Sat, 11 Jul 2026 00:14:41 +0200 Subject: [PATCH 02/11] feat(saas): checkout buffer scaffold, config, and the eligibility gate Co-Authored-By: Claude Fable 5 Signed-off-by: Jannes Stubbemann --- internal/saas/controlplane/checkout.go | 92 +++++++++++++++++++++ internal/saas/controlplane/checkout_test.go | 62 ++++++++++++++ internal/saas/controlplane/controlplane.go | 33 ++++++++ 3 files changed, 187 insertions(+) create mode 100644 internal/saas/controlplane/checkout.go create mode 100644 internal/saas/controlplane/checkout_test.go diff --git a/internal/saas/controlplane/checkout.go b/internal/saas/controlplane/checkout.go new file mode 100644 index 00000000..d8ed669a --- /dev/null +++ b/internal/saas/controlplane/checkout.go @@ -0,0 +1,92 @@ +package controlplane + +import ( + "sync" + "time" +) + +// BufferedLabelKey marks a pre-created, org-less sandbox held by the gateway +// checkout buffer. Removing it, atomically with stamping the org labels, IS +// the checkout (see docs/superpowers/specs/2026-07-11-preclaimed-checkout-design.md). +const BufferedLabelKey = "mitos.run/buffered" + +// bufferedPoolLabelKey carries the pool identity on a buffered Sandbox so the +// reconcile loop can LIST per pool (spec.source.poolRef is not selectable). +const bufferedPoolLabelKey = "mitos.run/pool" + +// CheckoutConfig configures the pre-claimed checkout buffer. Pools empty +// means the feature is off. +type CheckoutConfig struct { + Pools []string + Floor int + Cap int + MaxAge time.Duration +} + +func (c CheckoutConfig) enabledFor(pool string) bool { + for _, p := range c.Pools { + if p == pool { + return true + } + } + return false +} + +// bufferedSandbox is one cached, already-Ready, org-less sandbox. The token +// value stays in memory and the controller-owned Secret only, exactly the +// custody chain a classic create has; it is never logged. +type bufferedSandbox struct { + name string + pool string + endpoint string + token string + resourceVersion string + podName string + createdAt time.Time +} + +// checkoutBuffer serves eligible creates from pre-activated sandboxes. The +// cluster (label state on the CRs) is the source of truth; this struct is a +// cache plus the refill/janitor loop. +type checkoutBuffer struct { + k *K8sControlPlane + cfg CheckoutConfig + + mu sync.Mutex + entries map[string][]bufferedSandbox + // nextRefill backs off refill attempts per pool after failures (the #894 + // shape: a refill that cannot succeed must not retry in a tight loop). + nextRefill map[string]time.Time + refillFails map[string]int +} + +func newCheckoutBuffer(k *K8sControlPlane, cfg CheckoutConfig) *checkoutBuffer { + return &checkoutBuffer{ + k: k, + cfg: cfg, + entries: make(map[string][]bufferedSandbox), + nextRefill: make(map[string]time.Time), + refillFails: make(map[string]int), + } +} + +// eligible reports whether this create may be served from the buffer: a +// checkout-enabled pool and NOTHING tenant-specific that the activation +// handshake would have had to deliver (env, secrets, workspace), no fan-out, +// and no lifetime knobs (a buffered sandbox's TTL clock would predate the +// claim). +func (b *checkoutBuffer) eligible(body createBody, pool string) bool { + if !b.cfg.enabledFor(pool) { + return false + } + if len(body.Env) > 0 || len(body.Secrets) > 0 || body.Workspace != "" { + return false + } + if body.Replicas > 1 { + return false + } + if body.TTL != "" || body.Timeout != "" { + return false + } + return true +} diff --git a/internal/saas/controlplane/checkout_test.go b/internal/saas/controlplane/checkout_test.go new file mode 100644 index 00000000..da931a7e --- /dev/null +++ b/internal/saas/controlplane/checkout_test.go @@ -0,0 +1,62 @@ +package controlplane + +import ( + "testing" + "time" +) + +// checkoutCP builds a control plane with the checkout feature on for the +// "python" pool in single-tenant mode, the shape hosted prod runs. +func checkoutCP(t *testing.T) *K8sControlPlane { + t.Helper() + return New(newFakeClient(t), + WithPollInterval(5*time.Millisecond), + WithReadyTimeout(2*time.Second), + WithSingleTenantNamespace("mitos"), + WithCheckout(CheckoutConfig{Pools: []string{"python"}, Floor: 2, Cap: 4, MaxAge: 10 * time.Minute}), + ) +} + +// TestCheckoutEligibilityGate pins which creates may be served from the +// buffer: a checkout-enabled pool with NO env, secrets, workspace, fan-out, +// or lifetime knobs. Everything else must take the classic path, because a +// buffered sandbox's activation handshake already ran with empty tenant +// inputs and its TTL clock would predate the claim. +func TestCheckoutEligibilityGate(t *testing.T) { + cp := checkoutCP(t) + b := cp.checkout + if b == nil { + t.Fatal("checkout buffer not constructed despite WithCheckout + single-tenant mode") + } + cases := []struct { + name string + body createBody + pool string + want bool + }{ + {"plain create on a checkout pool", createBody{}, "python", true}, + {"pool not enabled", createBody{}, "default", false}, + {"env set", createBody{Env: map[string]string{"A": "b"}}, "python", false}, + {"secrets set", createBody{Secrets: []secretMountReq{{Name: "s"}}}, "python", false}, + {"workspace set", createBody{Workspace: "ws"}, "python", false}, + {"replicas fan-out", createBody{Replicas: 2}, "python", false}, + {"ttl set", createBody{TTL: "5m"}, "python", false}, + {"timeout set", createBody{Timeout: "5m"}, "python", false}, + } + for _, tc := range cases { + if got := b.eligible(tc.body, tc.pool); got != tc.want { + t.Errorf("%s: eligible = %v, want %v", tc.name, got, tc.want) + } + } +} + +// TestCheckoutRequiresSingleTenantNamespace asserts the buffer is NOT +// constructed under per-org namespacing: the design leans on the shared +// namespace (see the spec's migration note) and must fail safe to the +// classic path everywhere else. +func TestCheckoutRequiresSingleTenantNamespace(t *testing.T) { + cp := New(newFakeClient(t), WithCheckout(CheckoutConfig{Pools: []string{"python"}})) + if cp.checkout != nil { + t.Fatal("checkout buffer constructed without single-tenant namespace mode") + } +} diff --git a/internal/saas/controlplane/controlplane.go b/internal/saas/controlplane/controlplane.go index ff30500c..e1e4b3a4 100644 --- a/internal/saas/controlplane/controlplane.go +++ b/internal/saas/controlplane/controlplane.go @@ -64,6 +64,14 @@ type K8sControlPlane struct { // forward.go). Absence is never stored. poolSeenMu sync.Mutex poolSeen map[string]time.Time + + // checkout serves eligible creates from a buffer of pre-activated + // sandboxes (nil = feature off). Constructed by New only when BOTH + // WithCheckout and single-tenant mode are set; the design leans on the + // shared namespace, see the spec's per-org-namespace migration note + // (docs/superpowers/specs/2026-07-11-preclaimed-checkout-design.md). + checkoutCfg *CheckoutConfig + checkout *checkoutBuffer } // Option configures a K8sControlPlane. @@ -157,5 +165,30 @@ func New(c client.Client, opts ...Option) *K8sControlPlane { for _, o := range opts { o(k) } + if k.checkoutCfg != nil && k.singleTenantNamespace != "" { + k.checkout = newCheckoutBuffer(k, *k.checkoutCfg) + } return k } + +// WithCheckout enables the pre-claimed checkout buffer for the named pools. +// Ignored (feature off) when Pools is empty or the control plane is not in +// single-tenant namespace mode. Zero Floor, Cap, or MaxAge take the defaults +// 2, twice the floor, and 10m. +func WithCheckout(cfg CheckoutConfig) Option { + return func(k *K8sControlPlane) { + if len(cfg.Pools) == 0 { + return + } + if cfg.Floor <= 0 { + cfg.Floor = 2 + } + if cfg.Cap < cfg.Floor { + cfg.Cap = cfg.Floor * 2 + } + if cfg.MaxAge <= 0 { + cfg.MaxAge = 10 * time.Minute + } + k.checkoutCfg = &cfg + } +} From 2bbeeebf6a7d55d4d1b22a9942d3ddc2992dc21b Mon Sep 17 00:00:00 2001 From: Jannes Stubbemann Date: Sat, 11 Jul 2026 00:15:34 +0200 Subject: [PATCH 03/11] refactor(saas): extract createSandboxAndAwait for reuse by the checkout refill Co-Authored-By: Claude Fable 5 Signed-off-by: Jannes Stubbemann --- internal/saas/controlplane/forward.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/internal/saas/controlplane/forward.go b/internal/saas/controlplane/forward.go index f9c86f45..dfdf9066 100644 --- a/internal/saas/controlplane/forward.go +++ b/internal/saas/controlplane/forward.go @@ -251,6 +251,17 @@ func (k *K8sControlPlane) create(ctx context.Context, req saas.ForwardRequest) ( sb.Spec.Lifetime = &v1.SandboxLifetime{TTL: &metav1.Duration{Duration: d}} } + return k.createSandboxAndAwait(ctx, sb, startedAt) +} + +// createSandboxAndAwait writes sb and blocks for its terminal outcome. It is +// shared by the tenant create path and the checkout buffer's refill, which is +// why it takes a fully built Sandbox rather than a request body. The org id +// used by error envelopes comes from sb's org label (empty for a buffered +// refill, whose failures are logged, not returned to a tenant). +func (k *K8sControlPlane) createSandboxAndAwait(ctx context.Context, sb *v1.Sandbox, startedAt time.Time) (saas.ForwardResponse, error) { + ns, name := sb.Namespace, sb.Name + // Establish the ready watch BEFORE the Create: the create event then // arrives on the stream itself, which both closes the flip-before-watch // window (no authoritative re-read needed) and takes one serialized @@ -277,7 +288,7 @@ func (k *K8sControlPlane) create(ctx context.Context, req saas.ForwardRequest) ( return errResp(withStatus(apierr.Get(apierr.CodeInternal). WithCause("a sandbox with the generated name already exists; retry"), http.StatusConflict)), nil case isNamespaceMissing(err, ns): - return errResp(namespaceMissingErr(req.OrgID, ns)), nil + return errResp(namespaceMissingErr(sb.Labels[tenant.OrgLabelKey], ns)), nil case apierrors.IsInvalid(err), apierrors.IsBadRequest(err): // The caller's JSON decoded fine; the OBJECT the gateway built from // it failed api server validation (for example an org id that is not From 3b5105a74329c97f27c2beca16ce06a5b526dc60 Mon Sep 17 00:00:00 2001 From: Jannes Stubbemann Date: Sat, 11 Jul 2026 00:17:53 +0200 Subject: [PATCH 04/11] feat(saas): serve eligible creates from the checkout buffer with one attribution patch Co-Authored-By: Claude Fable 5 Signed-off-by: Jannes Stubbemann --- internal/saas/controlplane/checkout.go | 90 +++++++++++ internal/saas/controlplane/checkout_test.go | 162 ++++++++++++++++++++ internal/saas/controlplane/forward.go | 11 ++ 3 files changed, 263 insertions(+) diff --git a/internal/saas/controlplane/checkout.go b/internal/saas/controlplane/checkout.go index d8ed669a..9ed21619 100644 --- a/internal/saas/controlplane/checkout.go +++ b/internal/saas/controlplane/checkout.go @@ -1,8 +1,19 @@ package controlplane import ( + "context" + "log/slog" + "net/http" "sync" "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + v1 "mitos.run/mitos/api/v1" + "mitos.run/mitos/internal/saas" + "mitos.run/mitos/internal/tenant" ) // BufferedLabelKey marks a pre-created, org-less sandbox held by the gateway @@ -70,6 +81,85 @@ func newCheckoutBuffer(k *K8sControlPlane, cfg CheckoutConfig) *checkoutBuffer { } } +// add caches one buffered sandbox (refill, adopt, tests). +func (b *checkoutBuffer) add(e bufferedSandbox) { + b.mu.Lock() + defer b.mu.Unlock() + b.entries[e.pool] = append(b.entries[e.pool], e) +} + +// pop removes and returns the oldest cached entry for pool. +func (b *checkoutBuffer) pop(pool string) (bufferedSandbox, bool) { + b.mu.Lock() + defer b.mu.Unlock() + q := b.entries[pool] + if len(q) == 0 { + return bufferedSandbox{}, false + } + e := q[0] + b.entries[pool] = q[1:] + return e, true +} + +// claim serves one eligible create from the buffer. ok=false means the +// caller must take the classic path (empty buffer, or every candidate was +// lost to the other replica or already reaped). On success the CR already +// carries the org labels, so runtime authz (getOwned) is satisfied before +// the caller's first exec can arrive. +func (b *checkoutBuffer) claim(ctx context.Context, ns, org, pool string, startedAt time.Time) (saas.ForwardResponse, bool) { + for { + e, ok := b.pop(pool) + if !ok { + return saas.ForwardResponse{}, false + } + if !b.stampOrg(ctx, ns, org, e) { + continue + } + payload := map[string]any{ + "id": e.name, + "endpoint": e.endpoint, + "token": e.token, + "phase": string(v1.SandboxReady), + "template_id": e.pool, + "fork_time_ms": float64(b.k.now().Sub(startedAt).Milliseconds()), + } + resp := jsonResp(http.StatusCreated, payload) + // The same telemetry contract as readyResponse: the gateway attaches + // the non-identifying pool name to sandbox.created events. + resp.Header.Set("X-Mitos-Pool", e.pool) + return resp, true + } +} + +// stampOrg is THE checkout write: one resourceVersion-guarded patch that +// atomically drops the buffered label and stamps the org labels, making it +// both the mutual exclusion between gateway replicas and the claim-time org +// attribution. A conflict or a vanished CR means this entry is not ours (the +// other replica won, or a reaper got it): report false so claim tries the +// next entry. +func (b *checkoutBuffer) stampOrg(ctx context.Context, ns, org string, e bufferedSandbox) bool { + old := &v1.Sandbox{ObjectMeta: metav1.ObjectMeta{ + Name: e.name, + Namespace: ns, + ResourceVersion: e.resourceVersion, + Labels: map[string]string{ + BufferedLabelKey: "true", + bufferedPoolLabelKey: e.pool, + }, + }} + claimed := old.DeepCopy() + claimed.Labels = tenant.OrgLabels(org) + err := b.k.c.Patch(ctx, claimed, client.MergeFromWithOptions(old, client.MergeFromWithOptimisticLock{})) + if err == nil { + return true + } + if !apierrors.IsConflict(err) && !apierrors.IsNotFound(err) { + slog.Warn("checkout stamp failed; entry dropped, create falls back", + "sandbox", e.name, "error", err.Error()) + } + return false +} + // eligible reports whether this create may be served from the buffer: a // checkout-enabled pool and NOTHING tenant-specific that the activation // handshake would have had to deliver (env, secrets, workspace), no fan-out, diff --git a/internal/saas/controlplane/checkout_test.go b/internal/saas/controlplane/checkout_test.go index da931a7e..e39eb9c0 100644 --- a/internal/saas/controlplane/checkout_test.go +++ b/internal/saas/controlplane/checkout_test.go @@ -1,10 +1,58 @@ package controlplane import ( + "context" + "net/http" + "sync/atomic" "testing" "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + v1 "mitos.run/mitos/api/v1" + "mitos.run/mitos/internal/saas" + "mitos.run/mitos/internal/tenant" ) +// poolInNs builds a SandboxPool in an explicit namespace (the single-tenant +// shape, unlike poolIn's per-org namespace derivation). +func poolInNs(ns, name string) *v1.SandboxPool { + return &v1.SandboxPool{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}} +} + +// seedBuffered creates a Ready buffered sandbox CR in ns "mitos" and returns +// the entry the gateway would cache for it. +func seedBuffered(t *testing.T, c client.Client, name string) bufferedSandbox { + t.Helper() + sb := &v1.Sandbox{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, Namespace: "mitos", + Labels: map[string]string{BufferedLabelKey: "true", bufferedPoolLabelKey: "python"}, + }, + Spec: v1.SandboxSpec{Source: v1.SandboxSource{PoolRef: &v1.LocalObjectReference{Name: "python"}}}, + } + if err := c.Create(context.Background(), sb); err != nil { + t.Fatalf("seed buffered sandbox: %v", err) + } + sb.Status.Phase = v1.SandboxReady + sb.Status.Endpoint = "10.0.0.9:9091" + sb.Status.Pod = name + if err := c.Status().Update(context.Background(), sb); err != nil { + t.Fatalf("seed status: %v", err) + } + var cur v1.Sandbox + if err := c.Get(context.Background(), client.ObjectKey{Namespace: "mitos", Name: name}, &cur); err != nil { + t.Fatalf("re-get: %v", err) + } + return bufferedSandbox{ + name: name, pool: "python", endpoint: "10.0.0.9:9091", token: "tok-" + name, + resourceVersion: cur.ResourceVersion, podName: name, createdAt: time.Now(), + } +} + // checkoutCP builds a control plane with the checkout feature on for the // "python" pool in single-tenant mode, the shape hosted prod runs. func checkoutCP(t *testing.T) *K8sControlPlane { @@ -50,6 +98,120 @@ func TestCheckoutEligibilityGate(t *testing.T) { } } +// TestCheckoutServesCreateFromBuffer asserts an eligible create is served +// from the buffer: 201 with the cached endpoint and token, NO Sandbox Create +// on the hot path, and the CR atomically loses the buffered label and gains +// the caller's org labels BEFORE the response returns (runtime authz getOwned +// re-checks that label on the first exec). +func TestCheckoutServesCreateFromBuffer(t *testing.T) { + var creates atomic.Int64 + base := fakeclient.NewClientBuilder(). + WithScheme(newScheme(t)). + WithStatusSubresource(&v1.Sandbox{}). + Build() + c := interceptor.NewClient(base, interceptor.Funcs{ + Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + if _, ok := obj.(*v1.Sandbox); ok { + creates.Add(1) + } + return cl.Create(ctx, obj, opts...) + }, + }) + cp := New(c, + WithPollInterval(5*time.Millisecond), WithReadyTimeout(2*time.Second), + WithSingleTenantNamespace("mitos"), + WithCheckout(CheckoutConfig{Pools: []string{"python"}})) + cp.checkout.add(seedBuffered(t, c, "sb-buffered1")) + creates.Store(0) + + resp, err := cp.Forward(context.Background(), saas.ForwardRequest{ + OrgID: orgA, Op: "sandbox.create", Body: []byte(`{"pool":"python"}`), + }) + if err != nil { + t.Fatalf("Forward: %v", err) + } + if resp.Status != http.StatusCreated { + t.Fatalf("status = %d, body = %s", resp.Status, resp.Body) + } + m := decodeBody(t, resp.Body) + if m["id"] != "sb-buffered1" || m["endpoint"] != "10.0.0.9:9091" || m["token"] != "tok-sb-buffered1" { + t.Fatalf("payload = %v, want the buffered sandbox's identity", m) + } + if n := creates.Load(); n != 0 { + t.Fatalf("checkout performed %d Sandbox Create(s) on the hot path, want 0", n) + } + var sb v1.Sandbox + if err := c.Get(context.Background(), client.ObjectKey{Namespace: "mitos", Name: "sb-buffered1"}, &sb); err != nil { + t.Fatalf("get: %v", err) + } + if _, still := sb.Labels[BufferedLabelKey]; still { + t.Error("buffered label survived the checkout") + } + if sb.Labels[tenant.OrgLabelKey] != orgA { + t.Errorf("org label = %q, want %q", sb.Labels[tenant.OrgLabelKey], orgA) + } +} + +// TestCheckoutIsExclusiveAcrossReplicas asserts two gateways holding the same +// cached entry cannot both hand it out: the resourceVersion-guarded patch lets +// exactly one win, and the loser falls back to the classic path. +func TestCheckoutIsExclusiveAcrossReplicas(t *testing.T) { + c := newFakeClient(t, poolInNs("mitos", "python")) + mk := func() *K8sControlPlane { + return New(c, + WithPollInterval(5*time.Millisecond), WithReadyTimeout(2*time.Second), + WithSingleTenantNamespace("mitos"), + WithCheckout(CheckoutConfig{Pools: []string{"python"}})) + } + cpA, cpB := mk(), mk() + e := seedBuffered(t, c, "sb-shared") + cpA.checkout.add(e) + cpB.checkout.add(e) + + stop := flipToReadyWhenCreatedInNs(t, c, "mitos", "10.9.9.9:9091", "tok-classic") + defer stop() + + respA, errA := cpA.Forward(context.Background(), saas.ForwardRequest{OrgID: orgA, Op: "sandbox.create", Body: []byte(`{"pool":"python"}`)}) + respB, errB := cpB.Forward(context.Background(), saas.ForwardRequest{OrgID: orgB, Op: "sandbox.create", Body: []byte(`{"pool":"python"}`)}) + if errA != nil || errB != nil { + t.Fatalf("Forward: %v / %v", errA, errB) + } + if respA.Status != http.StatusCreated || respB.Status != http.StatusCreated { + t.Fatalf("statuses = %d/%d, bodies %s / %s", respA.Status, respB.Status, respA.Body, respB.Body) + } + idA := decodeBody(t, respA.Body)["id"] + idB := decodeBody(t, respB.Body)["id"] + got := 0 + if idA == "sb-shared" { + got++ + } + if idB == "sb-shared" { + got++ + } + if got != 1 { + t.Fatalf("the shared buffered sandbox was handed out %d times (ids %v, %v), want exactly 1", got, idA, idB) + } +} + +// TestCheckoutEmptyBufferFallsBackClassic asserts an eligible create with no +// buffered entry takes the classic path and still succeeds. +func TestCheckoutEmptyBufferFallsBackClassic(t *testing.T) { + c := newFakeClient(t, poolInNs("mitos", "python")) + cp := New(c, + WithPollInterval(5*time.Millisecond), WithReadyTimeout(2*time.Second), + WithSingleTenantNamespace("mitos"), + WithCheckout(CheckoutConfig{Pools: []string{"python"}})) + stop := flipToReadyWhenCreatedInNs(t, c, "mitos", "10.9.9.9:9091", "tok-classic") + defer stop() + resp, err := cp.Forward(context.Background(), saas.ForwardRequest{OrgID: orgA, Op: "sandbox.create", Body: []byte(`{"pool":"python"}`)}) + if err != nil { + t.Fatalf("Forward: %v", err) + } + if resp.Status != http.StatusCreated { + t.Fatalf("status = %d, body = %s", resp.Status, resp.Body) + } +} + // TestCheckoutRequiresSingleTenantNamespace asserts the buffer is NOT // constructed under per-org namespacing: the design leans on the shared // namespace (see the spec's migration note) and must fail safe to the diff --git a/internal/saas/controlplane/forward.go b/internal/saas/controlplane/forward.go index dfdf9066..86e81d7b 100644 --- a/internal/saas/controlplane/forward.go +++ b/internal/saas/controlplane/forward.go @@ -181,6 +181,17 @@ func (k *K8sControlPlane) create(ctx context.Context, req saas.ForwardRequest) ( ns := k.namespaceForOrg(req.OrgID) + // Pre-claimed checkout: an eligible create is served from the buffer of + // already-activated sandboxes, paying ONE label patch instead of the whole + // claim round trip. Any miss (feature off, ineligible body, empty buffer, + // or a lost race) falls through to the classic path below. A buffered + // entry also proves the pool exists, so a hit skips the pre-check too. + if k.checkout != nil && k.checkout.eligible(body, pool) { + if resp, ok := k.checkout.claim(ctx, ns, req.OrgID, pool, startedAt); ok { + return resp, nil + } + } + // Fast-fail on an unknown pool. In the hosted control plane pools are // pre-provisioned and stable, so a create naming a pool that does not exist // in the tenant namespace is a typo, not a manifest-ordering race: return an From cc572be53c04d13b903bc3e2e7bfcb6252e64121 Mon Sep 17 00:00:00 2001 From: Jannes Stubbemann Date: Sat, 11 Jul 2026 00:19:48 +0200 Subject: [PATCH 05/11] feat(saas): checkout refill to a floor with capped backoff on failure Co-Authored-By: Claude Fable 5 Signed-off-by: Jannes Stubbemann --- internal/saas/controlplane/checkout.go | 95 +++++++++++++++++++++ internal/saas/controlplane/checkout_test.go | 86 +++++++++++++++++++ 2 files changed, 181 insertions(+) diff --git a/internal/saas/controlplane/checkout.go b/internal/saas/controlplane/checkout.go index 9ed21619..7b00e5dc 100644 --- a/internal/saas/controlplane/checkout.go +++ b/internal/saas/controlplane/checkout.go @@ -2,6 +2,8 @@ package controlplane import ( "context" + "encoding/json" + "fmt" "log/slog" "net/http" "sync" @@ -160,6 +162,99 @@ func (b *checkoutBuffer) stampOrg(ctx context.Context, ns, org string, e buffere return false } +// refillBackoffBase and refillBackoffCap bound the retry cadence after +// consecutive refill failures (the #894 lesson: a refill that cannot succeed +// must never spin at full cadence). +const ( + refillBackoffBase = time.Second + refillBackoffCap = time.Minute +) + +// reconcilePool is one pass of the buffer loop for one pool: count the +// cluster's buffered sandboxes and refill toward the floor, at most one +// create per pass so two replicas converge without a thundering herd. +func (b *checkoutBuffer) reconcilePool(ctx context.Context, pool string) { + ns := b.k.singleTenantNamespace + var list v1.SandboxList + if err := b.k.c.List(ctx, &list, client.InNamespace(ns), + client.MatchingLabels{BufferedLabelKey: "true", bufferedPoolLabelKey: pool}); err != nil { + slog.Warn("checkout buffer list failed", "pool", pool, "error", err.Error()) + return + } + count := len(list.Items) + if count >= b.cfg.Floor || count >= b.cfg.Cap { + return + } + b.mu.Lock() + wait := b.k.now().Before(b.nextRefill[pool]) + b.mu.Unlock() + if wait { + return + } + if err := b.refillOne(ctx, pool); err != nil { + b.mu.Lock() + b.refillFails[pool]++ + backoff := refillBackoffBase << (b.refillFails[pool] - 1) + if backoff <= 0 || backoff > refillBackoffCap { + backoff = refillBackoffCap + } + b.nextRefill[pool] = b.k.now().Add(backoff) + b.mu.Unlock() + slog.Warn("checkout refill failed; backing off", "pool", pool, "error", err.Error()) + return + } + b.mu.Lock() + b.refillFails[pool] = 0 + delete(b.nextRefill, pool) + b.mu.Unlock() +} + +// refillOne creates ONE buffered sandbox through the normal create machinery +// (watch-before-create and all): buffered + pool labels, NO org labels, so it +// bills nobody and matches no caller until checkout. +func (b *checkoutBuffer) refillOne(ctx context.Context, pool string) error { + ns := b.k.singleTenantNamespace + sb := &v1.Sandbox{ + ObjectMeta: metav1.ObjectMeta{ + Name: generateName(), + Namespace: ns, + Labels: map[string]string{BufferedLabelKey: "true", bufferedPoolLabelKey: pool}, + }, + Spec: v1.SandboxSpec{Source: v1.SandboxSource{PoolRef: &v1.LocalObjectReference{Name: pool}}}, + } + resp, err := b.k.createSandboxAndAwait(ctx, sb, b.k.now()) + if err != nil { + return err + } + if resp.Status != http.StatusCreated { + return fmt.Errorf("buffered create for pool %q did not become ready (status %d)", pool, resp.Status) + } + var payload struct { + ID string `json:"id"` + Endpoint string `json:"endpoint"` + Token string `json:"token"` + } + if err := json.Unmarshal(resp.Body, &payload); err != nil { + return fmt.Errorf("parse buffered create payload: %w", err) + } + // One off-hot-path read to snapshot the resourceVersion (the checkout + // patch's optimistic lock) and the backing pod name. + var cur v1.Sandbox + if err := b.k.c.Get(ctx, client.ObjectKey{Namespace: ns, Name: payload.ID}, &cur); err != nil { + return fmt.Errorf("snapshot buffered sandbox: %w", err) + } + b.add(bufferedSandbox{ + name: payload.ID, + pool: pool, + endpoint: payload.Endpoint, + token: payload.Token, + resourceVersion: cur.ResourceVersion, + podName: cur.Status.Pod, + createdAt: cur.CreationTimestamp.Time, + }) + return nil +} + // eligible reports whether this create may be served from the buffer: a // checkout-enabled pool and NOTHING tenant-specific that the activation // handshake would have had to deliver (env, secrets, workspace), no fan-out, diff --git a/internal/saas/controlplane/checkout_test.go b/internal/saas/controlplane/checkout_test.go index e39eb9c0..fdba0658 100644 --- a/internal/saas/controlplane/checkout_test.go +++ b/internal/saas/controlplane/checkout_test.go @@ -2,6 +2,7 @@ package controlplane import ( "context" + "errors" "net/http" "sync/atomic" "testing" @@ -212,6 +213,91 @@ func TestCheckoutEmptyBufferFallsBackClassic(t *testing.T) { } } +// TestRefillFillsToFloorAndStopsAtIt asserts reconcilePool creates buffered +// sandboxes (org-less, buffered+pool labels) until the cluster count reaches +// the floor, one per pass so two replicas converge without a thundering herd, +// and a pass at the floor creates none. +func TestRefillFillsToFloorAndStopsAtIt(t *testing.T) { + c := newFakeClient(t, poolInNs("mitos", "python")) + cp := New(c, + WithPollInterval(5*time.Millisecond), WithReadyTimeout(2*time.Second), + WithSingleTenantNamespace("mitos"), + WithCheckout(CheckoutConfig{Pools: []string{"python"}, Floor: 2, Cap: 4, MaxAge: 10 * time.Minute})) + + countBuffered := func() int { + var list v1.SandboxList + if err := c.List(context.Background(), &list, client.InNamespace("mitos"), + client.MatchingLabels{BufferedLabelKey: "true"}); err != nil { + t.Fatalf("list: %v", err) + } + return len(list.Items) + } + + for i := 0; i < 2; i++ { + stop := flipToReadyWhenCreatedInNs(t, c, "mitos", "10.0.0.5:9091", "tok-refill") + cp.checkout.reconcilePool(context.Background(), "python") + stop() + } + if n := countBuffered(); n != 2 { + t.Fatalf("buffered count after two passes = %d, want floor 2", n) + } + cp.checkout.reconcilePool(context.Background(), "python") + if n := countBuffered(); n != 2 { + t.Fatalf("buffered count after an at-floor pass = %d, want 2 (no over-refill)", n) + } + e, ok := cp.checkout.pop("python") + if !ok { + t.Fatal("refill did not cache a claimable entry") + } + if e.token != "tok-refill" || e.endpoint != "10.0.0.5:9091" || e.resourceVersion == "" { + t.Fatalf("cached entry incomplete: %+v", e) + } + // The buffered CRs carry NO org label: they bill nobody and match no caller. + var list v1.SandboxList + _ = c.List(context.Background(), &list, client.InNamespace("mitos"), client.MatchingLabels{BufferedLabelKey: "true"}) + for _, sb := range list.Items { + if sb.Labels[tenant.OrgLabelKey] != "" { + t.Fatalf("buffered sandbox %s carries an org label %q", sb.Name, sb.Labels[tenant.OrgLabelKey]) + } + } +} + +// TestRefillBacksOffAfterFailure asserts a failed refill sets a backoff: the +// next pass makes NO create attempt until the backoff expires (the #894 +// lesson: a refill that cannot succeed must never spin at full cadence). The +// create itself errors so no CR lingers to satisfy the floor by accident; +// only the backoff can explain the second pass staying quiet. +func TestRefillBacksOffAfterFailure(t *testing.T) { + var creates atomic.Int64 + base := fakeclient.NewClientBuilder(). + WithScheme(newScheme(t)). + WithStatusSubresource(&v1.Sandbox{}). + WithObjects(poolInNs("mitos", "python")). + Build() + c := interceptor.NewClient(base, interceptor.Funcs{ + Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + if _, ok := obj.(*v1.Sandbox); ok { + creates.Add(1) + return errors.New("apiserver unavailable") + } + return cl.Create(ctx, obj, opts...) + }, + }) + cp := New(c, + WithPollInterval(5*time.Millisecond), WithReadyTimeout(50*time.Millisecond), + WithSingleTenantNamespace("mitos"), + WithCheckout(CheckoutConfig{Pools: []string{"python"}, Floor: 1, Cap: 2, MaxAge: 10 * time.Minute})) + + cp.checkout.reconcilePool(context.Background(), "python") + if n := creates.Load(); n != 1 { + t.Fatalf("first pass made %d create attempts, want 1", n) + } + cp.checkout.reconcilePool(context.Background(), "python") + if n := creates.Load(); n != 1 { + t.Fatalf("pass during backoff made a create attempt (total %d), want none", n) + } +} + // TestCheckoutRequiresSingleTenantNamespace asserts the buffer is NOT // constructed under per-org namespacing: the design leans on the shared // namespace (see the spec's migration note) and must fail safe to the From c49fe58d618c568790a2df1c42b3cfc9595a1059 Mon Sep 17 00:00:00 2001 From: Jannes Stubbemann Date: Sat, 11 Jul 2026 00:26:23 +0200 Subject: [PATCH 06/11] feat(saas): checkout adopt-on-restart, staleness reap, and the reconcile loop A zero CreationTimestamp (which a real apiserver never produces) reads as age-unknown, not infinitely old, so the reap can never fire on it. Co-Authored-By: Claude Fable 5 Signed-off-by: Jannes Stubbemann --- internal/saas/controlplane/checkout.go | 106 +++++++++++++++++++- internal/saas/controlplane/checkout_test.go | 97 ++++++++++++++++++ 2 files changed, 199 insertions(+), 4 deletions(-) diff --git a/internal/saas/controlplane/checkout.go b/internal/saas/controlplane/checkout.go index 7b00e5dc..0acf9519 100644 --- a/internal/saas/controlplane/checkout.go +++ b/internal/saas/controlplane/checkout.go @@ -170,9 +170,10 @@ const ( refillBackoffCap = time.Minute ) -// reconcilePool is one pass of the buffer loop for one pool: count the -// cluster's buffered sandboxes and refill toward the floor, at most one -// create per pass so two replicas converge without a thundering herd. +// reconcilePool is one pass of the buffer loop for one pool: sync the cache +// against the cluster (the CRs are the truth: adopt unknown Ready entries, +// prune failed ones, reap over-age ones), then refill toward the floor, at +// most one create per pass so two replicas converge without a thundering herd. func (b *checkoutBuffer) reconcilePool(ctx context.Context, pool string) { ns := b.k.singleTenantNamespace var list v1.SandboxList @@ -181,7 +182,36 @@ func (b *checkoutBuffer) reconcilePool(ctx context.Context, pool string) { slog.Warn("checkout buffer list failed", "pool", pool, "error", err.Error()) return } - count := len(list.Items) + + known := make(map[string]bool) + b.mu.Lock() + for _, e := range b.entries[pool] { + known[e.name] = true + } + b.mu.Unlock() + + count := 0 + for i := range list.Items { + sb := &list.Items[i] + age := b.k.now().Sub(sb.CreationTimestamp.Time) + switch { + case !sb.CreationTimestamp.IsZero() && age > b.cfg.MaxAge: + // Bounded staleness: a buffered VM runs live while it waits, so + // entries past MaxAge are recycled, never handed to a tenant. + b.dropAndDelete(ctx, pool, sb) + case sb.Status.Phase == v1.SandboxFailed: + b.dropAndDelete(ctx, pool, sb) + case sb.Status.Phase != v1.SandboxReady: + // In flight (a refill mid-activation): neither adoptable nor + // reapable yet; it counts toward the floor so no over-refill. + count++ + default: + count++ + if !known[sb.Name] { + b.adopt(ctx, pool, sb) + } + } + } if count >= b.cfg.Floor || count >= b.cfg.Cap { return } @@ -209,6 +239,74 @@ func (b *checkoutBuffer) reconcilePool(ctx context.Context, pool string) { b.mu.Unlock() } +// dropAndDelete removes sb from the cache and deletes the CR (recycle). The +// delete tolerates NotFound: an idle or lifetime reaper getting there first +// is fine, the pool refills either way. +func (b *checkoutBuffer) dropAndDelete(ctx context.Context, pool string, sb *v1.Sandbox) { + b.mu.Lock() + q := b.entries[pool][:0] + for _, e := range b.entries[pool] { + if e.name != sb.Name { + q = append(q, e) + } + } + b.entries[pool] = q + b.mu.Unlock() + if err := b.k.c.Delete(ctx, sb); err != nil && !apierrors.IsNotFound(err) { + slog.Warn("checkout recycle failed", "sandbox", sb.Name, "error", err.Error()) + } +} + +// adopt rebuilds a cache entry from the cluster after a gateway restart (or +// for a CR the other replica refilled): endpoint and pod from status, token +// from the controller-owned Secret. A failed Secret read leaves the CR for +// the next pass rather than caching an unusable entry. +func (b *checkoutBuffer) adopt(ctx context.Context, pool string, sb *v1.Sandbox) { + token, err := b.k.readToken(ctx, sb.Namespace, sb.Name+tokenSecretSuffix) + if err != nil { + slog.Warn("checkout adopt could not read the token secret; leaving for the next pass", + "sandbox", sb.Name, "error", err.Error()) + return + } + b.add(bufferedSandbox{ + name: sb.Name, + pool: pool, + endpoint: sb.Status.Endpoint, + token: token, + resourceVersion: sb.ResourceVersion, + podName: sb.Status.Pod, + createdAt: sb.CreationTimestamp.Time, + }) +} + +// checkoutReconcileInterval paces the buffer loop; the hot path never waits +// on it (claim reads only the cache). +const checkoutReconcileInterval = 15 * time.Second + +// StartCheckout starts the buffer's reconcile loop for the server's +// lifetime. A no-op when the checkout feature is off. +func (k *K8sControlPlane) StartCheckout(ctx context.Context) { + if k.checkout == nil { + return + } + go k.checkout.run(ctx) +} + +func (b *checkoutBuffer) run(ctx context.Context) { + t := time.NewTicker(checkoutReconcileInterval) + defer t.Stop() + for { + for _, pool := range b.cfg.Pools { + b.reconcilePool(ctx, pool) + } + select { + case <-ctx.Done(): + return + case <-t.C: + } + } +} + // refillOne creates ONE buffered sandbox through the normal create machinery // (watch-before-create and all): buffered + pool labels, NO org labels, so it // bills nobody and matches no caller until checkout. diff --git a/internal/saas/controlplane/checkout_test.go b/internal/saas/controlplane/checkout_test.go index fdba0658..41bb08d5 100644 --- a/internal/saas/controlplane/checkout_test.go +++ b/internal/saas/controlplane/checkout_test.go @@ -8,6 +8,8 @@ import ( "testing" "time" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -32,6 +34,9 @@ func seedBuffered(t *testing.T, c client.Client, name string) bufferedSandbox { ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: "mitos", Labels: map[string]string{BufferedLabelKey: "true", bufferedPoolLabelKey: "python"}, + // The fake client does not stamp CreationTimestamp (a real + // apiserver always does), and the reap path needs a real age. + CreationTimestamp: metav1.Now(), }, Spec: v1.SandboxSpec{Source: v1.SandboxSource{PoolRef: &v1.LocalObjectReference{Name: "python"}}}, } @@ -298,6 +303,98 @@ func TestRefillBacksOffAfterFailure(t *testing.T) { } } +// TestReconcileAdoptsExistingBufferedSandboxes asserts a replica whose memory +// is empty (a restart, or the other replica refilled) rebuilds a claimable +// entry from the cluster: endpoint and pod from status, token from the +// controller-owned Secret. +func TestReconcileAdoptsExistingBufferedSandboxes(t *testing.T) { + c := newFakeClient(t, poolInNs("mitos", "python")) + cp := New(c, + WithPollInterval(5*time.Millisecond), WithReadyTimeout(2*time.Second), + WithSingleTenantNamespace("mitos"), + WithCheckout(CheckoutConfig{Pools: []string{"python"}, Floor: 1, Cap: 4, MaxAge: 10 * time.Minute})) + + e := seedBuffered(t, c, "sb-adopt") + sec := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "sb-adopt" + tokenSecretSuffix, Namespace: "mitos"}, + Data: map[string][]byte{"token": []byte("tok-sb-adopt"), "endpoint": []byte(e.endpoint)}, + } + if err := c.Create(context.Background(), sec); err != nil { + t.Fatalf("seed secret: %v", err) + } + + cp.checkout.reconcilePool(context.Background(), "python") + + got, ok := cp.checkout.pop("python") + if !ok { + t.Fatal("nothing adopted") + } + if got.name != "sb-adopt" || got.token != "tok-sb-adopt" || got.endpoint != e.endpoint || got.resourceVersion == "" { + t.Fatalf("adopted entry incomplete: %+v", got) + } +} + +// TestReconcilePrunesFailedBufferedSandboxes asserts a buffered CR that is +// not Ready and not merely in flight (Failed) is deleted and never cached. +func TestReconcilePrunesFailedBufferedSandboxes(t *testing.T) { + c := newFakeClient(t, poolInNs("mitos", "python")) + cp := New(c, + WithPollInterval(5*time.Millisecond), WithReadyTimeout(2*time.Second), + WithSingleTenantNamespace("mitos"), + WithCheckout(CheckoutConfig{Pools: []string{"python"}, Floor: 0, Cap: 4, MaxAge: 10 * time.Minute})) + // Floor 0 keeps this pass from refilling, isolating the prune assertion. + cp.checkout.cfg.Floor = 0 + + fail := &v1.Sandbox{ObjectMeta: metav1.ObjectMeta{ + Name: "sb-failed", Namespace: "mitos", + Labels: map[string]string{BufferedLabelKey: "true", bufferedPoolLabelKey: "python"}, + }, Spec: v1.SandboxSpec{Source: v1.SandboxSource{PoolRef: &v1.LocalObjectReference{Name: "python"}}}} + if err := c.Create(context.Background(), fail); err != nil { + t.Fatalf("seed: %v", err) + } + fail.Status.Phase = v1.SandboxFailed + if err := c.Status().Update(context.Background(), fail); err != nil { + t.Fatalf("status: %v", err) + } + + cp.checkout.reconcilePool(context.Background(), "python") + + var gone v1.Sandbox + if err := c.Get(context.Background(), client.ObjectKey{Namespace: "mitos", Name: "sb-failed"}, &gone); !apierrors.IsNotFound(err) { + t.Fatalf("failed buffered sandbox not deleted: err=%v", err) + } + if _, ok := cp.checkout.pop("python"); ok { + t.Fatal("a failed buffered sandbox was cached as claimable") + } +} + +// TestReconcileReapsOverAgeBufferedSandboxes asserts a buffered CR older than +// MaxAge is recycled (deleted and dropped), bounding how stale a handed-out +// sandbox can be. +func TestReconcileReapsOverAgeBufferedSandboxes(t *testing.T) { + c := newFakeClient(t, poolInNs("mitos", "python")) + cp := New(c, + WithPollInterval(5*time.Millisecond), WithReadyTimeout(2*time.Second), + WithSingleTenantNamespace("mitos"), + WithCheckout(CheckoutConfig{Pools: []string{"python"}, Floor: 1, Cap: 4, MaxAge: 10 * time.Minute})) + cp.checkout.cfg.Floor = 0 // no refill noise + + e := seedBuffered(t, c, "sb-old") + cp.checkout.add(e) + // Age is CreationTimestamp-based; step the control plane clock past MaxAge. + cp.now = func() time.Time { return time.Now().Add(11 * time.Minute) } + + cp.checkout.reconcilePool(context.Background(), "python") + + var gone v1.Sandbox + if err := c.Get(context.Background(), client.ObjectKey{Namespace: "mitos", Name: "sb-old"}, &gone); !apierrors.IsNotFound(err) { + t.Fatalf("over-age buffered sandbox not reaped: err=%v", err) + } + if _, ok := cp.checkout.pop("python"); ok { + t.Fatal("an over-age buffered sandbox stayed claimable") + } +} + // TestCheckoutRequiresSingleTenantNamespace asserts the buffer is NOT // constructed under per-org namespacing: the design leans on the shared // namespace (see the spec's migration note) and must fail safe to the From 9d3687cf3d82e4fab8d6d046c87bde47ab970040 Mon Sep 17 00:00:00 2001 From: Jannes Stubbemann Date: Sat, 11 Jul 2026 00:34:46 +0200 Subject: [PATCH 07/11] feat(controller): propagate claim org label to the husk pod for checkout billing The gateway's checkout patch stamps the org on a formerly buffered sandbox's CR; the reconcile that patch triggers copies it to the backing husk pod, whose TRUSTED label is what the usage scraper bills. Idempotent and a no-op for classic claims labeled at claim time; an org-less (still-buffered) claim never stamps anything. Co-Authored-By: Claude Fable 5 Signed-off-by: Jannes Stubbemann --- .../controller/husk_orgpropagation_test.go | 70 +++++++++++++++++++ internal/controller/huskdrain.go | 25 +++++++ .../controller/sandboxclaim_controller.go | 14 ++++ 3 files changed, 109 insertions(+) create mode 100644 internal/controller/husk_orgpropagation_test.go diff --git a/internal/controller/husk_orgpropagation_test.go b/internal/controller/husk_orgpropagation_test.go new file mode 100644 index 00000000..7c041f81 --- /dev/null +++ b/internal/controller/husk_orgpropagation_test.go @@ -0,0 +1,70 @@ +package controller + +import ( + "testing" + + "mitos.run/mitos/internal/tenant" +) + +// TestPropagateOrgToHuskPod is the claim-time org attribution unit for the +// pre-claimed checkout: the gateway stamps the org on a formerly buffered +// sandbox's CR labels, and the reconcile triggered by that patch must +// propagate the org to the backing husk pod, whose TRUSTED label is what the +// usage scraper bills. It must be idempotent and a no-op for classic claims +// whose pod was labeled at claim time. +func TestPropagateOrgToHuskPod(t *testing.T) { + // A checked-out claim: CR has the org, pod does not -> propagate. + c := rdyClaim() + c.Labels = tenant.OrgLabels("acme") + p := rdyPod(true) + if !propagateOrgToHuskPod(c, p) { + t.Fatal("org-labeled claim with an org-less pod must propagate") + } + if p.Labels[tenant.OrgLabelKey] != "acme" { + t.Fatalf("pod org label = %q, want acme", p.Labels[tenant.OrgLabelKey]) + } + + // Idempotent: a second pass changes nothing. + if propagateOrgToHuskPod(c, p) { + t.Error("already-propagated pod must not re-patch") + } + + // Classic claim: pod already labeled at claim time -> no-op. + c2 := rdyClaim() + c2.Labels = tenant.OrgLabels("acme") + p2 := rdyPod(true) + p2.Labels = map[string]string{tenant.OrgLabelKey: "acme"} + if propagateOrgToHuskPod(c2, p2) { + t.Error("matching labels must not report a change") + } + + // An org-less claim (a still-buffered sandbox) must never stamp anything. + c3 := rdyClaim() + p3 := rdyPod(true) + if propagateOrgToHuskPod(c3, p3) { + t.Error("org-less claim must not propagate") + } + if _, has := p3.Labels[tenant.OrgLabelKey]; has { + t.Error("org-less claim stamped a pod label") + } + + // A nil pod (lost, being repended) is a no-op, never a panic. + if propagateOrgToHuskPod(c, nil) { + t.Error("nil pod must be a no-op") + } + + // The pod org label is TRUSTED for billing: an existing DIFFERENT org on + // the pod is corrected to the claim's (the claim label is the authority + // the controller itself stamps and the gateway sets only at creation or + // checkout, both org-authenticated). + c4 := rdyClaim() + c4.Labels = tenant.OrgLabels("acme") + p4 := rdyPod(true) + p4.Labels = map[string]string{tenant.OrgLabelKey: "other"} + if !propagateOrgToHuskPod(c4, p4) { + t.Fatal("mismatched pod org must be corrected") + } + if p4.Labels[tenant.OrgLabelKey] != "acme" { + t.Fatalf("pod org label = %q, want acme", p4.Labels[tenant.OrgLabelKey]) + } +} diff --git a/internal/controller/huskdrain.go b/internal/controller/huskdrain.go index b5087a74..f79129ba 100644 --- a/internal/controller/huskdrain.go +++ b/internal/controller/huskdrain.go @@ -12,6 +12,8 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" + + "mitos.run/mitos/internal/tenant" ) // Husk pod loss and drain (issue #18, slice 4b). @@ -299,3 +301,26 @@ func reflectHuskBackingReadiness(claim *v1.Sandbox, pod *corev1.Pod, now time.Ti } return false } + +// propagateOrgToHuskPod copies a Ready claim's org label onto its backing +// husk pod when they differ, mutating pod and reporting whether it changed +// anything (so the caller patches only when needed). This is the claim-time +// org attribution leg of the pre-claimed checkout: the gateway's checkout +// patch stamps the CR, and the reconcile it triggers lands the org on the +// pod, whose TRUSTED label is what the usage scraper bills. An org-less +// claim (a still-buffered sandbox) never stamps anything, and a nil pod +// (lost, being repended) is a no-op. +func propagateOrgToHuskPod(claim *v1.Sandbox, pod *corev1.Pod) bool { + if pod == nil { + return false + } + org := claim.Labels[tenant.OrgLabelKey] + if org == "" || pod.Labels[tenant.OrgLabelKey] == org { + return false + } + if pod.Labels == nil { + pod.Labels = map[string]string{} + } + pod.Labels[tenant.OrgLabelKey] = org + return true +} diff --git a/internal/controller/sandboxclaim_controller.go b/internal/controller/sandboxclaim_controller.go index 08c279ed..1eafe8be 100644 --- a/internal/controller/sandboxclaim_controller.go +++ b/internal/controller/sandboxclaim_controller.go @@ -444,6 +444,20 @@ func (r *SandboxReconciler) reconcilePoolRef(ctx context.Context, claim *v1.Sand return ctrl.Result{}, err } } + // Claim-time org attribution (pre-claimed checkout): the gateway + // stamps the org on a formerly buffered sandbox's CR labels, and + // the reconcile that very patch triggers propagates the org to the + // backing husk pod, whose TRUSTED label is what the usage scraper + // bills. Idempotent, and a no-op for classic claims whose pod was + // labeled at claim time. + if lostPod != nil { + beforeOrg := lostPod.DeepCopy() + if propagateOrgToHuskPod(claim, lostPod) { + if err := r.Patch(ctx, lostPod, client.MergeFrom(beforeOrg)); err != nil { + return ctrl.Result{}, err + } + } + } } // A Ready claim bound to a workspace hydrates its head into the sandbox // exactly once (idempotent via the hydrated annotation). A transient From 7575fbaedc02ef616fd58444d763ca76388a110a Mon Sep 17 00:00:00 2001 From: Jannes Stubbemann Date: Sat, 11 Jul 2026 00:36:46 +0200 Subject: [PATCH 08/11] feat(saas): gateway flags and startup wiring for the pre-claimed checkout Co-Authored-By: Claude Fable 5 Signed-off-by: Jannes Stubbemann --- cmd/gateway/main.go | 63 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 61 insertions(+), 2 deletions(-) diff --git a/cmd/gateway/main.go b/cmd/gateway/main.go index a2f479b0..32ca93c7 100644 --- a/cmd/gateway/main.go +++ b/cmd/gateway/main.go @@ -27,6 +27,8 @@ import ( "net/http" "os" "os/signal" + "strconv" + "strings" "sync/atomic" "syscall" "time" @@ -75,7 +77,7 @@ func newScheme() *runtime.Scheme { // the poll interval governs only the fail-open fallback loop. The client is // returned alongside so main can wire the SAME client into the quota // enforcer's live sandbox counter. -func newControlPlane(readyTimeout, readyPollInterval time.Duration, defaultPool, singleTenantNS string) (saas.ControlPlane, client.Client, error) { +func newControlPlane(readyTimeout, readyPollInterval time.Duration, defaultPool, singleTenantNS string, checkout controlplane.CheckoutConfig) (*controlplane.K8sControlPlane, client.Client, error) { cfg, err := ctrl.GetConfig() if err != nil { return nil, nil, fmt.Errorf("load kubeconfig for the control-plane client: %w", err) @@ -92,9 +94,44 @@ func newControlPlane(readyTimeout, readyPollInterval time.Duration, defaultPool, opts = append(opts, controlplane.WithDefaultPool(defaultPool)) } opts = append(opts, controlplane.WithSingleTenantNamespace(singleTenantNS)) + if len(checkout.Pools) > 0 { + opts = append(opts, controlplane.WithCheckout(checkout)) + } return controlplane.New(c, opts...), c, nil } +// splitNonEmpty splits a comma-separated flag value into trimmed, non-empty +// items. +func splitNonEmpty(s string) []string { + var out []string + for _, p := range strings.Split(s, ",") { + if p = strings.TrimSpace(p); p != "" { + out = append(out, p) + } + } + return out +} + +// envInt and envDuration read an env default for a numeric flag; any parse +// problem keeps the compiled-in default. +func envInt(key string, def int) int { + if v := os.Getenv(key); v != "" { + if n, err := strconv.Atoi(v); err == nil { + return n + } + } + return def +} + +func envDuration(key string, def time.Duration) time.Duration { + if v := os.Getenv(key); v != "" { + if d, err := time.ParseDuration(v); err == nil { + return d + } + } + return def +} + func main() { addr := flag.String("addr", ":8080", "public listen address") metricsAddr := flag.String("metrics-addr", ":9100", "Prometheus metrics listen address (a SEPARATE cluster-internal listener; /metrics is never mounted on the public mux). Empty disables the metrics listener.") @@ -108,6 +145,10 @@ func main() { databaseDSN := flag.String("database-dsn", "", "Postgres DSN for durable persistence (accounts, orgs, memberships, API keys). Falls back to the "+pgstore.EnvDSN+" env var. Empty means in-memory persistence (DEV ONLY). The value is a secret and is never logged.") enforceQuota := flag.Bool("enforce-quota", true, "enforce per-organization quotas, rate limits, and the abuse kill-switch before forwarding. Default on (the hosted profile). Set to false only for a trusted single-tenant deployment; the bypass is logged at startup.") trustedProxyHops := flag.Int("trusted-proxy-hops", 0, "number of trusted reverse-proxy hops in front of the gateway for client-IP resolution. 0 (the default) does NOT trust X-Forwarded-For and uses the connection RemoteAddr. Set to the count of trusted proxies (for example 1 behind a single ingress) so the per-IP rate limit keys on the real client; a too-short or spoofed X-Forwarded-For fails closed to RemoteAddr.") + checkoutPools := flag.String("checkout-pools", os.Getenv("MITOS_GATEWAY_CHECKOUT_POOLS"), "comma-separated pool names served by the pre-claimed checkout buffer (empty disables the feature); requires --single-tenant-namespace") + checkoutFloor := flag.Int("checkout-floor", envInt("MITOS_GATEWAY_CHECKOUT_FLOOR", 2), "buffered sandboxes to keep ready per checkout pool") + checkoutCap := flag.Int("checkout-cap", envInt("MITOS_GATEWAY_CHECKOUT_CAP", 4), "hard ceiling of buffered sandboxes per checkout pool") + checkoutMaxAge := flag.Duration("checkout-max-age", envDuration("MITOS_GATEWAY_CHECKOUT_MAX_AGE", 10*time.Minute), "buffered sandboxes older than this are recycled") flag.Parse() logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelInfo})) @@ -144,11 +185,29 @@ func main() { logger.Warn("gateway running with the DEV stub control plane; no sandboxes are created (--allow-stub)") cp = stubControlPlane{} } else { - real, k8sClient, err := newControlPlane(*readyTimeout, *readyPollInterval, *defaultPool, *singleTenantNS) + checkoutCfg := controlplane.CheckoutConfig{ + Pools: splitNonEmpty(*checkoutPools), + Floor: *checkoutFloor, + Cap: *checkoutCap, + MaxAge: *checkoutMaxAge, + } + real, k8sClient, err := newControlPlane(*readyTimeout, *readyPollInterval, *defaultPool, *singleTenantNS, checkoutCfg) if err != nil { log.Fatalf("build control plane: %v", err) } cp = real + // The checkout buffer's refill/janitor loop runs for the server's + // lifetime; the hot path only ever reads its cache. Not being silent + // about a half-configured feature: --checkout-pools without + // --single-tenant-namespace leaves the buffer off by design (see the + // spec's per-org-namespace migration note), and that must be loud. + switch { + case len(checkoutCfg.Pools) > 0 && *singleTenantNS == "": + logger.Warn("checkout pools configured but --single-tenant-namespace is unset; the pre-claimed checkout stays OFF", "pools", *checkoutPools) + case len(checkoutCfg.Pools) > 0: + real.StartCheckout(context.Background()) + logger.Info("pre-claimed checkout enabled", "pools", *checkoutPools, "floor", *checkoutFloor, "cap", *checkoutCap, "max_age", checkoutMaxAge.String()) + } // The counter shares the control plane's client and namespace model // (per-org, or the pinned single-tenant namespace) so it counts exactly // where the control plane creates. From 1144336d2e5bd2bdf68fb7f708c88a29821e2759 Mon Sep 17 00:00:00 2001 From: Jannes Stubbemann Date: Sat, 11 Jul 2026 00:40:42 +0200 Subject: [PATCH 09/11] fix(controller): org propagation fills only an EMPTY pod label, never overrides The threat-model invariant is that control-plane-derived attribution (namespace-derived, or stamped at claim time) is never overridden by a claim label; a tenant-set CR label must not rebill a pod to another org. Co-Authored-By: Claude Fable 5 Signed-off-by: Jannes Stubbemann --- internal/controller/husk_orgpropagation_test.go | 16 ++++++++-------- internal/controller/huskdrain.go | 17 +++++++++++------ 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/internal/controller/husk_orgpropagation_test.go b/internal/controller/husk_orgpropagation_test.go index 7c041f81..8daff5f5 100644 --- a/internal/controller/husk_orgpropagation_test.go +++ b/internal/controller/husk_orgpropagation_test.go @@ -53,18 +53,18 @@ func TestPropagateOrgToHuskPod(t *testing.T) { t.Error("nil pod must be a no-op") } - // The pod org label is TRUSTED for billing: an existing DIFFERENT org on - // the pod is corrected to the claim's (the claim label is the authority - // the controller itself stamps and the gateway sets only at creation or - // checkout, both org-authenticated). + // The threat-model invariant: a claim label can FILL attribution where + // none derives, but can never OVERRIDE an existing pod org label (which + // is control-plane identity: namespace-derived or stamped at claim time). + // A tenant-set CR label must not rebill a pod to another org. c4 := rdyClaim() c4.Labels = tenant.OrgLabels("acme") p4 := rdyPod(true) p4.Labels = map[string]string{tenant.OrgLabelKey: "other"} - if !propagateOrgToHuskPod(c4, p4) { - t.Fatal("mismatched pod org must be corrected") + if propagateOrgToHuskPod(c4, p4) { + t.Fatal("an existing pod org label must never be rewritten") } - if p4.Labels[tenant.OrgLabelKey] != "acme" { - t.Fatalf("pod org label = %q, want acme", p4.Labels[tenant.OrgLabelKey]) + if p4.Labels[tenant.OrgLabelKey] != "other" { + t.Fatalf("pod org label = %q, want the original untouched", p4.Labels[tenant.OrgLabelKey]) } } diff --git a/internal/controller/huskdrain.go b/internal/controller/huskdrain.go index f79129ba..7d049809 100644 --- a/internal/controller/huskdrain.go +++ b/internal/controller/huskdrain.go @@ -303,11 +303,16 @@ func reflectHuskBackingReadiness(claim *v1.Sandbox, pod *corev1.Pod, now time.Ti } // propagateOrgToHuskPod copies a Ready claim's org label onto its backing -// husk pod when they differ, mutating pod and reporting whether it changed -// anything (so the caller patches only when needed). This is the claim-time -// org attribution leg of the pre-claimed checkout: the gateway's checkout -// patch stamps the CR, and the reconcile it triggers lands the org on the -// pod, whose TRUSTED label is what the usage scraper bills. An org-less +// husk pod when the pod has NONE, mutating pod and reporting whether it +// changed anything (so the caller patches only when needed). This is the +// claim-time org attribution leg of the pre-claimed checkout: the gateway's +// checkout patch stamps the CR, and the reconcile it triggers lands the org +// on the pod, whose TRUSTED label is what the usage scraper bills. +// +// It ONLY fills an empty pod label, never rewrites one: an existing pod org +// label is control-plane identity (namespace-derived, or stamped at claim +// time), and the threat-model invariant is that a claim label can fill +// attribution where none derives but can never override it. An org-less // claim (a still-buffered sandbox) never stamps anything, and a nil pod // (lost, being repended) is a no-op. func propagateOrgToHuskPod(claim *v1.Sandbox, pod *corev1.Pod) bool { @@ -315,7 +320,7 @@ func propagateOrgToHuskPod(claim *v1.Sandbox, pod *corev1.Pod) bool { return false } org := claim.Labels[tenant.OrgLabelKey] - if org == "" || pod.Labels[tenant.OrgLabelKey] == org { + if org == "" || pod.Labels[tenant.OrgLabelKey] != "" { return false } if pod.Labels == nil { From 4d6db6445fb2250bc9ab9dd455155b726c37a5ad Mon Sep 17 00:00:00 2001 From: Jannes Stubbemann Date: Sat, 11 Jul 2026 00:40:42 +0200 Subject: [PATCH 10/11] feat(chart): gateway.checkout values, RBAC patch verb, and the threat-model row Co-Authored-By: Claude Fable 5 Signed-off-by: Jannes Stubbemann --- .../charts/mitos/templates/gateway-rbac.yaml | 6 ++++ deploy/charts/mitos/templates/gateway.yaml | 6 ++++ deploy/charts/mitos/values.schema.json | 28 +++++++++++++++++ deploy/charts/mitos/values.yaml | 11 +++++++ .../2026-07-11-preclaimed-checkout-design.md | 30 +++++++++++-------- docs/threat-model.md | 1 + 6 files changed, 70 insertions(+), 12 deletions(-) diff --git a/deploy/charts/mitos/templates/gateway-rbac.yaml b/deploy/charts/mitos/templates/gateway-rbac.yaml index 50c5ee3d..c47a66cb 100644 --- a/deploy/charts/mitos/templates/gateway-rbac.yaml +++ b/deploy/charts/mitos/templates/gateway-rbac.yaml @@ -37,6 +37,12 @@ rules: - get - list - watch + # patch is the pre-claimed checkout's ONE hot-path write: the + # resourceVersion-guarded label patch that atomically drops the buffered + # label and stamps the org labels (attribution plus replica mutual + # exclusion). The gateway still gets no pod writes and no Secret writes; + # the controller propagates the org to the husk pod. + - patch - delete - apiGroups: - mitos.run diff --git a/deploy/charts/mitos/templates/gateway.yaml b/deploy/charts/mitos/templates/gateway.yaml index d28acab2..ff064bf3 100644 --- a/deploy/charts/mitos/templates/gateway.yaml +++ b/deploy/charts/mitos/templates/gateway.yaml @@ -40,6 +40,12 @@ spec: - --metrics-addr=:9100 - --enforce-quota={{ .Values.gateway.enforce.enabled }} - --trusted-proxy-hops={{ int .Values.gateway.enforce.trustedProxyHops }} + {{- if .Values.gateway.checkout.pools }} + - --checkout-pools={{ join "," .Values.gateway.checkout.pools }} + - --checkout-floor={{ int .Values.gateway.checkout.floor }} + - --checkout-cap={{ int .Values.gateway.checkout.cap }} + - --checkout-max-age={{ .Values.gateway.checkout.maxAge }} + {{- end }} {{- if or .Values.database.dsnSecretRef.name (and .Values.telemetry.enabled .Values.telemetry.endpoint) .Values.gateway.singleTenantNamespace .Values.saas.apiKeyPepperSecret.name }} env: {{- include "mitos.database.env" . | nindent 12 }} diff --git a/deploy/charts/mitos/values.schema.json b/deploy/charts/mitos/values.schema.json index 1f06266e..f5e9bce3 100644 --- a/deploy/charts/mitos/values.schema.json +++ b/deploy/charts/mitos/values.schema.json @@ -938,6 +938,34 @@ "description": "When non-empty, pins all sandbox operations to this fixed namespace instead of per-org namespaces.", "type": "string" }, + "checkout": { + "description": "Pre-claimed checkout: a buffer of already-activated sandboxes served to eligible creates with one attribution patch. Requires singleTenantNamespace; empty pools disables it.", + "type": "object", + "additionalProperties": false, + "properties": { + "pools": { + "description": "Pool names served by the checkout buffer; empty disables the feature.", + "type": "array", + "items": { + "type": "string" + } + }, + "floor": { + "description": "Buffered sandboxes to keep ready per checkout pool.", + "type": "integer", + "minimum": 1 + }, + "cap": { + "description": "Hard ceiling of buffered sandboxes per checkout pool.", + "type": "integer", + "minimum": 1 + }, + "maxAge": { + "description": "Buffered sandboxes older than this are recycled (Go duration).", + "type": "string" + } + } + }, "ingress": { "$ref": "#/definitions/ingress" } diff --git a/deploy/charts/mitos/values.yaml b/deploy/charts/mitos/values.yaml index 1d261569..49f6598b 100644 --- a/deploy/charts/mitos/values.yaml +++ b/deploy/charts/mitos/values.yaml @@ -766,6 +766,17 @@ gateway: # platform Secret or another tenant's object; GHSA-pgv2-9w24-j7wh). Use per-org # tenancy to enable secretRef and workspaces safely. singleTenantNamespace: "" + # Pre-claimed checkout: keep a small buffer of already-activated sandboxes + # per listed pool and serve eligible creates (no env, secrets, workspace, + # fan-out, or TTL) from it with a single attribution patch instead of the + # whole claim round trip. Requires singleTenantNamespace. Buffered sandboxes + # carry no org and bill nobody until claimed: deliberate platform cost, + # bounded by cap and maxAge. Default off (empty pools). + checkout: + pools: [] + floor: 2 + cap: 4 + maxAge: 10m ingress: enabled: false className: "" diff --git a/docs/superpowers/specs/2026-07-11-preclaimed-checkout-design.md b/docs/superpowers/specs/2026-07-11-preclaimed-checkout-design.md index 6c8aa108..de42a0ee 100644 --- a/docs/superpowers/specs/2026-07-11-preclaimed-checkout-design.md +++ b/docs/superpowers/specs/2026-07-11-preclaimed-checkout-design.md @@ -54,16 +54,21 @@ attribution patch. labels, so the usage scraper (trusted pod label) bills nobody and the console shows it to nobody; that unbilled window is deliberate platform cost. At checkout the CR patch stamps the org synchronously (runtime authz getOwned - needs it before the first exec arrives); the husk POD org label (billing) is - patched asynchronously with retry, and the janitor sweeps for org-labeled CRs - whose pod is still org-less so no claimed sandbox can dodge metering for more - than one sweep interval. Quota is checked before checkout, exactly where the - classic path checks it before the CR create. -7. **Reaping: the buffered label exempts an entry from idle reap; the janitor - terminates buffered sandboxes older than maxAge.** A buffered sandbox is idle - by definition, so without the exemption the idle reaper would drain the - buffer. maxAge bounds staleness (clock and CRNG were stepped at activation; - the VM runs live while buffered) and bounds leak size if refill misbehaves. + needs it before the first exec arrives). (Amended during implementation:) + the husk POD org label (billing) is propagated by the CONTROLLER, not the + gateway: the reconcile the checkout patch itself triggers copies the claim's + org onto the pod (`propagateOrgToHuskPod`), which ONLY fills an empty pod + label so control-plane-derived attribution is never overridden. The gateway + therefore needs no pod RBAC at all, and the trusted-label chain (controller + writes pod labels) is unchanged. Quota is checked before checkout, exactly + where the classic path checks it before the CR create. +7. **Reaping: the janitor terminates buffered sandboxes older than maxAge; no + idle-reap exemption is needed.** (Amended during implementation.) An idle or + lifetime reaper deleting a buffered sandbox is HARMLESS: the checkout patch + tolerates NotFound and moves to the next entry, the reconcile prunes the + cache, and the refill replaces the capacity. maxAge bounds staleness (clock + and CRNG were stepped at activation; the VM runs live while buffered) and + bounds leak size if refill misbehaves. 8. **Health: the reconcile prunes non-Ready buffer entries.** A husk that dies while buffered is dropped from the buffer at the next 15 s pass; a pop that races a just-died sandbox surfaces to the tenant like any post-create death @@ -158,7 +163,8 @@ feature is capability-neutral (no console surface change). prunes non-Ready, reaps over-age, re-patches org-less pods; refill respects floor/cap and backs off on failures; org labels land on CR before the response returns. -- kind e2e: a checkout-enabled pool serves a create from the buffer (observable - via the buffered label lifecycle) and a create with env falls back classic. +- (Amended during implementation:) no kind gateway job exists and standing one + up is out of scope; the hosted surface's proof is the unit suite plus prod + acceptance, matching how the rest of the gateway shipped. - Prod acceptance: bench/tti-latency.py before/after, plus one burst run to measure the fallback cost honestly. diff --git a/docs/threat-model.md b/docs/threat-model.md index 023bc225..005f1785 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -1774,6 +1774,7 @@ deliberate follow-ups, not this change. | Guest kernel integrity at stage time | partial | The Helm kernel-stage init container downloads the guest kernel and, when `kernelProvisioner.kernelSha256` is set, verifies the staged file against that digest and FAILS CLOSED on mismatch (covering both a fresh download and an already-cached file), so a swapped upstream object or a MITMed fetch cannot boot a backdoored kernel fleet-wide. The digest is empty by default (a warning is logged); operators should pin it. The privileged forkd, kvm-device-plugin, and kernel-stage pods set `automountServiceAccountToken: false` since none call the Kubernetes API. | | Admission-time signature enforcement | open | The project ships signatures; requiring them at admission (policy-controller/Kyverno) is a documented operator choice, not a default. | | Org-scoped usage attribution (billing) | **mitigated** | The org a sandbox bills to is a trust boundary: it is derived only from control-plane identity, never from client input. The controller stamps the `mitos.run/org` label on a sandbox's husk pod from the sandbox's hard-isolation namespace `mitos-org-` via `tenant.OrgFromNamespace` (`internal/controller/huskpod.go`); a client-set `mitos.run/org` on the input object is overwritten, so a tenant cannot relabel its workload to another org (an adversarial test asserts the namespace value wins). In a SHARED (non-org) namespace, where pod creation derives no org, the controller additionally copies the Sandbox object's own `mitos.run/org` label onto the pod at CLAIM time (issue #602; `stampClaimOrgLabel`), and releases it again when a failed activation returns the pod to the dormant pool: that label is stamped by the hosted gateway on every Sandbox it creates, so in the hosted single-tenant deployment it is control-plane identity too. The precedence is fail-safe: a namespace-derived org is NEVER overridden by a claim label (unit-tested), so a tenant with direct access to its org namespace cannot bill another org via a Sandbox label; the claim label only fills attribution where the namespace derives none. The live usage scraper (`internal/usage/livesource.go`, run behind the off-by-default `--usage-collector` flag) reads org only through that controller-stamped label (`LabelOrgResolver`), and the per-sandbox metering report it pulls carries only ids, byte counts, and seconds, never secret values, argv, env, or file content. A self-host sandbox in a non-org namespace is left unattributed rather than misbilled to a default org. Usage aggregation is idempotent on (org, sandbox, window), so a re-scrape or a credit drawdown cannot double-count. The per-org Prometheus series is labeled by org only (no per-sandbox cardinality, no identifiers beyond the org id). The org/pool-labeled metering gauges (`mitos_usage_{vcpu_seconds,mem_gib_seconds,egress_bytes,gpu_seconds}_total`) are fed from the SAME store-fed cumulative, so the dashboard figure and the bill are one number; their only label is org (pool is a documented follow-up since the store keys on (org, sandbox, window)). Residual: the durable Postgres usage store (issue #211, `pgstore.PgUsageStore`) is the billing system of record and is wired via the chart's `database.dsnSecretRef`; without a DSN the in-memory store is bounded by a retention window and survives no controller restart (DEV ONLY). The real billing-provider push remains the SaaS follow-up. | +| Pre-claimed checkout buffer (gateway, org-less Ready sandboxes) | **mitigated (default off)** | When `gateway.checkout.pools` names a pool (requires single-tenant namespace mode; default off), the gateway keeps a small buffer of REAL, already-activated Sandboxes that carry `mitos.run/buffered: "true"` and NO org labels, and serves eligible creates from it. New states this adds, and why each fails safe: (1) BEFORE checkout a buffered sandbox matches no caller's org, so every gateway runtime and lifecycle op collapses to `not_found` (`getOwned` re-checks the org label); its endpoint is cluster-internal and bearer-token gated exactly like every Ready sandbox, and the token exists only in the controller-owned Secret and gateway memory (never logged), so the buffered window widens token CUSTODY DURATION (bounded by `checkout.maxAge`, default 10m) but not its exposure class. (2) NO tenant data can exist in a buffered VM: eligibility excludes env, secrets, and workspaces, and the fork-correctness activation handshake already ran with empty tenant inputs (fresh entropy and clock step included). (3) The checkout itself is ONE resourceVersion-guarded label patch that atomically drops the buffered label and stamps the org labels, so two gateway replicas can never hand the same sandbox to two orgs (the loser 409s and falls back), and runtime authz is satisfied before the caller's first exec can arrive. The gateway ClusterRole gains `patch` on sandboxes for exactly this write; it still has NO pod writes and NO Secret writes. (4) BILLING attribution at claim time: a buffered sandbox bills nobody (the usage scraper reads only the trusted pod org label, absent by construction), which is deliberate bounded platform cost (`floor`/`cap`/`maxAge`); at checkout the controller reconcile triggered by the org patch propagates the org to the husk pod via `propagateOrgToHuskPod`, which ONLY fills an EMPTY pod label, preserving the invariant above that control-plane-derived attribution is never overridden by a claim label. (5) A gateway restart re-adopts buffered CRs by LIST (the CRs are the source of truth); over-age or non-Ready entries are recycled by the janitor, so nothing outlives `maxAge`. Residual: the eligibility gate and the org-less window exist only under the single-tenant namespace; the per-org-namespace migration story is recorded in the design spec (docs/superpowers/specs/2026-07-11-preclaimed-checkout-design.md) and the buffer refuses to construct outside single-tenant mode. | | Internal usage API (controller `GET /internal/usage`, issue #602) | **mitigated** | The machine-to-machine HTTP surface the hosted console reads per-org usage through, now deployable via the chart (`controller.usage.collector`, off by default: a self-host install renders no listener, no Service, no env). Exposure is a ClusterIP Service (`mitos-controller-usage`) only; the chart never exposes it outside the cluster. AUTH fails closed: the endpoint requires the `MITOS_USAGE_API_TOKEN` bearer (constant-time compare, `internal/usage/internalapi.go`), and an EMPTY token refuses every request rather than serving unauthenticated. The token is a SECRET sourced via secretKeyRef on both the controller and the console (`controller.usage.tokenSecret`); it is never rendered inline, never logged. The org comes from the `X-Mitos-Org` header set by the console AFTER it verified the session (the same trust pattern as the identity-resolve endpoint), and the store scopes every read to that org, so even a forged header only ever returns that one org's records. PAYLOAD hygiene: records carry ids, byte counts, and seconds only; never secret values, argv, env, or file content. The listener serves on every controller replica (not leader-gated) so the Service round-robins safely; with the in-memory store (DEV ONLY) a non-leader read can be empty, which is a staleness, never a cross-org bleed. Residual: the hop is plaintext HTTP inside the cluster (the same in-cluster access class as the other internal M2M endpoints); in-cluster mTLS for it is a documented follow-up. | | Guest vitals telemetry surface (issue #164 Phase 1.a) | **mitigated** | The guest health metrics (`mitos_guest_{cpu_steal_percent,mem_balloon_bytes,mem_used_bytes,process_count}`) are sampled by an off-by-default control-plane runnable (`internal/controller/vitals_sampler.go`, `--vitals-sampler` flag). A NEW node-scoped operational endpoint `GET /v1/vitals/node` (`internal/daemon/vitals_api.go`, mounted on the forkd operational mux next to `/v1/metering`, `/metrics`, `/healthz`) is UNAUTHENTICATED (the same access class as `/metrics`), so it returns one numeric-only entry per sandbox on that node. SECRET HYGIENE: each entry carries ONLY the control-plane labels (claim, pool, workspace, namespace; all k8s object names), the numeric guest vitals (steal, balloon, used/total memory), and a numeric process COUNT (`process_count`). It carries NO per-process table at all: no process command name, pid, state, rss, argv, or env crosses this unauthenticated wire. This is enforced at the source by a dedicated node-batch struct (`NodeVitalsNumbers`) that has no field for a process object, so a per-process string CANNOT be serialized here; `handleNodeVitals` writes only `len(processes)` as the count. The FULL per-process table (program names, pid, state, rss) remains available ONLY behind the per-sandbox bearer-authenticated Connect `sandbox.v1.Sandbox.Vitals` RPC (used by `kubectl mitos ps --processes`; the legacy `/v1/vitals` JSON route was removed in #358), never on this node batch. The sampler decodes ONLY the numeric fields including `process_count`, and NEVER receives or relies on a per-process command line, argv, pid, env value, or any free-form string. The published metric label set is EXACTLY {org, pool}, both bounded trusted control-plane values, so there is no per-sandbox/per-pid cardinality and no identifier beyond org+pool. Org is resolved via the SAME trusted `mitos.run/org` husk-pod label (`LabelOrgResolver`) the usage scraper uses; a sandbox with no resolvable org is left unattributed (counted), never attributed to a guessed or empty org. The node endpoint is NOT per-sandbox traffic, so it carries no per-sandbox bearer token (the controller holds none); it shares the access class of `/v1/metering`. A sandbox whose guest is unreachable is skipped and counted, never failing the report or the sample cycle. Residual: the endpoint is plaintext on the operational mux today (an https operational mux is the same documented follow-up as `/v1/metering`); the cpu_steal aggregation is the per-bucket MAX and memory/process_count the SUM, documented in each gauge's Help text. | From 32317c2ed779836207b0751aa95ced8102c45c3d Mon Sep 17 00:00:00 2001 From: Jannes Stubbemann Date: Sat, 11 Jul 2026 01:48:04 +0200 Subject: [PATCH 11/11] test(saas): review follow-ups, flag-helper tests, error wrap, spec data-flow sync Co-Authored-By: Claude Fable 5 Signed-off-by: Jannes Stubbemann --- cmd/gateway/checkout_flags_test.go | 56 +++++++++++++++++++ .../2026-07-11-preclaimed-checkout-design.md | 6 +- internal/saas/controlplane/checkout.go | 2 +- 3 files changed, 61 insertions(+), 3 deletions(-) create mode 100644 cmd/gateway/checkout_flags_test.go diff --git a/cmd/gateway/checkout_flags_test.go b/cmd/gateway/checkout_flags_test.go new file mode 100644 index 00000000..a0d9623c --- /dev/null +++ b/cmd/gateway/checkout_flags_test.go @@ -0,0 +1,56 @@ +package main + +import ( + "testing" + "time" +) + +// TestSplitNonEmpty pins the checkout-pools flag parsing: comma separation, +// whitespace trimming, and empty items dropped (so a trailing comma or a bare +// value never enables a pool named ""). +func TestSplitNonEmpty(t *testing.T) { + cases := []struct { + in string + want []string + }{ + {"", nil}, + {" , ,", nil}, + {"python", []string{"python"}}, + {"python, node ,", []string{"python", "node"}}, + } + for _, tc := range cases { + got := splitNonEmpty(tc.in) + if len(got) != len(tc.want) { + t.Fatalf("splitNonEmpty(%q) = %v, want %v", tc.in, got, tc.want) + } + for i := range got { + if got[i] != tc.want[i] { + t.Fatalf("splitNonEmpty(%q) = %v, want %v", tc.in, got, tc.want) + } + } + } +} + +// TestEnvHelpersKeepDefaultsOnGarbage pins that a malformed env value keeps +// the compiled-in default instead of zeroing a limit. +func TestEnvHelpersKeepDefaultsOnGarbage(t *testing.T) { + t.Setenv("TEST_CHECKOUT_INT", "not-a-number") + if got := envInt("TEST_CHECKOUT_INT", 7); got != 7 { + t.Fatalf("envInt(garbage) = %d, want the default 7", got) + } + t.Setenv("TEST_CHECKOUT_INT", "3") + if got := envInt("TEST_CHECKOUT_INT", 7); got != 3 { + t.Fatalf("envInt(3) = %d, want 3", got) + } + t.Setenv("TEST_CHECKOUT_DUR", "soon") + if got := envDuration("TEST_CHECKOUT_DUR", time.Minute); got != time.Minute { + t.Fatalf("envDuration(garbage) = %s, want the default 1m", got) + } + t.Setenv("TEST_CHECKOUT_DUR", "90s") + if got := envDuration("TEST_CHECKOUT_DUR", time.Minute); got != 90*time.Second { + t.Fatalf("envDuration(90s) = %s, want 90s", got) + } + if got := envInt("TEST_CHECKOUT_UNSET_KEY", 5); got != 5 { + t.Fatalf("envInt(unset) = %d, want 5", got) + } +} diff --git a/docs/superpowers/specs/2026-07-11-preclaimed-checkout-design.md b/docs/superpowers/specs/2026-07-11-preclaimed-checkout-design.md index de42a0ee..e6b7a058 100644 --- a/docs/superpowers/specs/2026-07-11-preclaimed-checkout-design.md +++ b/docs/superpowers/specs/2026-07-11-preclaimed-checkout-design.md @@ -94,13 +94,15 @@ Checkout (the hot path): path. 3. Return 201 {id, endpoint, token, phase Ready} from cache. fork_time_ms is the observed wall time of this request, honestly tiny. -4. Async: patch the husk pod org label (billing); kick refill. +4. The CR patch's watch event triggers the controller's reconcile, which + propagates the org onto the backing husk pod (billing); refill happens on + the buffer's own periodic reconcile pass, not from this request. Adopt (gateway start): LIST buffered sandboxes, read each token Secret, rebuild the memory cache. Janitor (each reconcile pass): prune non-Ready entries; terminate entries older -than maxAge; re-patch pods for claimed CRs still missing the pod org label. +than maxAge. ## Security diff --git a/internal/saas/controlplane/checkout.go b/internal/saas/controlplane/checkout.go index 0acf9519..b59c4082 100644 --- a/internal/saas/controlplane/checkout.go +++ b/internal/saas/controlplane/checkout.go @@ -322,7 +322,7 @@ func (b *checkoutBuffer) refillOne(ctx context.Context, pool string) error { } resp, err := b.k.createSandboxAndAwait(ctx, sb, b.k.now()) if err != nil { - return err + return fmt.Errorf("buffered create for pool %q: %w", pool, err) } if resp.Status != http.StatusCreated { return fmt.Errorf("buffered create for pool %q did not become ready (status %d)", pool, resp.Status)