From da6ed5f9b3d87f11b802d7b7da4fc71990c15695 Mon Sep 17 00:00:00 2001 From: sumansaurabh Date: Sat, 18 Jul 2026 12:47:52 +0530 Subject: [PATCH] feat(isolate): per-sandbox egress attribution via per-slot service pool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flip isolate egress from fail-closed deny-all to enforced per-sandbox allowlists. workerd only accepts a static service binding as a loaded worker's globalOutbound — a JS-object Fetcher ("not of type 'Fetcher'") and a dynamically-loaded worker stub ("cannot be transferred") are both rejected — so attribution is carried by WHICH static egress service the sandbox binds. The group config pre-declares a pool of K per-slot egress services (one host UDS each); the host assigns each egressing sandbox a slot and returns egress_slot in the bundle spec; the controller sets globalOutbound: env["EGRESS_"+slot]. The loaded worker is given no other binding, so it can reach only its own slot — attribution is structural and unforgeable (a forged x-sb-id on the outbound is irrelevant). Block-all and pool-exhausted sandboxes bind a fail-closed EGRESS_DENY service; spill is logged, never silent. Mechanism proven by spike vs real workerd, validated offline + against real workerd + live on AWS (allow->200/deny->403/block->403 per-sandbox in one group; SSRF guard blocks EC2 IAM metadata + loopback for an allow-all sandbox while public egress flows). SB_ISOLATE_EGRESS_POOL_SIZE (default 64) caps concurrently-egressing sandboxes per group; workerd dials external services lazily, so the pool is cheap config and the host binds a slot socket only on assignment. Co-Authored-By: Claude Opus 4.8 --- internal/config/config.go | 6 + internal/runtime/isolate/config.go | 4 + internal/runtime/isolate/hostsupervisor.go | 21 +-- .../isolate/p0_gate_integration_test.go | 11 +- .../isolate/phase3_integration_test.go | 94 ++++++----- pkg/isolate/config.go | 80 ++++++---- pkg/isolate/egress.go | 151 ++++++++++++++---- pkg/isolate/egress_test.go | 60 +++++-- pkg/isolate/host.go | 112 ++++++++++--- pkg/isolate/host_test.go | 117 ++++++++++---- plans/isolate-runtime.md | 52 ++++-- 11 files changed, 515 insertions(+), 193 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index ac8ad373..efc74cb1 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -674,6 +674,11 @@ type Config struct { // IsolateBundleGCInterval is how often unreferenced content-addressed // bundles are swept. Zero disables. Default 15m. SB_ISOLATE_BUNDLE_GC_INTERVAL. IsolateBundleGCInterval time.Duration + // IsolateEgressPoolSize is the per-group egress-slot pool size (§4): the cap + // on concurrently-egressing sandboxes in a group (block-all sandboxes cost no + // slot). A sandbox beyond the pool falls back to deny-all egress (logged). + // Default 64. SB_ISOLATE_EGRESS_POOL_SIZE. + IsolateEgressPoolSize int // FirecrackerBinary is the absolute path to the `firecracker` VMM // binary on this host. Required only when EnableFirecracker is true. // Default /usr/local/bin/firecracker matches a typical install. @@ -1631,6 +1636,7 @@ func Load() (Config, error) { IsolatePoolDepthDefault: getEnvInt("SB_ISOLATE_POOL_DEPTH_DEFAULT", 2), IsolatePoolRefillInterval: getEnvDuration("SB_ISOLATE_POOL_REFILL_INTERVAL", 5*time.Second), IsolateBundleGCInterval: getEnvDuration("SB_ISOLATE_BUNDLE_GC_INTERVAL", 15*time.Minute), + IsolateEgressPoolSize: getEnvInt("SB_ISOLATE_EGRESS_POOL_SIZE", 64), FirecrackerBinary: getEnv("SB_FIRECRACKER_BINARY", "/usr/local/bin/firecracker"), JailerBinary: getEnv("SB_JAILER_BINARY", "/usr/local/bin/jailer"), FirecrackerKernelImage: getEnv("SB_FIRECRACKER_KERNEL", "/var/lib/sandboxd/firecracker/vmlinux"), diff --git a/internal/runtime/isolate/config.go b/internal/runtime/isolate/config.go index 1f6696ce..9db1b80d 100644 --- a/internal/runtime/isolate/config.go +++ b/internal/runtime/isolate/config.go @@ -37,6 +37,9 @@ type Config struct { // IdleTTL is how long a group may sit without Create/Invoke before the // idle reaper tears it down. Zero disables the reaper. IdleTTL time.Duration + // EgressPoolSize is the per-group egress-slot pool size (§4): the cap on + // concurrently-egressing sandboxes in a group. Zero → pkg/isolate default. + EgressPoolSize int } // FromDaemonConfig projects the isolate slice of daemon config into driver config. @@ -51,5 +54,6 @@ func FromDaemonConfig(cfg config.Config) Config { JailGID: cfg.IsolateJailGID, Jitless: cfg.IsolateJitless, IdleTTL: cfg.IsolateGroupIdleTTL, + EgressPoolSize: cfg.IsolateEgressPoolSize, } } diff --git a/internal/runtime/isolate/hostsupervisor.go b/internal/runtime/isolate/hostsupervisor.go index f71b1c32..bbfdc37f 100644 --- a/internal/runtime/isolate/hostsupervisor.go +++ b/internal/runtime/isolate/hostsupervisor.go @@ -12,25 +12,28 @@ import ( // bundle-server). It lives in the driver package so pkg/isolate need not know // the driver's seam. type workerdSupervisor struct { - workerdPath string - runDir string - useJail bool + workerdPath string + runDir string + useJail bool + egressPoolSize int } // NewHostSupervisor builds the production supervisor over the isolate config. func NewHostSupervisor(cfg Config) HostSupervisor { return &workerdSupervisor{ - workerdPath: cfg.WorkerdPath, - runDir: cfg.RunDir, - useJail: cfg.UseJail, + workerdPath: cfg.WorkerdPath, + runDir: cfg.RunDir, + useJail: cfg.UseJail, + egressPoolSize: cfg.EgressPoolSize, } } func (s *workerdSupervisor) SpawnGroup(ctx context.Context, spec JailSpec) (GroupHost, error) { host, err := pkgisolate.NewHost(pkgisolate.HostConfig{ - WorkerdPath: s.workerdPath, - GroupKey: spec.GroupKey, - RunDir: filepath.Join(s.runDir, spec.GroupKey), + WorkerdPath: s.workerdPath, + GroupKey: spec.GroupKey, + RunDir: filepath.Join(s.runDir, spec.GroupKey), + EgressPoolSize: s.egressPoolSize, // When UseJail is set, the host MUST realize this spec or fail closed. Jail: pkgisolate.JailConfig{ Require: s.useJail, diff --git a/internal/runtime/isolate/p0_gate_integration_test.go b/internal/runtime/isolate/p0_gate_integration_test.go index f3868be2..7c97af27 100644 --- a/internal/runtime/isolate/p0_gate_integration_test.go +++ b/internal/runtime/isolate/p0_gate_integration_test.go @@ -126,11 +126,12 @@ func TestP0HostileIsolateContainment(t *testing.T) { t.Fatalf("create good: %v", err) } - // (b) Undeclared egress is refused: the Go proxy denies with 403 (surfaced to - // the isolate as "egress-ok:403") or the fetch throws ("egress-err:*"). A 2xx - // upstream ("egress-ok:2xx") would mean egress actually leaked. (Egress is - // deny-all today — attribution is a §4 follow-up — so every destination is - // refused, which subsumes this allowlist check.) + // (b) Undeclared egress is refused: egress is attributed per-sandbox by slot + // (§4), so this sandbox's outbound is enforced against ITS allowlist + // (127.0.0.1/32) — example.invalid is not in it, so the Go proxy denies with + // 403 (surfaced as "egress-ok:403") or the fetch throws ("egress-err:*"). And + // even a 127.0.0.1 destination would be refused by the SSRF IP-range guard. A + // 2xx upstream ("egress-ok:2xx") would mean the capability boundary leaked. req, _ := http.NewRequestWithContext(ctx, http.MethodGet, "http://isolate/egress", nil) resp, err := d.InvokeHTTP(ctx, "sb-hostile", req) if err != nil { diff --git a/internal/runtime/isolate/phase3_integration_test.go b/internal/runtime/isolate/phase3_integration_test.go index c9cfc886..1e8161ce 100644 --- a/internal/runtime/isolate/phase3_integration_test.go +++ b/internal/runtime/isolate/phase3_integration_test.go @@ -8,6 +8,7 @@ import ( "io" "net/http" "net/http/httptest" + "net/url" "os" "path/filepath" "strings" @@ -116,65 +117,80 @@ func TestPhase3IngressPortGatewayEndToEnd(t *testing.T) { } } -// TestPhase3EgressDeniedByDefault: an isolate's outbound fetch must never reach -// upstream. Per-sandbox x-sb-id attribution via a loaded shim is not currently -// expressible (workerd rejects a dynamically-loaded worker's entrypoint as -// another worker's globalOutbound), so globalOutbound points at the static -// egress service and the Go proxy fail-closed DENIES every request (no -// attribution) — the safe Phase-2 deny-all posture. A denied fetch surfaces to -// the isolate as a 403 upstream status (body "ok:403") or a thrown error -// ("err:*"); a 2xx upstream would mean egress leaked. -func TestPhase3EgressAttributionEndToEnd(t *testing.T) { +// TestPhase3PerSandboxEgressEndToEnd proves the §4 redesign live against real +// workerd: egress is attributed PER SANDBOX by slot, so two sandboxes in the +// SAME tenant group get DIFFERENT policies enforced. An allow-listed sandbox +// reaches an allowed host and is refused a non-allowed one; a block-all sandbox +// in the same group is refused everything. Attribution is the slot socket — a +// forged x-sb-id on the outbound is irrelevant. +// +// A policy denial surfaces to the isolate as an upstream 403 from the egress +// proxy (body "status=403"); an allowed fetch is 200 (reachable) or 502/throw +// (allowed but unreachable), never 403 — so the allow assertion holds even where +// the runner has no outbound internet. +func TestPhase3PerSandboxEgressEndToEnd(t *testing.T) { workerd := requireWorkerd(t) runDir := shortRunDir(t) d := phase3Driver(t, workerd, runDir) - ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() bundleDir := t.TempDir() src := `export default { async fetch(req) { - const url = new URL(req.url); - if (url.pathname === "/egress") { - try { - const r = await fetch("https://example.invalid/"); - return new Response("ok:" + r.status); - } catch (e) { - return new Response("err:" + (e && e.message ? e.message : String(e)), { status: 502 }); - } + const u = new URL(req.url); + try { + const r = await fetch(u.searchParams.get("t")); + return new Response("status=" + r.status); + } catch (e) { + return new Response("throw=" + (e && e.message ? e.message : String(e))); } - return new Response("hi"); }};` p := filepath.Join(bundleDir, "eg.js") if err := os.WriteFile(p, []byte(src), 0o644); err != nil { t.Fatal(err) } + ref := "file://" + p + // Two sandboxes in ONE tenant group with DIFFERENT egress policies. if _, err := d.Create(ctx, models.CreateSandboxRequest{ - Runtime: models.RuntimeIsolate, - ModuleRef: "file://" + p, - TenantID: "eg", - MemoryMB: 128, + Runtime: models.RuntimeIsolate, ModuleRef: ref, TenantID: "eg-tenant", MemoryMB: 128, + NetworkAllowOut: []string{"example.com"}, + }, "sb-allow", "", nil); err != nil { + t.Fatalf("create allow: %v", err) + } + t.Cleanup(func() { _ = d.Destroy(context.Background(), &models.Sandbox{ID: "sb-allow"}) }) + if _, err := d.Create(ctx, models.CreateSandboxRequest{ + Runtime: models.RuntimeIsolate, ModuleRef: ref, TenantID: "eg-tenant", MemoryMB: 128, NetworkBlockAll: true, - }, "sb-eg", "", nil); err != nil { - t.Fatalf("create: %v", err) + }, "sb-block", "", nil); err != nil { + t.Fatalf("create block: %v", err) } - t.Cleanup(func() { _ = d.Destroy(context.Background(), &models.Sandbox{ID: "sb-eg"}) }) + t.Cleanup(func() { _ = d.Destroy(context.Background(), &models.Sandbox{ID: "sb-block"}) }) - req, _ := http.NewRequestWithContext(ctx, http.MethodGet, "http://isolate/egress", nil) - resp, err := d.InvokeHTTP(ctx, "sb-eg", req) - if err != nil { - t.Fatalf("invoke: %v", err) + invoke := func(id, target string) string { + u := "http://isolate/?t=" + url.QueryEscape(target) + req, _ := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + resp, err := d.InvokeHTTP(ctx, id, req) + if err != nil { + t.Fatalf("invoke %s: %v", id, err) + } + b, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + return string(b) } - body, _ := io.ReadAll(resp.Body) - _ = resp.Body.Close() - // Blocked egress is "ok:403" (proxy denied) or "err:*" (fetch threw). Only a - // 2xx upstream status means egress actually leaked to example.invalid. - got := string(body) - if strings.HasPrefix(got, "ok:2") { - t.Fatalf("egress reached upstream (body=%q) — deny-all policy broken", got) + + // Allowed sandbox → allowed host: PERMITTED (not a policy 403). + if got := invoke("sb-allow", "https://example.com/"); strings.Contains(got, "status=403") { + t.Fatalf("allow→example.com = %q, want it permitted (not a 403 policy denial)", got) + } + // Allowed sandbox → NON-allowed host: DENIED by its own allowlist. + if got := invoke("sb-allow", "https://not-allowed.example/"); !strings.Contains(got, "status=403") { + t.Fatalf("allow→not-allowed = %q, want status=403 (allowlist enforced)", got) } - if !strings.HasPrefix(got, "ok:") && !strings.HasPrefix(got, "err:") { - t.Fatalf("unexpected egress body %q", got) + // Block-all sandbox in the SAME group → allowed host still DENIED (proving + // per-sandbox differentiation, not a group-wide policy). + if got := invoke("sb-block", "https://example.com/"); !strings.Contains(got, "status=403") { + t.Fatalf("block→example.com = %q, want status=403 (block-all)", got) } } diff --git a/pkg/isolate/config.go b/pkg/isolate/config.go index 8aa0a594..83336c0f 100644 --- a/pkg/isolate/config.go +++ b/pkg/isolate/config.go @@ -30,10 +30,11 @@ const controllerModuleName = "controller.js" // - rejects requests with no x-sb-id (the driver always sets it); // - dynamically loads the isolate for that id, fetching the bundle from the // HOST binding on a cache miss; -// - attributes egress per sandbox: each sandbox's globalOutbound is a tiny -// shim isolate (also via LOADER) that stamps x-sb-id before hitting the -// shared EGRESS service, so the Go proxy knows ownership at accept time -// (plans/isolate-runtime.md §4 Phase 3); +// - attributes egress per sandbox by SLOT: the host assigns each egressing +// sandbox an egress slot and returns it in the bundle spec, so the loaded +// worker's globalOutbound binds to that slot's dedicated static egress +// service (env["EGRESS_"+slot]) — one host UDS per slot, ownership known at +// accept time (plans/isolate-runtime.md §4); // - strips the control header before handing the request to the isolate. const controllerJS = `export default { async fetch(request, env) { @@ -49,21 +50,26 @@ const controllerJS = `export default { if (probe.status === 404) return new Response("no such sandbox", { status: 404 }); if (!probe.ok) return new Response("isolate bundle fetch " + probe.status, { status: 502 }); const spec = await probe.json(); - // Egress: the sandbox worker's globalOutbound is the STATIC egress service - // (a real Fetcher). It cannot be a per-sandbox shim loaded via env.LOADER — - // workerd rejects a dynamically-loaded worker's stub/entrypoint as - // globalOutbound of ANOTHER dynamic worker ("Entrypoints to dynamically- - // loaded workers cannot be transferred"). So per-request x-sb-id - // attribution via a shim is not expressible this way; the Go egress server - // therefore sees no attribution and fail-closed DENIES all egress (the - // Phase-2 deny-all posture). Restoring per-sandbox allowlists needs a - // workerd-supported attribution mechanism (plans/isolate-runtime.md §4 - // follow-up) — until then egress is deny-all, which is safe. + // Egress attribution is by SOCKET, not header. workerd only accepts a real + // static service binding as globalOutbound — a JS-object Fetcher is rejected + // ("not of type 'Fetcher'") and a dynamically-loaded worker stub is rejected + // ("Entrypoints to dynamically-loaded workers cannot be transferred"). So the + // host pre-declares a pool of per-slot egress services (EGRESS_0..) each on + // its own UDS, assigns this sandbox a slot, and returns egress_slot here; we + // bind globalOutbound to that slot's service. The loaded worker is given NO + // other bindings, so it can reach ONLY its own slot — attribution is + // structural and unforgeable. A sandbox with no slot (block-all, or the pool + // is exhausted) binds EGRESS_DENY, which fail-closed 403s (spike-proven + // 2026-07-18; plans/isolate-runtime.md §4). + const slot = spec.egress_slot; + const outbound = (slot === undefined || slot === null || slot < 0) + ? env.EGRESS_DENY + : env["EGRESS_" + slot]; const worker = env.LOADER.get(id, async () => ({ compatibilityDate: spec.compatibility_date, mainModule: spec.main_module, modules: spec.modules, - globalOutbound: env.EGRESS, + globalOutbound: outbound, })); const fwd = new Request(request); fwd.headers.delete("x-sb-id"); @@ -76,24 +82,41 @@ const controllerJS = `export default { // dir: the controller listens on the first (driver → isolate traffic), the Go // host serves the bundle-server on the second (controller → host bundle fetch). const ( - controlSocketName = "control.sock" - hostSocketName = "host.sock" - egressSocketName = "egress.sock" + controlSocketName = "control.sock" + hostSocketName = "host.sock" + egressDenySocketName = "egress-deny.sock" ) +// egressSlotSocketName is the host UDS for egress slot n (§4). The pool is +// pre-declared in the group config, but the host binds a slot's socket only when +// it assigns that slot to a sandbox — so runtime cost tracks egressing +// sandboxes, not pool size. +func egressSlotSocketName(n int) string { return fmt.Sprintf("egress-%d.sock", n) } + // capnpConfig renders the workerd config for one group. controlSock is where // workerd listens for sandbox traffic; hostSock is the Go bundle-server the -// controller's provider fetches from; egressSock is the deny-all (Phase 2) / -// per-sandbox (Phase 3) egress service. loaderID scopes the workerLoader cache -// (per-group, so two groups never share loaded isolates). -func capnpConfig(controlSock, hostSock, egressSock, loaderID string) string { +// controller's provider fetches from; egressSocks is the pre-declared pool of +// per-slot egress services (§4) — one static service per slot, each bound to a +// host UDS the driver assigns per egressing sandbox; denySock backs EGRESS_DENY, +// the fail-closed service block-all / no-slot sandboxes bind. loaderID scopes +// the workerLoader cache (per-group, so two groups never share loaded isolates). +// +// External services are dialed lazily by workerd (spike-confirmed 2026-07-18): +// declaring K egress services whose UDS have no listener yet is fine — workerd +// only connects on a subrequest, so an unassigned slot is never dialed. +func capnpConfig(controlSock, hostSock string, egressSocks []string, denySock, loaderID string) string { + var svcs, binds strings.Builder + for i, sock := range egressSocks { + fmt.Fprintf(&svcs, " (name = \"egress%d\", external = (address = \"unix:%s\", http = ())),\n", i, sock) + fmt.Fprintf(&binds, " (name = \"EGRESS_%d\", service = \"egress%d\"),\n", i, i) + } return fmt.Sprintf(`using Workerd = import "/workerd/workerd.capnp"; const config :Workerd.Config = ( services = [ (name = "controller", worker = .controller), (name = "host", external = (address = "unix:%s", http = ())), - (name = "egress", external = (address = "unix:%s", http = ())), - ], + (name = "egressDeny", external = (address = "unix:%s", http = ())), +%s ], sockets = [ (name = "control", address = "unix:%s", http = (), service = "controller"), ] @@ -105,10 +128,10 @@ const controller :Workerd.Worker = ( bindings = [ (name = "LOADER", workerLoader = (id = %s)), (name = "HOST", service = "host"), - (name = "EGRESS", service = "egress"), - ], + (name = "EGRESS_DENY", service = "egressDeny"), +%s ], ); -`, hostSock, egressSock, controlSock, controllerModuleName, controllerModuleName, strconv.Quote(loaderID)) +`, hostSock, denySock, svcs.String(), controlSock, controllerModuleName, controllerModuleName, strconv.Quote(loaderID), binds.String()) } // bundleWireJSON is the shape the bundle-server returns and the controller @@ -117,6 +140,9 @@ type bundleWireJSON struct { CompatibilityDate string `json:"compatibility_date"` MainModule string `json:"main_module"` Modules map[string]string `json:"modules"` + // EgressSlot is the sandbox's assigned egress slot (§4). Nil (omitted) means + // no slot — the controller binds EGRESS_DENY (block-all or pool exhausted). + EgressSlot *int `json:"egress_slot,omitempty"` } // validateLoaderID guards the workerLoader cache id (it is embedded in the diff --git a/pkg/isolate/egress.go b/pkg/isolate/egress.go index 8ce7ba5a..3ca8032c 100644 --- a/pkg/isolate/egress.go +++ b/pkg/isolate/egress.go @@ -5,6 +5,7 @@ import ( "io" "net" "net/http" + "os" "strings" "syscall" "time" @@ -19,9 +20,12 @@ type EgressPolicy struct { Deny []string } -// SetEgressPolicy registers (or replaces) the outbound policy for a sandbox. -// Called after Load. Unload clears it. Until a policy is set, egress for that -// sandbox id is denied (fail-closed). +// SetEgressPolicy registers (or replaces) the outbound policy for a sandbox and +// (re)assigns its egress slot (§4). A non-block-all policy claims a free slot +// and lazily binds that slot's listener; block-all releases any slot so the +// sandbox binds EGRESS_DENY. Until a policy is set — or when the pool is +// exhausted — the sandbox has no slot and its egress is denied (fail-closed). +// Called after Load; Unload clears both policy and slot. func (h *Host) SetEgressPolicy(id string, p EgressPolicy) { if id == "" { return @@ -31,43 +35,133 @@ func (h *Host) SetEgressPolicy(id string, p EgressPolicy) { h.egressPolicy = make(map[string]EgressPolicy) } h.egressPolicy[id] = p + + if p.BlockAll { + // No slot for block-all: it binds EGRESS_DENY. Drop any prior slot. + if slot, ok := h.slotByID[id]; ok { + h.freeSlotLocked(id, slot) + } + h.mu.Unlock() + return + } + if _, ok := h.slotByID[id]; ok { + h.mu.Unlock() // already assigned; the policy replacement above suffices + return + } + slot := -1 + for i, occ := range h.idBySlot { + if occ == "" { + slot = i + break + } + } + if slot < 0 { + h.mu.Unlock() + // No silent caps: a sandbox beyond the pool falls back to deny-all. + h.logger.Warn("isolate egress pool exhausted; sandbox falls back to deny-all egress", + "group", h.cfg.GroupKey, "sandbox", id, "pool_size", h.cfg.EgressPoolSize) + return + } + if err := h.startSlotServerLocked(slot); err != nil { + h.mu.Unlock() + h.logger.Error("isolate: failed to bind egress slot listener; sandbox falls back to deny-all", + "group", h.cfg.GroupKey, "sandbox", id, "slot", slot, "err", err) + return + } + h.slotByID[id] = slot + h.idBySlot[slot] = id h.mu.Unlock() } -// startEgressServer is the Phase-3 attributed egress boundary: every outbound -// fetch an isolate makes hits this socket. The controller stamps x-sb-id on -// the request (via a per-sandbox outbound shim isolate), so ownership is known -// at accept/handler time — the same connection-ownership lesson that delayed -// the WASM resident-host flag. Undeclared destinations are refused. -func (h *Host) startEgressServer() error { - ln, err := net.Listen("unix", h.egressSock) +// startSlotServerLocked binds the per-slot egress listener. Caller holds h.mu; +// net.Listen on a local UDS is fast enough to hold the lock across. +func (h *Host) startSlotServerLocked(slot int) error { + sock := h.egressSocks[slot] + _ = os.Remove(sock) + ln, err := net.Listen("unix", sock) if err != nil { - return fmt.Errorf("isolate: listen egress socket: %w", err) + return fmt.Errorf("isolate: listen egress slot %d socket: %w", slot, err) } - h.egressSrv = &http.Server{Handler: http.HandlerFunc(h.serveEgress)} - go func() { _ = h.egressSrv.Serve(ln) }() + srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + h.serveEgressSlot(slot, w, r) + })} + h.slotSrv[slot] = srv + go func() { _ = srv.Serve(ln) }() return nil } -func (h *Host) serveEgress(w http.ResponseWriter, r *http.Request) { - id := strings.TrimSpace(r.Header.Get("x-sb-id")) - if id == "" { - http.Error(w, "egress denied: missing x-sb-id attribution", http.StatusForbidden) - return +// freeSlotLocked releases a sandbox's slot and tears its listener down; a +// subsequent outbound on that (now unbound) socket is refused at connect until +// the slot is reassigned. Caller holds h.mu. +func (h *Host) freeSlotLocked(id string, slot int) { + delete(h.slotByID, id) + if slot >= 0 && slot < len(h.idBySlot) && h.idBySlot[slot] == id { + h.idBySlot[slot] = "" + } + if slot >= 0 && slot < len(h.slotSrv) && h.slotSrv[slot] != nil { + _ = h.slotSrv[slot].Close() + h.slotSrv[slot] = nil + _ = os.Remove(h.egressSocks[slot]) } +} + +// startEgressDenyServer starts the always-on EGRESS_DENY service. Block-all and +// pool-exhausted sandboxes bind it; it fail-closed 403s every request. +func (h *Host) startEgressDenyServer() error { + ln, err := net.Listen("unix", h.egressDenySock) + if err != nil { + return fmt.Errorf("isolate: listen egress-deny socket: %w", err) + } + h.egressDenySrv = &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "egress denied: sandbox has no egress slot (block-all or pool exhausted)", http.StatusForbidden) + })} + go func() { _ = h.egressDenySrv.Serve(ln) }() + return nil +} + +// serveEgressSlot is the per-slot egress handler: the SOCKET identifies the +// sandbox (idBySlot[slot]), so no header trust is involved — a forged header on +// the outbound request is irrelevant. It applies that sandbox's policy + SSRF +// guard, then proxies. A slot with no current owner (a teardown race) fails +// closed. +func (h *Host) serveEgressSlot(slot int, w http.ResponseWriter, r *http.Request) { h.mu.RLock() + var id string + if slot >= 0 && slot < len(h.idBySlot) { + id = h.idBySlot[slot] + } p, ok := h.egressPolicy[id] h.mu.RUnlock() - if !ok { - http.Error(w, "egress denied: no policy for sandbox", http.StatusForbidden) + if id == "" || !ok { + http.Error(w, "egress denied: slot has no attributed sandbox", http.StatusForbidden) return } - host := r.URL.Hostname() + h.proxyEgress(w, r, p) +} + +// proxyEgress enforces p (allowlist/denylist + SSRF IP-range block) and proxies +// the request. The isolate reaches this only via its own slot socket, so p is +// unambiguously this sandbox's policy. +// +// workerd delivers an external egress service the request with the target +// authority in the Host header and only path+query in the URL — and it does NOT +// convey the original scheme (spike-observed: http:// and https:// arrive +// identically with an empty scheme). So we reconstruct the absolute upstream URL +// from the Host header and force https: an isolate cannot make a plaintext +// egress call, which is the safe default for an allowlist proxy and the only +// scheme we can honor unambiguously. +func (h *Host) proxyEgress(w http.ResponseWriter, r *http.Request, p EgressPolicy) { + authority := r.Host + if authority == "" { + authority = r.URL.Host + } + host := authority + if hname, _, err := net.SplitHostPort(authority); err == nil { + host = hname + } if host == "" { - host = r.Host - if hname, _, err := net.SplitHostPort(host); err == nil { - host = hname - } + http.Error(w, "egress denied: no destination host", http.StatusForbidden) + return } if !egressAllowed(p, host) { http.Error(w, "egress denied by sandbox policy", http.StatusForbidden) @@ -86,10 +180,9 @@ func (h *Host) serveEgress(w http.ResponseWriter, r *http.Request) { } outReq := r.Clone(r.Context()) outReq.RequestURI = "" - outReq.Header.Del("x-sb-id") - if outReq.URL.Scheme == "" { - outReq.URL.Scheme = "https" - } + outReq.URL.Scheme = "https" + outReq.URL.Host = authority + outReq.Host = authority resp, err := egressTransport.RoundTrip(outReq) if err != nil { http.Error(w, "egress proxy: "+err.Error(), http.StatusBadGateway) diff --git a/pkg/isolate/egress_test.go b/pkg/isolate/egress_test.go index 6a5443e2..51ca6d4c 100644 --- a/pkg/isolate/egress_test.go +++ b/pkg/isolate/egress_test.go @@ -62,15 +62,14 @@ func TestIsBlockedEgressIP(t *testing.T) { } } -// TestServeEgressBlocksLiteralSSRF proves an isolate whose policy is the default +// TestProxyEgressBlocksLiteralSSRF proves an isolate whose policy is the default // allow-all still cannot reach a loopback/metadata IP literal through the proxy. -func TestServeEgressBlocksLiteralSSRF(t *testing.T) { - h := &Host{egressPolicy: map[string]EgressPolicy{"sb-1": {}}} // allow-all policy +func TestProxyEgressBlocksLiteralSSRF(t *testing.T) { + h := &Host{} for _, target := range []string{"http://127.0.0.1:21212/v1/sandboxes", "http://169.254.169.254/latest/meta-data/"} { req := httptest.NewRequest(http.MethodGet, target, nil) - req.Header.Set("x-sb-id", "sb-1") rec := httptest.NewRecorder() - h.serveEgress(rec, req) + h.proxyEgress(rec, req, EgressPolicy{}) // allow-all policy if rec.Code != http.StatusForbidden { t.Fatalf("egress to %s = %d, want 403 (blocked)", target, rec.Code) } @@ -80,21 +79,50 @@ func TestServeEgressBlocksLiteralSSRF(t *testing.T) { } } -func TestServeEgressRequiresAttribution(t *testing.T) { - h := &Host{egressPolicy: map[string]EgressPolicy{}} - // No x-sb-id → forbidden (attribution missing). - req := httptest.NewRequest(http.MethodGet, "http://example.com/", nil) +// TestProxyEgressPolicyDeny covers the offline decision branches of proxyEgress: +// a host outside the allowlist is refused, and a request with no destination +// authority is refused (both without touching the network). +func TestProxyEgressPolicyDeny(t *testing.T) { + h := &Host{} + + // Host not in the allowlist → 403 by policy. + req := httptest.NewRequest(http.MethodGet, "http://other.example/x", nil) + rec := httptest.NewRecorder() + h.proxyEgress(rec, req, EgressPolicy{Allow: []string{"only.example"}}) + if rec.Code != http.StatusForbidden { + t.Fatalf("non-allowlisted host = %d, want 403", rec.Code) + } + if body, _ := io.ReadAll(rec.Result().Body); !strings.Contains(string(body), "policy") { + t.Fatalf("deny body = %q, want a policy denial", body) + } + + // No destination authority → 403 (cannot attribute a target). + req = httptest.NewRequest(http.MethodGet, "http://x/y", nil) + req.Host = "" + req.URL.Host = "" + rec = httptest.NewRecorder() + h.proxyEgress(rec, req, EgressPolicy{}) + if rec.Code != http.StatusForbidden { + t.Fatalf("no-host = %d, want 403", rec.Code) + } +} + +// TestServeEgressSlotFailsClosed proves the per-slot handler denies when the +// slot has no attributed owner (a teardown race) or the owner has no registered +// policy — the socket, not a header, is the attribution. +func TestServeEgressSlotFailsClosed(t *testing.T) { + h := &Host{idBySlot: []string{""}, egressPolicy: map[string]EgressPolicy{}} + // Slot 0 has no owner → forbidden. rec := httptest.NewRecorder() - h.serveEgress(rec, req) + h.serveEgressSlot(0, rec, httptest.NewRequest(http.MethodGet, "http://example.com/", nil)) if rec.Code != http.StatusForbidden { - t.Fatalf("egress without x-sb-id = %d, want 403", rec.Code) + t.Fatalf("unowned slot = %d, want 403", rec.Code) } - // Unknown sandbox id (no policy) → forbidden (fail-closed). - req = httptest.NewRequest(http.MethodGet, "http://example.com/", nil) - req.Header.Set("x-sb-id", "unknown") + // Owner present but no policy registered → forbidden (fail-closed). + h.idBySlot[0] = "sb-x" rec = httptest.NewRecorder() - h.serveEgress(rec, req) + h.serveEgressSlot(0, rec, httptest.NewRequest(http.MethodGet, "http://example.com/", nil)) if rec.Code != http.StatusForbidden { - t.Fatalf("egress for unknown sandbox = %d, want 403 (fail-closed)", rec.Code) + t.Fatalf("owner without policy = %d, want 403 (fail-closed)", rec.Code) } } diff --git a/pkg/isolate/host.go b/pkg/isolate/host.go index 47190366..2a4b1584 100644 --- a/pkg/isolate/host.go +++ b/pkg/isolate/host.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "log/slog" "net" "net/http" "os" @@ -17,6 +18,12 @@ import ( "github.com/aerol-ai/microvm/pkg/jsbundle" ) +// defaultEgressPoolSize is the per-group egress-slot pool size when unset (§4). +// It caps concurrently-egressing sandboxes in a group; block-all sandboxes cost +// no slot. Sized generously because sockets are cheap and only bound per active +// egress sandbox; operators tune it via SB_ISOLATE_EGRESS_POOL_SIZE. +const defaultEgressPoolSize = 64 + // HostConfig configures one workerd group host. type HostConfig struct { // WorkerdPath is the workerd binary. @@ -31,17 +38,26 @@ type HostConfig struct { // true, Start applies it before exec and FAILS CLOSED if it cannot be // realized on this platform — the group never runs unconfined. Jail JailConfig + // EgressPoolSize is the number of per-slot egress services pre-declared in + // the group config (§4). Caps concurrently-egressing sandboxes; zero → + // defaultEgressPoolSize. + EgressPoolSize int + // Logger records egress pool-exhaustion spill (no silent caps). Nil → + // slog.Default(). + Logger *slog.Logger } // Host owns one workerd process for an isolate group and the Go-side // bundle-server the controller fetches bundles from. It is safe for concurrent // use: Load/Unload/Invoke may race with each other. type Host struct { - cfg HostConfig + cfg HostConfig + logger *slog.Logger - controlSock string - hostSock string - egressSock string + controlSock string + hostSock string + egressDenySock string + egressSocks []string // slot → per-slot egress UDS path (len == pool size) // started is the fast-path liveness flag Invoke checks; atomic so a // concurrent Stop (last-member teardown / idle reaper) racing an in-flight @@ -52,12 +68,19 @@ type Host struct { mu sync.RWMutex bundles map[string]*jsbundle.Bundle // sandbox id → pinned bundle egressPolicy map[string]EgressPolicy // sandbox id → outbound policy + // Egress slot allocation (§4): a sandbox with a non-block-all policy is + // assigned a slot; its dedicated egress listener (slotSrv[slot]) is bound + // lazily on assignment and torn down on Unload. Attribution is the socket, + // so idBySlot[slot] is the authoritative owner the slot handler enforces. + slotByID map[string]int // sandbox id → assigned slot + idBySlot []string // slot → sandbox id ("" == free) + slotSrv []*http.Server // slot → lazily-started listener (nil == unbound) // cmd/ctrlClient/servers are set once in Start and cleared in Stop; guarded // by mu because Invoke reads ctrlClient while Stop nils cmd concurrently. - cmd *exec.Cmd - bundleSrv *http.Server - egressSrv *http.Server - ctrlClient *http.Client + cmd *exec.Cmd + bundleSrv *http.Server + egressDenySrv *http.Server + ctrlClient *http.Client } // NewHost prepares (does not start) a group host. @@ -74,13 +97,29 @@ func NewHost(cfg HostConfig) (*Host, error) { if cfg.StartTimeout == 0 { cfg.StartTimeout = 10 * time.Second } + if cfg.EgressPoolSize <= 0 { + cfg.EgressPoolSize = defaultEgressPoolSize + } + logger := cfg.Logger + if logger == nil { + logger = slog.Default() + } + egressSocks := make([]string, cfg.EgressPoolSize) + for i := range egressSocks { + egressSocks[i] = filepath.Join(cfg.RunDir, egressSlotSocketName(i)) + } return &Host{ - cfg: cfg, - controlSock: filepath.Join(cfg.RunDir, controlSocketName), - hostSock: filepath.Join(cfg.RunDir, hostSocketName), - egressSock: filepath.Join(cfg.RunDir, egressSocketName), - bundles: make(map[string]*jsbundle.Bundle), - egressPolicy: make(map[string]EgressPolicy), + cfg: cfg, + logger: logger, + controlSock: filepath.Join(cfg.RunDir, controlSocketName), + hostSock: filepath.Join(cfg.RunDir, hostSocketName), + egressDenySock: filepath.Join(cfg.RunDir, egressDenySocketName), + egressSocks: egressSocks, + bundles: make(map[string]*jsbundle.Bundle), + egressPolicy: make(map[string]EgressPolicy), + slotByID: make(map[string]int), + idBySlot: make([]string, cfg.EgressPoolSize), + slotSrv: make([]*http.Server, cfg.EgressPoolSize), }, nil } @@ -109,6 +148,9 @@ func (h *Host) Unload(id string) int { h.mu.Lock() delete(h.bundles, id) delete(h.egressPolicy, id) + if slot, ok := h.slotByID[id]; ok { + h.freeSlotLocked(id, slot) + } n := len(h.bundles) h.mu.Unlock() return n @@ -129,14 +171,18 @@ func (h *Host) Start(ctx context.Context) error { return fmt.Errorf("isolate: mkdir run dir: %w", err) } // Stale sockets from a crashed predecessor would make workerd's bind fail. - for _, s := range []string{h.controlSock, h.hostSock, h.egressSock} { + stale := append([]string{h.controlSock, h.hostSock, h.egressDenySock}, h.egressSocks...) + for _, s := range stale { _ = os.Remove(s) } if err := h.startBundleServer(); err != nil { return err } - if err := h.startEgressServer(); err != nil { + // The deny service (EGRESS_DENY) is always up: block-all and pool-exhausted + // sandboxes bind it. Per-slot egress listeners are bound lazily when the + // driver assigns a slot (SetEgressPolicy), so an idle group holds none. + if err := h.startEgressDenyServer(); err != nil { _ = h.stopServers() return err } @@ -182,7 +228,7 @@ func (h *Host) writeConfig() error { if err := os.WriteFile(filepath.Join(h.cfg.RunDir, controllerModuleName), []byte(controllerJS), 0o600); err != nil { return fmt.Errorf("isolate: write controller: %w", err) } - cfg := capnpConfig(h.controlSock, h.hostSock, h.egressSock, h.cfg.GroupKey) + cfg := capnpConfig(h.controlSock, h.hostSock, h.egressSocks, h.egressDenySock, h.cfg.GroupKey) if err := os.WriteFile(filepath.Join(h.cfg.RunDir, "config.capnp"), []byte(cfg), 0o600); err != nil { return fmt.Errorf("isolate: write config: %w", err) } @@ -202,17 +248,25 @@ func (h *Host) startBundleServer() error { id := strings.TrimPrefix(r.URL.Path, "/bundle/") h.mu.RLock() b := h.bundles[id] + slot, hasSlot := h.slotByID[id] h.mu.RUnlock() if b == nil { http.Error(w, "no such sandbox", http.StatusNotFound) return } - w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(bundleWireJSON{ + wire := bundleWireJSON{ CompatibilityDate: b.CompatibilityDate, MainModule: b.MainModule, Modules: b.Modules, - }) + } + // Carry the assigned egress slot so the controller binds this sandbox's + // globalOutbound to its dedicated slot service; no slot → EGRESS_DENY. + if hasSlot { + s := slot + wire.EgressSlot = &s + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(wire) }) h.bundleSrv = &http.Server{Handler: mux} go func() { _ = h.bundleSrv.Serve(ln) }() @@ -268,15 +322,25 @@ func (h *Host) Invoke(ctx context.Context, id string, r *http.Request) (*http.Re return client.Do(out) } -// stopServers shuts down the bundle + egress servers and removes their sockets. +// stopServers shuts down the bundle + egress (deny + all per-slot) servers and +// removes their sockets. func (h *Host) stopServers() error { + h.mu.Lock() if h.bundleSrv != nil { _ = h.bundleSrv.Close() } - if h.egressSrv != nil { - _ = h.egressSrv.Close() + if h.egressDenySrv != nil { + _ = h.egressDenySrv.Close() } - for _, s := range []string{h.hostSock, h.egressSock, h.controlSock} { + for i, srv := range h.slotSrv { + if srv != nil { + _ = srv.Close() + h.slotSrv[i] = nil + } + } + h.mu.Unlock() + socks := append([]string{h.hostSock, h.egressDenySock, h.controlSock}, h.egressSocks...) + for _, s := range socks { _ = os.Remove(s) } return nil diff --git a/pkg/isolate/host_test.go b/pkg/isolate/host_test.go index 607c76f1..189f758a 100644 --- a/pkg/isolate/host_test.go +++ b/pkg/isolate/host_test.go @@ -42,14 +42,19 @@ func shortRunDir(t *testing.T) string { } func TestCapnpConfigContents(t *testing.T) { - cfg := capnpConfig("/run/control.sock", "/run/host.sock", "/run/egress.sock", "acme") + cfg := capnpConfig("/run/control.sock", "/run/host.sock", + []string{"/run/egress-0.sock", "/run/egress-1.sock"}, "/run/egress-deny.sock", "acme") for _, want := range []string{ `address = "unix:/run/control.sock"`, `external = (address = "unix:/run/host.sock"`, - `external = (address = "unix:/run/egress.sock"`, + `external = (address = "unix:/run/egress-deny.sock"`, + `(name = "egress0", external = (address = "unix:/run/egress-0.sock"`, + `(name = "egress1", external = (address = "unix:/run/egress-1.sock"`, `workerLoader = (id = "acme")`, `(name = "HOST", service = "host")`, - `(name = "EGRESS", service = "egress")`, + `(name = "EGRESS_DENY", service = "egressDeny")`, + `(name = "EGRESS_0", service = "egress0")`, + `(name = "EGRESS_1", service = "egress1")`, `compatibilityFlags = ["experimental"]`, `embed "controller.js"`, } { @@ -57,6 +62,10 @@ func TestCapnpConfigContents(t *testing.T) { t.Fatalf("config missing %q:\n%s", want, cfg) } } + // The old single shared "egress" service must be gone. + if strings.Contains(cfg, `(name = "EGRESS", service = "egress")`) { + t.Fatalf("config still wires the removed single EGRESS service:\n%s", cfg) + } } func TestValidateLoaderID(t *testing.T) { @@ -163,50 +172,88 @@ func TestBundleServerServesPinnedBundle(t *testing.T) { } } -// TestEgressServerAttributedFailClosed asserts the Phase-3 egress boundary -// refuses unattributed traffic and sandboxes without a registered policy. -func TestEgressServerAttributedFailClosed(t *testing.T) { - h, _ := NewHost(HostConfig{WorkerdPath: "/w", GroupKey: "acme", RunDir: shortRunDir(t)}) - if err := h.startEgressServer(); err != nil { +// TestEgressDenyServerAndSlotLifecycle covers the §4 slot model end to end +// (offline, no workerd): the always-on EGRESS_DENY service, slot assignment for +// allow-all sandboxes, no-slot for block-all, per-slot SSRF enforcement, pool +// exhaustion falling back to deny (never a shared slot), and Unload freeing a +// slot for reuse. +func TestEgressDenyServerAndSlotLifecycle(t *testing.T) { + h, _ := NewHost(HostConfig{WorkerdPath: "/w", GroupKey: "acme", RunDir: shortRunDir(t), EgressPoolSize: 2}) + if err := h.startEgressDenyServer(); err != nil { t.Fatal(err) } defer h.stopServers() - client := unixHTTPClient(h.egressSock) - // No x-sb-id → deny. - resp, err := client.Get("http://egress/anything") + // EGRESS_DENY always 403s — this is what block-all / no-slot sandboxes bind. + deny := unixHTTPClient(h.egressDenySock) + resp, err := deny.Get("http://egress/anything") if err != nil { t.Fatal(err) } + body, _ := io.ReadAll(resp.Body) _ = resp.Body.Close() - if resp.StatusCode != http.StatusForbidden { - t.Fatalf("unattributed egress status = %d, want 403", resp.StatusCode) + if resp.StatusCode != http.StatusForbidden || !strings.Contains(string(body), "no egress slot") { + t.Fatalf("deny status/body = %d %q", resp.StatusCode, body) } - // x-sb-id present but no policy registered → deny. - req, _ := http.NewRequest(http.MethodGet, "http://example.com/", nil) - req.Header.Set("x-sb-id", "sb-1") - resp, err = client.Do(req) - if err != nil { - t.Fatal(err) + // Block-all sandbox claims no slot (binds EGRESS_DENY). + h.SetEgressPolicy("sb-block", EgressPolicy{BlockAll: true}) + if _, ok := h.slotByID["sb-block"]; ok { + t.Fatal("block-all sandbox must not claim a slot") } - body, _ := io.ReadAll(resp.Body) - _ = resp.Body.Close() - if resp.StatusCode != http.StatusForbidden || !strings.Contains(string(body), "no policy") { - t.Fatalf("no-policy status/body = %d %q", resp.StatusCode, body) + + // Allow-all sandbox claims slot 0 and binds its dedicated socket. + h.SetEgressPolicy("sb-1", EgressPolicy{}) + slot, ok := h.slotByID["sb-1"] + if !ok { + t.Fatal("sb-1 got no slot") + } + if h.idBySlot[slot] != "sb-1" { + t.Fatalf("idBySlot[%d] = %q, want sb-1", slot, h.idBySlot[slot]) + } + if _, err := os.Stat(h.egressSocks[slot]); err != nil { + t.Fatalf("slot socket not bound: %v", err) } - // BlockAll policy → deny even with attribution. - h.SetEgressPolicy("sb-1", EgressPolicy{BlockAll: true}) - req, _ = http.NewRequest(http.MethodGet, "http://example.com/", nil) - req.Header.Set("x-sb-id", "sb-1") - resp, err = client.Do(req) + // The slot socket still blocks SSRF even under an allow-all policy: the + // socket attributes the request to sb-1, whose policy allows all HOSTS but + // the IP-range guard refuses loopback/metadata. + slotClient := unixHTTPClient(h.egressSocks[slot]) + resp, err = slotClient.Get("http://127.0.0.1:21212/v1/sandboxes") if err != nil { t.Fatal(err) } + sb, _ := io.ReadAll(resp.Body) _ = resp.Body.Close() - if resp.StatusCode != http.StatusForbidden { - t.Fatalf("block-all status = %d, want 403", resp.StatusCode) + if resp.StatusCode != http.StatusForbidden || !strings.Contains(string(sb), "blocked") { + t.Fatalf("SSRF via slot = %d %q, want 403 blocked", resp.StatusCode, sb) + } + + // Second allow-all sandbox gets a DISTINCT slot; the pool (size 2) is now full. + h.SetEgressPolicy("sb-2", EgressPolicy{}) + if _, ok := h.slotByID["sb-2"]; !ok || h.slotByID["sb-2"] == slot { + t.Fatalf("sb-2 slot = %v (ok=%v), want a distinct slot", h.slotByID["sb-2"], ok) + } + + // Third sandbox: pool exhausted → no slot (deny-all fallback, logged) — it + // must NOT share another sandbox's slot. + h.SetEgressPolicy("sb-3", EgressPolicy{}) + if _, ok := h.slotByID["sb-3"]; ok { + t.Fatal("sb-3 must not get a slot from an exhausted pool") + } + + // Unload frees sb-1's slot and removes its socket; the slot is then reusable. + sock1 := h.egressSocks[slot] + h.Unload("sb-1") + if _, ok := h.slotByID["sb-1"]; ok { + t.Fatal("Unload did not release the slot") + } + if _, err := os.Stat(sock1); !os.IsNotExist(err) { + t.Fatalf("slot socket not removed after Unload: %v", err) + } + h.SetEgressPolicy("sb-3", EgressPolicy{}) + if _, ok := h.slotByID["sb-3"]; !ok { + t.Fatal("sb-3 should reclaim the freed slot") } } @@ -223,6 +270,10 @@ func TestWriteConfigProducesFiles(t *testing.T) { if err != nil || !strings.Contains(string(ctrl), "x-sb-id") { t.Fatalf("controller.js = %q err=%v", ctrl, err) } + // The controller binds globalOutbound to the sandbox's egress slot service. + if !strings.Contains(string(ctrl), `env["EGRESS_" + slot]`) || !strings.Contains(string(ctrl), "EGRESS_DENY") { + t.Fatalf("controller.js missing slot-based egress binding: %q", ctrl) + } cfg, err := os.ReadFile(filepath.Join(dir, "config.capnp")) if err != nil || !strings.Contains(string(cfg), `workerLoader = (id = "acme")`) { t.Fatalf("config.capnp missing loader id: %q err=%v", cfg, err) @@ -230,6 +281,10 @@ func TestWriteConfigProducesFiles(t *testing.T) { if !strings.Contains(string(cfg), h.hostSock) { t.Fatalf("config.capnp does not wire host sock %q", h.hostSock) } + // The egress pool + deny service are declared (default pool size). + if !strings.Contains(string(cfg), "EGRESS_DENY") || !strings.Contains(string(cfg), `(name = "EGRESS_0"`) { + t.Fatalf("config.capnp missing egress pool/deny wiring: %q", cfg) + } } // TestStopIdempotent covers Stop with no running process: it tears down servers @@ -239,7 +294,7 @@ func TestStopIdempotent(t *testing.T) { if err := h.startBundleServer(); err != nil { t.Fatal(err) } - if err := h.startEgressServer(); err != nil { + if err := h.startEgressDenyServer(); err != nil { t.Fatal(err) } if err := h.Stop(); err != nil { diff --git a/plans/isolate-runtime.md b/plans/isolate-runtime.md index 0c6f2339..f6ffffe8 100644 --- a/plans/isolate-runtime.md +++ b/plans/isolate-runtime.md @@ -28,7 +28,8 @@ with Deno Sandbox (microVMs). - **2026-07-18 — Phases 2 leftovers + Phase 3 landed.** Idle-TTL reaper; unreferenced bundle GC; guest HTTP PortGateway + expose_port (HTTP-only, - no TCP pool); per-sandbox attributed egress (`x-sb-id` shim); toolbox exec + no TCP pool); per-sandbox attributed egress (per-slot egress-service pool — + §4, superseding the rejected `x-sb-id` shim); toolbox exec = invoke-handler; blank-host warm pool with boot prewarm; P0 gate executes (tag-gated); `docs/.../isolate-sandbox.mdx` + 5-SDK `runtime:"isolate"` / `tenant_id`. Demand pitch still open (§10.1) — 0/3. @@ -295,16 +296,38 @@ already exposes on its `WorkerClient`: - outbound `fetch()` is routed through a **host-side proxy** the driver controls, so egress allowlist / CIDR policy / byte-counting still apply — the same knobs iptables gives Docker, enforced at the proxy layer; -- **per-sandbox egress attribution (Phase-3 deliverable):** capability - manifests are per-sandbox but the proxy is shared per group, so each - worker's `globalOutbound` is bound to a **per-sandbox UDS endpoint** on the - host proxy — ownership known at accept time; the proxy applies that - sandbox's allowlist/CIDR/byte-count. This is the exact connection-ownership - problem that delayed the WASM resident-host flag; it lands with a regression - test, not mid-implementation. Byte counters are per-connection atomics — no - shared mutex on the data path (the netrules-Manager head-of-line lesson, - PR #306). Non-network grants (env, KV, timers) are config-time workerd - bindings, not proxy-enforced; +- **per-sandbox egress attribution (§4 redesign — mechanism proven by spike + 2026-07-18):** capability manifests are per-sandbox but one workerd process + serves a whole group, so attribution has to survive dynamic loading. The + obvious designs are both **rejected by workerd**: a per-sandbox shim worker as + `globalOutbound` ("Entrypoints to dynamically-loaded workers cannot be + transferred") and an inline JS-object `globalOutbound` ("the provided value is + not of type 'Fetcher'"). workerd only accepts a **real static service binding** + as `globalOutbound`. So attribution is carried by **which static egress + service the sandbox is bound to**: the group config pre-declares a **pool of + `K` egress services** (`EGRESS_0..EGRESS_{K-1}`), each on its own host UDS + (`egress-.sock`); the host assigns each egressing sandbox a **slot** and + returns `egress_slot` in the bundle spec, so the controller's loader spec sets + `globalOutbound: env["EGRESS_"+slot]`. The loaded worker is given **no other + bindings**, so it can reach only its own slot — attribution is **structural and + unforgeable** (the sandbox cannot name a sibling slot; a forged `x-sb-id` on the + outbound request is irrelevant because ownership is the socket, not a header). + The per-slot Go listener applies that sandbox's allowlist/CIDR/SSRF-block/ + byte-count; ownership is known at accept time. Spike-confirmed properties: + `env["EGRESS_"+slot]` loads and routes; alpha's traffic reaches only + `egress-0.sock`, beta's only `egress-1.sock`, persisting across warm re-hits; + the sandbox env exposes no sibling binding; and **external services are dialed + lazily** — workerd boots and serves with declared-but-unbound egress sockets, + so the pool config is cheap and the host listens only on **assigned** slots + (runtime cost ∝ egressing sandboxes, not `K`). Block-all sandboxes and + pool-exhaustion spill bind a shared **`EGRESS_DENY`** service that fail-closed + 403s (spill is **logged**, never silent — CLAUDE.md no-silent-caps rule). `K` + is a per-group cap on *concurrently-egressing* sandboxes (block-all costs no + slot); it is configurable (`SB_ISOLATE_EGRESS_POOL_SIZE`, default 64) and a + known density limit that Phase 4 can raise via policy-interned slot sharing. + Byte counters are per-connection atomics — no shared mutex on the data path + (the netrules-Manager head-of-line lesson, PR #306). Non-network grants (env, + KV, timers) are config-time workerd bindings, not proxy-enforced; - inbound HTTP is proxied by the driver to the isolate's `fetch` handler via driver-attributed routing — per-worker sockets where the injection path supports them, or a trusted dispatcher entrypoint otherwise; never client @@ -553,8 +576,11 @@ Every new package ships with `_test.go` next to it at the ~85% bar (§11). walked); basic per-tenant warm pool; **P0 gate executes**; capability-denied egress test. - **LANDED 2026-07-18:** host-mediated `PortGateway` + `expose_port` (HTTP - only, `UpsertPortRouteWithDial`); attributed egress (`x-sb-id` shim + - per-sandbox policy on the Go proxy); toolbox exec = invoke-handler (501 + only, `UpsertPortRouteWithDial`); per-sandbox attributed egress (per-slot + egress-service pool — §4, superseding the rejected `x-sb-id` shim: workerd + only accepts a static service binding as globalOutbound, so attribution is + by which slot socket the outbound arrives on; block-all/pool-exhausted → + EGRESS_DENY); toolbox exec = invoke-handler (501 otherwise); `internal/pool/isolate` blank-host warm pool with boot prewarm; P0 gate implemented as tag-gated integration test; docs page `isolate-sandbox.mdx` + 5-SDK `runtime:"isolate"` / `tenant_id` constants.