feat(isolate): per-sandbox egress attribution via per-slot service pool

internal/config/config.go

@@ -674,6 +674,11 @@ type Config struct {
 	// IsolateBundleGCInterval is how often unreferenced content-addressed
 	// bundles are swept. Zero disables. Default 15m. SB_ISOLATE_BUNDLE_GC_INTERVAL.
 	IsolateBundleGCInterval time.Duration
+	// IsolateEgressPoolSize is the per-group egress-slot pool size (§4): the cap
+	// on concurrently-egressing sandboxes in a group (block-all sandboxes cost no
+	// slot). A sandbox beyond the pool falls back to deny-all egress (logged).
+	// Default 64. SB_ISOLATE_EGRESS_POOL_SIZE.
+	IsolateEgressPoolSize int
 	// FirecrackerBinary is the absolute path to the `firecracker` VMM
 	// binary on this host. Required only when EnableFirecracker is true.
 	// Default /usr/local/bin/firecracker matches a typical install.
@@ -1631,6 +1636,7 @@ func Load() (Config, error) {
 		IsolatePoolDepthDefault:      getEnvInt("SB_ISOLATE_POOL_DEPTH_DEFAULT", 2),
 		IsolatePoolRefillInterval:    getEnvDuration("SB_ISOLATE_POOL_REFILL_INTERVAL", 5*time.Second),
 		IsolateBundleGCInterval:      getEnvDuration("SB_ISOLATE_BUNDLE_GC_INTERVAL", 15*time.Minute),
+		IsolateEgressPoolSize:        getEnvInt("SB_ISOLATE_EGRESS_POOL_SIZE", 64),
 		FirecrackerBinary:            getEnv("SB_FIRECRACKER_BINARY", "/usr/local/bin/firecracker"),
 		JailerBinary:                 getEnv("SB_JAILER_BINARY", "/usr/local/bin/jailer"),
 		FirecrackerKernelImage:       getEnv("SB_FIRECRACKER_KERNEL", "/var/lib/sandboxd/firecracker/vmlinux"),

internal/runtime/isolate/config.go

@@ -37,6 +37,9 @@ type Config struct {
 	// IdleTTL is how long a group may sit without Create/Invoke before the
 	// idle reaper tears it down. Zero disables the reaper.
 	IdleTTL time.Duration
+	// EgressPoolSize is the per-group egress-slot pool size (§4): the cap on
+	// concurrently-egressing sandboxes in a group. Zero → pkg/isolate default.
+	EgressPoolSize int
 }
 
 // FromDaemonConfig projects the isolate slice of daemon config into driver config.
@@ -51,5 +54,6 @@ func FromDaemonConfig(cfg config.Config) Config {
 		JailGID:          cfg.IsolateJailGID,
 		Jitless:          cfg.IsolateJitless,
 		IdleTTL:          cfg.IsolateGroupIdleTTL,
+		EgressPoolSize:   cfg.IsolateEgressPoolSize,
 	}
 }

internal/runtime/isolate/hostsupervisor.go

@@ -12,25 +12,28 @@ import (
 // bundle-server). It lives in the driver package so pkg/isolate need not know
 // the driver's seam.
 type workerdSupervisor struct {
-	workerdPath string
-	runDir      string
-	useJail     bool
+	workerdPath    string
+	runDir         string
+	useJail        bool
+	egressPoolSize int
 }
 
 // NewHostSupervisor builds the production supervisor over the isolate config.
 func NewHostSupervisor(cfg Config) HostSupervisor {
 	return &workerdSupervisor{
-		workerdPath: cfg.WorkerdPath,
-		runDir:      cfg.RunDir,
-		useJail:     cfg.UseJail,
+		workerdPath:    cfg.WorkerdPath,
+		runDir:         cfg.RunDir,
+		useJail:        cfg.UseJail,
+		egressPoolSize: cfg.EgressPoolSize,
 	}
 }
 
 func (s *workerdSupervisor) SpawnGroup(ctx context.Context, spec JailSpec) (GroupHost, error) {
 	host, err := pkgisolate.NewHost(pkgisolate.HostConfig{
-		WorkerdPath: s.workerdPath,
-		GroupKey:    spec.GroupKey,
-		RunDir:      filepath.Join(s.runDir, spec.GroupKey),
+		WorkerdPath:    s.workerdPath,
+		GroupKey:       spec.GroupKey,
+		RunDir:         filepath.Join(s.runDir, spec.GroupKey),
+		EgressPoolSize: s.egressPoolSize,
 		// When UseJail is set, the host MUST realize this spec or fail closed.
 		Jail: pkgisolate.JailConfig{
 			Require:       s.useJail,

internal/runtime/isolate/p0_gate_integration_test.go

@@ -126,11 +126,12 @@ func TestP0HostileIsolateContainment(t *testing.T) {
 		t.Fatalf("create good: %v", err)
 	}
 
-	// (b) Undeclared egress is refused: the Go proxy denies with 403 (surfaced to
-	// the isolate as "egress-ok:403") or the fetch throws ("egress-err:*"). A 2xx
-	// upstream ("egress-ok:2xx") would mean egress actually leaked. (Egress is
-	// deny-all today — attribution is a §4 follow-up — so every destination is
-	// refused, which subsumes this allowlist check.)
+	// (b) Undeclared egress is refused: egress is attributed per-sandbox by slot
+	// (§4), so this sandbox's outbound is enforced against ITS allowlist
+	// (127.0.0.1/32) — example.invalid is not in it, so the Go proxy denies with
+	// 403 (surfaced as "egress-ok:403") or the fetch throws ("egress-err:*"). And
+	// even a 127.0.0.1 destination would be refused by the SSRF IP-range guard. A
+	// 2xx upstream ("egress-ok:2xx") would mean the capability boundary leaked.
 	req, _ := http.NewRequestWithContext(ctx, http.MethodGet, "http://isolate/egress", nil)
 	resp, err := d.InvokeHTTP(ctx, "sb-hostile", req)
 	if err != nil {

internal/runtime/isolate/phase3_integration_test.go

@@ -8,6 +8,7 @@ import (
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
 	"os"
 	"path/filepath"
 	"strings"
@@ -116,65 +117,80 @@ func TestPhase3IngressPortGatewayEndToEnd(t *testing.T) {
 	}
 }
 
-// TestPhase3EgressDeniedByDefault: an isolate's outbound fetch must never reach
-// upstream. Per-sandbox x-sb-id attribution via a loaded shim is not currently
-// expressible (workerd rejects a dynamically-loaded worker's entrypoint as
-// another worker's globalOutbound), so globalOutbound points at the static
-// egress service and the Go proxy fail-closed DENIES every request (no
-// attribution) — the safe Phase-2 deny-all posture. A denied fetch surfaces to
-// the isolate as a 403 upstream status (body "ok:403") or a thrown error
-// ("err:*"); a 2xx upstream would mean egress leaked.
-func TestPhase3EgressAttributionEndToEnd(t *testing.T) {
+// TestPhase3PerSandboxEgressEndToEnd proves the §4 redesign live against real
+// workerd: egress is attributed PER SANDBOX by slot, so two sandboxes in the
+// SAME tenant group get DIFFERENT policies enforced. An allow-listed sandbox
+// reaches an allowed host and is refused a non-allowed one; a block-all sandbox
+// in the same group is refused everything. Attribution is the slot socket — a
+// forged x-sb-id on the outbound is irrelevant.
+//
+// A policy denial surfaces to the isolate as an upstream 403 from the egress
+// proxy (body "status=403"); an allowed fetch is 200 (reachable) or 502/throw
+// (allowed but unreachable), never 403 — so the allow assertion holds even where
+// the runner has no outbound internet.
+func TestPhase3PerSandboxEgressEndToEnd(t *testing.T) {
 	workerd := requireWorkerd(t)
 	runDir := shortRunDir(t)
 	d := phase3Driver(t, workerd, runDir)
-	ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
 	defer cancel()
 
 	bundleDir := t.TempDir()
 	src := `export default { async fetch(req) {
-  const url = new URL(req.url);
-  if (url.pathname === "/egress") {
-    try {
-      const r = await fetch("https://example.invalid/");
-      return new Response("ok:" + r.status);
-    } catch (e) {
-      return new Response("err:" + (e && e.message ? e.message : String(e)), { status: 502 });
-    }
+  const u = new URL(req.url);
+  try {
+    const r = await fetch(u.searchParams.get("t"));
+    return new Response("status=" + r.status);
+  } catch (e) {
+    return new Response("throw=" + (e && e.message ? e.message : String(e)));
   }
-  return new Response("hi");
 }};`
 	p := filepath.Join(bundleDir, "eg.js")
 	if err := os.WriteFile(p, []byte(src), 0o644); err != nil {
 		t.Fatal(err)
 	}
+	ref := "file://" + p
 
+	// Two sandboxes in ONE tenant group with DIFFERENT egress policies.
 	if _, err := d.Create(ctx, models.CreateSandboxRequest{
-		Runtime:         models.RuntimeIsolate,
-		ModuleRef:       "file://" + p,
-		TenantID:        "eg",
-		MemoryMB:        128,
+		Runtime: models.RuntimeIsolate, ModuleRef: ref, TenantID: "eg-tenant", MemoryMB: 128,
+		NetworkAllowOut: []string{"example.com"},
+	}, "sb-allow", "", nil); err != nil {
+		t.Fatalf("create allow: %v", err)
+	}
+	t.Cleanup(func() { _ = d.Destroy(context.Background(), &models.Sandbox{ID: "sb-allow"}) })
+	if _, err := d.Create(ctx, models.CreateSandboxRequest{
+		Runtime: models.RuntimeIsolate, ModuleRef: ref, TenantID: "eg-tenant", MemoryMB: 128,
 		NetworkBlockAll: true,
-	}, "sb-eg", "", nil); err != nil {
-		t.Fatalf("create: %v", err)
+	}, "sb-block", "", nil); err != nil {
+		t.Fatalf("create block: %v", err)
 	}
-	t.Cleanup(func() { _ = d.Destroy(context.Background(), &models.Sandbox{ID: "sb-eg"}) })
+	t.Cleanup(func() { _ = d.Destroy(context.Background(), &models.Sandbox{ID: "sb-block"}) })
 
-	req, _ := http.NewRequestWithContext(ctx, http.MethodGet, "http://isolate/egress", nil)
-	resp, err := d.InvokeHTTP(ctx, "sb-eg", req)
-	if err != nil {
-		t.Fatalf("invoke: %v", err)
+	invoke := func(id, target string) string {
+		u := "http://isolate/?t=" + url.QueryEscape(target)
+		req, _ := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
+		resp, err := d.InvokeHTTP(ctx, id, req)
+		if err != nil {
+			t.Fatalf("invoke %s: %v", id, err)
+		}
+		b, _ := io.ReadAll(resp.Body)
+		_ = resp.Body.Close()
+		return string(b)
 	}
-	body, _ := io.ReadAll(resp.Body)
-	_ = resp.Body.Close()
-	// Blocked egress is "ok:403" (proxy denied) or "err:*" (fetch threw). Only a
-	// 2xx upstream status means egress actually leaked to example.invalid.
-	got := string(body)
-	if strings.HasPrefix(got, "ok:2") {
-		t.Fatalf("egress reached upstream (body=%q) — deny-all policy broken", got)
+
+	// Allowed sandbox → allowed host: PERMITTED (not a policy 403).
+	if got := invoke("sb-allow", "https://example.com/"); strings.Contains(got, "status=403") {
+		t.Fatalf("allow→example.com = %q, want it permitted (not a 403 policy denial)", got)
+	}
+	// Allowed sandbox → NON-allowed host: DENIED by its own allowlist.
+	if got := invoke("sb-allow", "https://not-allowed.example/"); !strings.Contains(got, "status=403") {
+		t.Fatalf("allow→not-allowed = %q, want status=403 (allowlist enforced)", got)
 	}
-	if !strings.HasPrefix(got, "ok:") && !strings.HasPrefix(got, "err:") {
-		t.Fatalf("unexpected egress body %q", got)
+	// Block-all sandbox in the SAME group → allowed host still DENIED (proving
+	// per-sandbox differentiation, not a group-wide policy).
+	if got := invoke("sb-block", "https://example.com/"); !strings.Contains(got, "status=403") {
+		t.Fatalf("block→example.com = %q, want status=403 (block-all)", got)
 	}
 }
 

pkg/isolate/config.go

@@ -30,10 +30,11 @@ const controllerModuleName = "controller.js"
 //   - rejects requests with no x-sb-id (the driver always sets it);
 //   - dynamically loads the isolate for that id, fetching the bundle from the
 //     HOST binding on a cache miss;
-//   - attributes egress per sandbox: each sandbox's globalOutbound is a tiny
-//     shim isolate (also via LOADER) that stamps x-sb-id before hitting the
-//     shared EGRESS service, so the Go proxy knows ownership at accept time
-//     (plans/isolate-runtime.md §4 Phase 3);
+//   - attributes egress per sandbox by SLOT: the host assigns each egressing
+//     sandbox an egress slot and returns it in the bundle spec, so the loaded
+//     worker's globalOutbound binds to that slot's dedicated static egress
+//     service (env["EGRESS_"+slot]) — one host UDS per slot, ownership known at
+//     accept time (plans/isolate-runtime.md §4);
 //   - strips the control header before handing the request to the isolate.
 const controllerJS = `export default {
   async fetch(request, env) {
@@ -49,21 +50,26 @@ const controllerJS = `export default {
     if (probe.status === 404) return new Response("no such sandbox", { status: 404 });
     if (!probe.ok) return new Response("isolate bundle fetch " + probe.status, { status: 502 });
     const spec = await probe.json();
-    // Egress: the sandbox worker's globalOutbound is the STATIC egress service
-    // (a real Fetcher). It cannot be a per-sandbox shim loaded via env.LOADER —
-    // workerd rejects a dynamically-loaded worker's stub/entrypoint as
-    // globalOutbound of ANOTHER dynamic worker ("Entrypoints to dynamically-
-    // loaded workers cannot be transferred"). So per-request x-sb-id
-    // attribution via a shim is not expressible this way; the Go egress server
-    // therefore sees no attribution and fail-closed DENIES all egress (the
-    // Phase-2 deny-all posture). Restoring per-sandbox allowlists needs a
-    // workerd-supported attribution mechanism (plans/isolate-runtime.md §4
-    // follow-up) — until then egress is deny-all, which is safe.
+    // Egress attribution is by SOCKET, not header. workerd only accepts a real
+    // static service binding as globalOutbound — a JS-object Fetcher is rejected
+    // ("not of type 'Fetcher'") and a dynamically-loaded worker stub is rejected
+    // ("Entrypoints to dynamically-loaded workers cannot be transferred"). So the
+    // host pre-declares a pool of per-slot egress services (EGRESS_0..) each on
+    // its own UDS, assigns this sandbox a slot, and returns egress_slot here; we
+    // bind globalOutbound to that slot's service. The loaded worker is given NO
+    // other bindings, so it can reach ONLY its own slot — attribution is
+    // structural and unforgeable. A sandbox with no slot (block-all, or the pool
+    // is exhausted) binds EGRESS_DENY, which fail-closed 403s (spike-proven
+    // 2026-07-18; plans/isolate-runtime.md §4).
+    const slot = spec.egress_slot;
+    const outbound = (slot === undefined || slot === null || slot < 0)
+      ? env.EGRESS_DENY
+      : env["EGRESS_" + slot];
     const worker = env.LOADER.get(id, async () => ({
       compatibilityDate: spec.compatibility_date,
       mainModule: spec.main_module,
       modules: spec.modules,
-      globalOutbound: env.EGRESS,
+      globalOutbound: outbound,
     }));
     const fwd = new Request(request);
     fwd.headers.delete("x-sb-id");
@@ -76,24 +82,41 @@ const controllerJS = `export default {
 // dir: the controller listens on the first (driver → isolate traffic), the Go
 // host serves the bundle-server on the second (controller → host bundle fetch).
 const (
-	controlSocketName = "control.sock"
-	hostSocketName    = "host.sock"
-	egressSocketName  = "egress.sock"
+	controlSocketName    = "control.sock"
+	hostSocketName       = "host.sock"
+	egressDenySocketName = "egress-deny.sock"
 )
 
+// egressSlotSocketName is the host UDS for egress slot n (§4). The pool is
+// pre-declared in the group config, but the host binds a slot's socket only when
+// it assigns that slot to a sandbox — so runtime cost tracks egressing
+// sandboxes, not pool size.
+func egressSlotSocketName(n int) string { return fmt.Sprintf("egress-%d.sock", n) }
+
 // capnpConfig renders the workerd config for one group. controlSock is where
 // workerd listens for sandbox traffic; hostSock is the Go bundle-server the
-// controller's provider fetches from; egressSock is the deny-all (Phase 2) /
-// per-sandbox (Phase 3) egress service. loaderID scopes the workerLoader cache
-// (per-group, so two groups never share loaded isolates).
-func capnpConfig(controlSock, hostSock, egressSock, loaderID string) string {
+// controller's provider fetches from; egressSocks is the pre-declared pool of
+// per-slot egress services (§4) — one static service per slot, each bound to a
+// host UDS the driver assigns per egressing sandbox; denySock backs EGRESS_DENY,
+// the fail-closed service block-all / no-slot sandboxes bind. loaderID scopes
+// the workerLoader cache (per-group, so two groups never share loaded isolates).
+//
+// External services are dialed lazily by workerd (spike-confirmed 2026-07-18):
+// declaring K egress services whose UDS have no listener yet is fine — workerd
+// only connects on a subrequest, so an unassigned slot is never dialed.
+func capnpConfig(controlSock, hostSock string, egressSocks []string, denySock, loaderID string) string {
+	var svcs, binds strings.Builder
+	for i, sock := range egressSocks {
+		fmt.Fprintf(&svcs, "    (name = \"egress%d\", external = (address = \"unix:%s\", http = ())),\n", i, sock)
+		fmt.Fprintf(&binds, "    (name = \"EGRESS_%d\", service = \"egress%d\"),\n", i, i)
+	}
 	return fmt.Sprintf(`using Workerd = import "/workerd/workerd.capnp";
 const config :Workerd.Config = (
   services = [
     (name = "controller", worker = .controller),
     (name = "host", external = (address = "unix:%s", http = ())),
-    (name = "egress", external = (address = "unix:%s", http = ())),
-  ],
+    (name = "egressDeny", external = (address = "unix:%s", http = ())),
+%s  ],
   sockets = [
     (name = "control", address = "unix:%s", http = (), service = "controller"),
   ]
@@ -105,10 +128,10 @@ const controller :Workerd.Worker = (
   bindings = [
     (name = "LOADER", workerLoader = (id = %s)),
     (name = "HOST", service = "host"),
-    (name = "EGRESS", service = "egress"),
-  ],
+    (name = "EGRESS_DENY", service = "egressDeny"),
+%s  ],
 );
-`, hostSock, egressSock, controlSock, controllerModuleName, controllerModuleName, strconv.Quote(loaderID))
+`, hostSock, denySock, svcs.String(), controlSock, controllerModuleName, controllerModuleName, strconv.Quote(loaderID), binds.String())
 }
 
 // bundleWireJSON is the shape the bundle-server returns and the controller
@@ -117,6 +140,9 @@ type bundleWireJSON struct {
 	CompatibilityDate string            `json:"compatibility_date"`
 	MainModule        string            `json:"main_module"`
 	Modules           map[string]string `json:"modules"`
+	// EgressSlot is the sandbox's assigned egress slot (§4). Nil (omitted) means
+	// no slot — the controller binds EGRESS_DENY (block-all or pool exhausted).
+	EgressSlot *int `json:"egress_slot,omitempty"`
 }
 
 // validateLoaderID guards the workerLoader cache id (it is embedded in the