P5 privileged networking and full data plane

Makefile

@@ -22,6 +22,11 @@ build: ## Build the binary into bin/ with version stamping
 	@mkdir -p $(BIN_DIR)
 	go build -ldflags "$(LDFLAGS)" -o $(BIN_DIR)/$(BINARY) $(CMD)
 
+.PHONY: netd
+netd: ## Build the privileged network helper daemon into bin/macvz-netd
+	@mkdir -p $(BIN_DIR)
+	go build -ldflags "$(LDFLAGS)" -o $(BIN_DIR)/macvz-netd ./cmd/macvz-netd
+
 .PHONY: bench
 bench: ## Build the density/RAM benchmark harness into bin/mvz-bench
 	@mkdir -p $(BIN_DIR)

README.md

@@ -130,4 +130,5 @@ macvz/
 - **Density is bounded by RAM, not by container-style kernel sharing.** Each micro-VM carries its own kernel and a fixed memory floor. Validate the real per-host concurrent-VM ceiling and per-VM overhead early in Phase 1 — this defines the project's practical capacity.
 - **Image architecture.** Guests are arm64. Pulling amd64 images requires the arm64 variant or Rosetta-for-Linux support; surface this clearly to users.
 - **Security.** The `macvz-kubelet` ↔ API server channel must use the cluster's normal mTLS/RBAC. Do not expose the runtime service or node ports publicly. Image-registry credentials and any secrets come from Kubernetes Secrets / environment, never hardcoded.
+- **Privileged networking needs root tools, but the kubelet runs as your user.** The cross-Mac data plane (WireGuard mesh + pf/route/sysctl) needs root, yet `apple/container` refuses to run as root — so `macvz-kubelet` runs as your user and delegates the privileged commands to the `macvz-netd` helper daemon over a unix socket. You install the helper once with `sudo`; day-to-day kubelet starts need no elevation. See [docs/PRIVILEGED_NETWORKING.md](docs/PRIVILEGED_NETWORKING.md) for the full setup and recovery runbook.
 - **Entitlements & signing.** `macvz-kubelet` runs as a resident process needing the virtualization entitlement; it must be signed appropriately (and notarized for distribution).

README.zh-TW.md

@@ -130,4 +130,5 @@ macvz/
 - **密度受限於 RAM，而非容器式的 kernel 共享。** 每個 micro-VM 自帶 kernel 與固定記憶體下限。請在階段一就驗證單機實際的並發 VM 上限與每 VM 開銷——這決定專案的實用容量。
 - **映像架構。** Guest 為 arm64。拉取 amd64 映像需使用 arm64 variant 或 Rosetta-for-Linux 支援；應向使用者明確說明。
 - **安全。** `macvz-kubelet` ↔ API server 的通道必須使用叢集既有的 mTLS/RBAC。不要對外公開 runtime 服務或節點埠口。映像倉庫憑證與任何機密均來自 Kubernetes Secrets／環境變數，絕不硬編碼。
+- **特權網路需要 root 工具，但 kubelet 以你的使用者身分執行。** 跨 Mac 資料平面（WireGuard 網格 + pf／route／sysctl）需要 root，但 `apple/container` 拒絕以 root 執行——因此 `macvz-kubelet` 以使用者身分執行，並透過 unix socket 將特權命令委派給 `macvz-netd` 輔助常駐程式。輔助程式只需用 `sudo` 安裝一次；日常啟動 kubelet 不需提權。完整的安裝與復原手冊見 [docs/PRIVILEGED_NETWORKING.md](docs/PRIVILEGED_NETWORKING.md)。
 - **Entitlement 與簽章。** `macvz-kubelet` 以常駐行程執行，需要虛擬化 entitlement，並須正確簽章（對外分發還需 notarization）。

cmd/macvz-kubelet/main.go

@@ -17,6 +17,7 @@ import (
 
 	"github.com/chimerakang/macvz/internal/version"
 	"github.com/chimerakang/macvz/pkg/config"
+	"github.com/chimerakang/macvz/pkg/network/privhelper"
 	"github.com/chimerakang/macvz/pkg/provider"
 	"github.com/chimerakang/macvz/pkg/runtime/container"
 	vknode "github.com/virtual-kubelet/virtual-kubelet/node"
@@ -93,15 +94,13 @@ func run(ctx context.Context, configPath, runtimeBinary string) error {
 		klog.InfoS("apple/container runtime is ready")
 	}
 
-	// Build the Kubernetes client from kubeconfig / in-cluster config.
+	// Resolve Kubernetes client config now, but delay constructing the clientset
+	// until after the data plane has mutated host routes. That avoids carrying a
+	// client-go transport across WireGuard/vmnet route changes on remote-API nodes.
 	restCfg, err := cfg.RestConfig()
 	if err != nil {
 		return fmt.Errorf("kubernetes client config: %w", err)
 	}
-	clientset, err := kubernetes.NewForConfig(restCfg)
-	if err != nil {
-		return fmt.Errorf("build kubernetes client: %w", err)
-	}
 
 	internalIP := cfg.Node.InternalIP
 	if internalIP == "" {
@@ -116,6 +115,10 @@ func run(ctx context.Context, configPath, runtimeBinary string) error {
 			Root:                    cfg.Node.Volumes.Root,
 			HostPathAllowedPrefixes: cfg.Node.Volumes.HostPathAllowedPrefixes,
 		}),
+		provider.WithDNS(provider.DNSConfig{
+			ClusterDNS:    cfg.Node.ClusterDNS,
+			ClusterDomain: cfg.Node.ClusterDomain,
+		}),
 	)
 
 	// Resolve the configured node shape (capacity/taints validated at load).
@@ -171,6 +174,51 @@ func run(ctx context.Context, configPath, runtimeBinary string) error {
 		vknode.WithNodeStatusUpdateInterval(statusInterval),
 		vknode.WithNodeStatusUpdateErrorHandler(statusErrHandler),
 	}
+	// Bring the host data plane up BEFORE the node controller connects to the API
+	// server. The mesh/route changes perturb host routing, and doing them after the
+	// long-lived API (HTTP/2) connection is established severs it on a node whose
+	// API server is remote — client-go then wedges on "no route to host" and the
+	// node flaps NotReady. Establishing the API connection over the final routing
+	// avoids that (the data plane needs only static config, not registration).
+	if cfg.PrivilegedHelperSocket != "" && (cfg.Mesh.Enabled || cfg.PodNetwork.Enabled) {
+		hc := privhelper.NewClient(cfg.PrivilegedHelperSocket)
+		st, err := hc.Status(ctx)
+		if err != nil {
+			return fmt.Errorf("privileged network helper not reachable at %s (start macvz-netd as root): %w", cfg.PrivilegedHelperSocket, err)
+		}
+		// Confirm the exec path itself works (the socket may answer status while the
+		// daemon lacks the privileges to run commands).
+		if err := hc.Ping(ctx); err != nil {
+			return fmt.Errorf("privileged network helper at %s answered status but cannot run commands: %w", cfg.PrivilegedHelperSocket, err)
+		}
+		if !st.PolicyEnforced {
+			return fmt.Errorf("privileged network helper at %s is running without per-request policy validation; start macvz-netd with --config", cfg.PrivilegedHelperSocket)
+		}
+		klog.InfoS("privileged network helper reachable",
+			"socket", cfg.PrivilegedHelperSocket, "version", st.Version, "protocol", st.Protocol,
+			"policyEnforced", st.PolicyEnforced, "allow", st.AllowedCommands, "uptime", st.Uptime)
+	}
+
+	stopMesh, err := setupMesh(ctx, cfg, configPath)
+	if err != nil {
+		return fmt.Errorf("setup mesh: %w", err)
+	}
+	defer stopMesh()
+
+	podNetRouter, stopPodNet, err := setupPodNetwork(ctx, cfg, p)
+	if err != nil {
+		return fmt.Errorf("setup pod network: %w", err)
+	}
+	defer stopPodNet()
+
+	clientset, err := kubernetes.NewForConfig(restCfg)
+	if err != nil {
+		return fmt.Errorf("build kubernetes client after data-plane setup: %w", err)
+	}
+	if err := waitForAPIServer(ctx, clientset); err != nil {
+		return fmt.Errorf("kubernetes API not reachable after data-plane setup: %w", err)
+	}
+
 	if cfg.Node.EnableLease {
 		// The Lease in kube-node-lease is the modern node heartbeat; Kubernetes
 		// marks the node NotReady if it is not renewed within the lease duration.
@@ -211,21 +259,11 @@ func run(ctx context.Context, configPath, runtimeBinary string) error {
 		return fmt.Errorf("setup pod IPAM: %w", err)
 	}
 
-	// Bring up the cross-host WireGuard mesh (if enabled) so remote Pod CIDRs are
-	// routed through encrypted tunnels before Pods start landing on this node.
-	stopMesh, err := setupMesh(ctx, cfg)
-	if err != nil {
-		return fmt.Errorf("setup mesh: %w", err)
-	}
-	defer stopMesh()
-
-	// Start the host Pod network path so each micro-VM is reachable at its Pod IP
-	// across the mesh, and attach it to the provider before Pods are reconciled.
-	stopPodNet, err := setupPodNetwork(ctx, cfg, p)
-	if err != nil {
-		return fmt.Errorf("setup pod network: %w", err)
-	}
-	defer stopPodNet()
+	// Program ClusterIP Service routing into the Pod network anchor so micro-VMs
+	// can reach Services (#37). The Pod network path was brought up before node
+	// registration (see above); the router is reused here. No-op when disabled.
+	stopSvc := startServiceController(ctx, cfg, clientset, podNetRouter)
+	defer stopSvc()
 
 	// Start the Pod lifecycle controller so scheduled Pods become micro-VMs.
 	stopPods, err := startPodController(ctx, cfg, clientset, p, runtime.NumCPU())

cmd/macvz-kubelet/mesh_reconcile.go

@@ -0,0 +1,86 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/chimerakang/macvz/pkg/config"
+	"github.com/chimerakang/macvz/pkg/network/privhelper"
+	"github.com/chimerakang/macvz/pkg/network/wireguard"
+	"k8s.io/klog/v2"
+)
+
+// reconcileMeshPeers reloads the config from disk and re-applies its peer set to
+// the running mesh, adding/removing WireGuard peers and their Pod-CIDR routes in
+// place (#42). It never recreates the interface, so existing tunnels and the
+// host Pod network attachments (owned by a separate podnet.Router) are
+// undisturbed. wireguard.Mesh.Sync is idempotent, so reconciling an unchanged
+// peer set is a no-op beyond re-applying the wg config.
+func reconcileMeshPeers(ctx context.Context, mesh *wireguard.Mesh, configPath string) error {
+	peers, err := loadMeshPeers(configPath)
+	if err != nil {
+		return err
+	}
+	before := mesh.Peers()
+	if err := mesh.Sync(ctx, peers); err != nil {
+		return fmt.Errorf("sync mesh peers: %w", err)
+	}
+	klog.InfoS("mesh peers reconciled",
+		"interface", mesh.InterfaceName(),
+		"before", before, "after", mesh.Peers(),
+		"routes", mesh.InstalledRoutes())
+	return nil
+}
+
+// loadMeshPeers reloads the config and resolves the desired peer set for
+// reconciliation. It refuses a config that fails validation or disables the
+// mesh, so a bad edit or an accidental `enabled: false` cannot silently tear
+// down a live data plane on SIGHUP — the caller keeps the running peers instead.
+func loadMeshPeers(configPath string) ([]wireguard.Peer, error) {
+	cfg, err := config.Load(configPath)
+	if err != nil {
+		return nil, fmt.Errorf("reload config %q: %w", configPath, err)
+	}
+	if err := cfg.Validate(); err != nil {
+		return nil, fmt.Errorf("reloaded config is invalid: %w", err)
+	}
+	if !cfg.Mesh.Enabled {
+		return nil, fmt.Errorf("reloaded config disables the mesh; restart to tear it down")
+	}
+	peers, err := cfg.MeshPeers()
+	if err != nil {
+		return nil, fmt.Errorf("resolve mesh peers: %w", err)
+	}
+	return peers, nil
+}
+
+// watchMeshReload reconciles the mesh peer set whenever SIGHUP arrives, so an
+// operator can add or remove MacVz nodes by editing the config and signalling
+// the kubelet (`kill -HUP <pid>`) — no restart, no dropped Pod attachments. It
+// runs until ctx is done. A failed reconcile is logged and the current peer set
+// is kept, so a bad edit never takes the mesh down.
+func watchMeshReload(ctx context.Context, mesh *wireguard.Mesh, configPath, helperSocket string) {
+	sig := make(chan os.Signal, 1)
+	signal.Notify(sig, syscall.SIGHUP)
+	defer signal.Stop(sig)
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-sig:
+			klog.InfoS("SIGHUP received; reconciling mesh peers from config", "config", configPath)
+			if helperSocket != "" {
+				if err := privhelper.NewClient(helperSocket).ReloadPolicy(ctx); err != nil {
+					klog.ErrorS(err, "privileged helper policy reload failed; keeping current mesh peers")
+					continue
+				}
+			}
+			if err := reconcileMeshPeers(ctx, mesh, configPath); err != nil {
+				klog.ErrorS(err, "mesh peer reconciliation failed; keeping current peers")
+			}
+		}
+	}
+}

cmd/macvz-kubelet/mesh_reconcile_test.go

@@ -0,0 +1,97 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/chimerakang/macvz/pkg/network/wireguard"
+)
+
+// writeMeshConfig writes a minimal valid kubelet config with a mesh stanza and
+// the given peers, returning its path. Each peer is (name, podCIDR, publicKey).
+func writeMeshConfig(t *testing.T, enabled bool, peers [][3]string) string {
+	t.Helper()
+	var b strings.Builder
+	fmt.Fprintf(&b, "nodeName: mac-test\n")
+	fmt.Fprintf(&b, "mesh:\n  enabled: %t\n  interface: utun7\n  privateKeyFile: %s\n  address: 10.99.0.1/32\n  listenPort: 51820\n",
+		enabled, filepath.Join(t.TempDir(), "wg.key"))
+	if len(peers) > 0 {
+		b.WriteString("  peers:\n")
+		for _, p := range peers {
+			fmt.Fprintf(&b, "    - name: %s\n      publicKey: %s\n      podCIDR: %s\n", p[0], p[2], p[1])
+		}
+	}
+	path := filepath.Join(t.TempDir(), "config.yaml")
+	if err := os.WriteFile(path, []byte(b.String()), 0o600); err != nil {
+		t.Fatalf("write config: %v", err)
+	}
+	return path
+}
+
+func pubKey(t *testing.T) string {
+	t.Helper()
+	k, err := wireguard.GenerateKey()
+	if err != nil {
+		t.Fatalf("GenerateKey: %v", err)
+	}
+	return k.PublicKey().String()
+}
+
+func TestLoadMeshPeersResolvesConfiguredPeers(t *testing.T) {
+	keyA, keyB := pubKey(t), pubKey(t)
+	path := writeMeshConfig(t, true, [][3]string{
+		{"mac-02", "10.244.2.0/24", keyA},
+		{"mac-03", "10.244.3.0/24", keyB},
+	})
+
+	peers, err := loadMeshPeers(path)
+	if err != nil {
+		t.Fatalf("loadMeshPeers: %v", err)
+	}
+	if len(peers) != 2 {
+		t.Fatalf("peers = %d, want 2", len(peers))
+	}
+	got := map[string]bool{}
+	for _, p := range peers {
+		got[p.Name] = true
+		if len(p.AllowedIPs) == 0 {
+			t.Errorf("peer %q has no AllowedIPs", p.Name)
+		}
+	}
+	if !got["mac-02"] || !got["mac-03"] {
+		t.Errorf("missing peers, got %v", got)
+	}
+}
+
+func TestLoadMeshPeersRefusesDisabledMesh(t *testing.T) {
+	// A reload that turns the mesh off must not silently drop every peer.
+	path := writeMeshConfig(t, false, nil)
+	if _, err := loadMeshPeers(path); err == nil {
+		t.Fatal("expected error when reloaded config disables the mesh")
+	}
+}
+
+func TestLoadMeshPeersRejectsInvalidConfig(t *testing.T) {
+	// A peer with an unparseable public key is invalid and must be rejected, so a
+	// bad edit keeps the running peer set rather than applying garbage.
+	path := writeMeshConfig(t, true, [][3]string{{"mac-02", "10.244.2.0/24", "not-a-key"}})
+	if _, err := loadMeshPeers(path); err == nil {
+		t.Fatal("expected error for an invalid peer public key")
+	}
+}
+
+func TestLoadMeshPeersEmptyPeerSet(t *testing.T) {
+	// Removing the last peer is a valid reconcile target (the node has no peers),
+	// not an error — Sync would then remove all peers and their routes.
+	path := writeMeshConfig(t, true, nil)
+	peers, err := loadMeshPeers(path)
+	if err != nil {
+		t.Fatalf("loadMeshPeers: %v", err)
+	}
+	if len(peers) != 0 {
+		t.Errorf("peers = %d, want 0", len(peers))
+	}
+}