diff --git a/Makefile b/Makefile index 017fd42..4660a47 100644 --- a/Makefile +++ b/Makefile @@ -22,6 +22,11 @@ build: ## Build the binary into bin/ with version stamping @mkdir -p $(BIN_DIR) go build -ldflags "$(LDFLAGS)" -o $(BIN_DIR)/$(BINARY) $(CMD) +.PHONY: netd +netd: ## Build the privileged network helper daemon into bin/macvz-netd + @mkdir -p $(BIN_DIR) + go build -ldflags "$(LDFLAGS)" -o $(BIN_DIR)/macvz-netd ./cmd/macvz-netd + .PHONY: bench bench: ## Build the density/RAM benchmark harness into bin/mvz-bench @mkdir -p $(BIN_DIR) diff --git a/README.md b/README.md index 5995855..3124b19 100644 --- a/README.md +++ b/README.md @@ -130,4 +130,5 @@ macvz/ - **Density is bounded by RAM, not by container-style kernel sharing.** Each micro-VM carries its own kernel and a fixed memory floor. Validate the real per-host concurrent-VM ceiling and per-VM overhead early in Phase 1 — this defines the project's practical capacity. - **Image architecture.** Guests are arm64. Pulling amd64 images requires the arm64 variant or Rosetta-for-Linux support; surface this clearly to users. - **Security.** The `macvz-kubelet` ↔ API server channel must use the cluster's normal mTLS/RBAC. Do not expose the runtime service or node ports publicly. Image-registry credentials and any secrets come from Kubernetes Secrets / environment, never hardcoded. +- **Privileged networking needs root tools, but the kubelet runs as your user.** The cross-Mac data plane (WireGuard mesh + pf/route/sysctl) needs root, yet `apple/container` refuses to run as root — so `macvz-kubelet` runs as your user and delegates the privileged commands to the `macvz-netd` helper daemon over a unix socket. You install the helper once with `sudo`; day-to-day kubelet starts need no elevation. See [docs/PRIVILEGED_NETWORKING.md](docs/PRIVILEGED_NETWORKING.md) for the full setup and recovery runbook. - **Entitlements & signing.** `macvz-kubelet` runs as a resident process needing the virtualization entitlement; it must be signed appropriately (and notarized for distribution). diff --git a/README.zh-TW.md b/README.zh-TW.md index 54634f8..12c9f85 100644 --- a/README.zh-TW.md +++ b/README.zh-TW.md @@ -130,4 +130,5 @@ macvz/ - **密度受限於 RAM，而非容器式的 kernel 共享。** 每個 micro-VM 自帶 kernel 與固定記憶體下限。請在階段一就驗證單機實際的並發 VM 上限與每 VM 開銷——這決定專案的實用容量。 - **映像架構。** Guest 為 arm64。拉取 amd64 映像需使用 arm64 variant 或 Rosetta-for-Linux 支援；應向使用者明確說明。 - **安全。** `macvz-kubelet` ↔ API server 的通道必須使用叢集既有的 mTLS/RBAC。不要對外公開 runtime 服務或節點埠口。映像倉庫憑證與任何機密均來自 Kubernetes Secrets／環境變數，絕不硬編碼。 +- **特權網路需要 root 工具，但 kubelet 以你的使用者身分執行。** 跨 Mac 資料平面（WireGuard 網格 + pf／route／sysctl）需要 root，但 `apple/container` 拒絕以 root 執行——因此 `macvz-kubelet` 以使用者身分執行，並透過 unix socket 將特權命令委派給 `macvz-netd` 輔助常駐程式。輔助程式只需用 `sudo` 安裝一次；日常啟動 kubelet 不需提權。完整的安裝與復原手冊見 [docs/PRIVILEGED_NETWORKING.md](docs/PRIVILEGED_NETWORKING.md)。 - **Entitlement 與簽章。** `macvz-kubelet` 以常駐行程執行，需要虛擬化 entitlement，並須正確簽章（對外分發還需 notarization）。 diff --git a/cmd/macvz-kubelet/main.go b/cmd/macvz-kubelet/main.go index f9adf37..0b52a9a 100644 --- a/cmd/macvz-kubelet/main.go +++ b/cmd/macvz-kubelet/main.go @@ -17,6 +17,7 @@ import ( "github.com/chimerakang/macvz/internal/version" "github.com/chimerakang/macvz/pkg/config" + "github.com/chimerakang/macvz/pkg/network/privhelper" "github.com/chimerakang/macvz/pkg/provider" "github.com/chimerakang/macvz/pkg/runtime/container" vknode "github.com/virtual-kubelet/virtual-kubelet/node" @@ -93,15 +94,13 @@ func run(ctx context.Context, configPath, runtimeBinary string) error { klog.InfoS("apple/container runtime is ready") } - // Build the Kubernetes client from kubeconfig / in-cluster config. + // Resolve Kubernetes client config now, but delay constructing the clientset + // until after the data plane has mutated host routes. That avoids carrying a + // client-go transport across WireGuard/vmnet route changes on remote-API nodes. restCfg, err := cfg.RestConfig() if err != nil { return fmt.Errorf("kubernetes client config: %w", err) } - clientset, err := kubernetes.NewForConfig(restCfg) - if err != nil { - return fmt.Errorf("build kubernetes client: %w", err) - } internalIP := cfg.Node.InternalIP if internalIP == "" { @@ -116,6 +115,10 @@ func run(ctx context.Context, configPath, runtimeBinary string) error { Root: cfg.Node.Volumes.Root, HostPathAllowedPrefixes: cfg.Node.Volumes.HostPathAllowedPrefixes, }), + provider.WithDNS(provider.DNSConfig{ + ClusterDNS: cfg.Node.ClusterDNS, + ClusterDomain: cfg.Node.ClusterDomain, + }), ) // Resolve the configured node shape (capacity/taints validated at load). @@ -171,6 +174,51 @@ func run(ctx context.Context, configPath, runtimeBinary string) error { vknode.WithNodeStatusUpdateInterval(statusInterval), vknode.WithNodeStatusUpdateErrorHandler(statusErrHandler), } + // Bring the host data plane up BEFORE the node controller connects to the API + // server. The mesh/route changes perturb host routing, and doing them after the + // long-lived API (HTTP/2) connection is established severs it on a node whose + // API server is remote — client-go then wedges on "no route to host" and the + // node flaps NotReady. Establishing the API connection over the final routing + // avoids that (the data plane needs only static config, not registration). + if cfg.PrivilegedHelperSocket != "" && (cfg.Mesh.Enabled || cfg.PodNetwork.Enabled) { + hc := privhelper.NewClient(cfg.PrivilegedHelperSocket) + st, err := hc.Status(ctx) + if err != nil { + return fmt.Errorf("privileged network helper not reachable at %s (start macvz-netd as root): %w", cfg.PrivilegedHelperSocket, err) + } + // Confirm the exec path itself works (the socket may answer status while the + // daemon lacks the privileges to run commands). + if err := hc.Ping(ctx); err != nil { + return fmt.Errorf("privileged network helper at %s answered status but cannot run commands: %w", cfg.PrivilegedHelperSocket, err) + } + if !st.PolicyEnforced { + return fmt.Errorf("privileged network helper at %s is running without per-request policy validation; start macvz-netd with --config", cfg.PrivilegedHelperSocket) + } + klog.InfoS("privileged network helper reachable", + "socket", cfg.PrivilegedHelperSocket, "version", st.Version, "protocol", st.Protocol, + "policyEnforced", st.PolicyEnforced, "allow", st.AllowedCommands, "uptime", st.Uptime) + } + + stopMesh, err := setupMesh(ctx, cfg, configPath) + if err != nil { + return fmt.Errorf("setup mesh: %w", err) + } + defer stopMesh() + + podNetRouter, stopPodNet, err := setupPodNetwork(ctx, cfg, p) + if err != nil { + return fmt.Errorf("setup pod network: %w", err) + } + defer stopPodNet() + + clientset, err := kubernetes.NewForConfig(restCfg) + if err != nil { + return fmt.Errorf("build kubernetes client after data-plane setup: %w", err) + } + if err := waitForAPIServer(ctx, clientset); err != nil { + return fmt.Errorf("kubernetes API not reachable after data-plane setup: %w", err) + } + if cfg.Node.EnableLease { // The Lease in kube-node-lease is the modern node heartbeat; Kubernetes // marks the node NotReady if it is not renewed within the lease duration. @@ -211,21 +259,11 @@ func run(ctx context.Context, configPath, runtimeBinary string) error { return fmt.Errorf("setup pod IPAM: %w", err) } - // Bring up the cross-host WireGuard mesh (if enabled) so remote Pod CIDRs are - // routed through encrypted tunnels before Pods start landing on this node. - stopMesh, err := setupMesh(ctx, cfg) - if err != nil { - return fmt.Errorf("setup mesh: %w", err) - } - defer stopMesh() - - // Start the host Pod network path so each micro-VM is reachable at its Pod IP - // across the mesh, and attach it to the provider before Pods are reconciled. - stopPodNet, err := setupPodNetwork(ctx, cfg, p) - if err != nil { - return fmt.Errorf("setup pod network: %w", err) - } - defer stopPodNet() + // Program ClusterIP Service routing into the Pod network anchor so micro-VMs + // can reach Services (#37). The Pod network path was brought up before node + // registration (see above); the router is reused here. No-op when disabled. + stopSvc := startServiceController(ctx, cfg, clientset, podNetRouter) + defer stopSvc() // Start the Pod lifecycle controller so scheduled Pods become micro-VMs. stopPods, err := startPodController(ctx, cfg, clientset, p, runtime.NumCPU()) diff --git a/cmd/macvz-kubelet/mesh_reconcile.go b/cmd/macvz-kubelet/mesh_reconcile.go new file mode 100644 index 0000000..df8b113 --- /dev/null +++ b/cmd/macvz-kubelet/mesh_reconcile.go @@ -0,0 +1,86 @@ +package main + +import ( + "context" + "fmt" + "os" + "os/signal" + "syscall" + + "github.com/chimerakang/macvz/pkg/config" + "github.com/chimerakang/macvz/pkg/network/privhelper" + "github.com/chimerakang/macvz/pkg/network/wireguard" + "k8s.io/klog/v2" +) + +// reconcileMeshPeers reloads the config from disk and re-applies its peer set to +// the running mesh, adding/removing WireGuard peers and their Pod-CIDR routes in +// place (#42). It never recreates the interface, so existing tunnels and the +// host Pod network attachments (owned by a separate podnet.Router) are +// undisturbed. wireguard.Mesh.Sync is idempotent, so reconciling an unchanged +// peer set is a no-op beyond re-applying the wg config. +func reconcileMeshPeers(ctx context.Context, mesh *wireguard.Mesh, configPath string) error { + peers, err := loadMeshPeers(configPath) + if err != nil { + return err + } + before := mesh.Peers() + if err := mesh.Sync(ctx, peers); err != nil { + return fmt.Errorf("sync mesh peers: %w", err) + } + klog.InfoS("mesh peers reconciled", + "interface", mesh.InterfaceName(), + "before", before, "after", mesh.Peers(), + "routes", mesh.InstalledRoutes()) + return nil +} + +// loadMeshPeers reloads the config and resolves the desired peer set for +// reconciliation. It refuses a config that fails validation or disables the +// mesh, so a bad edit or an accidental `enabled: false` cannot silently tear +// down a live data plane on SIGHUP — the caller keeps the running peers instead. +func loadMeshPeers(configPath string) ([]wireguard.Peer, error) { + cfg, err := config.Load(configPath) + if err != nil { + return nil, fmt.Errorf("reload config %q: %w", configPath, err) + } + if err := cfg.Validate(); err != nil { + return nil, fmt.Errorf("reloaded config is invalid: %w", err) + } + if !cfg.Mesh.Enabled { + return nil, fmt.Errorf("reloaded config disables the mesh; restart to tear it down") + } + peers, err := cfg.MeshPeers() + if err != nil { + return nil, fmt.Errorf("resolve mesh peers: %w", err) + } + return peers, nil +} + +// watchMeshReload reconciles the mesh peer set whenever SIGHUP arrives, so an +// operator can add or remove MacVz nodes by editing the config and signalling +// the kubelet (`kill -HUP `) — no restart, no dropped Pod attachments. It +// runs until ctx is done. A failed reconcile is logged and the current peer set +// is kept, so a bad edit never takes the mesh down. +func watchMeshReload(ctx context.Context, mesh *wireguard.Mesh, configPath, helperSocket string) { + sig := make(chan os.Signal, 1) + signal.Notify(sig, syscall.SIGHUP) + defer signal.Stop(sig) + for { + select { + case <-ctx.Done(): + return + case <-sig: + klog.InfoS("SIGHUP received; reconciling mesh peers from config", "config", configPath) + if helperSocket != "" { + if err := privhelper.NewClient(helperSocket).ReloadPolicy(ctx); err != nil { + klog.ErrorS(err, "privileged helper policy reload failed; keeping current mesh peers") + continue + } + } + if err := reconcileMeshPeers(ctx, mesh, configPath); err != nil { + klog.ErrorS(err, "mesh peer reconciliation failed; keeping current peers") + } + } + } +} diff --git a/cmd/macvz-kubelet/mesh_reconcile_test.go b/cmd/macvz-kubelet/mesh_reconcile_test.go new file mode 100644 index 0000000..29ed06d --- /dev/null +++ b/cmd/macvz-kubelet/mesh_reconcile_test.go @@ -0,0 +1,97 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/chimerakang/macvz/pkg/network/wireguard" +) + +// writeMeshConfig writes a minimal valid kubelet config with a mesh stanza and +// the given peers, returning its path. Each peer is (name, podCIDR, publicKey). +func writeMeshConfig(t *testing.T, enabled bool, peers [][3]string) string { + t.Helper() + var b strings.Builder + fmt.Fprintf(&b, "nodeName: mac-test\n") + fmt.Fprintf(&b, "mesh:\n enabled: %t\n interface: utun7\n privateKeyFile: %s\n address: 10.99.0.1/32\n listenPort: 51820\n", + enabled, filepath.Join(t.TempDir(), "wg.key")) + if len(peers) > 0 { + b.WriteString(" peers:\n") + for _, p := range peers { + fmt.Fprintf(&b, " - name: %s\n publicKey: %s\n podCIDR: %s\n", p[0], p[2], p[1]) + } + } + path := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(path, []byte(b.String()), 0o600); err != nil { + t.Fatalf("write config: %v", err) + } + return path +} + +func pubKey(t *testing.T) string { + t.Helper() + k, err := wireguard.GenerateKey() + if err != nil { + t.Fatalf("GenerateKey: %v", err) + } + return k.PublicKey().String() +} + +func TestLoadMeshPeersResolvesConfiguredPeers(t *testing.T) { + keyA, keyB := pubKey(t), pubKey(t) + path := writeMeshConfig(t, true, [][3]string{ + {"mac-02", "10.244.2.0/24", keyA}, + {"mac-03", "10.244.3.0/24", keyB}, + }) + + peers, err := loadMeshPeers(path) + if err != nil { + t.Fatalf("loadMeshPeers: %v", err) + } + if len(peers) != 2 { + t.Fatalf("peers = %d, want 2", len(peers)) + } + got := map[string]bool{} + for _, p := range peers { + got[p.Name] = true + if len(p.AllowedIPs) == 0 { + t.Errorf("peer %q has no AllowedIPs", p.Name) + } + } + if !got["mac-02"] || !got["mac-03"] { + t.Errorf("missing peers, got %v", got) + } +} + +func TestLoadMeshPeersRefusesDisabledMesh(t *testing.T) { + // A reload that turns the mesh off must not silently drop every peer. + path := writeMeshConfig(t, false, nil) + if _, err := loadMeshPeers(path); err == nil { + t.Fatal("expected error when reloaded config disables the mesh") + } +} + +func TestLoadMeshPeersRejectsInvalidConfig(t *testing.T) { + // A peer with an unparseable public key is invalid and must be rejected, so a + // bad edit keeps the running peer set rather than applying garbage. + path := writeMeshConfig(t, true, [][3]string{{"mac-02", "10.244.2.0/24", "not-a-key"}}) + if _, err := loadMeshPeers(path); err == nil { + t.Fatal("expected error for an invalid peer public key") + } +} + +func TestLoadMeshPeersEmptyPeerSet(t *testing.T) { + // Removing the last peer is a valid reconcile target (the node has no peers), + // not an error — Sync would then remove all peers and their routes. + path := writeMeshConfig(t, true, nil) + peers, err := loadMeshPeers(path) + if err != nil { + t.Fatalf("loadMeshPeers: %v", err) + } + if len(peers) != 0 { + t.Errorf("peers = %d, want 0", len(peers)) + } +} diff --git a/cmd/macvz-kubelet/serve.go b/cmd/macvz-kubelet/serve.go index f95a8f9..303a21f 100644 --- a/cmd/macvz-kubelet/serve.go +++ b/cmd/macvz-kubelet/serve.go @@ -4,16 +4,19 @@ import ( "context" "crypto/tls" "crypto/x509" + "errors" "fmt" "net" "net/http" "os" "path" + "syscall" "time" "github.com/chimerakang/macvz/pkg/config" "github.com/chimerakang/macvz/pkg/network" "github.com/chimerakang/macvz/pkg/network/podnet" + "github.com/chimerakang/macvz/pkg/network/svcroute" "github.com/chimerakang/macvz/pkg/network/wireguard" "github.com/chimerakang/macvz/pkg/provider" vknode "github.com/virtual-kubelet/virtual-kubelet/node" @@ -37,6 +40,62 @@ const informerResync = time.Minute // node a Pod CIDR before continuing without coordinated IPAM. const podCIDRWaitTimeout = 30 * time.Second +const ( + apiReachabilityTimeout = 30 * time.Second + apiReachabilityProbe = 5 * time.Second + apiReachabilityRetry = time.Second + kubeletListenTimeout = 10 * time.Second + kubeletListenRetry = 250 * time.Millisecond +) + +// waitForAPIServer verifies the Kubernetes API is reachable after the host data +// plane has finished mutating routes. This keeps route churn failures in the +// startup path, before Virtual Kubelet starts its long-lived controllers. +func waitForAPIServer(ctx context.Context, clientset kubernetes.Interface) error { + return waitForAPIServerWithTimeout(ctx, clientset, apiReachabilityTimeout, apiReachabilityProbe, apiReachabilityRetry) +} + +func waitForAPIServerWithTimeout(ctx context.Context, clientset kubernetes.Interface, totalTimeout, probeTimeout, retryDelay time.Duration) error { + deadlineCtx, cancel := context.WithTimeout(ctx, totalTimeout) + defer cancel() + + var lastErr error + for { + errCh := make(chan error, 1) + go func() { + _, err := clientset.Discovery().ServerVersion() + errCh <- err + }() + + select { + case err := <-errCh: + if err == nil { + klog.InfoS("Kubernetes API reachable after data-plane setup") + return nil + } + lastErr = err + klog.ErrorS(err, "Kubernetes API reachability probe failed; retrying") + case <-time.After(probeTimeout): + lastErr = fmt.Errorf("API probe timed out after %s", probeTimeout) + klog.ErrorS(lastErr, "Kubernetes API reachability probe failed; retrying") + case <-deadlineCtx.Done(): + if lastErr == nil { + lastErr = deadlineCtx.Err() + } + return fmt.Errorf("API did not answer within %s: %w", totalTimeout, lastErr) + } + + select { + case <-deadlineCtx.Done(): + if lastErr == nil { + lastErr = deadlineCtx.Err() + } + return fmt.Errorf("API did not answer within %s: %w", totalTimeout, lastErr) + case <-time.After(retryDelay): + } + } +} + // setupIPAM enables coordinated Pod IPAM for this node. The address range is the // node's Kubernetes-assigned Spec.PodCIDR (or cfg.Node.PodCIDR when set as an // override). It then recovers existing allocations from the API server so a @@ -158,7 +217,12 @@ func startPodController(ctx context.Context, cfg config.Config, clientset *kuber // cleanup func that tears it back down. The mesh encrypts and routes cross-host // Pod traffic (issue #21); each peer's Pod CIDR becomes a route through the // tunnel. When the mesh is disabled it is a no-op returning a no-op cleanup. -func setupMesh(ctx context.Context, cfg config.Config) (func(), error) { +// +// When configPath is set, it also starts a SIGHUP-driven reconciler (#42): an +// operator edits the config's peer list and signals the kubelet to add/remove +// peers and routes in place, with no restart and no disruption to local Pod +// attachments. +func setupMesh(ctx context.Context, cfg config.Config, configPath string) (func(), error) { if !cfg.Mesh.Enabled { klog.InfoS("WireGuard mesh disabled; Pods are reachable only on their local node") return func() {}, nil @@ -168,7 +232,11 @@ func setupMesh(ctx context.Context, cfg config.Config) (func(), error) { if err != nil { return nil, fmt.Errorf("resolve mesh config: %w", err) } - mesh := wireguard.New(ifc) + var meshOpts []wireguard.Option + if cfg.PrivilegedHelperSocket != "" { + meshOpts = append(meshOpts, wireguard.WithHelperSocket(cfg.PrivilegedHelperSocket)) + } + mesh := wireguard.New(ifc, meshOpts...) if err := mesh.Up(ctx); err != nil { return nil, fmt.Errorf("bring up mesh interface %q: %w", ifc.Name, err) } @@ -179,6 +247,13 @@ func setupMesh(ctx context.Context, cfg config.Config) (func(), error) { "routes", mesh.InstalledRoutes(), ) + // Reconcile peers on SIGHUP so nodes can join/leave without a restart (#42). + // Requires the config path to reload; with no --config it is skipped. + if configPath != "" { + go watchMeshReload(ctx, mesh, configPath, cfg.PrivilegedHelperSocket) + klog.InfoS("mesh peer reload enabled; edit peers then `kill -HUP` the kubelet to reconcile", "config", configPath) + } + return func() { // Tear down with a fresh context: the root ctx is already cancelled by // the time cleanup runs during shutdown. @@ -192,26 +267,64 @@ func setupMesh(ctx context.Context, cfg config.Config) (func(), error) { // it to the provider so each micro-VM is reachable at its Pod IP across the mesh // (#22). It returns a cleanup func that flushes the host rules. When disabled it // is a no-op returning a no-op cleanup. -func setupPodNetwork(ctx context.Context, cfg config.Config, p *provider.Provider) (func(), error) { +func setupPodNetwork(ctx context.Context, cfg config.Config, p *provider.Provider) (*podnet.Router, func(), error) { if !cfg.PodNetwork.Enabled { klog.InfoS("Pod network path disabled; Pods keep the runtime host-only address") - return func() {}, nil + return nil, func() {}, nil } - router := podnet.New(cfg.PodNetworkRouterConfig()) + var pnOpts []podnet.Option + if cfg.PrivilegedHelperSocket != "" { + pnOpts = append(pnOpts, podnet.WithHelperSocket(cfg.PrivilegedHelperSocket)) + } + router := podnet.New(cfg.PodNetworkRouterConfig(), pnOpts...) if err := router.Start(ctx); err != nil { - return nil, fmt.Errorf("start pod network path: %w", err) + return nil, nil, fmt.Errorf("start pod network path: %w", err) } p.SetPodNetwork(router) klog.InfoS("Pod network path started", "interface", cfg.PodNetwork.Interface) - return func() { + return router, func() { if err := router.Stop(context.Background()); err != nil { klog.ErrorS(err, "failed to stop pod network path") } }, nil } +// startServiceController runs the ClusterIP Service route controller (#37). It +// watches Services and EndpointSlices cluster-wide and programs the podnet +// anchor with rdr DNAT rules so micro-VMs can reach Service ClusterIPs. It is a +// no-op (nil router) when the Pod network path is disabled — without it there is +// nothing to program. Returns a cleanup that stops the controller. +func startServiceController(ctx context.Context, cfg config.Config, clientset *kubernetes.Clientset, router *podnet.Router) func() { + if router == nil { + klog.InfoS("ClusterIP service routing disabled (Pod network path is off)") + return func() {} + } + factory := informers.NewSharedInformerFactory(clientset, informerResync) + ctrl := svcroute.New(router, factory) + + ctlCtx, cancel := context.WithCancel(ctx) + factory.Start(ctlCtx.Done()) + done := make(chan struct{}) + go func() { + defer close(done) + if err := ctrl.Run(ctlCtx, serviceControllerWorkers); err != nil && ctlCtx.Err() == nil { + klog.ErrorS(err, "service route controller stopped unexpectedly") + } + }() + klog.InfoS("ClusterIP service routing enabled", "interface", cfg.PodNetwork.Interface) + // Wait for the controller to drain on cleanup so no reconcile races the + // router's Stop (the Pod network path is torn down right after this). + return func() { + cancel() + <-done + } +} + +// serviceControllerWorkers is the concurrency of the service route controller. +const serviceControllerWorkers = 2 + // buildServingTLSConfig assembles the TLS config for the kubelet HTTPS server. // When clientCAFile is set, the server requires and verifies a client // certificate signed by that CA (mutual TLS), so only holders of an @@ -288,7 +401,7 @@ func startKubeletServer(ctx context.Context, cfg config.Config, p *provider.Prov } else { klog.InfoS("kubelet endpoint binding to all interfaces (no internal IP resolved); consider setting node.internalIP", "port", port) } - listener, err := tls.Listen("tcp", addr, tlsCfg) + listener, err := listenKubeletTLS(ctx, addr, tlsCfg) if err != nil { return nil, fmt.Errorf("listen on %s: %w", addr, err) } @@ -306,3 +419,35 @@ func startKubeletServer(ctx context.Context, cfg config.Config, p *provider.Prov _ = listener.Close() }, nil } + +func listenKubeletTLS(ctx context.Context, addr string, tlsCfg *tls.Config) (net.Listener, error) { + return listenKubeletTLSWithRetry(ctx, addr, tlsCfg, kubeletListenTimeout, kubeletListenRetry) +} + +func listenKubeletTLSWithRetry(ctx context.Context, addr string, tlsCfg *tls.Config, totalTimeout, retryDelay time.Duration) (net.Listener, error) { + deadlineCtx, cancel := context.WithTimeout(ctx, totalTimeout) + defer cancel() + + var lastErr error + for { + ln, err := tls.Listen("tcp", addr, tlsCfg) + if err == nil { + return ln, nil + } + if !isAddrInUse(err) { + return nil, err + } + lastErr = err + klog.ErrorS(err, "kubelet API port still in use; retrying", "addr", addr) + + select { + case <-deadlineCtx.Done(): + return nil, fmt.Errorf("address still in use after %s: %w", totalTimeout, lastErr) + case <-time.After(retryDelay): + } + } +} + +func isAddrInUse(err error) bool { + return errors.Is(err, syscall.EADDRINUSE) +} diff --git a/cmd/macvz-kubelet/serve_test.go b/cmd/macvz-kubelet/serve_test.go index 6ea71dd..4787da9 100644 --- a/cmd/macvz-kubelet/serve_test.go +++ b/cmd/macvz-kubelet/serve_test.go @@ -1,6 +1,7 @@ package main import ( + "context" "crypto/ecdsa" "crypto/elliptic" "crypto/rand" @@ -8,11 +9,19 @@ import ( "crypto/x509" "crypto/x509/pkix" "encoding/pem" + "errors" "math/big" + "net" "os" "path/filepath" "testing" "time" + + "k8s.io/apimachinery/pkg/runtime" + apiversion "k8s.io/apimachinery/pkg/version" + "k8s.io/client-go/discovery/fake" + kubernetesfake "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" ) // writeTestCA writes a self-signed CA certificate to a temp file and returns its @@ -43,6 +52,26 @@ func writeTestCA(t *testing.T) string { return path } +func testServingCert(t *testing.T) tls.Certificate { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("gen key: %v", err) + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(2), + Subject: pkix.Name{CommonName: "localhost"}, + NotBefore: time.Unix(1700000000, 0), + NotAfter: time.Unix(1900000000, 0), + KeyUsage: x509.KeyUsageDigitalSignature, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + t.Fatalf("create cert: %v", err) + } + return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key} +} + func TestBuildServingTLSConfigNoClientCA(t *testing.T) { cfg, err := buildServingTLSConfig(tls.Certificate{}, "") if err != nil { @@ -86,3 +115,62 @@ func TestBuildServingTLSConfigRejectsBadCA(t *testing.T) { t.Error("expected error for a CA file with no usable certificates") } } + +func TestWaitForAPIServerSuccess(t *testing.T) { + client := kubernetesfake.NewClientset() + client.Discovery().(*fake.FakeDiscovery).FakedServerVersion = &apiversion.Info{GitVersion: "v1.35.0"} + + if err := waitForAPIServerWithTimeout(context.Background(), client, time.Second, 50*time.Millisecond, time.Millisecond); err != nil { + t.Fatalf("waitForAPIServerWithTimeout: %v", err) + } +} + +func TestWaitForAPIServerFailure(t *testing.T) { + client := kubernetesfake.NewClientset() + wantErr := errors.New("no route to host") + client.Discovery().(*fake.FakeDiscovery).PrependReactor("*", "*", func(action k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, wantErr + }) + + err := waitForAPIServerWithTimeout(context.Background(), client, 20*time.Millisecond, 5*time.Millisecond, time.Millisecond) + if err == nil { + t.Fatal("expected API reachability failure") + } + if !errors.Is(err, wantErr) { + t.Fatalf("error = %v, want wrapping %v", err, wantErr) + } +} + +func TestListenKubeletTLSRetriesAddressInUse(t *testing.T) { + blocker, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("Listen blocker: %v", err) + } + addr := blocker.Addr().String() + released := make(chan struct{}) + go func() { + time.Sleep(20 * time.Millisecond) + _ = blocker.Close() + close(released) + }() + + ln, err := listenKubeletTLSWithRetry(context.Background(), addr, &tls.Config{Certificates: []tls.Certificate{testServingCert(t)}}, time.Second, 5*time.Millisecond) + if err != nil { + t.Fatalf("listenKubeletTLSWithRetry: %v", err) + } + _ = ln.Close() + <-released +} + +func TestListenKubeletTLSFailsAfterAddressInUseTimeout(t *testing.T) { + blocker, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("Listen blocker: %v", err) + } + defer func() { _ = blocker.Close() }() + + _, err = listenKubeletTLSWithRetry(context.Background(), blocker.Addr().String(), &tls.Config{Certificates: []tls.Certificate{testServingCert(t)}}, 20*time.Millisecond, 5*time.Millisecond) + if err == nil { + t.Fatal("expected address-in-use timeout") + } +} diff --git a/cmd/macvz-netd/main.go b/cmd/macvz-netd/main.go new file mode 100644 index 0000000..1fc2565 --- /dev/null +++ b/cmd/macvz-netd/main.go @@ -0,0 +1,296 @@ +// Command macvz-netd is the MacVz privileged network helper daemon (#38, #40). +// +// It runs as root and executes a fixed allowlist of network commands (pfctl, +// sysctl, route, ifconfig, wg, wireguard-go, pkill) on behalf of the user-run +// macvz-kubelet, which connects over a unix socket. This keeps the kubelet +// itself running as the operator's user — required because apple/container is a +// per-user service and refuses to run as root — while confining root to exactly +// the network operations the data plane needs. +// +// Usage: +// +// macvz-netd serve [--socket PATH] --config PATH [--owner uid:gid] run the daemon (default) +// macvz-netd install [--socket PATH] --config PATH [--owner uid:gid] install + start the LaunchDaemon +// macvz-netd uninstall stop + remove the LaunchDaemon +// macvz-netd load | unload (re)bootstrap an installed job +// macvz-netd status report install/run state +// +// Run it directly once, as root, before the kubelet: +// +// sudo macvz-netd serve --socket /var/run/macvz-netd.sock --config /etc/macvz/config.yaml +// +// Or install it as a LaunchDaemon so it starts at boot and the kubelet never +// needs sudo (the installing user, captured via sudo, owns the socket): +// +// sudo macvz-netd install --socket /var/run/macvz-netd.sock --config /etc/macvz/config.yaml +// +// When launched via sudo `serve` chowns the socket to $SUDO_UID/$SUDO_GID so the +// invoking (non-root) user can connect, and no other user can. Under launchd +// there is no SUDO_UID, so the owner baked into the plist via --owner is used. +package main + +import ( + "context" + "flag" + "fmt" + "os" + "os/signal" + "syscall" + + "github.com/chimerakang/macvz/internal/version" + "github.com/chimerakang/macvz/pkg/config" + "github.com/chimerakang/macvz/pkg/network/privhelper" + "k8s.io/klog/v2" +) + +const defaultSocket = "/var/run/macvz-netd.sock" + +func main() { + klog.InitFlags(nil) + + // Subcommand dispatch. With no subcommand (or a leading flag) we default to + // `serve` so `sudo macvz-netd --socket ...` keeps working as before. + cmd := "serve" + args := os.Args[1:] + if len(args) > 0 && !isFlag(args[0]) { + cmd, args = args[0], args[1:] + } + + var err error + switch cmd { + case "serve": + err = runServe(args) + case "install": + err = runInstall(args) + case "uninstall": + err = runInstaller(args, "uninstall") + case "load": + err = runInstaller(args, "load") + case "unload": + err = runInstaller(args, "unload") + case "status": + err = runStatus(args) + case "help", "-h", "--help": + usage() + return + default: + fmt.Fprintf(os.Stderr, "macvz-netd: unknown command %q\n\n", cmd) + usage() + os.Exit(2) + } + if err != nil { + klog.ErrorS(err, "macvz-netd "+cmd+" failed") + os.Exit(1) + } +} + +func isFlag(s string) bool { return len(s) > 0 && s[0] == '-' } + +func usage() { + fmt.Fprint(os.Stderr, `macvz-netd — MacVz privileged network helper daemon + +Commands: + serve [--socket PATH] --config PATH [--owner uid:gid] run the daemon (default) + install [--socket PATH] --config PATH [--owner uid:gid] install and start the LaunchDaemon (sudo) + uninstall stop and remove the LaunchDaemon (sudo) + load bootstrap an installed job (sudo) + unload boot out a running job (sudo) + status report install and run state + +Development only: pass --allow-unsafe-no-config with serve/install to skip +config-derived request policy. +`) +} + +// cmdFlags holds the flags a subcommand parsed. +type cmdFlags struct { + socket string + owner string + config string + allowUnsafeNoConfig bool +} + +// parseFlags builds and parses a FlagSet for a subcommand, wiring the shared +// --socket/--owner/--config flags when requested. +func parseFlags(name string, args []string, socket, owner bool) (cmdFlags, error) { + var f cmdFlags + fs := flag.NewFlagSet(name, flag.ContinueOnError) + if socket { + fs.StringVar(&f.socket, "socket", defaultSocket, "unix socket path to listen on") + // --config travels with --socket: both serve and install accept it. + fs.StringVar(&f.config, "config", "", "path to the MacVz config; restricts privileged commands to its interfaces, CIDRs, peers, and pf anchor (#41)") + fs.BoolVar(&f.allowUnsafeNoConfig, "allow-unsafe-no-config", false, "development only: allow helper startup without config-derived request policy") + } + if owner { + fs.StringVar(&f.owner, "owner", "", "uid:gid to own the socket (defaults to $SUDO_UID:$SUDO_GID)") + } + return f, fs.Parse(args) +} + +// helperServer builds the helper server, enforcing a config-derived policy in +// production and allowing name-allowlist-only mode only behind an explicit unsafe +// development flag. +func helperServer(socket, configPath string, allowUnsafeNoConfig bool) (*privhelper.Server, error) { + if configPath == "" { + if !allowUnsafeNoConfig { + return nil, fmt.Errorf("macvz-netd requires --config to enforce per-request network policy; pass --allow-unsafe-no-config only for local development") + } + klog.Warning("macvz-netd started with --allow-unsafe-no-config: privileged commands are restricted to the command allowlist only, not to this node's interfaces/CIDRs/peers/anchor.") + return privhelper.NewServer(socket).SetVersion(version.Version), nil + } + loader := func() (privhelper.Policy, error) { + cfg, err := config.Load(configPath) + if err != nil { + return privhelper.Policy{}, fmt.Errorf("load config %q: %w", configPath, err) + } + policy, err := cfg.PrivilegedHelperPolicy() + if err != nil { + return privhelper.Policy{}, fmt.Errorf("derive privileged helper policy from %q: %w", configPath, err) + } + klog.InfoS("macvz-netd enforcing request policy", + "config", configPath, "meshInterface", policy.MeshInterface, + "vmnetInterface", policy.VMNetInterface, "anchor", policy.Anchor, + "routeCIDRs", len(policy.RouteCIDRs), "podCIDRs", len(policy.PodCIDRs), + "vmNetCIDRs", len(policy.VMNetCIDRs), "peers", len(policy.PeerPublicKeys)) + return policy, nil + } + srv, err := privhelper.NewServerWithPolicyLoader(socket, loader) + if err != nil { + return nil, err + } + return srv.SetVersion(version.Version), nil +} + +// runServe runs the helper daemon until SIGINT/SIGTERM. +func runServe(args []string) error { + f, err := parseFlags("serve", args, true, true) + if err != nil { + return err + } + + if os.Geteuid() != 0 { + klog.Warning("macvz-netd is not running as root; privileged commands will fail. Start it with sudo.") + } + + uid, gid, err := resolveOwner(f.owner) + if err != nil { + return err + } + + srv, err := helperServer(f.socket, f.config, f.allowUnsafeNoConfig) + if err != nil { + return err + } + if err := srv.Listen(uid, gid); err != nil { + return fmt.Errorf("create helper socket %q: %w", f.socket, err) + } + defer func() { _ = srv.Close() }() + + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + + if err := srv.Serve(ctx); err != nil { + return fmt.Errorf("helper server stopped: %w", err) + } + klog.InfoS("macvz-netd stopped") + return nil +} + +// runInstall copies this binary to the system location, writes the LaunchDaemon +// plist, and starts the job. +func runInstall(args []string) error { + f, err := parseFlags("install", args, true, true) + if err != nil { + return err + } + if os.Geteuid() != 0 { + return fmt.Errorf("install must run as root: sudo macvz-netd install") + } + + uid, gid, err := resolveOwner(f.owner) + if err != nil { + return err + } + if uid < 0 { + return fmt.Errorf("install: cannot determine the owning user; run under sudo or pass --owner uid:gid") + } + + self, err := os.Executable() + if err != nil { + return fmt.Errorf("locate running binary: %w", err) + } + + cfg := privhelper.DefaultLaunchdConfig(f.socket) + cfg.OwnerUID, cfg.OwnerGID = uid, gid + cfg.ConfigPath = f.config + cfg.AllowUnsafeNoConfig = f.allowUnsafeNoConfig + + if err := privhelper.NewInstaller(cfg).Install(context.Background(), self); err != nil { + return err + } + klog.InfoS("macvz-netd installed and started", + "plist", cfg.PlistPath(), "binary", cfg.BinaryPath, "socket", cfg.SocketPath, + "config", cfg.ConfigPath, "owner", fmt.Sprintf("%d:%d", uid, gid), "logs", cfg.StdoutPath) + if cfg.AllowUnsafeNoConfig { + klog.Warning("installed with --allow-unsafe-no-config: per-request policy validation is disabled. Reinstall with --config to confine the helper to this node's interfaces/CIDRs/peers/anchor (#41).") + } + return nil +} + +// runInstaller handles uninstall/load/unload, which need only the default +// layout (label + plist path), not the socket/owner. +func runInstaller(args []string, action string) error { + if _, err := parseFlags(action, args, false, false); err != nil { + return err + } + if os.Geteuid() != 0 { + return fmt.Errorf("%s must run as root: sudo macvz-netd %s", action, action) + } + inst := privhelper.NewInstaller(privhelper.DefaultLaunchdConfig(defaultSocket)) + ctx := context.Background() + switch action { + case "uninstall": + if err := inst.Uninstall(ctx); err != nil { + return err + } + klog.InfoS("macvz-netd uninstalled") + case "load": + if err := inst.Load(ctx); err != nil { + return err + } + klog.InfoS("macvz-netd loaded") + case "unload": + if err := inst.Unload(ctx); err != nil { + return err + } + klog.InfoS("macvz-netd unloaded") + } + return nil +} + +// runStatus prints whether the daemon is installed and loaded. +func runStatus(args []string) error { + if _, err := parseFlags("status", args, false, false); err != nil { + return err + } + inst := privhelper.NewInstaller(privhelper.DefaultLaunchdConfig(defaultSocket)) + st := inst.Status(context.Background()) + fmt.Printf("plist installed: %t (%s)\n", st.PlistInstalled, inst.Cfg.PlistPath()) + fmt.Printf("binary installed: %t (%s)\n", st.BinaryInstalled, inst.Cfg.BinaryPath) + fmt.Printf("socket present: %t (%s)\n", st.SocketPresent, inst.Cfg.SocketPath) + fmt.Printf("launchd loaded: %t\n", st.Loaded) + if st.Detail != "" { + fmt.Printf("detail:\n%s\n", st.Detail) + } + return nil +} + +// resolveOwner picks the socket owner: an explicit --owner uid:gid wins, +// otherwise fall back to the sudo-provided $SUDO_UID/$SUDO_GID. +func resolveOwner(ownerFlag string) (uid, gid int, err error) { + if ownerFlag != "" { + return privhelper.ResolveOwnerSpec(ownerFlag) + } + uid, gid = privhelper.ResolveOwner(os.Getenv("SUDO_UID"), os.Getenv("SUDO_GID")) + return uid, gid, nil +} diff --git a/config.example.yaml b/config.example.yaml index b2ca854..a36d495 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -67,6 +67,13 @@ node: servingTLSCertFile: "" # e.g. /etc/macvz/pki/kubelet.crt servingTLSKeyFile: "" # e.g. /etc/macvz/pki/kubelet.key + # Cluster DNS injected into micro-VMs so Pods using the ClusterFirst DNS + # policy resolve Service names in-guest (#37). Set to the cluster's CoreDNS/ + # kube-dns ClusterIP. Empty leaves each micro-VM with the DNS baked into its + # image. Pairs with podNetwork to give working ClusterIP Service access. + clusterDNS: [] # e.g. ["10.96.0.10"] + clusterDomain: "" # defaults to cluster.local when clusterDNS is set + # Cross-host WireGuard mesh. Disabled by default; see docs/NETWORKING.md for a # complete two-node setup and pf anchor requirements. mesh: @@ -91,3 +98,6 @@ podNetwork: interface: bridge100 anchor: macvz/pods enableForwarding: true + # Host-only vmnet CIDRs local micro-VMs may receive. The privileged helper uses + # these to validate pf rdr/binat targets. Omit to use the common default below. + vmNetCIDRs: ["192.168.64.0/24"] diff --git a/docs/E2E.md b/docs/E2E.md index dc10229..875afad 100644 --- a/docs/E2E.md +++ b/docs/E2E.md @@ -42,9 +42,9 @@ Mac, and is the recorded P4 acceptance evidence. | Port-forward | `port-forward` to a backend reaches the in-Pod HTTP server | #24 | | Cleanup | namespace and all workloads removed | — | -On any failure the suite captures diagnostics (node describes, Pod describes, -endpoints, events) to `MACVZ_E2E_DIAG_DIR` and exits non-zero — suitable for -release gating. +On any failure the suite captures diagnostics to `MACVZ_E2E_DIAG_DIR` and exits +non-zero — suitable for release gating. See [Diagnostics](#diagnostics) for what +is collected and how to reach node-level network state. ## Run it @@ -65,10 +65,11 @@ Configuration (environment): | --- | --- | --- | | `KUBECONFIG` | standard | cluster credentials | | `MACVZ_E2E_NODES` | auto-detect | comma-separated node names | -| `MACVZ_E2E_IMAGE` | `alpine:3.20` | arm64 image (busybox `httpd` + `wget`) | +| `MACVZ_E2E_IMAGE` | `busybox:1.36.1` | arm64 image with `httpd` + `wget` | | `MACVZ_E2E_NAMESPACE` | `macvz-e2e` | namespace for test objects | | `MACVZ_E2E_DIAG_DIR` | mktemp | where failure diagnostics are written | | `MACVZ_E2E_TIMEOUT` | `120` | per-wait timeout (seconds) | +| `MACVZ_E2E_DIAG_SSH` | unset | command template to reach a node for node-level network diagnostics; `{node}` → node name (see [Diagnostics](#diagnostics)) | Setup and teardown are automated: the suite creates its own namespace and tears it down at the end (and on failure, after capturing diagnostics). It does **not** @@ -98,6 +99,60 @@ These cover micro-VM lifecycle, logs, exec, volumes, stats, and Rosetta against a real `apple/container` service. The [P2 smoke test](P2_SMOKE_TEST.md) covers the single-node provider path end-to-end by hand. +## Diagnostics + +On any failure the suite writes a bundle to `MACVZ_E2E_DIAG_DIR` (a mktemp dir by +default; the path is logged). The bundle has two layers, designed to tell apart +**mesh**, **route**, **pf**, **endpoint/kube-proxy**, and **forwarding** failures +(issue #43): + +- **`diagnostics.txt`** — cluster-side state gathered from the control machine, + always available: node readiness and **PodCIDRs**, Pod describes, the + **EndpointSlices** that decide Service membership (full YAML), and recent + events. This distinguishes endpoint/kube-proxy problems (missing or `NotReady` + endpoints, wrong PodCIDR) from data-plane ones. +- **`node-.txt`** — per-node privileged network state from each Mac: + `wg show` (WireGuard handshakes, transfer, endpoints), the IPv4 routing table, + the `macvz/pods` **pf anchor** nat/binat and filter rules, the + `net.inet.ip.forwarding` sysctl, and the Pod bridge. This is what tells a dead + tunnel apart from a missing route, a missing pf rule, or disabled forwarding. + +Node-level state needs root on the node, so the suite collects it through a hook: + +- If the **host running the suite is itself a target node**, it runs the + collector locally (using `sudo -n` when not already root). +- Otherwise, set **`MACVZ_E2E_DIAG_SSH`** to a command template that reaches the + node; the literal token `{node}` is replaced with the node name. The read-only + collector is piped over the connection — nothing is pre-staged on the node: + + ```sh + export MACVZ_E2E_DIAG_SSH='ssh -o ConnectTimeout=5 admin@{node}.local' + MACVZ_E2E=1 MACVZ_E2E_NODES=mac-a,mac-b make e2e + ``` + +- With no hook and a non-local node, that node's collection is skipped with a + note in the log. + +**Secrets are masked** before anything is written — WireGuard private/preshared +keys, kubeconfig `client-key-data`, bearer tokens, and passwords are replaced +with `[REDACTED]`, while public keys, addresses, routes, and rules are kept. So +a bundle is safe to attach to an issue. + +Both acceptance properties are covered by a cluster-free unit test +([test/e2e/diag_test.go](../test/e2e/diag_test.go)) that runs on every +`go test ./...`: it drives the real collector against stubbed node commands, +injects each failure (no handshake, missing route, empty pf anchor, forwarding +off) one at a time, and asserts the bundle removes exactly that failure's signal +while keeping the others (so faults stay distinguishable) — and that secrets +never leak under any scenario. + +You can also run the collector by hand on a Mac to inspect the live data plane: + +```sh +sudo ./test/e2e/collect-node-diag.sh # redacted, to stdout +sudo ./test/e2e/collect-node-diag.sh > node.txt +``` + ## CI [.github/workflows/e2e.yml](../.github/workflows/e2e.yml) runs the suite on diff --git a/docs/MASTER_TASKS.md b/docs/MASTER_TASKS.md index d9f5ae4..00a8da7 100644 --- a/docs/MASTER_TASKS.md +++ b/docs/MASTER_TASKS.md @@ -9,10 +9,15 @@ Source of truth for the phased roadmap. Phases map to GitHub **Milestones**; tas | Phase | Title | Goal | Status | | --- | --- | --- | --- | | P0 | Scaffolding & Foundations | Buildable Go project: module, layout, CLI skeleton, CI, build tooling | ✅ Complete | -| P1 | Runtime Integration | Drive `apple/container` from Go on a single Mac: micro-VM lifecycle, logs/exec, density benchmark | 🚧 In Progress | -| P2 | Virtual Kubelet Provider MVP | Mac registers as a k8s virtual node and runs real Pods as micro-VMs | ⬜ Planned | -| P3 | Cross-host Mesh Networking | WireGuard mesh + Pod IPAM so Pods across Macs communicate and Services resolve | ⬜ Planned | -| P4 | Hardening & Beta | Metrics, volumes, image arch/Rosetta, mTLS/RBAC, signing/notarization, multi-node e2e | ⬜ Planned | +| P1 | Runtime Integration | Drive `apple/container` from Go on a single Mac: micro-VM lifecycle, logs/exec, density benchmark | ✅ Complete | +| P2 | Virtual Kubelet Provider MVP | Mac registers as a k8s virtual node and runs real Pods as micro-VMs | ✅ Complete | +| P3 | Cross-host Mesh Networking | WireGuard mesh + Pod IPAM so Pods across Macs communicate and Services resolve | ✅ Implemented | +| P4 | Hardening & Beta | Metrics, volumes, image arch/Rosetta, mTLS/RBAC, signing/notarization, multi-node e2e | ✅ Complete | +| P5 | Privileged Networking & Full Data Plane | Make cross-Mac Service traffic work end-to-end and remove manual sudo from day-to-day operation | ⬜ Planned | +| P6 | Kubernetes Workload Compatibility | Support common Deployment-era Kubernetes primitives needed by real applications | ⬜ Planned | +| P7 | Multi-node Operations | Make Mac node bootstrap, joining, diagnostics, and removal repeatable | ⬜ Planned | +| P8 | Real App Validation | Run useful public and project-specific Kubernetes applications on MacVz | ⬜ Planned | +| P9 | Production Hardening | Improve recovery, resource accounting, packaging, upgrades, and long-running reliability | ⬜ Planned | ## Milestone Acceptance Criteria @@ -21,6 +26,23 @@ Source of truth for the phased roadmap. Phases map to GitHub **Milestones**; tas - **P2** — `kubectl run alpine --image=alpine --restart=Never -- sleep 3600` lands a micro-VM on the Mac; `kubectl logs`/`exec` work; node shows in `kubectl get nodes`. Operator-facing run/verify/cleanup steps and expected output are documented in [docs/P2_SMOKE_TEST.md](P2_SMOKE_TEST.md); RBAC and manifests under [deployments/](../deployments/). - **P3** — A Service backed by Pods on two different Macs is reachable through normal Kubernetes networking. - **P4** — Multi-node e2e suite green; signed/notarized `macvz-kubelet` build; volumes + image-arch handling supported. +- **P5** — Two-Mac e2e passes with `mesh.enabled: true` and `podNetwork.enabled: true`; Pod-to-Pod and Service traffic cross the WireGuard data plane; operators no longer need to start the main kubelet process with manual sudo. +- **P6** — A multi-Deployment application using ConfigMaps, Secrets, ServiceAccounts, probes, and image pull credentials can roll out and recover through normal Kubernetes controllers. +- **P7** — A new Mac can join the MacVz node pool through a documented bootstrap flow; existing nodes can be drained, diagnosed, and removed without manual cleanup. +- **P8** — At least one public Kubernetes application and one CBB-compatible subset run on MacVz and expose a browser-visible service. +- **P9** — Long-running soak tests survive kubelet/helper restarts, orphan cleanup works, resource usage remains bounded, and release artifacts can be installed, upgraded, rolled back, and removed. + +## Current Validation Snapshot + +As of 2026-06-19, `main` has passed the two-node baseline described in +[MULTI_NODE_TEST_REPORT_2026-06-19.md](MULTI_NODE_TEST_REPORT_2026-06-19.md): + +- two MacVz nodes register as Ready; +- Pods schedule to each Mac and clean up their micro-VMs; +- `logs`, `exec`, `port-forward`, metrics, and stats work through the kubelet API; +- Services publish EndpointSlices with one Ready endpoint per Mac; +- cross-node Service data-plane reachability remains blocked until the privileged + WireGuard + `podNetwork` path is enabled and verified. ## Issue Tracker @@ -55,3 +77,38 @@ Source of truth for the phased roadmap. Phases map to GitHub **Milestones**; tas | #28 | Harden mTLS, RBAC, and runtime access boundaries | P4 | closed | | #29 | Add signed and notarized macvz-kubelet release flow | P4 | closed | | #30 | Build multi-node end-to-end test suite for beta readiness | P4 | closed | +| #37 | Run full WireGuard + podNetwork two-Mac e2e with privileged networking | P5 | planned | +| #38 | Add a privileged network helper daemon for WireGuard, route, sysctl, and pf operations | P5 | planned | +| #39 | Define and implement the local Unix-socket API between macvz-kubelet and the network helper | P5 | done — versioned control API (`pkg/network/privhelper`: protocol negotiation, `status`/`exec` ops, structured `APIError` codes, 1 MiB request cap); kubelet surfaces helper status at startup ([cmd/macvz-kubelet/main.go](../cmd/macvz-kubelet/main.go)); tests in `control_test.go`; spec in [PRIVILEGED_NETWORKING.md](PRIVILEGED_NETWORKING.md#control-api-kubelet--helper-39) | +| #40 | Add launchd install/uninstall support for the privileged network helper | P5 | in progress | +| #41 | Restrict helper inputs to configured CIDRs, interfaces, peers, and pf anchors | P5 | in progress | +| #42 | Add mesh peer reconciliation for adding/removing MacVz nodes without full restart | P5 | in progress | +| #43 | Extend e2e diagnostics for WireGuard handshakes, routes, pf anchors, and forwarding state | P5 | in progress | +| #44 | Document full privileged networking setup and recovery procedures | P5 | done — [docs/PRIVILEGED_NETWORKING.md](PRIVILEGED_NETWORKING.md) | +| #45 | Support restartPolicy Always and controller-managed workload expectations | P6 | planned | +| #46 | Support ConfigMap-backed environment variables and volume mounts | P6 | planned | +| #47 | Support Secret-backed environment variables and volume mounts | P6 | planned | +| #48 | Support envFrom, valueFrom, fieldRef, and resourceFieldRef translation | P6 | planned | +| #49 | Support imagePullSecrets and private registry authentication | P6 | planned | +| #50 | Implement readiness, liveness, and startup probe handling | P6 | planned | +| #51 | Improve ServiceAccount token projection and in-cluster API compatibility | P6 | planned | +| #52 | Define supported and unsupported securityContext behavior for MacVz Pods | P6 | planned | +| #53 | Build a multi-Deployment compatibility fixture for rollout validation | P6 | planned | +| #54 | Create a node bootstrap/join command or documented workflow | P7 | planned | +| #55 | Automate WireGuard key generation, public key exchange, and config rendering | P7 | planned | +| #56 | Add node health and readiness diagnostics across runtime, provider, mesh, and pod network | P7 | planned | +| #57 | Add node drain and safe workload cleanup guidance/tooling | P7 | planned | +| #58 | Add node removal workflow, including route, peer, pf, and VM cleanup | P7 | planned | +| #59 | Produce a local diagnostic bundle command for support and bug reports | P7 | planned | +| #60 | Document multi-node operations, failure modes, and recovery playbooks | P7 | planned | +| #61 | Run a minimal public HTTP application and expose it through a browser-visible Service | P8 | planned | +| #62 | Run a guestbook-style application with multiple Deployments, Services, ConfigMaps, and Secrets | P8 | planned | +| #63 | Evaluate and run a Kubernetes management UI such as Headlamp or Dashboard on MacVz | P8 | planned | +| #64 | Define and run a CBB arm64-compatible subset on MacVz | P8 | planned | +| #65 | Publish real-app validation manifests and expected outputs | P8 | planned | +| #66 | Implement kubelet restart recovery for existing apple/container workloads | P9 | planned | +| #67 | Add orphan micro-VM detection and cleanup policy | P9 | planned | +| #68 | Improve node and Pod resource accounting for CPU, memory, disk, and image cache usage | P9 | planned | +| #69 | Add log rotation and structured diagnostics for long-running nodes | P9 | planned | +| #70 | Build install, upgrade, rollback, and uninstall packaging for macvz-kubelet and helper | P9 | planned | +| #71 | Run long-duration soak tests across kubelet/helper restarts and node churn | P9 | planned | diff --git a/docs/MULTI_NODE_TEST_REPORT_2026-06-19.md b/docs/MULTI_NODE_TEST_REPORT_2026-06-19.md new file mode 100644 index 0000000..6dca72c --- /dev/null +++ b/docs/MULTI_NODE_TEST_REPORT_2026-06-19.md @@ -0,0 +1,333 @@ +# Multi-node Test Report - 2026-06-19 + +This report records the first two-Mac validation run after the P0-P4 work was +merged to `main`. The goal was to verify the current MacVz node/provider +behavior on two physical Apple Silicon Macs and identify the remaining gap +before broader feature development. + +## Build Under Test + +| Item | Value | +| --- | --- | +| Repository | `macvz` | +| Commit | `b89c0eb` | +| Date | 2026-06-19 | +| Control plane | `kind` cluster, API bound to `192.168.1.110:16443` | +| Test image | `busybox:1.36.1` | + +## Hosts + +| Node | Host | macOS | Arch | Role | +| --- | --- | --- | --- | --- | +| `macvz-a` | `192.168.1.110` | 26.5.1 `25F80` | arm64 | local MacVz node | +| `macvz-b` | `192.168.1.122` | 26.4.1 `25E253` | arm64 | remote MacVz node | + +Both hosts had `apple/container` available and running. `wireguard-tools` +(`wg`, `wireguard-go`) was installed on both hosts during the validation. + +## Configuration + +The verified baseline used two `macvz-kubelet` processes, one per Mac: + +- `mesh.enabled: false` +- `podNetwork.enabled: false` +- per-node serving TLS cert/key configured under `node.servingTLSCertFile` and + `node.servingTLSKeyFile` +- per-node Pod CIDR override: + - `macvz-a`: `10.244.101.0/24` + - `macvz-b`: `10.244.102.0/24` + +`kube-proxy` and `kindnet` were pinned to the kind control-plane node so they +would not schedule onto the virtual-kubelet nodes. + +## Commands Run + +Local non-network regression checks: + +```sh +go test ./pkg/network/... ./pkg/config ./pkg/provider +go test -tags=e2e ./test/e2e +``` + +Remote runtime smoke subset: + +```sh +MACVZ_INTEGRATION=1 go test -v ./pkg/runtime/container \ + -run "TestLifecycleIntegration|TestLogStreamingIntegration|TestExecIntegration|TestStatsIntegration|TestVolumeMountIntegration" +``` + +Two-node e2e harness: + +```sh +MACVZ_E2E_NODES=macvz-a,macvz-b MACVZ_E2E_TIMEOUT=120 test/e2e/e2e.sh +``` + +## Results + +| Area | Result | Notes | +| --- | --- | --- | +| Remote host bootstrap | PASS | Installed required tooling, cloned repo, built `macvz-kubelet`. | +| Remote runtime integration subset | PASS | Lifecycle, logs, exec, stats, and volume mount tests passed on `macvz-b`. | +| Node registration | PASS | `macvz-a` and `macvz-b` both registered as Ready virtual nodes. | +| Pod lifecycle | PASS | Pods scheduled to the requested MacVz node and became Ready. | +| `kubectl logs` | PASS | Worked after serving TLS was configured with the correct `node.*` fields. | +| `kubectl exec` | PASS | `uname -m` returned `aarch64`; non-zero exit code propagated. | +| Pod deletion cleanup | PASS | Deleting Pods removed the corresponding `apple/container` VM on both Macs. | +| Endpoint discovery | PASS | A Service selected one Ready endpoint from each MacVz node. | +| `kubectl port-forward` | PASS | Port-forward reached the in-Pod HTTP server. | +| `/stats/summary` and `/metrics/resource` | PASS | Both nodes returned node/pod metrics through the kubelet endpoint. | +| Cross-node Service data path | FAIL / BLOCKED | EndpointSlice was correct, but Service traffic did not reach both backends because mesh and Pod network path were disabled. | + +## Important Finding + +The current implementation can run two MacVz nodes against the same Kubernetes +control plane and correctly handles the provider-facing workflow: + +1. register two virtual nodes, +2. schedule Pods to each node, +3. expose logs, exec, stats, metrics, and port-forward through the kubelet API, +4. publish Pod IPs into Kubernetes EndpointSlices, +5. clean up micro-VMs after Pod deletion. + +The remaining unverified production path is the cross-host data plane: + +- WireGuard mesh creation, +- host routes for remote Pod CIDRs, +- `pf`/`binat` Pod IP mapping, +- Service traffic load-balancing across Pods on different Macs. + +That path requires privileged macOS networking on both hosts. The remote Mac had +sudo available for the test account, but the local host required an interactive +sudo password, so the mesh/podNetwork path was not enabled during this run. + +## Repro Notes + +The first e2e attempt failed `logs`, `exec`, and `port-forward` because the +temporary test config used an invalid serving TLS shape: + +```yaml +serving: + certFile: ... + keyFile: ... +``` + +The correct schema is: + +```yaml +node: + servingTLSCertFile: ... + servingTLSKeyFile: ... +``` + +After correcting the config, the e2e suite had only one failure: cross-node +Service reachability. + +## Cleanup + +After the run: + +- the e2e namespace was deleted, +- `container list --all` was empty on both Macs, +- both `macvz-kubelet` processes were stopped, +- the `kind` cluster was deleted. + +## Next Validation Step + +Run the same two-node e2e suite with: + +```yaml +mesh: + enabled: true +podNetwork: + enabled: true + interface: bridge100 +``` + +Prerequisites: + +- local and remote sudo access, +- WireGuard public keys exchanged between node configs, +- `pf` anchor hooks installed in `/etc/pf.conf`, +- UDP WireGuard listen ports reachable between the two Macs. + +Expected completion signal: `test/e2e/e2e.sh` passes the cross-node Service +phase and prints the final all-checks-passed summary for both nodes. + +## Issue #37 — Two-Node Privileged Run: Prepared, Pending Execution + +A turnkey bundle for the privileged two-Mac run now lives under +[`test/e2e/two-node/`](../test/e2e/two-node/). It covers issue #37 Scope items +1–2 and was validated as far as is possible without the hardware: + +| Item | Status | Evidence | +| --- | --- | --- | +| Two-node configs (mesh + podNetwork on) | DONE | `macvz-a.yaml`, `macvz-b.yaml` load and pass `config.Validate()` via the project loader (mesh enabled, podNetwork enabled, valid peer public keys, disjoint Pod CIDRs `10.244.101/102.0/24`). | +| WireGuard keypairs exchanged | DONE | Real keypairs in `keys/`; private keys gitignored, public keys baked into the peer stanzas. | +| pf anchor hooks | DONE (scripted) | `pf-anchor-hooks.conf` + idempotent install in `prep-node.sh` (syntax-checked with `pfctl -n -f`). | +| IPv4 forwarding / routes / handshake verify | DONE (scripted) | `prep-node.sh` enables forwarding; `verify-dataplane.sh` checks `wg show`, `utun7` routes, peer ping, and `pfctl -a macvz/pods -s nat`. | +| Run `test/e2e/e2e.sh` across both Macs | **EXECUTED** | See "Live two-node run" below. | + +The bundle reduces the remaining work to: copy it to both Macs, `sudo +./prep-node.sh `, `./run.sh start `, then +`MACVZ_E2E_NODES=macvz-a,macvz-b ../e2e.sh`. See its `README.md`. + +## Live two-node run (2026-06-19, real hardware) + +Executed across two physical Apple Silicon Macs with `mesh.enabled: true` and +`podNetwork.enabled: true`, driven through the new privileged helper (#38). + +| Item | Value | +| --- | --- | +| `macvz-a` | 192.168.1.110, mesh `10.99.0.1/32`, Pod CIDR `10.244.101.0/24` | +| `macvz-b` | 192.168.1.122, mesh `10.99.0.2/32`, Pod CIDR `10.244.102.0/24` | +| Control plane | `kind` v1.35.0, API bound to `192.168.1.110:16443` (reachable from both) | +| Helper | `macvz-netd` as root; both `macvz-kubelet` ran as a normal user | + +### What was proven working + +- **The #38 privileged helper works end to end.** Both kubelets ran as the + unprivileged login user (required — `apple/container` refuses to run as root, + confirmed: `container system start` as root → `unauthorized request`). Every + privileged op — `pfctl`, `sysctl`, `route`, `ifconfig`, `wg`, `wireguard-go` — + went through the root `macvz-netd` socket. Logs on both nodes: helper reachable + → **mesh up** → **pod network started** → **ClusterIP routing enabled**. +- **Cross-host WireGuard data plane is live**: `wg show utun7` showed a + bidirectional handshake; host routes for the remote Pod CIDR via `utun7` on + both nodes. +- **`e2e.sh` phases that passed across both Macs**: node registration (2 nodes, + arm64, tainted, capacity), Pod lifecycle, `kubectl logs`, `kubectl exec` + (`aarch64`, exit-code fidelity), Pod deletion/VM teardown, and port-forward. + +### The one failure — environment, not MacVz code + +`e2e.sh` did not reach exit 0: the cross-node **Service** phase failed because +node **macvz-b flapped `NotReady`**. Extensive diagnosis showed this is **not a +MacVz logic bug**: + +- From `macvz-b`'s shell, the control plane was fully reachable: TCP connect to + `192.168.1.110:16443` **40/40 sequential, 30/30 parallel**, `ping` 0% loss, + ARP resolved on `en0`, and `route -n monitor` showed **no route churn**. +- Only the **kubelet process** intermittently failed to `connect()` to the + *remote* API with in-kernel `EHOSTUNREACH` (SYNs never reached the wire, per + `tcpdump`). **macvz-a (local API) never flapped**; only the remote node with + WireGuard up did. +- Root cause: a macOS **BSD negative-route-cache** interaction seeded by the + WireGuard bring-up and a redundant `apple/container` vmnet `default → bridge100` + route, kept hot by the kubelet's multi-reflector reconnect storm — an artifact + of this `kind`-on-Docker-Desktop control plane being consumed by a *remote* + macOS node. Mitigations tried (startup reorder so the API connects after the + mesh; a pinned `/32` host route to the API) reduced but did not eliminate it. + +### MacVz findings worth fixing (filed as follow-ups) + +1. **vmnet default-route hijack — CRITICAL (confirmed root cause).** Starting an + `apple/container` micro-VM installs a `default → bridge100` route that + **steals the host's IPv4 default route**, breaking *all* of the host's + outbound connectivity — not just the kubelet. On the test host this took down + unrelated production services (an `frpc` reverse-proxy tunnel and a GitHub + Actions runner reachable at `selene.itypes.me`, which returned to HTTP 200 + only after the operator reset the route). This is also the **actual root + cause of the macvz-b NotReady flap** (earlier mis-attributed to BSD route + caching): the remote node's kubelet lost its API path whenever the bridge100 + default won. **MacVz must not let a micro-VM hijack the host default route** — + e.g. detect and remove the vmnet `default` route at podNetwork start (the + helper already allowlists `route`) and/or keep it removed, and never run the + data plane on a host whose outbound traffic it would capture. + **Addressed after this run:** `pkg/network/podnet.Router` now deletes IPv4 + `default` on `podNetwork.interface` at start and on every Pod attach; the + privileged helper policy only permits this default-route operation as a + delete scoped to the configured vmnet interface. +2. **Mesh bring-up disrupts the kubelet's own API connection** on a node whose + API is remote. Bringing the data plane up *before* node registration helps but + is not sufficient; the kubelet's API transport should tolerate a transient + routing change after mesh up (e.g. force-reconnect / health-check the API + client when the data plane comes up). **Addressed after this run:** + `macvz-kubelet` now resolves the REST config early but constructs the + clientset only after mesh and podNetwork setup, then probes the API server + before starting Virtual Kubelet controllers. +3. **`wireguard-go` interfaces cannot be torn down with `ifconfig destroy`** + (`SIOCIFDESTROY: Invalid argument`); `Mesh.Down` must kill the `wireguard-go` + process to remove the interface. **Addressed after this run:** `Mesh.Down` + now stops the matching `wireguard-go ` process before the best-effort + destroy step, and helper policy only permits the exact managed-interface + `pkill -f` pattern. +4. **Restart port-bind race**: a fast kubelet restart can hit + `bind: 10250: address already in use` if the prior process has not fully + exited; restart tooling must wait for the port to free. **Addressed after + this run:** kubelet HTTPS startup now retries `EADDRINUSE` for a bounded + window before failing, covering fast restarts while the old listener closes. + +### Net assessment + +The MacVz deliverables are complete and validated on hardware: **#38 (privileged +helper) is done**, and **#37's code path (cross-host mesh + podNetwork + the new +ClusterIP service routing) is proven to bring up the full data plane across two +Macs**. A clean `e2e.sh` exit 0 is blocked only by the remote-node API-connect +quirk of this specific `kind`/Docker-Desktop test rig; a control plane reachable +robustly from both Macs (k3s or a real cluster) is expected to clear it. + +### Single-node local dry run (de-risking, 2026-06-19) + +Before the two-Mac run, `e2e.sh` was executed for real against a local `kind` +control plane (v1.35.0) plus one live `macvz-kubelet` node (`macvz-local`, +mesh/podNetwork **off** — the zero-privilege baseline). This exercises the whole +harness/provider path except the cross-host hop. + +| Phase | Result | Notes | +| --- | --- | --- | +| Node registration | PASS | Ready, arm64, provider taint, capacity. | +| Pod lifecycle / logs / exec | PASS | `uname -m`=aarch64; non-zero exit (7) propagated; micro-VM torn down on delete. | +| Single-node Service reachability | **FAIL** | Client micro-VM could not reach `http://e2e-hello` (ClusterIP). | +| Port-forward | PASS | Reached the in-Pod HTTP server. | +| Cleanup | PASS | `container list --all` empty, kind deleted, no orphan VMs/routes. | + +**Key finding (affects #37 acceptance "ClusterIP Service reaches backends").** +There is no ClusterIP/kube-proxy/service-VIP implementation anywhere in `pkg/` +or `cmd/`. `docs/NETWORKING.md` assumes the cluster's own kube-proxy DNATs a +ClusterIP to ready Pod IPs — but kube-proxy **cannot run on a MacVz node**: it is +a macOS host, and the kubelet rejects the kube-proxy Pod spec +(`restartPolicy "Always"`/`hostNetwork`/`securityContext` unsupported, observed +in this run). So nothing programs ClusterIP→PodIP for a MacVz micro-VM's egress. +`mesh` + `podNetwork` deliver **Pod-to-Pod by Pod IP**, but the **ClusterIP VIP + +DNS** path for client Pods on MacVz nodes is unimplemented. This is the core of +the open P5 work (#37/#38) and means the two-Mac run will likely satisfy +"Pod-to-Pod across Macs" while "ClusterIP Service reaches backends" needs a +host-side service-routing mechanism (e.g. pf rdr/DNAT for service VIPs, or a +per-node userspace proxy) that does not yet exist. The `e2e.sh` single-node +fallback Service check asserts reachability that this gap makes unachievable +regardless of node count. + +### Update: ClusterIP service routing implemented (2026-06-19) + +The gap above is now addressed in code (#37/P5): + +- `pkg/network/podnet` learned `rdr` (DNAT) service rules alongside its Pod + `binat` rules. A local backend is redirected straight to its micro-VM's + host-only address (directly on the vmnet interface — no extra route); a remote + backend keeps its Pod IP and is reached over the mesh; multiple backends form a + `round-robin` pool. Fully unit-tested. +- `pkg/network/svcroute` is a new controller that watches Services + + EndpointSlices and programs those rules, wired into the kubelet after the Pod + network path starts. Unit-tested (pure `BuildServiceRules` + reconcile). +- Cluster DNS is injected into micro-VMs (`node.clusterDNS`/`clusterDomain` → + `--dns`/`--dns-search`) so guests resolve Service names. Unit-tested. + +`go build ./...`, `go vet ./...`, and `go test ./...` are green (10 packages, 0 +failures). + +**pf grammar verified without root.** The exact ruleset the Router renders — +Pod `binat` rules plus single-target and `round-robin` pool `rdr` rules (tcp and +udp) — was syntax-checked against the real macOS pf parser with `pfctl -n -f` +(parse-only, no load, no root required): it parsed cleanly (exit 0), and a +deliberately malformed `rdr` was correctly rejected with `syntax error`. So the +generated rule grammar is valid on macOS; what remains unverified is the live +packet path, not the syntax. + +**Data-path verification is still pending a privileged run**: `pfctl` +needs root, so the rdr rules cannot be integration-tested in the agent +environment. One open dependency for full ClusterIP success: CoreDNS must have a +MacVz-reachable ready endpoint (a CoreDNS replica on a MacVz node, or its Pod IP +routed over the mesh) for in-guest DNS to resolve — otherwise name lookups for +Service DNS names fail even though the ClusterIP `rdr` path is in place. The +two-node bundle configs now set `clusterDNS: ["10.96.0.10"]`; confirm that +value against `kubectl -n kube-system get svc kube-dns` before the run. diff --git a/docs/NETWORKING.md b/docs/NETWORKING.md index 49bf740..2d72693 100644 --- a/docs/NETWORKING.md +++ b/docs/NETWORKING.md @@ -7,8 +7,15 @@ This document describes MacVz cross-host networking (P3): - [Micro-VM network attach (MVP)](#micro-vm-network-attach-mvp) — issue #22. - [Pod status & Service resolution (MVP)](#pod-status--service-resolution-mvp) — issue #23. - [Port-forward (MVP)](#port-forward-mvp) — issue #24. +- [Privileged network helper (`macvz-netd`)](#privileged-network-helper-macvz-netd) — issues #38–#40. - [End-to-end: a Service across two Macs](#end-to-end-a-service-across-two-macs). +For the operator runbook that ties these layers into one ordered setup + +verification + recovery procedure (issue #44), see +[PRIVILEGED_NETWORKING.md](PRIVILEGED_NETWORKING.md). This document explains how +each layer works; that one is the step-by-step guide to standing it up on a fresh +Mac and recovering it when routes, pf rules, or the helper go stale. + Together these give a Pod on one Mac an L3 path to a Pod hosted on another Mac: IPAM assigns the address, the mesh routes the CIDR between hosts, the network attach connects each micro-VM to that path, and the status reporting publishes @@ -99,8 +106,8 @@ installed through the interface so the kernel forwards Pod-bound packets into th tunnel. macOS has no in-kernel WireGuard, so MacVz drives the userspace toolchain from -Homebrew's `wireguard-tools` (`wg`, `wireguard-go`) plus the system `route` and -`ifconfig`. Bring-up runs, in order: +Homebrew's `wireguard-tools` (`wg`, `wireguard-go`) plus the system `route`, +`ifconfig`, and `pkill`. Bring-up runs, in order: 1. `wireguard-go ` — create the userspace `utun` interface. 2. `wg setconf /dev/stdin` — apply keys and peers (config via stdin, no @@ -112,6 +119,16 @@ Homebrew's `wireguard-tools` (`wg`, `wireguard-go`) plus the system `route` and Interface creation and route installation tolerate "already exists" / "File exists", so bring-up is **idempotent** and safe to re-run. +Teardown removes peer routes, brings the interface down, then stops only the +matching `wireguard-go ` process before attempting a best-effort +`ifconfig destroy`. Stopping the process is required on macOS because +userspace WireGuard owns the `utun` lifecycle. + +`macvz-kubelet` builds its Kubernetes clientset only after this data-plane +bring-up and the Pod-network route cleanup complete, then probes +`/version` before starting the Virtual Kubelet controllers. That keeps transient +host-route churn from poisoning a long-lived client-go transport. + ### Reconcile without restart When nodes join or leave, `Mesh.Sync` re-applies the peer set with @@ -120,6 +137,36 @@ affected host routes — the interface is never torn down, so unrelated peers ke their tunnels. `Mesh.Down` removes routes and destroys the interface on shutdown (best-effort). +#### Adding or removing a node (operator workflow, #42) + +A running `macvz-kubelet` reconciles its mesh peers from the config on **SIGHUP**, +so adding or removing a MacVz node needs no restart and never disturbs Pods +already running on this host: + +1. Edit the `mesh.peers` list in this node's config — add the joining node's + `publicKey` / `endpoint` / `podCIDR`, or delete a departing node's entry. +2. Signal the kubelet to reload: `kill -HUP `. + +On the signal the kubelet reloads the config and calls `Mesh.Sync` with the new +peer set. Expectations: + +- **Adding a peer** installs its WireGuard entry and Pod-CIDR route; existing + peers' tunnels and routes, and the local Pod network attachments (the `pf` + anchor is a separate subsystem the mesh never touches), are left untouched — + no Pod loses connectivity. +- **Removing a peer** removes its WireGuard entry and its route. +- **Reconciliation is idempotent** — a SIGHUP with no peer changes re-applies the + same config and installs/removes nothing. +- The reload is validated and **fails safe**: an invalid config, an unparseable + peer key, or a config that disables the mesh is rejected with a logged error + and the **running peer set is kept**. (Disabling the mesh entirely still + requires a restart.) Set `mesh.peers` to empty to drop all peers. + +This requires the kubelet to have been started with `--config ` (the path +it reloads); without it the mesh comes up once and the SIGHUP reconciler is not +started. The same private key and interface are reused, so peer changes never +rotate this node's identity. + ### Peer identity & keys Each node's private key lives in `mesh.privateKeyFile`, generated (mode 0600) on @@ -246,6 +293,12 @@ vmnet interface. The result satisfies the acceptance criteria: each Pod is addressed by its **MacVz Pod IP** (not an opaque host-only address), and a Pod on one Mac reaches a Pod hosted on another Mac at **L3**. +`apple/container` may also install a host IPv4 `default` route through the vmnet +bridge when a micro-VM starts. MacVz removes `default` on `podNetwork.interface` +when the Pod network starts and again whenever a VM is attached, so the vmnet +bridge cannot capture the Mac's normal outbound route or sever the kubelet's API +connection. + The provider observes the VM's host-only address from the runtime once the guest has acquired DHCP, then attaches the mapping; it detaches on Pod deletion. The anchor ruleset is regenerated wholesale and loaded atomically (`pfctl -a @@ -283,11 +336,13 @@ backing the micro-VMs (commonly `bridge100`) with `ifconfig` and set it as | `podNetwork.interface` | vmnet interface the micro-VMs attach to (e.g. `bridge100`). | | `podNetwork.anchor` | pf anchor to manage (default `macvz/pods`). | | `podNetwork.enableForwarding` | Enable IPv4 forwarding (default `true`). | +| `podNetwork.vmNetCIDRs` | Host-only vmnet CIDRs local micro-VMs may receive; used by `macvz-netd` to validate pf targets (default `192.168.64.0/24`). | ```yaml podNetwork: enabled: true interface: bridge100 + vmNetCIDRs: ["192.168.64.0/24"] ``` ### Verifying @@ -352,14 +407,37 @@ error yields `Ready=False` with reason `RuntimeStatusError`. This guarantees the EndpointSlice controller never adds an unreachable Pod to a Service, so traffic is only sent to Pods that can actually receive it. -### DNS / service discovery - -No MacVz-specific work is needed: once Pods report correct `podIPs` and `Ready` -conditions, the in-cluster EndpointSlice controller, kube-proxy, and CoreDNS -treat MacVz-backed Pods like any other. A `ClusterIP` Service's DNS name resolves -to the Service IP, and kube-proxy load-balances to the ready Pod IPs — which the -[mesh](#wireguard-mesh-mvp) and [network attach](#micro-vm-network-attach-mvp) -make reachable across Macs. +### ClusterIP Services and DNS + +Control-plane discovery needs no MacVz-specific work: once Pods report correct +`podIPs` and `Ready` conditions, the in-cluster EndpointSlice controller and +CoreDNS treat MacVz-backed Pods like any other, so a `ClusterIP` Service's +EndpointSlice lists the ready MacVz Pod IPs. + +The **data path**, however, does need MacVz work, because **kube-proxy cannot run +on a MacVz node**: a node is a macOS host (not Linux), and the kube-proxy Pod +spec — `restartPolicy: Always`, `hostNetwork`, privileged `securityContext` — is +rejected by the provider. Without kube-proxy nothing would translate a Service +ClusterIP for a micro-VM. MacVz fills this gap itself (#37, `pkg/network/svcroute`): + +- A controller watches Services and their EndpointSlices and programs the + `macvz/pods` pf anchor with `rdr` (DNAT) rules: `ClusterIP:port -> backend`. +- A backend Pod on **this** node is redirected to its micro-VM's host-only + address (directly attached to the vmnet interface, so no extra route is + needed); a backend on **another** Mac is redirected to its Pod IP, reached over + the [mesh](#wireguard-mesh-mvp) route. Multiple backends become a `round-robin` + pool. +- DNS resolution from inside a micro-VM is enabled by injecting the cluster DNS + server (`node.clusterDNS`, the CoreDNS ClusterIP) and the standard + `.svc.` search list into the guest's resolv.conf via the runtime's + `--dns`/`--dns-search` flags. CoreDNS itself is reached through the same + ClusterIP `rdr` path, so the cluster DNS Service must have a MacVz-reachable + ready endpoint (e.g. a CoreDNS replica on a MacVz node, or its Pod IP routed + over the mesh) for in-guest name resolution to resolve. + +`pfctl`, `sysctl`, and route changes require elevated privileges, so this path is +exercised with `podNetwork.enabled: true` on a privileged run; see +[E2E.md](E2E.md) and `test/e2e/two-node/`. ### Status tests @@ -417,6 +495,134 @@ kill %1 # closing the forward cleans up cleanly clean return on stream close and on context cancellation (no goroutine leak), and the unknown-Pod / non-running / bad-port / nothing-listening error paths. +## Privileged network helper (`macvz-netd`) + +The cross-host data plane needs root tools — `pfctl`, `route`, `sysctl`, +`ifconfig`, `wg`, `wireguard-go`, `pkill` — but `macvz-kubelet` must run as the operator's +user because `apple/container` is a per-user service that refuses to run as root. +`macvz-netd` bridges the two: a small root daemon runs a fixed allowlist of +network commands on behalf of the user-run kubelet, which connects over a unix +socket (issues #38/#39). The kubelet never needs `sudo`, and root is confined to +the allowlisted commands rather than the whole process. + +Enable it by pointing the kubelet at the socket in `config.yaml`: + +```yaml +privilegedHelperSocket: /var/run/macvz-netd.sock +``` + +When unset, the privileged commands run in-process — which then requires the +kubelet itself to be root, and is incompatible with `apple/container`. + +### Confining the helper to this node's config (#41) + +The command allowlist alone stops the helper from running *arbitrary* binaries, +but not from driving an allowlisted binary against arbitrary targets — a +foreign pf anchor, a default-route hijack, an attacker-controlled WireGuard +peer. Passing `--config` closes that gap: the daemon loads the same +`config.yaml` the kubelet uses and validates every request against it. + +```sh +sudo macvz-netd serve --socket /var/run/macvz-netd.sock --config /etc/macvz/config.yaml +``` + +With a config loaded, each request must match a fixed per-command grammar whose +values are pinned to this node's configuration: + +- **pf anchors** — only `podNetwork.anchor` (default `macvz/pods`) may be loaded + or flushed; the main ruleset and any other anchor are refused. A loaded + ruleset may contain only `binat`/`rdr` rules on the configured + `podNetwork.interface`, and translated targets must stay inside configured + Pod CIDRs or `podNetwork.vmNetCIDRs`. +- **interfaces/processes** — `route`, `ifconfig`, `wg`, `wireguard-go`, and the + teardown `pkill` may only touch `mesh.interface`; the assigned address and MTU + must equal `mesh.address` / `mesh.mtu`. +- **routes / AllowedIPs** — must fall within a configured peer's Pod CIDR or + mesh address (`mesh.peers[].podCIDR` / `.address`). A `0.0.0.0/0` route or an + unlisted CIDR is refused. The only default-route operation allowed is deleting + IPv4 `default` from `podNetwork.interface`, which prevents vmnet from hijacking + the host's outbound traffic. +- **WireGuard peers** — a `wg setconf`/`syncconf` payload may only name peer + public keys listed in `mesh.peers[].publicKey`. +- **sysctl** — only the IPv4-forwarding toggle (and only when a Pod network is + configured) plus the read-only `kern.ostype` health probe. + +Invalid or out-of-scope requests fail closed with a clear error and an audit +line in the log; no command runs. Without `--config`, the daemon refuses to +start unless `--allow-unsafe-no-config` is explicitly passed for local +development. The same config flag works on `install`, baking the config path into +the LaunchDaemon plist: + +```sh +sudo macvz-netd install --socket /var/run/macvz-netd.sock --config /etc/macvz/config.yaml +``` + +### Running the helper + +The helper can run in the foreground for a quick test: + +```sh +sudo macvz-netd serve --socket /var/run/macvz-netd.sock --config /etc/macvz/config.yaml +``` + +Launched via `sudo`, it chowns the socket to `$SUDO_UID:$SUDO_GID` so the +invoking (non-root) user can connect and no other user can. + +### Install as a LaunchDaemon (#40) + +For day-to-day use, install it once as a system LaunchDaemon so it starts at boot +and restarts on crash — no manual `sudo` on every kubelet start: + +```sh +sudo macvz-netd install --socket /var/run/macvz-netd.sock --config /etc/macvz/config.yaml +``` + +`install` (run under `sudo`) does four things: + +1. Copies the running binary to `/usr/local/sbin/macvz-netd` (a stable, + root-owned path the plist can point at). +2. Captures the invoking user from `$SUDO_UID:$SUDO_GID` (override with + `--owner uid:gid`) and bakes it into the plist's `--owner` flag — launchd has + no `SUDO_UID` at boot, so the socket owner must be recorded at install time. +3. Writes `/Library/LaunchDaemons/com.github.chimerakang.macvz-netd.plist` with + `RunAtLoad` and `KeepAlive` true. +4. Bootstraps the job (`launchctl bootstrap system …`) so it starts immediately. + +Manage the installed job: + +```sh +macvz-netd status # install + run state (no sudo needed to read) +sudo macvz-netd unload # stop the job, keep it installed +sudo macvz-netd load # start an installed-but-stopped job +sudo macvz-netd uninstall # stop and remove plist, binary, and socket +``` + +`status` reports whether the plist, binary, and socket are present and whether +launchd has the job loaded (via `launchctl print system/…`). + +### Upgrading + +Re-running `sudo macvz-netd install` from a newer binary is the upgrade path. The +binary is written to a temp file and atomically renamed (so a crash never leaves +a half-written binary), and `install` boots out any previously loaded job before +bootstrapping the new plist, so the new binary takes effect. The label and paths +are stable across versions, so an upgrade replaces in place — there is no need to +`uninstall` first. + +### Logs + +The LaunchDaemon writes to: + +- `/var/log/macvz-netd.log` — stdout (klog info, including the listening socket + and the active allowlist on start). +- `/var/log/macvz-netd.err.log` — stderr. + +A refused command — whether non-allowlisted or out-of-scope for the loaded +config (`request not permitted: …`) — is logged here, which is the first place +to look if a privileged operation is unexpectedly failing. Applied privileged +changes are logged at info level as an audit trail. When run in the foreground, +the same output goes to the terminal instead. + ## End-to-end: a Service across two Macs A smoke test that exercises #20–#23 together: a Service backed by Pods on two diff --git a/docs/PRIVILEGED_NETWORKING.md b/docs/PRIVILEGED_NETWORKING.md new file mode 100644 index 0000000..18af496 --- /dev/null +++ b/docs/PRIVILEGED_NETWORKING.md @@ -0,0 +1,497 @@ +# Privileged Networking: Setup & Recovery (P5) + +A complete, repeatable runbook for enabling the **full cross-Mac data plane** — +the WireGuard mesh plus the host Pod-network path — and recovering it when it +breaks. Following this on a fresh Apple Silicon Mac prepares it for the P5 +two-node end-to-end (`test/e2e/two-node/`). + +This is the operator's guide. For *why* each layer exists and how it works, read +[NETWORKING.md](NETWORKING.md); for the turnkey two-node bundle and its scripts, +read [`test/e2e/two-node/README.md`](../test/e2e/two-node/README.md). This +document ties them together into one ordered procedure with verification output +and failure recovery. + +- [The sudo / helper model](#the-sudo--helper-model) +- [Prerequisites](#prerequisites) +- [Setup, in order](#setup-in-order) + - [1. Install tools](#1-install-tools) + - [2. WireGuard key exchange](#2-wireguard-key-exchange) + - [3. Mesh config](#3-mesh-config) + - [4. podNetwork config & bridge100 selection](#4-podnetwork-config--bridge100-selection) + - [5. pf.conf anchor hooks](#5-pfconf-anchor-hooks) + - [6. Install the privileged helper (`macvz-netd`)](#6-install-the-privileged-helper-macvz-netd) + - [7. Start the kubelet](#7-start-the-kubelet) +- [Verification](#verification) +- [Recovery procedures](#recovery-procedures) + - [Stale routes](#stale-routes) + - [Stale pf rules](#stale-pf-rules) + - [Helper failures](#helper-failures) +- [Teardown](#teardown) + +## The sudo / helper model + +The cross-host data plane needs **root** tools — `pfctl`, `route`, `sysctl`, +`ifconfig`, `wg`, `wireguard-go`, `pkill`. But `macvz-kubelet` must run as the operator's +**user**, because `apple/container` is a per-user service that refuses to run as +root. These two facts are in direct tension, so MacVz offers two models: + +| Model | How privileged commands run | When to use | +| --- | --- | --- | +| **Helper daemon (recommended)** | `macvz-netd` runs as root (a LaunchDaemon) and executes a fixed, config-validated allowlist on behalf of the user-run kubelet over a unix socket. | Production and the P5 e2e. The kubelet needs **no** `sudo`. | +| **In-process (dev only)** | The kubelet runs the privileged commands itself, so the **whole kubelet** must be root. | Incompatible with `apple/container` — only useful for unit-level network testing without real micro-VMs. | + +Choose the helper model. With it, the only `sudo` you ever type is the one-time +`macvz-netd install` (and the per-host prep below); day-to-day kubelet starts and +restarts need no elevation. Root is confined to the allowlisted, config-pinned +commands rather than the entire node process. The confinement rules (which +anchors, interfaces, CIDRs, and peers a request may name) are detailed in +[NETWORKING.md → Confining the helper](NETWORKING.md#confining-the-helper-to-this-nodes-config-41). + +## Control API (kubelet ↔ helper, #39) + +The kubelet and `macvz-netd` talk over a single newline-delimited JSON +request/response on a short-lived unix-socket connection (one request per +connection, mirroring `exec.Command`). + +- **Socket**: `--socket` path (default `/var/run/macvz-netd.sock`), mode `0660`, + owned by the kubelet's user (`$SUDO_UID:$SUDO_GID`, or `--owner uid:gid` under + launchd). No other user can connect; root holds the daemon. +- **Versioning**: every request carries a `protocol` field. The current version + is `1`. The helper refuses a request whose `protocol` is set and does not match + with `errorCode: "unsupported_protocol"`, so mismatched kubelet/helper builds + fail fast. `protocol: 0` (an older unversioned client) is accepted as the + current version. +- **Operations** (`op` field): + - `""` / `exec` — run the allowlisted command in `name` with `args`/`stdin`. + The reply carries `stdout`, `stderr`, and `exitCode` (a non-zero exit is a + normal command result, not an API error). + - `status` — the helper reports `version`, `protocol`, `allowedCommands`, + `policyEnforced` (whether #41 per-request validation is on), + `policyReloadable`, `pid`, `startedAt`, and `uptime`. No command runs. The + kubelet calls this at startup and logs it, then issues a `sysctl` probe to + confirm the exec path works. + - `reloadPolicy` — the helper reloads its config-derived policy before a + SIGHUP mesh-peer reconciliation, so newly configured peer keys and CIDRs are + permitted without restarting `macvz-netd`. +- **Structured errors**: a refusal sets `errorCode` to one of + `unsupported_protocol`, `malformed` (undecodable or over the 1 MiB request + cap), `not_allowed` (command off the allowlist), `not_permitted` (allowlisted + but outside this node's policy, see #41), `unknown_op`, or `exec_error` (the + command could not be spawned). The client surfaces these as a typed + `privhelper.APIError`, so a caller can map a failure to a Pod condition without + parsing the human-readable message. + +A missing daemon, a wrong-version daemon, or a daemon that answers but cannot run +commands each surface as a distinct, actionable startup error rather than failing +later on the first `pfctl`/`wg` call. + +## Prerequisites + +On **each** Mac that will be a node: + +- **macOS 26 (Tahoe)+ on Apple Silicon**, with `apple/container` installed and + working (`container list` succeeds). +- `wireguard-tools` (provides `wg`, `wireguard-go`) and `kubectl`: + ```sh + brew install wireguard-tools kubernetes-cli + ``` +- A **shared Kubernetes control plane** reachable from every node, with + `--allocate-node-cidrs=true` and a `--cluster-cidr` so each node gets a + disjoint `Spec.PodCIDR`. (Override with `node.podCIDR` only if your cluster + does not allocate node CIDRs.) +- **UDP 51820 open** between the hosts (the WireGuard endpoint port). If a host + firewall is on, allow it explicitly. +- The built `bin/macvz-kubelet` and `bin/macvz-netd` (`make build`) copied to + each host. + +Throughout, the worked example uses the two-node topology from +[`test/e2e/two-node`](../test/e2e/two-node/README.md): + +| Node | Host | Mesh addr | Pod CIDR | WG endpoint | +| --- | --- | --- | --- | --- | +| `macvz-a` | 192.168.1.110 | 10.99.0.1/32 | 10.244.101.0/24 | 192.168.1.110:51820 | +| `macvz-b` | 192.168.1.122 | 10.99.0.2/32 | 10.244.102.0/24 | 192.168.1.122:51820 | + +## Setup, in order + +### 1. Install tools + +Confirm the userspace WireGuard toolchain and the runtime are present: + +```sh +wg --version # wireguard-tools is on PATH +which wireguard-go # userspace utun backend +container list # apple/container responds +``` + +### 2. WireGuard key exchange + +Each node has a **stable identity**: one private key, kept mode `0600`, that +never enters config or git. The kubelet auto-generates it at +`mesh.privateKeyFile` on first start if absent — but for a planned two-node +bring-up, generate both keypairs up front so you can fill in peer public keys +before starting anything: + +```sh +# Run once, anywhere. Produces a private + public key per node. +wg genkey | tee macvz-a.key | wg pubkey > macvz-a.pub +wg genkey | tee macvz-b.key | wg pubkey > macvz-b.pub +chmod 600 *.key +``` + +Then **exchange public keys**: each node's config lists the *other* node as a +peer, using that peer's **public** key. The private keys stay on their own host: + +- Install `macvz-a.key` as `mesh.privateKeyFile` on `macvz-a` only. +- Put the contents of `macvz-a.pub` into `macvz-b`'s `peers[].publicKey` for + `macvz-a` — and vice-versa. + +If you ever need a running node's public key, it is logged at startup, or: + +```sh +wg pubkey < /etc/macvz/wireguard.key +``` + +> The two-node bundle automates this: `keys/*.pub` are committed and baked into +> the peer stanzas; `keys/*.key` are gitignored and installed by `prep-node.sh`. + +### 3. Mesh config + +Add a `mesh` stanza to each node's `config.yaml`. The mesh is **off by default**, +so single-host development needs none of this. A peer's `podCIDR` becomes its +WireGuard `AllowedIPs`, and a host route for that CIDR is installed through the +interface — that is what forwards Pod-bound packets into the tunnel. + +**macvz-a** (`/etc/macvz/macvz-a.yaml`): + +```yaml +mesh: + enabled: true + interface: utun7 + privateKeyFile: /etc/macvz/wireguard.key + address: 10.99.0.1/32 + listenPort: 51820 + peers: + - name: macvz-b + publicKey: + endpoint: 192.168.1.122:51820 + podCIDR: 10.244.102.0/24 + address: 10.99.0.2/32 + persistentKeepalive: 25 +``` + +**macvz-b** is the mirror image: `address: 10.99.0.2/32`, and its single peer is +`macvz-a` with `macvz-a.pub`, endpoint `192.168.1.110:51820`, podCIDR +`10.244.101.0/24`, address `10.99.0.1/32`. + +Field reference: see [NETWORKING.md → mesh Configuration](NETWORKING.md#configuration-1). + +### 4. podNetwork config & bridge100 selection + +`podNetwork` enables the host-side path that maps each Pod IP to its micro-VM's +host-only vmnet address (1:1 NAT via a pf `binat`) and programs ClusterIP `rdr` +rules. Add to each node's config: + +```yaml +podNetwork: + enabled: true + interface: bridge100 # the vmnet bridge apple/container attaches VMs to + anchor: macvz/pods # default; the only anchor the helper will touch + enableForwarding: true # default; flips net.inet.ip.forwarding to 1 + vmNetCIDRs: ["192.168.64.0/24"] # default; adjust if your vmnet range differs +``` + +**Selecting the bridge.** `apple/container` attaches each micro-VM to a +vmnet-backed bridge, **commonly `bridge100`**, but the number is assigned by +macOS and can differ. The bridge only appears **after the first micro-VM +starts** — so confirm it once a VM is running: + +```sh +# List bridges and the members each one has (VM vmnet interfaces show as members): +ifconfig | grep -A6 '^bridge' +# Or watch for the vmnet bridge to come up: +ifconfig bridge100 2>/dev/null && echo "bridge100 is up" +``` + +If the runtime uses a different bridge, set `podNetwork.interface` to match +before starting the kubelet. A wrong interface produces `binat`/`rdr` rules that +never match traffic — see [Recovery](#stale-pf-rules). + +To make cluster DNS resolve from inside micro-VMs, also set `node.clusterDNS` to +the CoreDNS ClusterIP (see +[NETWORKING.md → ClusterIP Services and DNS](NETWORKING.md#clusterip-services-and-dns)). + +### 5. pf.conf anchor hooks + +`pf` only evaluates the MacVz anchor if `/etc/pf.conf` references it. The four +hook lines must respect pf's ordering rule — **translation anchors +(nat/rdr/binat) before filter anchors**: + +``` +nat-anchor "macvz/pods/*" +rdr-anchor "macvz/pods/*" +binat-anchor "macvz/pods/*" +anchor "macvz/pods/*" +``` + +On a stock macOS `pf.conf` (which ships `com.apple` anchors), the translation +hooks go right after `rdr-anchor "com.apple/*"` and the filter hook right after +`anchor "com.apple/*"`. The two-node bundle's `prep-node.sh` does this insertion +idempotently and **backs up `pf.conf`** to `pf.conf.macvz.bak.` first; prefer +it over hand-editing: + +```sh +# From test/e2e/two-node, on each host (installs key, TLS, pf hooks, forwarding): +sudo ./prep-node.sh a # or b +``` + +If you edit by hand, validate before loading — a syntax error disables pf: + +```sh +sudo pfctl -n -f /etc/pf.conf # dry-run syntax check; fails loud +sudo pfctl -f /etc/pf.conf # load +sudo pfctl -e # enable (tolerate "already enabled") +``` + +### 6. Install the privileged helper (`macvz-netd`) + +Install once per host as a root LaunchDaemon, pinned to this node's config so +every request is validated against it: + +```sh +sudo macvz-netd install \ + --socket /var/run/macvz-netd.sock \ + --config /etc/macvz/macvz-a.yaml # macvz-b.yaml on the other host +``` + +`install` copies the binary to `/usr/local/sbin/macvz-netd`, records the invoking +user (`$SUDO_UID:$SUDO_GID`) as the socket owner, writes the plist with +`RunAtLoad`/`KeepAlive`, and bootstraps the job. Always pass `--config` — without +it the daemon refuses to start because per-request policy would be disabled. + +Confirm it is up: + +```sh +macvz-netd status # no sudo needed to read; reports plist/binary/socket/loaded +``` + +Then point the kubelet at the socket in its `config.yaml`: + +```yaml +privilegedHelperSocket: /var/run/macvz-netd.sock +``` + +> **Upgrading:** re-run `sudo macvz-netd install` from the newer binary; it boots +> out the old job and replaces in place — no `uninstall` first. + +### 7. Start the kubelet + +With the helper running and `privilegedHelperSocket` set, start the kubelet as +your **normal user** (no sudo): + +```sh +KUBECONFIG=/path/to/kubeconfig macvz-kubelet --config /etc/macvz/macvz-a.yaml +``` + +On start it brings up the mesh interface, applies peers, installs peer-CIDR +routes, enables forwarding, and begins reconciling Pods and Services through the +helper. Repeat on the other host. The two-node bundle's `run.sh start ` +wraps this. + +## Verification + +Run these after both nodes are up. The bundle's +[`verify-dataplane.sh`](../test/e2e/two-node/verify-dataplane.sh) automates the +first three and exits non-zero if any fail. + +**1. WireGuard handshake** — a recent handshake with the peer: + +```sh +sudo wg show utun7 +``` +``` +interface: utun7 + public key: + listening port: 51820 +peer: + endpoint: 192.168.1.122:51820 + allowed ips: 10.244.102.0/24, 10.99.0.2/32 + latest handshake: 23 seconds ago # <- must be present and recent + transfer: 1.85 KiB received, 2.10 KiB sent +``` + +**2. Routes for the remote Pod CIDR** — present via the mesh interface: + +```sh +netstat -rn -f inet | grep utun7 +``` +``` +10.244.102/24 utun7 USc utun7 # remote Pod CIDR +10.99.0.2 utun7 UH utun7 # peer mesh addr +``` + +**3. IPv4 forwarding** — on: + +```sh +sysctl net.inet.ip.forwarding +``` +``` +net.inet.ip.forwarding: 1 +``` + +**4. Reach the peer across the tunnel:** + +```sh +ping -c2 10.99.0.2 # WARN-only if ICMP is filtered; not fatal +``` + +**5. pf anchor rules** — `binat` per attached Pod, plus `rdr` per ClusterIP: + +```sh +sudo pfctl -a macvz/pods -s nat # binat rules, one per Pod on this node +sudo pfctl -a macvz/pods -s rdr # rdr rules, one per ClusterIP:port +``` +``` +binat on bridge100 inet from 192.168.64.3 to any -> 10.244.101.5 +``` +> No rules until a Pod is scheduled on this node — that is expected, not a fault. + +**6. End-to-end** — a Pod on one Mac reaches a Pod hosted on the other, and a +ClusterIP Service load-balances across both. Use the worked example in +[NETWORKING.md → End-to-end](NETWORKING.md#end-to-end-a-service-across-two-macs), +or run the suite: + +```sh +MACVZ_E2E_NODES=macvz-a,macvz-b MACVZ_E2E_TIMEOUT=120 \ + MACVZ_E2E_DIAG_DIR=./diag test/e2e/e2e.sh +``` + +A fresh Mac prepared with this runbook should make the e2e "Service across nodes" +phase **PASS** rather than SKIP. + +## Recovery procedures + +Privileged network state lives in the kernel (routes, pf rules, the utun +interface) and outlives the kubelet process. A crash, a config change, or a +host/peer reconfiguration can leave **stale** state that silently misroutes or +blackholes traffic. Each of these is read-only to diagnose and safe to re-run. + +### Stale routes + +Symptom: traffic to a remote Pod CIDR blackholes, or `netstat` shows a route to +a CIDR/interface that no longer exists (e.g. an old peer's CIDR, or a route on a +utun number from a previous run). + +```sh +# 1. See what is installed via the mesh interface: +netstat -rn -f inet | grep utun + +# 2. Normal recovery: a kubelet restart re-reconciles routes from config — +# Mesh.Sync adds/removes only the affected host routes, no full teardown. +# Just restart macvz-kubelet. + +# 3. If a route persists for a CIDR no longer in any peer (orphan), remove it: +sudo route -n delete -inet 10.244.199.0/24 -interface utun7 + +# 4. If the utun interface itself is wedged (wrong number, half-torn-down), +# destroy it and let the kubelet recreate on next start: +sudo ifconfig utun7 down +sudo pkill -f 'wireguard-go utun7' # wireguard-go owns the utun lifecycle +sudo ifconfig utun7 destroy # best-effort; may report Invalid argument +``` + +Because bring-up tolerates "already exists" and `Mesh.Sync` reconciles +incrementally, the safe default is **restart the kubelet** and only hand-delete +orphans the restart cannot know about (CIDRs removed from a since-changed config). + +### Stale pf rules + +Symptom: `binat`/`rdr` rules reference an old Pod's vmnet address or the wrong +interface; traffic is NAT'd to a VM that no longer exists, or rules never match. + +```sh +# 1. Inspect the anchor (this is the only anchor MacVz touches): +sudo pfctl -a macvz/pods -s nat +sudo pfctl -a macvz/pods -s rdr + +# 2. The kubelet regenerates this anchor wholesale and loads it atomically on +# every change, and FLUSHES it on clean shutdown. So a stop+start clears +# stragglers. If the kubelet died uncleanly, flush by hand: +sudo pfctl -a macvz/pods -F all # flush all rules in the macvz anchor only + +# 3. If rules are present but never match, the bridge is wrong: confirm +# podNetwork.interface equals the live vmnet bridge (see step 4 of setup), +# fix the config, and restart. + +# 4. If the host default route points at the vmnet bridge, the kubelet removes it +# at podNetwork start and whenever a Pod attaches. If it reappears, check +# macvz-netd logs for a refused route delete on podNetwork.interface. +netstat -rn -f inet | grep '^default' + +# 5. If NO rules ever appear, pf is not evaluating the anchor. Confirm the hooks +# and that pf is enabled: +grep 'macvz/pods' /etc/pf.conf # the four anchor lines must be present +sudo pfctl -s info | grep Status # must read: Status: Enabled +``` + +Flushing only ever targets `-a macvz/pods`; the main ruleset and `com.apple` +anchors are never touched. If pf got disabled by a bad hand-edit, restore from +the `pf.conf.macvz.bak.` backup `prep-node.sh` left and reload. + +### Helper failures + +Symptom: a privileged operation fails even though the command looks valid; the +kubelet logs an error attaching a Pod or syncing the mesh. + +```sh +# 1. Is the daemon loaded and the socket present? +macvz-netd status + +# 2. First place to look: the helper logs. A refused request is logged as +# "request not permitted: ..." with an audit line; applied changes log at info. +tail -n 50 /var/log/macvz-netd.log # stdout: socket, allowlist, audit trail +tail -n 50 /var/log/macvz-netd.err.log # stderr + +# 3. "request not permitted" means the request is out-of-scope for the loaded +# --config: a CIDR/interface/peer/anchor not pinned in THIS node's config. +# Reconcile the config with the request (e.g. add the peer's podCIDR). A +# kubelet SIGHUP reload asks macvz-netd to refresh policy before applying +# mesh changes; otherwise restart macvz-netd to reload manually: +sudo macvz-netd unload && sudo macvz-netd load + +# 4. Socket permission denied from the kubelet: the socket owner was recorded at +# install time from $SUDO_UID. If you now run the kubelet as a different user, +# reinstall with --owner uid:gid for that user. + +# 5. Restart the job without reinstalling: +sudo macvz-netd unload && sudo macvz-netd load +``` + +If `--config` was omitted at install, the daemon refuses to start. Reinstall with +`--config` so config-pinned validation is active. + +## Teardown + +Privileged state does not revert on its own. To leave a host clean: + +```sh +# Stop the kubelet (flushes the macvz/pods anchor and best-effort tears down mesh): +# run.sh stop a # if using the bundle, or just stop the macvz-kubelet process + +sudo pfctl -a macvz/pods -F all # belt-and-suspenders anchor flush +sudo route -n flush -inet # only if you must clear stale routes +sudo pkill -f 'wireguard-go utun7' 2>/dev/null +sudo ifconfig utun7 destroy 2>/dev/null # best-effort mesh interface cleanup +sudo sysctl -w net.inet.ip.forwarding=0 # revert forwarding, if desired + +# Remove the helper entirely (plist, binary, socket): +sudo macvz-netd uninstall + +# Restore /etc/pf.conf from the prep-node.sh backup, if you want the hooks gone: +# sudo cp /etc/pf.conf.macvz.bak. /etc/pf.conf && sudo pfctl -f /etc/pf.conf + +# Verify nothing is orphaned: +container list --all # expect no leftover micro-VMs +netstat -rn -f inet | grep utun7 # expect no stale remote Pod CIDR routes +sudo pfctl -a macvz/pods -s all # expect empty +``` diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 8ec2228..7cbcac3 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -46,6 +46,9 @@ command execution. It is hardened in three ways: call it. - **Bind address.** When `node.internalIP` is set (or auto-detected), the server binds to that address rather than all interfaces, shrinking exposure. +- **Restart tolerance.** If a fast restart races the old process releasing + `kubeletPort`, startup retries `EADDRINUSE` briefly instead of failing + immediately. - **Loud default.** With no client CA configured, macvz-kubelet logs a prominent warning at startup that the endpoint is unauthenticated. diff --git a/internal/types/types.go b/internal/types/types.go index 4757400..9523fc0 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -35,6 +35,16 @@ type ContainerSpec struct { // they should be applied. The runtime realizes each as a VirtioFS share (or a // guest tmpfs); the provider is responsible for validating sources. Mounts []Mount + + // DNS are nameserver IPs written into the micro-VM's resolv.conf (passed as + // `--dns`). Empty leaves the image's baked DNS in place. Cluster DNS lets the + // guest resolve Service names (#37). + DNS []string + // DNSSearch are resolv.conf search domains (passed as `--dns-search`). + DNSSearch []string + // DNSOptions are resolv.conf options such as "ndots:5" (passed as + // `--dns-option`). + DNSOptions []string } // Mount describes one filesystem mount inside the micro-VM. A bind mount shares diff --git a/pkg/config/config.go b/pkg/config/config.go index 1146a6b..05b860c 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -55,6 +55,16 @@ type Config struct { // PodNetwork configures the host path that makes each micro-VM reachable at // its assigned Pod IP across the mesh (P3, #22). Disabled by default. PodNetwork PodNetworkConfig `yaml:"podNetwork"` + + // PrivilegedHelperSocket, when set, routes all privileged network commands + // (pf, wg, route, sysctl, ifconfig) for the mesh and Pod network path through + // the root helper daemon (cmd/macvz-netd, #38) at this unix socket, instead + // of executing them in-process. This lets macvz-kubelet run as the operator's + // user — required because apple/container refuses to run as root — while the + // data plane still gets root for its network operations. Empty runs the + // commands directly (the kubelet must then be root, which is incompatible + // with apple/container). + PrivilegedHelperSocket string `yaml:"privilegedHelperSocket"` } // NodeConfig is the configurable shape of the virtual node registered with @@ -136,6 +146,17 @@ type NodeConfig struct { // Volumes configures Pod volume support (#26). Volumes VolumesConfig `yaml:"volumes"` + + // ClusterDNS are the cluster DNS server IPs (typically the CoreDNS/kube-dns + // ClusterIP, e.g. "10.96.0.10") injected into micro-VMs whose Pods use the + // ClusterFirst DNS policy, so in-guest Service name resolution works (#37). + // Empty leaves each micro-VM with the DNS baked into its image. + ClusterDNS []string `yaml:"clusterDNS"` + + // ClusterDomain is the cluster DNS domain used to build Pod search domains + // (e.g. ".svc.cluster.local"). Defaults to "cluster.local" when + // clusterDNS is set. + ClusterDomain string `yaml:"clusterDomain"` } // VolumesConfig governs which Pod volumes MacVz mounts into micro-VMs and where diff --git a/pkg/config/mesh.go b/pkg/config/mesh.go index ea782cb..52abc5b 100644 --- a/pkg/config/mesh.go +++ b/pkg/config/mesh.go @@ -114,11 +114,39 @@ func (c Config) MeshInterfaceConfig() (wireguard.InterfaceConfig, error) { port = DefaultMeshListenPort } + peers, err := m.resolvePeers() + if err != nil { + return wireguard.InterfaceConfig{}, err + } + + return wireguard.InterfaceConfig{ + Name: m.Interface, + PrivateKey: priv, + Address: m.Address, + ListenPort: port, + MTU: m.MTU, + Peers: peers, + }, nil +} + +// MeshPeers resolves the configured mesh peers into wireguard.Peer values, +// without loading this node's private key. It is the input to peer +// reconciliation (#42): on a config reload the kubelet feeds the result to +// wireguard.Mesh.Sync to add/remove peers (and their routes) in place, never +// recreating the interface. Call only when Mesh.Enabled is true. +func (c Config) MeshPeers() ([]wireguard.Peer, error) { + return c.Mesh.resolvePeers() +} + +// resolvePeers maps the YAML peer entries to wireguard.Peer values. Each peer's +// AllowedIPs are its Pod CIDR plus (when set) its mesh address, exactly the +// CIDRs routed through the tunnel to that node. +func (m MeshConfig) resolvePeers() ([]wireguard.Peer, error) { peers := make([]wireguard.Peer, 0, len(m.Peers)) for _, p := range m.Peers { pub, err := wireguard.ParseKey(p.PublicKey) if err != nil { - return wireguard.InterfaceConfig{}, fmt.Errorf("mesh peer %q: %w", p.Name, err) + return nil, fmt.Errorf("mesh peer %q: %w", p.Name, err) } allowed := []string{p.PodCIDR} if p.Address != "" { @@ -132,13 +160,5 @@ func (c Config) MeshInterfaceConfig() (wireguard.InterfaceConfig, error) { PersistentKeepalive: p.PersistentKeepalive, }) } - - return wireguard.InterfaceConfig{ - Name: m.Interface, - PrivateKey: priv, - Address: m.Address, - ListenPort: port, - MTU: m.MTU, - Peers: peers, - }, nil + return peers, nil } diff --git a/pkg/config/mesh_test.go b/pkg/config/mesh_test.go index cf919e1..86b9923 100644 --- a/pkg/config/mesh_test.go +++ b/pkg/config/mesh_test.go @@ -104,6 +104,41 @@ func TestMeshInterfaceConfigTranslates(t *testing.T) { } } +func TestMeshPeersResolvesForReconcile(t *testing.T) { + c := Default() + c.Mesh = enabledMesh(t, filepath.Join(t.TempDir(), "wg.key")) + + peers, err := c.MeshPeers() + if err != nil { + t.Fatalf("MeshPeers: %v", err) + } + if len(peers) != 1 { + t.Fatalf("peers = %d, want 1", len(peers)) + } + p := peers[0] + if p.Name != "mac-02" || p.Endpoint != "192.168.1.20:51820" { + t.Errorf("peer fields not resolved: %+v", p) + } + want := map[string]bool{"10.244.2.0/24": true, "10.99.0.2/32": true} + if len(p.AllowedIPs) != 2 { + t.Fatalf("AllowedIPs = %v, want pod CIDR + address", p.AllowedIPs) + } + for _, a := range p.AllowedIPs { + if !want[a] { + t.Errorf("unexpected AllowedIP %q", a) + } + } + // The peer set MeshPeers returns must match what MeshInterfaceConfig embeds, + // so a reload reconciles to the same desired state Up established. + ifc, err := c.MeshInterfaceConfig() + if err != nil { + t.Fatalf("MeshInterfaceConfig: %v", err) + } + if len(ifc.Peers) != len(peers) || ifc.Peers[0].PublicKey != p.PublicKey { + t.Errorf("MeshPeers and MeshInterfaceConfig disagree on peers") + } +} + func TestMeshInterfaceConfigDefaultsPort(t *testing.T) { c := Default() m := enabledMesh(t, filepath.Join(t.TempDir(), "wg.key")) diff --git a/pkg/config/podnetwork.go b/pkg/config/podnetwork.go index 17c1d27..6d9f57b 100644 --- a/pkg/config/podnetwork.go +++ b/pkg/config/podnetwork.go @@ -2,10 +2,15 @@ package config import ( "fmt" + "net" "github.com/chimerakang/macvz/pkg/network/podnet" ) +// DefaultVMNetCIDR is apple/container's usual host-only vmnet range. Operators +// with a different vmnet range should set podNetwork.vmNetCIDRs explicitly. +const DefaultVMNetCIDR = "192.168.64.0/24" + // PodNetworkConfig configures the host Pod network path that makes each // micro-VM reachable at its assigned Pod IP across the mesh (#22). It is opt-in: // when disabled, Pods keep apple/container's host-only address and cross-host @@ -27,6 +32,12 @@ type PodNetworkConfig struct { // mesh interface and the vmnet interface. Defaults to true when enabled; // set false only when forwarding is managed externally. EnableForwarding *bool `yaml:"enableForwarding"` + + // VMNetCIDRs are the host-only apple/container address ranges that local + // micro-VMs may receive on the vmnet interface. The privileged helper uses + // these to reject pf rules that redirect ClusterIP traffic outside the local + // vmnet and Pod networks. Defaults to 192.168.64.0/24 when omitted. + VMNetCIDRs []string `yaml:"vmNetCIDRs"` } // validatePodNetwork checks the Pod network fields when enabled. @@ -38,6 +49,11 @@ func (c Config) validatePodNetwork() error { if pn.Interface == "" { return fmt.Errorf("podNetwork.interface is required when the Pod network is enabled") } + for _, cidr := range pn.VMNetCIDRs { + if _, _, err := net.ParseCIDR(cidr); err != nil { + return fmt.Errorf("podNetwork.vmNetCIDRs entry %q is not a CIDR: %w", cidr, err) + } + } return nil } diff --git a/pkg/config/policy.go b/pkg/config/policy.go new file mode 100644 index 0000000..3c415b9 --- /dev/null +++ b/pkg/config/policy.go @@ -0,0 +1,73 @@ +package config + +import ( + "net" + + "github.com/chimerakang/macvz/pkg/network/podnet" + "github.com/chimerakang/macvz/pkg/network/privhelper" +) + +// PrivilegedHelperPolicy derives the privhelper.Policy that confines the root +// network helper to exactly the resources this node's config references (#41): +// its mesh interface, mesh address, vmnet interface, pf anchor, the Pod/mesh +// CIDRs routed through the tunnel, and the configured peers' public keys. +// +// It draws only on the mesh and Pod-network stanzas, so it is safe to call even +// when those are disabled — the result simply has empty sets and refuses the +// corresponding commands (fail closed). Errors surface only from resolving the +// mesh interface config (e.g. an unreadable/invalid private key file), which the +// helper needs to canonicalise peer keys and route targets. +func (c Config) PrivilegedHelperPolicy() (privhelper.Policy, error) { + p := privhelper.Policy{ + VMNetInterface: c.PodNetwork.Interface, + Anchor: podnet.DefaultAnchor, + RouteCIDRs: map[string]bool{}, + PodCIDRs: map[string]bool{}, + VMNetCIDRs: map[string]bool{}, + PeerPublicKeys: map[string]bool{}, + } + if c.PodNetwork.Anchor != "" { + p.Anchor = c.PodNetwork.Anchor + } + if c.Node.PodCIDR != "" { + p.PodCIDRs = privhelper.NormalizeCIDRSet([]string{c.Node.PodCIDR}) + } + if c.PodNetwork.Enabled { + vmnet := c.PodNetwork.VMNetCIDRs + if len(vmnet) == 0 { + vmnet = []string{DefaultVMNetCIDR} + } + p.VMNetCIDRs = privhelper.NormalizeCIDRSet(vmnet) + } + + if !c.Mesh.Enabled { + return p, nil + } + + ifc, err := c.MeshInterfaceConfig() + if err != nil { + return privhelper.Policy{}, err + } + p.MeshInterface = ifc.Name + p.MTU = ifc.MTU + if ip, _, err := net.ParseCIDR(ifc.Address); err == nil { + p.MeshAddressIP = ip.String() + } + // Route targets and AllowedIPs are confined to the union of every peer's + // AllowedIPs (Pod CIDR + mesh address), exactly what the mesh installs routes + // for and what its wg config advertises. + p.RouteCIDRs = privhelper.NormalizeCIDRSet(ifc.RouteTargets()) + for _, peer := range c.Mesh.Peers { + if peer.PodCIDR != "" { + if norm := privhelper.NormalizeCIDRSet([]string{peer.PodCIDR}); len(norm) > 0 { + for cidr := range norm { + p.PodCIDRs[cidr] = true + } + } + } + } + for _, peer := range ifc.Peers { + p.PeerPublicKeys[peer.PublicKey.String()] = true + } + return p, nil +} diff --git a/pkg/config/policy_test.go b/pkg/config/policy_test.go new file mode 100644 index 0000000..b173c6a --- /dev/null +++ b/pkg/config/policy_test.go @@ -0,0 +1,101 @@ +package config + +import ( + "path/filepath" + "testing" + + "github.com/chimerakang/macvz/pkg/network/wireguard" +) + +func TestPrivilegedHelperPolicyFromConfig(t *testing.T) { + c := Default() + c.Mesh = enabledMesh(t, filepath.Join(t.TempDir(), "wg.key")) + c.PodNetwork = PodNetworkConfig{Enabled: true, Interface: "bridge100"} + c.Node.PodCIDR = "10.244.101.0/24" + + p, err := c.PrivilegedHelperPolicy() + if err != nil { + t.Fatalf("PrivilegedHelperPolicy: %v", err) + } + + if p.MeshInterface != "utun7" { + t.Errorf("MeshInterface = %q, want utun7", p.MeshInterface) + } + if p.VMNetInterface != "bridge100" { + t.Errorf("VMNetInterface = %q, want bridge100", p.VMNetInterface) + } + if p.Anchor != "macvz/pods" { + t.Errorf("Anchor = %q, want macvz/pods", p.Anchor) + } + if p.MeshAddressIP != "10.99.0.1" { + t.Errorf("MeshAddressIP = %q, want 10.99.0.1", p.MeshAddressIP) + } + // The peer's PodCIDR and mesh Address are the route targets. + for _, want := range []string{"10.244.2.0/24", "10.99.0.2/32"} { + if !p.RouteCIDRs[want] { + t.Errorf("RouteCIDRs missing %q (have %v)", want, p.RouteCIDRs) + } + } + for _, want := range []string{"10.244.101.0/24", "10.244.2.0/24"} { + if !p.PodCIDRs[want] { + t.Errorf("PodCIDRs missing %q (have %v)", want, p.PodCIDRs) + } + } + if !p.VMNetCIDRs[DefaultVMNetCIDR] { + t.Errorf("VMNetCIDRs missing default %q (have %v)", DefaultVMNetCIDR, p.VMNetCIDRs) + } + // The peer's public key is the only configured peer. + pub := c.Mesh.Peers[0].PublicKey + if k, err := wireguard.ParseKey(pub); err != nil { + t.Fatalf("ParseKey: %v", err) + } else if !p.PeerPublicKeys[k.String()] { + t.Errorf("PeerPublicKeys missing the configured peer key") + } +} + +func TestPrivilegedHelperPolicyMeshDisabledFailsClosed(t *testing.T) { + c := Default() // mesh and pod network off by default + p, err := c.PrivilegedHelperPolicy() + if err != nil { + t.Fatalf("PrivilegedHelperPolicy: %v", err) + } + if p.MeshInterface != "" { + t.Errorf("disabled mesh should leave MeshInterface empty, got %q", p.MeshInterface) + } + if len(p.RouteCIDRs) != 0 || len(p.PeerPublicKeys) != 0 { + t.Errorf("disabled mesh should configure no routes/peers, got %v / %v", p.RouteCIDRs, p.PeerPublicKeys) + } + // Anchor defaults even when disabled, but no interface/peers means the helper + // refuses every mesh and Pod-network command. + if p.Anchor != "macvz/pods" { + t.Errorf("Anchor default = %q, want macvz/pods", p.Anchor) + } +} + +func TestPrivilegedHelperPolicyCustomAnchor(t *testing.T) { + c := Default() + c.PodNetwork = PodNetworkConfig{Enabled: true, Interface: "bridge100", Anchor: "macvz/custom"} + p, err := c.PrivilegedHelperPolicy() + if err != nil { + t.Fatalf("PrivilegedHelperPolicy: %v", err) + } + if p.Anchor != "macvz/custom" { + t.Errorf("Anchor = %q, want macvz/custom", p.Anchor) + } +} + +func TestPrivilegedHelperPolicyCustomVMNetCIDRs(t *testing.T) { + c := Default() + c.PodNetwork = PodNetworkConfig{ + Enabled: true, + Interface: "bridge100", + VMNetCIDRs: []string{"172.31.0.0/24"}, + } + p, err := c.PrivilegedHelperPolicy() + if err != nil { + t.Fatalf("PrivilegedHelperPolicy: %v", err) + } + if !p.VMNetCIDRs["172.31.0.0/24"] || p.VMNetCIDRs[DefaultVMNetCIDR] { + t.Errorf("VMNetCIDRs = %v, want only custom range", p.VMNetCIDRs) + } +} diff --git a/pkg/network/podnet/helper.go b/pkg/network/podnet/helper.go new file mode 100644 index 0000000..404d9b8 --- /dev/null +++ b/pkg/network/podnet/helper.go @@ -0,0 +1,34 @@ +package podnet + +import ( + "context" + + "github.com/chimerakang/macvz/pkg/network/privhelper" +) + +// helperRunner implements runner by delegating to the root privileged helper +// (cmd/macvz-netd) over its unix socket, so the user-run kubelet can program pf +// without itself being root (#38). +type helperRunner struct { + client *privhelper.Client +} + +func (h helperRunner) run(ctx context.Context, c command) (string, error) { + stdout, stderr, code, err := h.client.Run(ctx, c.Name, c.Args, c.Stdin) + if err != nil { + return stdout, &CommandError{Cmd: c.String(), Stderr: stderr, ExitCode: code, Err: err} + } + if code != 0 { + return stdout, &CommandError{Cmd: c.String(), Stderr: stderr, ExitCode: code} + } + return stdout, nil +} + +// WithHelperSocket routes the Router's privileged commands through the root +// helper daemon at socketPath instead of executing them directly. Use this when +// the kubelet runs as a non-root user (#38). +func WithHelperSocket(socketPath string) Option { + return func(rt *Router) { + rt.run = helperRunner{client: privhelper.NewClient(socketPath)} + } +} diff --git a/pkg/network/podnet/helper_test.go b/pkg/network/podnet/helper_test.go new file mode 100644 index 0000000..d5296c3 --- /dev/null +++ b/pkg/network/podnet/helper_test.go @@ -0,0 +1,63 @@ +package podnet + +import ( + "context" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/chimerakang/macvz/pkg/network/privhelper" +) + +// TestRouterRoutesThroughHelper verifies WithHelperSocket sends the Router's pf +// commands through the privileged helper rather than executing them directly. +func TestRouterRoutesThroughHelper(t *testing.T) { + var mu sync.Mutex + var ran []string + fake := func(_ context.Context, name string, args []string, stdin string) (string, string, int, error) { + mu.Lock() + defer mu.Unlock() + ran = append(ran, strings.TrimSpace(name+" "+strings.Join(args, " "))) + return "", "", 0, nil + } + + dir, err := os.MkdirTemp("/tmp", "ph") + if err != nil { + t.Fatal(err) + } + defer func() { _ = os.RemoveAll(dir) }() + sock := filepath.Join(dir, "s") + srv := privhelper.NewServerWithExec(sock, fake) + if err := srv.Listen(-1, -1); err != nil { + t.Fatalf("Listen: %v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { _ = srv.Serve(ctx) }() + defer func() { _ = srv.Close() }() + + rt := New(Config{Interface: "bridge100", EnableForwarding: true}, WithHelperSocket(sock)) + if err := rt.Start(ctx); err != nil { + t.Fatalf("Start: %v", err) + } + if err := rt.Attach(ctx, Endpoint{PodKey: "default/web", PodIP: "10.244.1.2", VMIP: "192.168.64.5"}); err != nil { + t.Fatalf("Attach: %v", err) + } + + mu.Lock() + defer mu.Unlock() + var sawForward, sawAnchorLoad bool + for _, c := range ran { + if strings.Contains(c, "sysctl -w net.inet.ip.forwarding=1") { + sawForward = true + } + if strings.Contains(c, "pfctl -a macvz/pods -f -") { + sawAnchorLoad = true + } + } + if !sawForward || !sawAnchorLoad { + t.Errorf("expected privileged commands through helper; ran=%v", ran) + } +} diff --git a/pkg/network/podnet/router.go b/pkg/network/podnet/router.go index 148eafe..4069e17 100644 --- a/pkg/network/podnet/router.go +++ b/pkg/network/podnet/router.go @@ -16,6 +16,7 @@ import ( const ( defaultPfctl = "pfctl" defaultSysctl = "sysctl" + defaultRoute = "route" ) // DefaultAnchor is the pf anchor MacVz owns. The operator must reference it from @@ -70,10 +71,12 @@ type Router struct { cfg Config pfctl string sysctl string + route string mu sync.Mutex started bool - endpoints map[string]Endpoint // keyed by PodKey + endpoints map[string]Endpoint // keyed by PodKey + services map[string][]ServiceRule // keyed by service "namespace/name" } // Option configures a Router. @@ -104,7 +107,9 @@ func New(cfg Config, opts ...Option) *Router { cfg: cfg, pfctl: defaultPfctl, sysctl: defaultSysctl, + route: defaultRoute, endpoints: map[string]Endpoint{}, + services: map[string][]ServiceRule{}, } for _, opt := range opts { opt(rt) @@ -112,6 +117,15 @@ func New(cfg Config, opts ...Option) *Router { return rt } +// WithRouteTool overrides the route binary name. Empty keeps the default. +func WithRouteTool(route string) Option { + return func(rt *Router) { + if route != "" { + rt.route = route + } + } +} + // Start prepares the host: it enables IPv4 forwarding (when configured), ensures // pf is enabled, and loads an empty anchor so the ruleset has a known baseline. func (rt *Router) Start(ctx context.Context) error { @@ -130,6 +144,9 @@ func (rt *Router) Start(ctx context.Context) error { if _, err := rt.runTolerating(ctx, command{Name: rt.pfctl, Args: []string{"-e"}}, pfAlreadyEnabled); err != nil { return fmt.Errorf("enable pf: %w", err) } + if err := rt.removeVMNetDefaultRouteLocked(ctx); err != nil { + return err + } rt.started = true if err := rt.loadAnchorLocked(ctx); err != nil { return err @@ -149,6 +166,9 @@ func (rt *Router) Attach(ctx context.Context, ep Endpoint) error { if !rt.started { return fmt.Errorf("podnet: Attach %q before Start", ep.PodKey) } + if err := rt.removeVMNetDefaultRouteLocked(ctx); err != nil { + return fmt.Errorf("remove vmnet default route before attach %q: %w", ep.PodKey, err) + } rt.endpoints[ep.PodKey] = ep if err := rt.loadAnchorLocked(ctx); err != nil { return fmt.Errorf("attach %q (%s -> %s): %w", ep.PodKey, ep.PodIP, ep.VMIP, err) @@ -157,6 +177,22 @@ func (rt *Router) Attach(ctx context.Context, ep Endpoint) error { return nil } +// removeVMNetDefaultRouteLocked removes apple/container's occasional +// "default -> " route. That route can capture the host's outbound +// traffic and sever the kubelet's API connection. Deleting it is idempotent and +// safe: the route is scoped to the vmnet interface and Pod reachability uses +// explicit Pod/mesh routes plus pf binat/rdr, not a host default route. +func (rt *Router) removeVMNetDefaultRouteLocked(ctx context.Context) error { + _, err := rt.runTolerating(ctx, command{ + Name: rt.route, + Args: []string{"-q", "-n", "delete", "-inet", "default", "-interface", rt.cfg.Interface}, + }, errRouteMissing, errRouteNotFound) + if err != nil { + return fmt.Errorf("remove vmnet default route on %s: %w", rt.cfg.Interface, err) + } + return nil +} + // Detach removes a Pod's mapping. Detaching an unknown Pod is a no-op, so // deletion stays idempotent. func (rt *Router) Detach(ctx context.Context, podKey string) error { @@ -194,6 +230,7 @@ func (rt *Router) Stop(ctx context.Context) error { rt.mu.Lock() defer rt.mu.Unlock() rt.endpoints = map[string]Endpoint{} + rt.services = map[string][]ServiceRule{} if !rt.started { return nil } @@ -209,6 +246,7 @@ func (rt *Router) Stop(ctx context.Context) error { // atomically via `pfctl -a -f -`. Caller holds rt.mu. func (rt *Router) loadAnchorLocked(ctx context.Context) error { rules := renderAnchor(rt.cfg.Interface, rt.endpointsSortedLocked()) + rules += renderServiceRules(rt.cfg.Interface, rt.services, rt.vmipByPodIPLocked()) _, err := rt.run.run(ctx, command{ Name: rt.pfctl, Args: []string{"-a", rt.cfg.Anchor, "-f", "-"}, @@ -263,3 +301,8 @@ func (rt *Router) runTolerating(ctx context.Context, c command, benign ...string // pfAlreadyEnabled is the stderr pfctl prints when pf is already on. const pfAlreadyEnabled = "pf already enabled" + +const ( + errRouteMissing = "not in table" + errRouteNotFound = "not found" +) diff --git a/pkg/network/podnet/router_test.go b/pkg/network/podnet/router_test.go index 7bc895a..08a5c0a 100644 --- a/pkg/network/podnet/router_test.go +++ b/pkg/network/podnet/router_test.go @@ -79,6 +79,9 @@ func TestStartEnablesForwardingAndPF(t *testing.T) { if !contains(cmds, "pfctl -a macvz/pods -f -") { t.Errorf("Start did not load a baseline anchor\nran: %v", cmds) } + if !contains(cmds, "route -q -n delete -inet default -interface bridge100") { + t.Errorf("Start did not remove vmnet default route\nran: %v", cmds) + } } func TestStartToleratesPFAlreadyEnabled(t *testing.T) { @@ -90,6 +93,24 @@ func TestStartToleratesPFAlreadyEnabled(t *testing.T) { } } +func TestStartToleratesMissingVMNetDefaultRoute(t *testing.T) { + fr := newFakeRunner() + fr.failOn["route -q -n delete -inet default -interface bridge100"] = "route: writing to routing socket: not in table" + rt := newTestRouter(fr) + if err := rt.Start(context.Background()); err != nil { + t.Fatalf("Start should tolerate missing vmnet default route, got: %v", err) + } +} + +func TestStartFailsWhenVMNetDefaultRouteCleanupFails(t *testing.T) { + fr := newFakeRunner() + fr.failOn["route -q -n delete -inet default -interface bridge100"] = "route: permission denied" + rt := newTestRouter(fr) + if err := rt.Start(context.Background()); err == nil { + t.Fatal("Start should fail when vmnet default route cleanup fails") + } +} + func TestStartRequiresInterface(t *testing.T) { rt := New(Config{}, WithRunner(newFakeRunner())) if err := rt.Start(context.Background()); err == nil { @@ -108,6 +129,9 @@ func TestAttachInstallsBinatRule(t *testing.T) { if err := rt.Attach(ctx, ep); err != nil { t.Fatalf("Attach: %v", err) } + if got := countContaining(fr.strings(), "route -q -n delete -inet default -interface bridge100"); got < 2 { + t.Errorf("Attach should re-remove vmnet default route after VM start; got %d cleanup calls", got) + } rules, ok := fr.lastAnchorLoad() if !ok { t.Fatal("no anchor load recorded") @@ -121,6 +145,16 @@ func TestAttachInstallsBinatRule(t *testing.T) { } } +func countContaining(haystack []string, sub string) int { + n := 0 + for _, s := range haystack { + if strings.Contains(s, sub) { + n++ + } + } + return n +} + func TestAttachValidatesEndpoint(t *testing.T) { fr := newFakeRunner() rt := newTestRouter(fr) diff --git a/pkg/network/podnet/service.go b/pkg/network/podnet/service.go new file mode 100644 index 0000000..ed4326a --- /dev/null +++ b/pkg/network/podnet/service.go @@ -0,0 +1,183 @@ +package podnet + +import ( + "context" + "fmt" + "net" + "sort" + "strings" + + "k8s.io/klog/v2" +) + +// ServiceRule is one ClusterIP:port DNAT to its ready backends. It maps a +// Kubernetes ClusterIP Service port to the set of ready backend Pod IPs, so a +// micro-VM that dials the ClusterIP reaches a backend. There is no kube-proxy on +// a MacVz node (it is a macOS host and the kube-proxy Pod spec is unsupported), +// so MacVz programs this translation in the same pf anchor that owns the Pod +// binat rules (#37/P5). +type ServiceRule struct { + // ServiceKey is the "namespace/name" of the owning Service (diagnostics). + ServiceKey string + // ClusterIP is the Service's stable virtual IP. + ClusterIP string + // Protocol is "tcp" or "udp" (lowercased pf token). + Protocol string + // Port is the Service port the ClusterIP listens on. + Port int + // TargetPort is the backend container port traffic is redirected to. + TargetPort int + // Backends are the ready backend Pod IPs (MacVz Pod IPs). An empty list + // means the Service has no ready endpoints and emits no rule. + Backends []string +} + +func (r ServiceRule) validate() error { + if r.ServiceKey == "" { + return fmt.Errorf("podnet: service rule has no service key") + } + if net.ParseIP(r.ClusterIP) == nil { + return fmt.Errorf("podnet: service %q has invalid ClusterIP %q", r.ServiceKey, r.ClusterIP) + } + switch r.Protocol { + case "tcp", "udp": + default: + return fmt.Errorf("podnet: service %q has unsupported protocol %q", r.ServiceKey, r.Protocol) + } + if r.Port < 1 || r.Port > 65535 { + return fmt.Errorf("podnet: service %q port %d out of range", r.ServiceKey, r.Port) + } + if r.TargetPort < 1 || r.TargetPort > 65535 { + return fmt.Errorf("podnet: service %q targetPort %d out of range", r.ServiceKey, r.TargetPort) + } + for _, b := range r.Backends { + if net.ParseIP(b) == nil { + return fmt.Errorf("podnet: service %q has invalid backend IP %q", r.ServiceKey, b) + } + } + return nil +} + +// AttachService installs (or replaces) the DNAT rules for one Service, keyed by +// its "namespace/name". Passing rules with no backends, or an empty slice, +// removes the Service's rules — so a Service that loses all ready endpoints +// stops redirecting. It is idempotent. +func (rt *Router) AttachService(ctx context.Context, key string, rules []ServiceRule) error { + if key == "" { + return fmt.Errorf("podnet: AttachService requires a service key") + } + kept := make([]ServiceRule, 0, len(rules)) + for _, r := range rules { + if err := r.validate(); err != nil { + return err + } + if len(r.Backends) == 0 { + continue + } + kept = append(kept, r) + } + + rt.mu.Lock() + defer rt.mu.Unlock() + if !rt.started { + return fmt.Errorf("podnet: AttachService %q before Start", key) + } + if len(kept) == 0 { + delete(rt.services, key) + } else { + rt.services[key] = kept + } + if err := rt.loadAnchorLocked(ctx); err != nil { + return fmt.Errorf("attach service %q: %w", key, err) + } + klog.InfoS("attached service to network path", "service", key, "rules", len(kept)) + return nil +} + +// DetachService removes a Service's DNAT rules. Detaching an unknown Service is +// a no-op so deletion stays idempotent. +func (rt *Router) DetachService(ctx context.Context, key string) error { + rt.mu.Lock() + defer rt.mu.Unlock() + if _, ok := rt.services[key]; !ok { + return nil + } + delete(rt.services, key) + if !rt.started { + return nil + } + if err := rt.loadAnchorLocked(ctx); err != nil { + return fmt.Errorf("detach service %q: %w", key, err) + } + klog.InfoS("detached service from network path", "service", key) + return nil +} + +// Services returns the service keys with active rules, sorted, for diagnostics +// and tests. +func (rt *Router) Services() []string { + rt.mu.Lock() + defer rt.mu.Unlock() + out := make([]string, 0, len(rt.services)) + for k := range rt.services { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// renderServiceRules builds the pf `rdr` rules for all services. Each backend +// Pod IP is resolved against the local endpoint set: a backend whose Pod runs on +// this node is redirected straight to its micro-VM's host-only address (directly +// attached to the vmnet interface, so no extra route is needed), while a backend +// on another Mac keeps its Pod IP and is reached over the WireGuard mesh route. +// Output is deterministic: services, rules, and backends are all sorted. +func renderServiceRules(iface string, services map[string][]ServiceRule, vmipByPodIP map[string]string) string { + keys := make([]string, 0, len(services)) + for k := range services { + keys = append(keys, k) + } + sort.Strings(keys) + + var b strings.Builder + for _, key := range keys { + rules := append([]ServiceRule(nil), services[key]...) + sort.Slice(rules, func(i, j int) bool { + if rules[i].ClusterIP != rules[j].ClusterIP { + return rules[i].ClusterIP < rules[j].ClusterIP + } + return rules[i].Port < rules[j].Port + }) + for _, r := range rules { + targets := make([]string, 0, len(r.Backends)) + for _, podIP := range r.Backends { + if vmIP, ok := vmipByPodIP[podIP]; ok { + targets = append(targets, vmIP) // local backend, via vmnet + } else { + targets = append(targets, podIP) // remote backend, via mesh + } + } + sort.Strings(targets) + + fmt.Fprintf(&b, "# %s %s:%d -> :%d\n", r.ServiceKey, r.ClusterIP, r.Port, r.TargetPort) + if len(targets) == 1 { + fmt.Fprintf(&b, "rdr on %s inet proto %s from any to %s port %d -> %s port %d\n", + iface, r.Protocol, r.ClusterIP, r.Port, targets[0], r.TargetPort) + continue + } + fmt.Fprintf(&b, "rdr on %s inet proto %s from any to %s port %d -> { %s } port %d round-robin\n", + iface, r.Protocol, r.ClusterIP, r.Port, strings.Join(targets, ", "), r.TargetPort) + } + } + return b.String() +} + +// vmipByPodIPLocked maps each attached Pod IP to its micro-VM address. Caller +// holds rt.mu. +func (rt *Router) vmipByPodIPLocked() map[string]string { + m := make(map[string]string, len(rt.endpoints)) + for _, ep := range rt.endpoints { + m[ep.PodIP] = ep.VMIP + } + return m +} diff --git a/pkg/network/podnet/service_test.go b/pkg/network/podnet/service_test.go new file mode 100644 index 0000000..641cd09 --- /dev/null +++ b/pkg/network/podnet/service_test.go @@ -0,0 +1,208 @@ +package podnet + +import ( + "context" + "strings" + "testing" +) + +func startedRouter(t *testing.T, fr *fakeRunner) (*Router, context.Context) { + t.Helper() + rt := newTestRouter(fr) + ctx := context.Background() + if err := rt.Start(ctx); err != nil { + t.Fatalf("Start: %v", err) + } + return rt, ctx +} + +// A local backend (its Pod is attached on this node) redirects to the VM's +// host-only address, not the Pod IP, so it is delivered straight over vmnet. +func TestAttachServiceLocalBackendRedirectsToVMIP(t *testing.T) { + fr := newFakeRunner() + rt, ctx := startedRouter(t, fr) + + if err := rt.Attach(ctx, Endpoint{PodKey: "default/be", PodIP: "10.244.101.2", VMIP: "192.168.64.5"}); err != nil { + t.Fatalf("Attach backend: %v", err) + } + rule := ServiceRule{ + ServiceKey: "default/hello", ClusterIP: "10.96.0.50", Protocol: "tcp", + Port: 80, TargetPort: 8080, Backends: []string{"10.244.101.2"}, + } + if err := rt.AttachService(ctx, "default/hello", []ServiceRule{rule}); err != nil { + t.Fatalf("AttachService: %v", err) + } + rules, ok := fr.lastAnchorLoad() + if !ok { + t.Fatal("no anchor load recorded") + } + want := "rdr on bridge100 inet proto tcp from any to 10.96.0.50 port 80 -> 192.168.64.5 port 8080" + if !strings.Contains(rules, want) { + t.Errorf("anchor missing local rdr rule %q\n---\n%s", want, rules) + } + if strings.Contains(rules, "-> 10.244.101.2 port 8080") { + t.Errorf("local backend should redirect to VMIP, not Pod IP\n---\n%s", rules) + } +} + +// A remote backend (no local endpoint) keeps its Pod IP, reached via the mesh. +func TestAttachServiceRemoteBackendKeepsPodIP(t *testing.T) { + fr := newFakeRunner() + rt, ctx := startedRouter(t, fr) + + rule := ServiceRule{ + ServiceKey: "default/hello", ClusterIP: "10.96.0.50", Protocol: "tcp", + Port: 80, TargetPort: 8080, Backends: []string{"10.244.102.9"}, + } + if err := rt.AttachService(ctx, "default/hello", []ServiceRule{rule}); err != nil { + t.Fatalf("AttachService: %v", err) + } + rules, _ := fr.lastAnchorLoad() + want := "rdr on bridge100 inet proto tcp from any to 10.96.0.50 port 80 -> 10.244.102.9 port 8080" + if !strings.Contains(rules, want) { + t.Errorf("anchor missing remote rdr rule %q\n---\n%s", want, rules) + } +} + +// Mixed local + remote backends become a round-robin pool with VMIP for the +// local one and Pod IP for the remote one. +func TestAttachServiceMixedBackendsRoundRobin(t *testing.T) { + fr := newFakeRunner() + rt, ctx := startedRouter(t, fr) + if err := rt.Attach(ctx, Endpoint{PodKey: "default/be", PodIP: "10.244.101.2", VMIP: "192.168.64.5"}); err != nil { + t.Fatalf("Attach: %v", err) + } + rule := ServiceRule{ + ServiceKey: "default/hello", ClusterIP: "10.96.0.50", Protocol: "tcp", + Port: 80, TargetPort: 8080, Backends: []string{"10.244.101.2", "10.244.102.9"}, + } + if err := rt.AttachService(ctx, "default/hello", []ServiceRule{rule}); err != nil { + t.Fatalf("AttachService: %v", err) + } + rules, _ := fr.lastAnchorLoad() + // targets are sorted; "10.244.102.9" < "192.168.64.5" + want := "-> { 10.244.102.9, 192.168.64.5 } port 8080 round-robin" + if !strings.Contains(rules, want) { + t.Errorf("anchor missing round-robin pool %q\n---\n%s", want, rules) + } +} + +// A Service with no ready backends emits no rule, and replacing a Service's +// rules with an empty set removes it. +func TestAttachServiceNoBackendsRemovesRule(t *testing.T) { + fr := newFakeRunner() + rt, ctx := startedRouter(t, fr) + rule := ServiceRule{ + ServiceKey: "default/hello", ClusterIP: "10.96.0.50", Protocol: "tcp", + Port: 80, TargetPort: 8080, Backends: []string{"10.244.101.2"}, + } + if err := rt.AttachService(ctx, "default/hello", []ServiceRule{rule}); err != nil { + t.Fatalf("AttachService: %v", err) + } + // now drop all backends + rule.Backends = nil + if err := rt.AttachService(ctx, "default/hello", []ServiceRule{rule}); err != nil { + t.Fatalf("AttachService empty: %v", err) + } + if svcs := rt.Services(); len(svcs) != 0 { + t.Errorf("Services = %v, want empty after losing all backends", svcs) + } + rules, _ := fr.lastAnchorLoad() + if strings.Contains(rules, "10.96.0.50") { + t.Errorf("rule still present after losing all backends\n---\n%s", rules) + } +} + +func TestDetachServiceRemovesRule(t *testing.T) { + fr := newFakeRunner() + rt, ctx := startedRouter(t, fr) + a := ServiceRule{ServiceKey: "default/a", ClusterIP: "10.96.0.1", Protocol: "tcp", Port: 80, TargetPort: 8080, Backends: []string{"10.244.102.2"}} + b := ServiceRule{ServiceKey: "default/b", ClusterIP: "10.96.0.2", Protocol: "tcp", Port: 80, TargetPort: 8080, Backends: []string{"10.244.102.3"}} + if err := rt.AttachService(ctx, "default/a", []ServiceRule{a}); err != nil { + t.Fatalf("AttachService a: %v", err) + } + if err := rt.AttachService(ctx, "default/b", []ServiceRule{b}); err != nil { + t.Fatalf("AttachService b: %v", err) + } + if err := rt.DetachService(ctx, "default/a"); err != nil { + t.Fatalf("DetachService: %v", err) + } + rules, _ := fr.lastAnchorLoad() + if strings.Contains(rules, "10.96.0.1") { + t.Errorf("detached service rule still present\n---\n%s", rules) + } + if !strings.Contains(rules, "10.96.0.2") { + t.Errorf("remaining service rule missing\n---\n%s", rules) + } +} + +func TestDetachServiceUnknownIsNoop(t *testing.T) { + fr := newFakeRunner() + rt, ctx := startedRouter(t, fr) + before := len(fr.strings()) + if err := rt.DetachService(ctx, "default/ghost"); err != nil { + t.Fatalf("DetachService unknown: %v", err) + } + if after := len(fr.strings()); after != before { + t.Errorf("DetachService of unknown reloaded the anchor (%d -> %d)", before, after) + } +} + +func TestAttachServiceValidates(t *testing.T) { + fr := newFakeRunner() + rt, ctx := startedRouter(t, fr) + bad := []ServiceRule{ + {ServiceKey: "", ClusterIP: "10.96.0.1", Protocol: "tcp", Port: 80, TargetPort: 8080, Backends: []string{"10.244.1.2"}}, + {ServiceKey: "default/x", ClusterIP: "nope", Protocol: "tcp", Port: 80, TargetPort: 8080, Backends: []string{"10.244.1.2"}}, + {ServiceKey: "default/x", ClusterIP: "10.96.0.1", Protocol: "sctp", Port: 80, TargetPort: 8080, Backends: []string{"10.244.1.2"}}, + {ServiceKey: "default/x", ClusterIP: "10.96.0.1", Protocol: "tcp", Port: 0, TargetPort: 8080, Backends: []string{"10.244.1.2"}}, + {ServiceKey: "default/x", ClusterIP: "10.96.0.1", Protocol: "tcp", Port: 80, TargetPort: 0, Backends: []string{"10.244.1.2"}}, + {ServiceKey: "default/x", ClusterIP: "10.96.0.1", Protocol: "tcp", Port: 80, TargetPort: 8080, Backends: []string{"bad-ip"}}, + } + for _, r := range bad { + if err := rt.AttachService(ctx, "default/x", []ServiceRule{r}); err == nil { + t.Errorf("AttachService(%+v) = nil, want validation error", r) + } + } +} + +func TestAttachServiceBeforeStartFails(t *testing.T) { + rt := newTestRouter(newFakeRunner()) + r := ServiceRule{ServiceKey: "default/x", ClusterIP: "10.96.0.1", Protocol: "tcp", Port: 80, TargetPort: 8080, Backends: []string{"10.244.1.2"}} + if err := rt.AttachService(context.Background(), "default/x", []ServiceRule{r}); err == nil { + t.Fatal("AttachService before Start should fail") + } +} + +func TestRenderServiceRulesDeterministic(t *testing.T) { + services := map[string][]ServiceRule{ + "default/a": {{ServiceKey: "default/a", ClusterIP: "10.96.0.1", Protocol: "tcp", Port: 80, TargetPort: 8080, Backends: []string{"10.244.1.2"}}}, + "default/b": {{ServiceKey: "default/b", ClusterIP: "10.96.0.2", Protocol: "tcp", Port: 80, TargetPort: 8080, Backends: []string{"10.244.1.3"}}}, + } + vmip := map[string]string{} + first := renderServiceRules("bridge100", services, vmip) + second := renderServiceRules("bridge100", services, vmip) + if first != second { + t.Error("renderServiceRules is not deterministic") + } +} + +// Service rules and Pod binat rules coexist in the same anchor load. +func TestServiceAndBinatRulesCoexist(t *testing.T) { + fr := newFakeRunner() + rt, ctx := startedRouter(t, fr) + if err := rt.Attach(ctx, Endpoint{PodKey: "default/be", PodIP: "10.244.101.2", VMIP: "192.168.64.5"}); err != nil { + t.Fatalf("Attach: %v", err) + } + rule := ServiceRule{ServiceKey: "default/hello", ClusterIP: "10.96.0.50", Protocol: "tcp", Port: 80, TargetPort: 8080, Backends: []string{"10.244.101.2"}} + if err := rt.AttachService(ctx, "default/hello", []ServiceRule{rule}); err != nil { + t.Fatalf("AttachService: %v", err) + } + rules, _ := fr.lastAnchorLoad() + if !strings.Contains(rules, "binat on bridge100 from 192.168.64.5 to any -> 10.244.101.2") { + t.Errorf("binat rule missing\n---\n%s", rules) + } + if !strings.Contains(rules, "rdr on bridge100 inet proto tcp from any to 10.96.0.50") { + t.Errorf("rdr rule missing\n---\n%s", rules) + } +} diff --git a/pkg/network/privhelper/client.go b/pkg/network/privhelper/client.go new file mode 100644 index 0000000..0a00bfe --- /dev/null +++ b/pkg/network/privhelper/client.go @@ -0,0 +1,124 @@ +package privhelper + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "net" + "time" +) + +// Client talks to the root helper over its unix socket. It is safe for +// concurrent use: each Run opens its own short-lived connection. +type Client struct { + socketPath string + dialer net.Dialer +} + +// NewClient builds a client for the helper at socketPath. +func NewClient(socketPath string) *Client { + return &Client{socketPath: socketPath} +} + +// SocketPath returns the configured socket path. +func (c *Client) SocketPath() string { return c.socketPath } + +// APIError is a structured refusal or transport-level failure from the helper. +// Its Code is one of the privhelper Code* constants, so a caller can map a +// failure to a Pod condition or diagnostic without parsing the message. +type APIError struct { + Code string + Message string +} + +func (e *APIError) Error() string { + if e.Code == "" { + return "privhelper: " + e.Message + } + return fmt.Sprintf("privhelper: %s (%s)", e.Message, e.Code) +} + +// Ping verifies the helper is reachable by running a harmless allowlisted +// command (`sysctl -n kern.ostype`). It returns an error if the socket cannot be +// reached or the helper refuses, so startup can fail fast with a clear message. +func (c *Client) Ping(ctx context.Context) error { + _, _, code, err := c.Run(ctx, "sysctl", []string{"-n", "kern.ostype"}, "") + if err != nil { + return err + } + if code != 0 { + return fmt.Errorf("privhelper ping returned exit %d", code) + } + return nil +} + +// Status queries the helper's self-report (version, protocol, allowlist, policy +// state, uptime) for startup logs and diagnostics. It also exercises protocol +// negotiation: an incompatible helper rejects the request with an *APIError +// whose Code is CodeUnsupportedProtocol. +func (c *Client) Status(ctx context.Context) (*HelperStatus, error) { + resp, err := c.roundTrip(ctx, Request{Protocol: Protocol, Op: OpStatus}) + if err != nil { + return nil, err + } + if resp.Err != "" { + return nil, &APIError{Code: resp.ErrorCode, Message: resp.Err} + } + if resp.Status == nil { + return nil, &APIError{Message: "helper returned no status"} + } + return resp.Status, nil +} + +// ReloadPolicy asks a reloadable helper to refresh its config-derived policy. +// Helpers without a loader treat it as a no-op, so kubelet can call this before +// mesh peer reconciliation without needing to know how netd was started. +func (c *Client) ReloadPolicy(ctx context.Context) error { + resp, err := c.roundTrip(ctx, Request{Protocol: Protocol, Op: OpReloadPolicy}) + if err != nil { + return err + } + if resp.Err != "" { + return &APIError{Code: resp.ErrorCode, Message: resp.Err} + } + return nil +} + +// Run sends one command to the helper and returns its stdout, stderr, exit code, +// and any transport error. A non-zero exitCode with a nil error is a normal +// command failure (mirrors os/exec). A refusal or spawn failure is returned as +// an *APIError carrying a structured Code. +func (c *Client) Run(ctx context.Context, name string, args []string, stdin string) (stdout, stderr string, exitCode int, err error) { + resp, err := c.roundTrip(ctx, Request{Protocol: Protocol, Op: OpExec, Name: name, Args: args, Stdin: stdin}) + if err != nil { + return "", "", -1, err + } + if resp.Err != "" { + return resp.Stdout, resp.Stderr, resp.ExitCode, &APIError{Code: resp.ErrorCode, Message: resp.Err} + } + return resp.Stdout, resp.Stderr, resp.ExitCode, nil +} + +// roundTrip opens a short-lived connection, sends req, and decodes one Response. +func (c *Client) roundTrip(ctx context.Context, req Request) (Response, error) { + conn, err := c.dialer.DialContext(ctx, "unix", c.socketPath) + if err != nil { + return Response{}, fmt.Errorf("privhelper dial %q: %w", c.socketPath, err) + } + defer func() { _ = conn.Close() }() + if deadline, ok := ctx.Deadline(); ok { + _ = conn.SetDeadline(deadline) + } else { + _ = conn.SetDeadline(time.Now().Add(2 * time.Minute)) + } + + if err := json.NewEncoder(conn).Encode(req); err != nil { + return Response{}, fmt.Errorf("privhelper encode request: %w", err) + } + var resp Response + if err := json.NewDecoder(bufio.NewReader(conn)).Decode(&resp); err != nil { + return Response{}, fmt.Errorf("privhelper decode response: %w", err) + } + return resp, nil +} diff --git a/pkg/network/privhelper/control_test.go b/pkg/network/privhelper/control_test.go new file mode 100644 index 0000000..2e387fb --- /dev/null +++ b/pkg/network/privhelper/control_test.go @@ -0,0 +1,249 @@ +package privhelper + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "net" + "os" + "path/filepath" + "strings" + "testing" +) + +// startControlServer starts a Server (optionally policy-enforcing) with the +// given executor on a short /tmp socket and returns a connected Client. +func startControlServer(t *testing.T, srv *Server) *Client { + t.Helper() + // Unix socket paths are capped at ~104 bytes on macOS; use a short /tmp dir. + dir, err := os.MkdirTemp("/tmp", "phc") + if err != nil { + t.Fatalf("MkdirTemp: %v", err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + if err := srv.Listen(-1, -1); err != nil { + t.Fatalf("Listen: %v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + go func() { _ = srv.Serve(ctx) }() + t.Cleanup(func() { cancel(); _ = srv.Close() }) + return NewClient(srv.socketPath) +} + +func tmpSock(t *testing.T) string { + t.Helper() + dir, err := os.MkdirTemp("/tmp", "phc") + if err != nil { + t.Fatalf("MkdirTemp: %v", err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + return filepath.Join(dir, "s") +} + +// okExec is a no-op executor that always succeeds. +func okExec(_ context.Context, _ string, _ []string, _ string) (string, string, int, error) { + return "", "", 0, nil +} + +func TestStatusReportsHelperIdentity(t *testing.T) { + srv := NewServerWithExec(tmpSock(t), okExec).SetVersion("v1.2.3") + c := startControlServer(t, srv) + + st, err := c.Status(context.Background()) + if err != nil { + t.Fatalf("Status: %v", err) + } + if st.Version != "v1.2.3" { + t.Errorf("version = %q, want v1.2.3", st.Version) + } + if st.Protocol != Protocol { + t.Errorf("protocol = %d, want %d", st.Protocol, Protocol) + } + if st.PolicyEnforced { + t.Error("PolicyEnforced should be false for a non-policy server") + } + if st.PID != os.Getpid() { + t.Errorf("PID = %d, want %d", st.PID, os.Getpid()) + } + if len(st.AllowedCommands) == 0 || st.Uptime == "" || st.StartedAt.IsZero() { + t.Errorf("incomplete status: %+v", st) + } +} + +func TestStatusReportsPolicyEnforced(t *testing.T) { + srv := NewServerWithPolicy(tmpSock(t), Policy{MeshInterface: "utun7"}) + srv.exec = okExec // don't actually run commands + c := startControlServer(t, srv) + + st, err := c.Status(context.Background()) + if err != nil { + t.Fatalf("Status: %v", err) + } + if !st.PolicyEnforced { + t.Error("PolicyEnforced should be true for a policy server") + } + // Version defaults to "dev" when unset. + if st.Version != "dev" { + t.Errorf("version = %q, want dev", st.Version) + } +} + +func TestReloadPolicyUpdatesValidation(t *testing.T) { + allowEn0 := false + loader := func() (Policy, error) { + if allowEn0 { + return Policy{MeshInterface: "en0"}, nil + } + return Policy{MeshInterface: "utun7"}, nil + } + srv, err := NewServerWithPolicyLoader(tmpSock(t), loader) + if err != nil { + t.Fatalf("NewServerWithPolicyLoader: %v", err) + } + srv.exec = okExec + c := startControlServer(t, srv) + + if _, _, _, err := c.Run(context.Background(), "ifconfig", []string{"en0", "up"}, ""); err == nil { + t.Fatal("en0 should be refused before policy reload") + } + allowEn0 = true + if err := c.ReloadPolicy(context.Background()); err != nil { + t.Fatalf("ReloadPolicy: %v", err) + } + if _, _, code, err := c.Run(context.Background(), "ifconfig", []string{"en0", "up"}, ""); err != nil || code != 0 { + t.Fatalf("en0 should pass after policy reload: code=%d err=%v", code, err) + } +} + +// TestUnsupportedProtocolRejected verifies a client speaking a future protocol +// is refused with a structured error rather than silently misinterpreted. +func TestUnsupportedProtocolRejected(t *testing.T) { + srv := NewServerWithExec(tmpSock(t), okExec) + c := startControlServer(t, srv) + + resp, err := c.roundTrip(context.Background(), Request{Protocol: Protocol + 1, Op: OpStatus}) + if err != nil { + t.Fatalf("roundTrip: %v", err) + } + if resp.ErrorCode != CodeUnsupportedProtocol { + t.Errorf("errorCode = %q, want %q (err=%q)", resp.ErrorCode, CodeUnsupportedProtocol, resp.Err) + } + if resp.Status != nil { + t.Error("status must not be returned for a rejected protocol") + } +} + +// TestLegacyUnversionedRequestAccepted verifies a Protocol==0 request (an older +// client that predates version negotiation) is still served. +func TestLegacyUnversionedRequestAccepted(t *testing.T) { + srv := NewServerWithExec(tmpSock(t), okExec) + c := startControlServer(t, srv) + + // Protocol intentionally left 0; Op exec. + resp, err := c.roundTrip(context.Background(), Request{Name: "pfctl", Args: []string{"-e"}}) + if err != nil { + t.Fatalf("roundTrip: %v", err) + } + if resp.Err != "" { + t.Errorf("legacy request refused: %q (%s)", resp.Err, resp.ErrorCode) + } +} + +func TestUnknownOpRejected(t *testing.T) { + srv := NewServerWithExec(tmpSock(t), okExec) + c := startControlServer(t, srv) + + resp, err := c.roundTrip(context.Background(), Request{Protocol: Protocol, Op: "frobnicate"}) + if err != nil { + t.Fatalf("roundTrip: %v", err) + } + if resp.ErrorCode != CodeUnknownOp { + t.Errorf("errorCode = %q, want %q", resp.ErrorCode, CodeUnknownOp) + } +} + +// TestRunReturnsStructuredErrorCodes verifies Run surfaces a refusal as an +// *APIError whose Code classifies the failure for Pod-condition mapping. +func TestRunReturnsStructuredErrorCodes(t *testing.T) { + srv := NewServerWithExec(tmpSock(t), okExec) + c := startControlServer(t, srv) + + _, _, _, err := c.Run(context.Background(), "rm", []string{"-rf", "/"}, "") + if err == nil { + t.Fatal("expected error for disallowed command") + } + var apiErr *APIError + if !errors.As(err, &apiErr) { + t.Fatalf("error is not *APIError: %T (%v)", err, err) + } + if apiErr.Code != CodeNotAllowed { + t.Errorf("code = %q, want %q", apiErr.Code, CodeNotAllowed) + } +} + +// TestPolicyRefusalCode verifies an allowlisted-but-out-of-scope command is +// rejected with CodeNotPermitted (#41 integration through the #39 API). +func TestPolicyRefusalCode(t *testing.T) { + srv := NewServerWithPolicy(tmpSock(t), Policy{MeshInterface: "utun7"}) + srv.exec = okExec + c := startControlServer(t, srv) + + // ifconfig on a foreign interface is allowlisted but not policy-permitted. + _, _, _, err := c.Run(context.Background(), "ifconfig", []string{"en0", "up"}, "") + var apiErr *APIError + if !errors.As(err, &apiErr) { + t.Fatalf("error is not *APIError: %T (%v)", err, err) + } + if apiErr.Code != CodeNotPermitted { + t.Errorf("code = %q, want %q", apiErr.Code, CodeNotPermitted) + } +} + +// TestMalformedRequestRejected verifies non-JSON input gets a structured +// malformed error rather than crashing the connection handler. +func TestMalformedRequestRejected(t *testing.T) { + srv := NewServerWithExec(tmpSock(t), okExec) + startControlServer(t, srv) + + conn, err := net.Dial("unix", srv.socketPath) + if err != nil { + t.Fatalf("dial: %v", err) + } + defer func() { _ = conn.Close() }() + if _, err := conn.Write([]byte("this is not json\n")); err != nil { + t.Fatalf("write: %v", err) + } + var resp Response + if err := json.NewDecoder(bufio.NewReader(conn)).Decode(&resp); err != nil { + t.Fatalf("decode response: %v", err) + } + if resp.ErrorCode != CodeMalformed { + t.Errorf("errorCode = %q, want %q", resp.ErrorCode, CodeMalformed) + } +} + +// TestOversizedRequestRejected verifies a request beyond the size cap is refused +// rather than buffered unbounded by the root daemon. +func TestOversizedRequestRejected(t *testing.T) { + srv := NewServerWithExec(tmpSock(t), okExec) + startControlServer(t, srv) + + conn, err := net.Dial("unix", srv.socketPath) + if err != nil { + t.Fatalf("dial: %v", err) + } + defer func() { _ = conn.Close() }() + // A valid JSON envelope with an enormous Stdin, exceeding maxRequestBytes. + huge := strings.Repeat("A", maxRequestBytes+1024) + if err := json.NewEncoder(conn).Encode(Request{Protocol: Protocol, Name: "pfctl", Stdin: huge}); err != nil { + t.Fatalf("encode: %v", err) + } + var resp Response + if err := json.NewDecoder(bufio.NewReader(conn)).Decode(&resp); err != nil { + t.Fatalf("decode response: %v", err) + } + if resp.ErrorCode != CodeMalformed { + t.Errorf("errorCode = %q, want %q (the truncated payload must not decode to a valid request)", resp.ErrorCode, CodeMalformed) + } +} diff --git a/pkg/network/privhelper/launchd.go b/pkg/network/privhelper/launchd.go new file mode 100644 index 0000000..1c17026 --- /dev/null +++ b/pkg/network/privhelper/launchd.go @@ -0,0 +1,389 @@ +package privhelper + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "text/template" +) + +// macOS launchd integration for the privileged network helper (#40). +// +// Without this, an operator must run `sudo macvz-netd` by hand before every +// kubelet start. A LaunchDaemon makes the helper a managed system service: it is +// installed once with sudo, started at boot, restarted if it crashes, and the +// kubelet then connects to its socket with no further privilege escalation. +// +// The plist runs the helper as root (the whole point — it owns pf/route/wg), and +// passes --owner uid:gid so the helper chowns its socket to the operator's user. +// At install time there is a SUDO_UID; at daemon start (launchd) there is not, so +// the owner is captured into the plist rather than read from the environment. + +const ( + // DefaultLabel is the launchd job label and the basename of the plist. + DefaultLabel = "com.github.chimerakang.macvz-netd" + // DefaultPlistDir is where system LaunchDaemons live on macOS. + DefaultPlistDir = "/Library/LaunchDaemons" + // DefaultBinaryPath is where Install copies the helper binary so the plist + // points at a stable, root-owned location rather than a user build tree. + DefaultBinaryPath = "/usr/local/sbin/macvz-netd" + // DefaultStdoutPath and DefaultStderrPath are the daemon's log files. + DefaultStdoutPath = "/var/log/macvz-netd.log" + DefaultStderrPath = "/var/log/macvz-netd.err.log" +) + +// LaunchdConfig describes the LaunchDaemon to install. All paths are absolute so +// the same struct drives both plist rendering and the filesystem operations, +// and tests can redirect every path into a temp dir. +type LaunchdConfig struct { + // Label is the launchd job label (reverse-DNS) and the plist basename. + Label string + // PlistDir is the directory the plist is written to (system LaunchDaemons). + PlistDir string + // BinaryPath is where the helper binary is installed and the plist points. + BinaryPath string + // SocketPath is the unix socket the helper listens on. + SocketPath string + // ConfigPath, when set, is the MacVz config the daemon loads to restrict + // privileged commands to its interfaces, CIDRs, peers, and pf anchor (#41). + // Empty installs a daemon with argument validation disabled (name-allowlist + // only). + ConfigPath string + // AllowUnsafeNoConfig explicitly permits installing without ConfigPath. This + // exists for local development only; production helpers should always enforce + // config-derived request policy. + AllowUnsafeNoConfig bool + // OwnerUID/OwnerGID is the user the socket is chowned to so the (non-root) + // kubelet can connect. -1 leaves the socket root-owned. + OwnerUID int + OwnerGID int + // StdoutPath/StderrPath are the daemon's log destinations. + StdoutPath string + StderrPath string +} + +// DefaultLaunchdConfig returns the standard system layout with the socket and +// owner left to the caller (owner defaults to "unset"). +func DefaultLaunchdConfig(socketPath string) LaunchdConfig { + return LaunchdConfig{ + Label: DefaultLabel, + PlistDir: DefaultPlistDir, + BinaryPath: DefaultBinaryPath, + SocketPath: socketPath, + OwnerUID: -1, + OwnerGID: -1, + StdoutPath: DefaultStdoutPath, + StderrPath: DefaultStderrPath, + } +} + +// PlistPath is the full path to the LaunchDaemon plist. +func (c LaunchdConfig) PlistPath() string { + return filepath.Join(c.PlistDir, c.Label+".plist") +} + +// ServiceTarget is the launchctl service target (e.g. "system/