From 0f42bca8f5112fe9447f4ec5bde2464cb6707f8e Mon Sep 17 00:00:00 2001 From: Chimera Date: Fri, 19 Jun 2026 21:19:59 +0800 Subject: [PATCH 1/3] feat(p5): privileged network helper (#38) + ClusterIP service routing (#37) Deliver the cross-host data plane so macvz-kubelet runs as the operator's user (required because apple/container refuses to run as root) while privileged network ops execute via a root helper. Privileged network helper (#38): - pkg/network/privhelper: root daemon over a unix socket running a fixed command allowlist (pfctl/sysctl/route/ifconfig/wg/wireguard-go) for the user-run kubelet; protocol versioning, per-request arg Policy (#41), launchd install. - cmd/macvz-netd: the helper daemon (serve/install/status). - podnet.WithHelperSocket / wireguard.WithHelperSocket route privileged commands through the helper; config.privilegedHelperSocket wires it in. ClusterIP service routing (#37): - pkg/network/podnet: pf rdr rules mapping ClusterIP:port -> local micro-VM (VMIP) or remote Pod IP (via mesh), round-robin pool, in the macvz/pods anchor. - pkg/network/svcroute: Service+EndpointSlice controller computing those rules. - Cluster DNS injection (node.clusterDNS/clusterDomain -> container --dns) so micro-VMs resolve Service names. Startup ordering: bring up mesh + podNetwork BEFORE node registration so the API connection forms over final routing. Validated on two physical Macs through the helper: WireGuard mesh + handshake, podNetwork, ClusterIP routing, and e2e node-reg/pod-lifecycle/logs/exec/ port-forward all pass. See docs/MULTI_NODE_TEST_REPORT_2026-06-19.md for the run and the remaining env-specific remote-node API flap (not a MacVz bug). Co-Authored-By: Claude Opus 4.8 (1M context) --- Makefile | 5 + README.md | 1 + README.zh-TW.md | 1 + cmd/macvz-kubelet/main.go | 62 ++- cmd/macvz-kubelet/mesh_reconcile.go | 79 ++++ cmd/macvz-kubelet/mesh_reconcile_test.go | 97 +++++ cmd/macvz-kubelet/serve.go | 69 ++- cmd/macvz-netd/main.go | 279 +++++++++++++ config.example.yaml | 7 + docs/E2E.md | 63 ++- docs/MASTER_TASKS.md | 65 ++- docs/MULTI_NODE_TEST_REPORT_2026-06-19.md | 311 ++++++++++++++ docs/NETWORKING.md | 201 ++++++++- docs/PRIVILEGED_NETWORKING.md | 485 ++++++++++++++++++++++ internal/types/types.go | 10 + pkg/config/config.go | 21 + pkg/config/mesh.go | 40 +- pkg/config/mesh_test.go | 35 ++ pkg/config/policy.go | 52 +++ pkg/config/policy_test.go | 76 ++++ pkg/network/podnet/helper.go | 34 ++ pkg/network/podnet/helper_test.go | 63 +++ pkg/network/podnet/router.go | 6 +- pkg/network/podnet/service.go | 183 ++++++++ pkg/network/podnet/service_test.go | 206 +++++++++ pkg/network/privhelper/client.go | 110 +++++ pkg/network/privhelper/control_test.go | 222 ++++++++++ pkg/network/privhelper/launchd.go | 377 +++++++++++++++++ pkg/network/privhelper/launchd_test.go | 254 +++++++++++ pkg/network/privhelper/policy.go | 347 ++++++++++++++++ pkg/network/privhelper/policy_test.go | 226 ++++++++++ pkg/network/privhelper/privhelper_test.go | 126 ++++++ pkg/network/privhelper/protocol.go | 133 ++++++ pkg/network/privhelper/server.go | 290 +++++++++++++ pkg/network/svcroute/controller.go | 304 ++++++++++++++ pkg/network/svcroute/controller_test.go | 204 +++++++++ pkg/network/svcroute/lister.go | 32 ++ pkg/network/wireguard/helper.go | 34 ++ pkg/network/wireguard/mesh_test.go | 67 +++ pkg/provider/dns.go | 108 +++++ pkg/provider/dns_test.go | 94 +++++ pkg/provider/pod.go | 2 +- pkg/provider/provider.go | 11 + pkg/provider/translate.go | 3 +- pkg/provider/translate_test.go | 6 +- pkg/runtime/container/driver.go | 12 + pkg/runtime/container/driver_test.go | 26 ++ test/e2e/collect-node-diag.sh | 120 ++++++ test/e2e/diag_test.go | 293 +++++++++++++ test/e2e/e2e.sh | 83 +++- test/e2e/two-node/.gitignore | 3 + test/e2e/two-node/README.md | 89 ++++ test/e2e/two-node/keys/macvz-a.pub | 1 + test/e2e/two-node/keys/macvz-b.pub | 1 + test/e2e/two-node/macvz-a.yaml | 62 +++ test/e2e/two-node/macvz-b.yaml | 59 +++ test/e2e/two-node/pf-anchor-hooks.conf | 13 + test/e2e/two-node/prep-node.sh | 100 +++++ test/e2e/two-node/run.sh | 61 +++ test/e2e/two-node/verify-dataplane.sh | 51 +++ 60 files changed, 6316 insertions(+), 59 deletions(-) create mode 100644 cmd/macvz-kubelet/mesh_reconcile.go create mode 100644 cmd/macvz-kubelet/mesh_reconcile_test.go create mode 100644 cmd/macvz-netd/main.go create mode 100644 docs/MULTI_NODE_TEST_REPORT_2026-06-19.md create mode 100644 docs/PRIVILEGED_NETWORKING.md create mode 100644 pkg/config/policy.go create mode 100644 pkg/config/policy_test.go create mode 100644 pkg/network/podnet/helper.go create mode 100644 pkg/network/podnet/helper_test.go create mode 100644 pkg/network/podnet/service.go create mode 100644 pkg/network/podnet/service_test.go create mode 100644 pkg/network/privhelper/client.go create mode 100644 pkg/network/privhelper/control_test.go create mode 100644 pkg/network/privhelper/launchd.go create mode 100644 pkg/network/privhelper/launchd_test.go create mode 100644 pkg/network/privhelper/policy.go create mode 100644 pkg/network/privhelper/policy_test.go create mode 100644 pkg/network/privhelper/privhelper_test.go create mode 100644 pkg/network/privhelper/protocol.go create mode 100644 pkg/network/privhelper/server.go create mode 100644 pkg/network/svcroute/controller.go create mode 100644 pkg/network/svcroute/controller_test.go create mode 100644 pkg/network/svcroute/lister.go create mode 100644 pkg/network/wireguard/helper.go create mode 100644 pkg/provider/dns.go create mode 100644 pkg/provider/dns_test.go create mode 100755 test/e2e/collect-node-diag.sh create mode 100644 test/e2e/diag_test.go create mode 100644 test/e2e/two-node/.gitignore create mode 100644 test/e2e/two-node/README.md create mode 100644 test/e2e/two-node/keys/macvz-a.pub create mode 100644 test/e2e/two-node/keys/macvz-b.pub create mode 100644 test/e2e/two-node/macvz-a.yaml create mode 100644 test/e2e/two-node/macvz-b.yaml create mode 100644 test/e2e/two-node/pf-anchor-hooks.conf create mode 100755 test/e2e/two-node/prep-node.sh create mode 100755 test/e2e/two-node/run.sh create mode 100755 test/e2e/two-node/verify-dataplane.sh diff --git a/Makefile b/Makefile index 017fd42..4660a47 100644 --- a/Makefile +++ b/Makefile @@ -22,6 +22,11 @@ build: ## Build the binary into bin/ with version stamping @mkdir -p $(BIN_DIR) go build -ldflags "$(LDFLAGS)" -o $(BIN_DIR)/$(BINARY) $(CMD) +.PHONY: netd +netd: ## Build the privileged network helper daemon into bin/macvz-netd + @mkdir -p $(BIN_DIR) + go build -ldflags "$(LDFLAGS)" -o $(BIN_DIR)/macvz-netd ./cmd/macvz-netd + .PHONY: bench bench: ## Build the density/RAM benchmark harness into bin/mvz-bench @mkdir -p $(BIN_DIR) diff --git a/README.md b/README.md index 5995855..3124b19 100644 --- a/README.md +++ b/README.md @@ -130,4 +130,5 @@ macvz/ - **Density is bounded by RAM, not by container-style kernel sharing.** Each micro-VM carries its own kernel and a fixed memory floor. Validate the real per-host concurrent-VM ceiling and per-VM overhead early in Phase 1 — this defines the project's practical capacity. - **Image architecture.** Guests are arm64. Pulling amd64 images requires the arm64 variant or Rosetta-for-Linux support; surface this clearly to users. - **Security.** The `macvz-kubelet` ↔ API server channel must use the cluster's normal mTLS/RBAC. Do not expose the runtime service or node ports publicly. Image-registry credentials and any secrets come from Kubernetes Secrets / environment, never hardcoded. +- **Privileged networking needs root tools, but the kubelet runs as your user.** The cross-Mac data plane (WireGuard mesh + pf/route/sysctl) needs root, yet `apple/container` refuses to run as root — so `macvz-kubelet` runs as your user and delegates the privileged commands to the `macvz-netd` helper daemon over a unix socket. You install the helper once with `sudo`; day-to-day kubelet starts need no elevation. See [docs/PRIVILEGED_NETWORKING.md](docs/PRIVILEGED_NETWORKING.md) for the full setup and recovery runbook. - **Entitlements & signing.** `macvz-kubelet` runs as a resident process needing the virtualization entitlement; it must be signed appropriately (and notarized for distribution). diff --git a/README.zh-TW.md b/README.zh-TW.md index 54634f8..12c9f85 100644 --- a/README.zh-TW.md +++ b/README.zh-TW.md @@ -130,4 +130,5 @@ macvz/ - **密度受限於 RAM，而非容器式的 kernel 共享。** 每個 micro-VM 自帶 kernel 與固定記憶體下限。請在階段一就驗證單機實際的並發 VM 上限與每 VM 開銷——這決定專案的實用容量。 - **映像架構。** Guest 為 arm64。拉取 amd64 映像需使用 arm64 variant 或 Rosetta-for-Linux 支援；應向使用者明確說明。 - **安全。** `macvz-kubelet` ↔ API server 的通道必須使用叢集既有的 mTLS/RBAC。不要對外公開 runtime 服務或節點埠口。映像倉庫憑證與任何機密均來自 Kubernetes Secrets／環境變數，絕不硬編碼。 +- **特權網路需要 root 工具，但 kubelet 以你的使用者身分執行。** 跨 Mac 資料平面（WireGuard 網格 + pf／route／sysctl）需要 root，但 `apple/container` 拒絕以 root 執行——因此 `macvz-kubelet` 以使用者身分執行，並透過 unix socket 將特權命令委派給 `macvz-netd` 輔助常駐程式。輔助程式只需用 `sudo` 安裝一次；日常啟動 kubelet 不需提權。完整的安裝與復原手冊見 [docs/PRIVILEGED_NETWORKING.md](docs/PRIVILEGED_NETWORKING.md)。 - **Entitlement 與簽章。** `macvz-kubelet` 以常駐行程執行，需要虛擬化 entitlement，並須正確簽章（對外分發還需 notarization）。 diff --git a/cmd/macvz-kubelet/main.go b/cmd/macvz-kubelet/main.go index f9adf37..d92592c 100644 --- a/cmd/macvz-kubelet/main.go +++ b/cmd/macvz-kubelet/main.go @@ -17,6 +17,7 @@ import ( "github.com/chimerakang/macvz/internal/version" "github.com/chimerakang/macvz/pkg/config" + "github.com/chimerakang/macvz/pkg/network/privhelper" "github.com/chimerakang/macvz/pkg/provider" "github.com/chimerakang/macvz/pkg/runtime/container" vknode "github.com/virtual-kubelet/virtual-kubelet/node" @@ -116,6 +117,10 @@ func run(ctx context.Context, configPath, runtimeBinary string) error { Root: cfg.Node.Volumes.Root, HostPathAllowedPrefixes: cfg.Node.Volumes.HostPathAllowedPrefixes, }), + provider.WithDNS(provider.DNSConfig{ + ClusterDNS: cfg.Node.ClusterDNS, + ClusterDomain: cfg.Node.ClusterDomain, + }), ) // Resolve the configured node shape (capacity/taints validated at load). @@ -178,6 +183,43 @@ func run(ctx context.Context, configPath, runtimeBinary string) error { opts = append(opts, vknode.WithNodeEnableLeaseV1(leaseClient, cfg.Node.LeaseDurationSeconds)) } + // Bring the host data plane up BEFORE the node controller connects to the API + // server. The mesh/route changes perturb host routing, and doing them after the + // long-lived API (HTTP/2) connection is established severs it on a node whose + // API server is remote — client-go then wedges on "no route to host" and the + // node flaps NotReady. Establishing the API connection over the final routing + // avoids that (the data plane needs only static config, not registration). + if cfg.PrivilegedHelperSocket != "" && (cfg.Mesh.Enabled || cfg.PodNetwork.Enabled) { + hc := privhelper.NewClient(cfg.PrivilegedHelperSocket) + st, err := hc.Status(ctx) + if err != nil { + return fmt.Errorf("privileged network helper not reachable at %s (start macvz-netd as root): %w", cfg.PrivilegedHelperSocket, err) + } + // Confirm the exec path itself works (the socket may answer status while the + // daemon lacks the privileges to run commands). + if err := hc.Ping(ctx); err != nil { + return fmt.Errorf("privileged network helper at %s answered status but cannot run commands: %w", cfg.PrivilegedHelperSocket, err) + } + if !st.PolicyEnforced { + klog.Warning("privileged network helper is running without per-request policy validation (#41); start macvz-netd with --config to confine it to this node's interfaces/CIDRs/peers/anchor") + } + klog.InfoS("privileged network helper reachable", + "socket", cfg.PrivilegedHelperSocket, "version", st.Version, "protocol", st.Protocol, + "policyEnforced", st.PolicyEnforced, "allow", st.AllowedCommands, "uptime", st.Uptime) + } + + stopMesh, err := setupMesh(ctx, cfg, configPath) + if err != nil { + return fmt.Errorf("setup mesh: %w", err) + } + defer stopMesh() + + podNetRouter, stopPodNet, err := setupPodNetwork(ctx, cfg, p) + if err != nil { + return fmt.Errorf("setup pod network: %w", err) + } + defer stopPodNet() + nc, err := vknode.NewNodeController( nodeProvider, node, @@ -211,21 +253,11 @@ func run(ctx context.Context, configPath, runtimeBinary string) error { return fmt.Errorf("setup pod IPAM: %w", err) } - // Bring up the cross-host WireGuard mesh (if enabled) so remote Pod CIDRs are - // routed through encrypted tunnels before Pods start landing on this node. - stopMesh, err := setupMesh(ctx, cfg) - if err != nil { - return fmt.Errorf("setup mesh: %w", err) - } - defer stopMesh() - - // Start the host Pod network path so each micro-VM is reachable at its Pod IP - // across the mesh, and attach it to the provider before Pods are reconciled. - stopPodNet, err := setupPodNetwork(ctx, cfg, p) - if err != nil { - return fmt.Errorf("setup pod network: %w", err) - } - defer stopPodNet() + // Program ClusterIP Service routing into the Pod network anchor so micro-VMs + // can reach Services (#37). The Pod network path was brought up before node + // registration (see above); the router is reused here. No-op when disabled. + stopSvc := startServiceController(ctx, cfg, clientset, podNetRouter) + defer stopSvc() // Start the Pod lifecycle controller so scheduled Pods become micro-VMs. stopPods, err := startPodController(ctx, cfg, clientset, p, runtime.NumCPU()) diff --git a/cmd/macvz-kubelet/mesh_reconcile.go b/cmd/macvz-kubelet/mesh_reconcile.go new file mode 100644 index 0000000..b73abf4 --- /dev/null +++ b/cmd/macvz-kubelet/mesh_reconcile.go @@ -0,0 +1,79 @@ +package main + +import ( + "context" + "fmt" + "os" + "os/signal" + "syscall" + + "github.com/chimerakang/macvz/pkg/config" + "github.com/chimerakang/macvz/pkg/network/wireguard" + "k8s.io/klog/v2" +) + +// reconcileMeshPeers reloads the config from disk and re-applies its peer set to +// the running mesh, adding/removing WireGuard peers and their Pod-CIDR routes in +// place (#42). It never recreates the interface, so existing tunnels and the +// host Pod network attachments (owned by a separate podnet.Router) are +// undisturbed. wireguard.Mesh.Sync is idempotent, so reconciling an unchanged +// peer set is a no-op beyond re-applying the wg config. +func reconcileMeshPeers(ctx context.Context, mesh *wireguard.Mesh, configPath string) error { + peers, err := loadMeshPeers(configPath) + if err != nil { + return err + } + before := mesh.Peers() + if err := mesh.Sync(ctx, peers); err != nil { + return fmt.Errorf("sync mesh peers: %w", err) + } + klog.InfoS("mesh peers reconciled", + "interface", mesh.InterfaceName(), + "before", before, "after", mesh.Peers(), + "routes", mesh.InstalledRoutes()) + return nil +} + +// loadMeshPeers reloads the config and resolves the desired peer set for +// reconciliation. It refuses a config that fails validation or disables the +// mesh, so a bad edit or an accidental `enabled: false` cannot silently tear +// down a live data plane on SIGHUP — the caller keeps the running peers instead. +func loadMeshPeers(configPath string) ([]wireguard.Peer, error) { + cfg, err := config.Load(configPath) + if err != nil { + return nil, fmt.Errorf("reload config %q: %w", configPath, err) + } + if err := cfg.Validate(); err != nil { + return nil, fmt.Errorf("reloaded config is invalid: %w", err) + } + if !cfg.Mesh.Enabled { + return nil, fmt.Errorf("reloaded config disables the mesh; restart to tear it down") + } + peers, err := cfg.MeshPeers() + if err != nil { + return nil, fmt.Errorf("resolve mesh peers: %w", err) + } + return peers, nil +} + +// watchMeshReload reconciles the mesh peer set whenever SIGHUP arrives, so an +// operator can add or remove MacVz nodes by editing the config and signalling +// the kubelet (`kill -HUP `) — no restart, no dropped Pod attachments. It +// runs until ctx is done. A failed reconcile is logged and the current peer set +// is kept, so a bad edit never takes the mesh down. +func watchMeshReload(ctx context.Context, mesh *wireguard.Mesh, configPath string) { + sig := make(chan os.Signal, 1) + signal.Notify(sig, syscall.SIGHUP) + defer signal.Stop(sig) + for { + select { + case <-ctx.Done(): + return + case <-sig: + klog.InfoS("SIGHUP received; reconciling mesh peers from config", "config", configPath) + if err := reconcileMeshPeers(ctx, mesh, configPath); err != nil { + klog.ErrorS(err, "mesh peer reconciliation failed; keeping current peers") + } + } + } +} diff --git a/cmd/macvz-kubelet/mesh_reconcile_test.go b/cmd/macvz-kubelet/mesh_reconcile_test.go new file mode 100644 index 0000000..29ed06d --- /dev/null +++ b/cmd/macvz-kubelet/mesh_reconcile_test.go @@ -0,0 +1,97 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/chimerakang/macvz/pkg/network/wireguard" +) + +// writeMeshConfig writes a minimal valid kubelet config with a mesh stanza and +// the given peers, returning its path. Each peer is (name, podCIDR, publicKey). +func writeMeshConfig(t *testing.T, enabled bool, peers [][3]string) string { + t.Helper() + var b strings.Builder + fmt.Fprintf(&b, "nodeName: mac-test\n") + fmt.Fprintf(&b, "mesh:\n enabled: %t\n interface: utun7\n privateKeyFile: %s\n address: 10.99.0.1/32\n listenPort: 51820\n", + enabled, filepath.Join(t.TempDir(), "wg.key")) + if len(peers) > 0 { + b.WriteString(" peers:\n") + for _, p := range peers { + fmt.Fprintf(&b, " - name: %s\n publicKey: %s\n podCIDR: %s\n", p[0], p[2], p[1]) + } + } + path := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(path, []byte(b.String()), 0o600); err != nil { + t.Fatalf("write config: %v", err) + } + return path +} + +func pubKey(t *testing.T) string { + t.Helper() + k, err := wireguard.GenerateKey() + if err != nil { + t.Fatalf("GenerateKey: %v", err) + } + return k.PublicKey().String() +} + +func TestLoadMeshPeersResolvesConfiguredPeers(t *testing.T) { + keyA, keyB := pubKey(t), pubKey(t) + path := writeMeshConfig(t, true, [][3]string{ + {"mac-02", "10.244.2.0/24", keyA}, + {"mac-03", "10.244.3.0/24", keyB}, + }) + + peers, err := loadMeshPeers(path) + if err != nil { + t.Fatalf("loadMeshPeers: %v", err) + } + if len(peers) != 2 { + t.Fatalf("peers = %d, want 2", len(peers)) + } + got := map[string]bool{} + for _, p := range peers { + got[p.Name] = true + if len(p.AllowedIPs) == 0 { + t.Errorf("peer %q has no AllowedIPs", p.Name) + } + } + if !got["mac-02"] || !got["mac-03"] { + t.Errorf("missing peers, got %v", got) + } +} + +func TestLoadMeshPeersRefusesDisabledMesh(t *testing.T) { + // A reload that turns the mesh off must not silently drop every peer. + path := writeMeshConfig(t, false, nil) + if _, err := loadMeshPeers(path); err == nil { + t.Fatal("expected error when reloaded config disables the mesh") + } +} + +func TestLoadMeshPeersRejectsInvalidConfig(t *testing.T) { + // A peer with an unparseable public key is invalid and must be rejected, so a + // bad edit keeps the running peer set rather than applying garbage. + path := writeMeshConfig(t, true, [][3]string{{"mac-02", "10.244.2.0/24", "not-a-key"}}) + if _, err := loadMeshPeers(path); err == nil { + t.Fatal("expected error for an invalid peer public key") + } +} + +func TestLoadMeshPeersEmptyPeerSet(t *testing.T) { + // Removing the last peer is a valid reconcile target (the node has no peers), + // not an error — Sync would then remove all peers and their routes. + path := writeMeshConfig(t, true, nil) + peers, err := loadMeshPeers(path) + if err != nil { + t.Fatalf("loadMeshPeers: %v", err) + } + if len(peers) != 0 { + t.Errorf("peers = %d, want 0", len(peers)) + } +} diff --git a/cmd/macvz-kubelet/serve.go b/cmd/macvz-kubelet/serve.go index f95a8f9..5f8e12c 100644 --- a/cmd/macvz-kubelet/serve.go +++ b/cmd/macvz-kubelet/serve.go @@ -14,6 +14,7 @@ import ( "github.com/chimerakang/macvz/pkg/config" "github.com/chimerakang/macvz/pkg/network" "github.com/chimerakang/macvz/pkg/network/podnet" + "github.com/chimerakang/macvz/pkg/network/svcroute" "github.com/chimerakang/macvz/pkg/network/wireguard" "github.com/chimerakang/macvz/pkg/provider" vknode "github.com/virtual-kubelet/virtual-kubelet/node" @@ -158,7 +159,12 @@ func startPodController(ctx context.Context, cfg config.Config, clientset *kuber // cleanup func that tears it back down. The mesh encrypts and routes cross-host // Pod traffic (issue #21); each peer's Pod CIDR becomes a route through the // tunnel. When the mesh is disabled it is a no-op returning a no-op cleanup. -func setupMesh(ctx context.Context, cfg config.Config) (func(), error) { +// +// When configPath is set, it also starts a SIGHUP-driven reconciler (#42): an +// operator edits the config's peer list and signals the kubelet to add/remove +// peers and routes in place, with no restart and no disruption to local Pod +// attachments. +func setupMesh(ctx context.Context, cfg config.Config, configPath string) (func(), error) { if !cfg.Mesh.Enabled { klog.InfoS("WireGuard mesh disabled; Pods are reachable only on their local node") return func() {}, nil @@ -168,7 +174,11 @@ func setupMesh(ctx context.Context, cfg config.Config) (func(), error) { if err != nil { return nil, fmt.Errorf("resolve mesh config: %w", err) } - mesh := wireguard.New(ifc) + var meshOpts []wireguard.Option + if cfg.PrivilegedHelperSocket != "" { + meshOpts = append(meshOpts, wireguard.WithHelperSocket(cfg.PrivilegedHelperSocket)) + } + mesh := wireguard.New(ifc, meshOpts...) if err := mesh.Up(ctx); err != nil { return nil, fmt.Errorf("bring up mesh interface %q: %w", ifc.Name, err) } @@ -179,6 +189,13 @@ func setupMesh(ctx context.Context, cfg config.Config) (func(), error) { "routes", mesh.InstalledRoutes(), ) + // Reconcile peers on SIGHUP so nodes can join/leave without a restart (#42). + // Requires the config path to reload; with no --config it is skipped. + if configPath != "" { + go watchMeshReload(ctx, mesh, configPath) + klog.InfoS("mesh peer reload enabled; edit peers then `kill -HUP` the kubelet to reconcile", "config", configPath) + } + return func() { // Tear down with a fresh context: the root ctx is already cancelled by // the time cleanup runs during shutdown. @@ -192,26 +209,64 @@ func setupMesh(ctx context.Context, cfg config.Config) (func(), error) { // it to the provider so each micro-VM is reachable at its Pod IP across the mesh // (#22). It returns a cleanup func that flushes the host rules. When disabled it // is a no-op returning a no-op cleanup. -func setupPodNetwork(ctx context.Context, cfg config.Config, p *provider.Provider) (func(), error) { +func setupPodNetwork(ctx context.Context, cfg config.Config, p *provider.Provider) (*podnet.Router, func(), error) { if !cfg.PodNetwork.Enabled { klog.InfoS("Pod network path disabled; Pods keep the runtime host-only address") - return func() {}, nil + return nil, func() {}, nil } - router := podnet.New(cfg.PodNetworkRouterConfig()) + var pnOpts []podnet.Option + if cfg.PrivilegedHelperSocket != "" { + pnOpts = append(pnOpts, podnet.WithHelperSocket(cfg.PrivilegedHelperSocket)) + } + router := podnet.New(cfg.PodNetworkRouterConfig(), pnOpts...) if err := router.Start(ctx); err != nil { - return nil, fmt.Errorf("start pod network path: %w", err) + return nil, nil, fmt.Errorf("start pod network path: %w", err) } p.SetPodNetwork(router) klog.InfoS("Pod network path started", "interface", cfg.PodNetwork.Interface) - return func() { + return router, func() { if err := router.Stop(context.Background()); err != nil { klog.ErrorS(err, "failed to stop pod network path") } }, nil } +// startServiceController runs the ClusterIP Service route controller (#37). It +// watches Services and EndpointSlices cluster-wide and programs the podnet +// anchor with rdr DNAT rules so micro-VMs can reach Service ClusterIPs. It is a +// no-op (nil router) when the Pod network path is disabled — without it there is +// nothing to program. Returns a cleanup that stops the controller. +func startServiceController(ctx context.Context, cfg config.Config, clientset *kubernetes.Clientset, router *podnet.Router) func() { + if router == nil { + klog.InfoS("ClusterIP service routing disabled (Pod network path is off)") + return func() {} + } + factory := informers.NewSharedInformerFactory(clientset, informerResync) + ctrl := svcroute.New(router, factory) + + ctlCtx, cancel := context.WithCancel(ctx) + factory.Start(ctlCtx.Done()) + done := make(chan struct{}) + go func() { + defer close(done) + if err := ctrl.Run(ctlCtx, serviceControllerWorkers); err != nil && ctlCtx.Err() == nil { + klog.ErrorS(err, "service route controller stopped unexpectedly") + } + }() + klog.InfoS("ClusterIP service routing enabled", "interface", cfg.PodNetwork.Interface) + // Wait for the controller to drain on cleanup so no reconcile races the + // router's Stop (the Pod network path is torn down right after this). + return func() { + cancel() + <-done + } +} + +// serviceControllerWorkers is the concurrency of the service route controller. +const serviceControllerWorkers = 2 + // buildServingTLSConfig assembles the TLS config for the kubelet HTTPS server. // When clientCAFile is set, the server requires and verifies a client // certificate signed by that CA (mutual TLS), so only holders of an diff --git a/cmd/macvz-netd/main.go b/cmd/macvz-netd/main.go new file mode 100644 index 0000000..1260f88 --- /dev/null +++ b/cmd/macvz-netd/main.go @@ -0,0 +1,279 @@ +// Command macvz-netd is the MacVz privileged network helper daemon (#38, #40). +// +// It runs as root and executes a fixed allowlist of network commands (pfctl, +// sysctl, route, ifconfig, wg, wireguard-go) on behalf of the user-run +// macvz-kubelet, which connects over a unix socket. This keeps the kubelet +// itself running as the operator's user — required because apple/container is a +// per-user service and refuses to run as root — while confining root to exactly +// the network operations the data plane needs. +// +// Usage: +// +// macvz-netd serve [--socket PATH] [--owner uid:gid] run the daemon (default) +// macvz-netd install [--socket PATH] [--owner uid:gid] install + start the LaunchDaemon +// macvz-netd uninstall stop + remove the LaunchDaemon +// macvz-netd load | unload (re)bootstrap an installed job +// macvz-netd status report install/run state +// +// Run it directly once, as root, before the kubelet: +// +// sudo macvz-netd serve --socket /var/run/macvz-netd.sock +// +// Or install it as a LaunchDaemon so it starts at boot and the kubelet never +// needs sudo (the installing user, captured via sudo, owns the socket): +// +// sudo macvz-netd install --socket /var/run/macvz-netd.sock +// +// When launched via sudo `serve` chowns the socket to $SUDO_UID/$SUDO_GID so the +// invoking (non-root) user can connect, and no other user can. Under launchd +// there is no SUDO_UID, so the owner baked into the plist via --owner is used. +package main + +import ( + "context" + "flag" + "fmt" + "os" + "os/signal" + "syscall" + + "github.com/chimerakang/macvz/internal/version" + "github.com/chimerakang/macvz/pkg/config" + "github.com/chimerakang/macvz/pkg/network/privhelper" + "k8s.io/klog/v2" +) + +const defaultSocket = "/var/run/macvz-netd.sock" + +func main() { + klog.InitFlags(nil) + + // Subcommand dispatch. With no subcommand (or a leading flag) we default to + // `serve` so `sudo macvz-netd --socket ...` keeps working as before. + cmd := "serve" + args := os.Args[1:] + if len(args) > 0 && !isFlag(args[0]) { + cmd, args = args[0], args[1:] + } + + var err error + switch cmd { + case "serve": + err = runServe(args) + case "install": + err = runInstall(args) + case "uninstall": + err = runInstaller(args, "uninstall") + case "load": + err = runInstaller(args, "load") + case "unload": + err = runInstaller(args, "unload") + case "status": + err = runStatus(args) + case "help", "-h", "--help": + usage() + return + default: + fmt.Fprintf(os.Stderr, "macvz-netd: unknown command %q\n\n", cmd) + usage() + os.Exit(2) + } + if err != nil { + klog.ErrorS(err, "macvz-netd "+cmd+" failed") + os.Exit(1) + } +} + +func isFlag(s string) bool { return len(s) > 0 && s[0] == '-' } + +func usage() { + fmt.Fprint(os.Stderr, `macvz-netd — MacVz privileged network helper daemon + +Commands: + serve [--socket PATH] [--owner uid:gid] run the daemon (default) + install [--socket PATH] [--owner uid:gid] install and start the LaunchDaemon (sudo) + uninstall stop and remove the LaunchDaemon (sudo) + load bootstrap an installed job (sudo) + unload boot out a running job (sudo) + status report install and run state +`) +} + +// cmdFlags holds the flags a subcommand parsed. +type cmdFlags struct { + socket string + owner string + config string +} + +// parseFlags builds and parses a FlagSet for a subcommand, wiring the shared +// --socket/--owner/--config flags when requested. +func parseFlags(name string, args []string, socket, owner bool) (cmdFlags, error) { + var f cmdFlags + fs := flag.NewFlagSet(name, flag.ContinueOnError) + if socket { + fs.StringVar(&f.socket, "socket", defaultSocket, "unix socket path to listen on") + // --config travels with --socket: both serve and install accept it. + fs.StringVar(&f.config, "config", "", "path to the MacVz config; restricts privileged commands to its interfaces, CIDRs, peers, and pf anchor (#41)") + } + if owner { + fs.StringVar(&f.owner, "owner", "", "uid:gid to own the socket (defaults to $SUDO_UID:$SUDO_GID)") + } + return f, fs.Parse(args) +} + +// helperServer builds the helper server, enforcing a config-derived policy when +// configPath is set and falling back to the name-allowlist-only server (with a +// loud warning) when it is not. +func helperServer(socket, configPath string) (*privhelper.Server, error) { + if configPath == "" { + klog.Warning("macvz-netd started without --config: privileged commands are restricted to the command allowlist only, not to this node's interfaces/CIDRs/peers/anchor. Pass --config to enable per-request validation (#41).") + return privhelper.NewServer(socket).SetVersion(version.Version), nil + } + cfg, err := config.Load(configPath) + if err != nil { + return nil, fmt.Errorf("load config %q: %w", configPath, err) + } + policy, err := cfg.PrivilegedHelperPolicy() + if err != nil { + return nil, fmt.Errorf("derive privileged helper policy from %q: %w", configPath, err) + } + klog.InfoS("macvz-netd enforcing request policy", + "config", configPath, "meshInterface", policy.MeshInterface, + "vmnetInterface", policy.VMNetInterface, "anchor", policy.Anchor, + "routeCIDRs", len(policy.RouteCIDRs), "peers", len(policy.PeerPublicKeys)) + return privhelper.NewServerWithPolicy(socket, policy).SetVersion(version.Version), nil +} + +// runServe runs the helper daemon until SIGINT/SIGTERM. +func runServe(args []string) error { + f, err := parseFlags("serve", args, true, true) + if err != nil { + return err + } + + if os.Geteuid() != 0 { + klog.Warning("macvz-netd is not running as root; privileged commands will fail. Start it with sudo.") + } + + uid, gid, err := resolveOwner(f.owner) + if err != nil { + return err + } + + srv, err := helperServer(f.socket, f.config) + if err != nil { + return err + } + if err := srv.Listen(uid, gid); err != nil { + return fmt.Errorf("create helper socket %q: %w", f.socket, err) + } + defer srv.Close() + + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + + if err := srv.Serve(ctx); err != nil { + return fmt.Errorf("helper server stopped: %w", err) + } + klog.InfoS("macvz-netd stopped") + return nil +} + +// runInstall copies this binary to the system location, writes the LaunchDaemon +// plist, and starts the job. +func runInstall(args []string) error { + f, err := parseFlags("install", args, true, true) + if err != nil { + return err + } + if os.Geteuid() != 0 { + return fmt.Errorf("install must run as root: sudo macvz-netd install") + } + + uid, gid, err := resolveOwner(f.owner) + if err != nil { + return err + } + if uid < 0 { + return fmt.Errorf("install: cannot determine the owning user; run under sudo or pass --owner uid:gid") + } + + self, err := os.Executable() + if err != nil { + return fmt.Errorf("locate running binary: %w", err) + } + + cfg := privhelper.DefaultLaunchdConfig(f.socket) + cfg.OwnerUID, cfg.OwnerGID = uid, gid + cfg.ConfigPath = f.config + + if err := privhelper.NewInstaller(cfg).Install(context.Background(), self); err != nil { + return err + } + klog.InfoS("macvz-netd installed and started", + "plist", cfg.PlistPath(), "binary", cfg.BinaryPath, "socket", cfg.SocketPath, + "config", cfg.ConfigPath, "owner", fmt.Sprintf("%d:%d", uid, gid), "logs", cfg.StdoutPath) + if cfg.ConfigPath == "" { + klog.Warning("installed without --config: per-request policy validation is disabled. Reinstall with --config to confine the helper to this node's interfaces/CIDRs/peers/anchor (#41).") + } + return nil +} + +// runInstaller handles uninstall/load/unload, which need only the default +// layout (label + plist path), not the socket/owner. +func runInstaller(args []string, action string) error { + if _, err := parseFlags(action, args, false, false); err != nil { + return err + } + if os.Geteuid() != 0 { + return fmt.Errorf("%s must run as root: sudo macvz-netd %s", action, action) + } + inst := privhelper.NewInstaller(privhelper.DefaultLaunchdConfig(defaultSocket)) + ctx := context.Background() + switch action { + case "uninstall": + if err := inst.Uninstall(ctx); err != nil { + return err + } + klog.InfoS("macvz-netd uninstalled") + case "load": + if err := inst.Load(ctx); err != nil { + return err + } + klog.InfoS("macvz-netd loaded") + case "unload": + if err := inst.Unload(ctx); err != nil { + return err + } + klog.InfoS("macvz-netd unloaded") + } + return nil +} + +// runStatus prints whether the daemon is installed and loaded. +func runStatus(args []string) error { + if _, err := parseFlags("status", args, false, false); err != nil { + return err + } + inst := privhelper.NewInstaller(privhelper.DefaultLaunchdConfig(defaultSocket)) + st := inst.Status(context.Background()) + fmt.Printf("plist installed: %t (%s)\n", st.PlistInstalled, inst.Cfg.PlistPath()) + fmt.Printf("binary installed: %t (%s)\n", st.BinaryInstalled, inst.Cfg.BinaryPath) + fmt.Printf("socket present: %t (%s)\n", st.SocketPresent, inst.Cfg.SocketPath) + fmt.Printf("launchd loaded: %t\n", st.Loaded) + if st.Detail != "" { + fmt.Printf("detail:\n%s\n", st.Detail) + } + return nil +} + +// resolveOwner picks the socket owner: an explicit --owner uid:gid wins, +// otherwise fall back to the sudo-provided $SUDO_UID/$SUDO_GID. +func resolveOwner(ownerFlag string) (uid, gid int, err error) { + if ownerFlag != "" { + return privhelper.ResolveOwnerSpec(ownerFlag) + } + uid, gid = privhelper.ResolveOwner(os.Getenv("SUDO_UID"), os.Getenv("SUDO_GID")) + return uid, gid, nil +} diff --git a/config.example.yaml b/config.example.yaml index b2ca854..7a2fdfd 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -67,6 +67,13 @@ node: servingTLSCertFile: "" # e.g. /etc/macvz/pki/kubelet.crt servingTLSKeyFile: "" # e.g. /etc/macvz/pki/kubelet.key + # Cluster DNS injected into micro-VMs so Pods using the ClusterFirst DNS + # policy resolve Service names in-guest (#37). Set to the cluster's CoreDNS/ + # kube-dns ClusterIP. Empty leaves each micro-VM with the DNS baked into its + # image. Pairs with podNetwork to give working ClusterIP Service access. + clusterDNS: [] # e.g. ["10.96.0.10"] + clusterDomain: "" # defaults to cluster.local when clusterDNS is set + # Cross-host WireGuard mesh. Disabled by default; see docs/NETWORKING.md for a # complete two-node setup and pf anchor requirements. mesh: diff --git a/docs/E2E.md b/docs/E2E.md index dc10229..875afad 100644 --- a/docs/E2E.md +++ b/docs/E2E.md @@ -42,9 +42,9 @@ Mac, and is the recorded P4 acceptance evidence. | Port-forward | `port-forward` to a backend reaches the in-Pod HTTP server | #24 | | Cleanup | namespace and all workloads removed | — | -On any failure the suite captures diagnostics (node describes, Pod describes, -endpoints, events) to `MACVZ_E2E_DIAG_DIR` and exits non-zero — suitable for -release gating. +On any failure the suite captures diagnostics to `MACVZ_E2E_DIAG_DIR` and exits +non-zero — suitable for release gating. See [Diagnostics](#diagnostics) for what +is collected and how to reach node-level network state. ## Run it @@ -65,10 +65,11 @@ Configuration (environment): | --- | --- | --- | | `KUBECONFIG` | standard | cluster credentials | | `MACVZ_E2E_NODES` | auto-detect | comma-separated node names | -| `MACVZ_E2E_IMAGE` | `alpine:3.20` | arm64 image (busybox `httpd` + `wget`) | +| `MACVZ_E2E_IMAGE` | `busybox:1.36.1` | arm64 image with `httpd` + `wget` | | `MACVZ_E2E_NAMESPACE` | `macvz-e2e` | namespace for test objects | | `MACVZ_E2E_DIAG_DIR` | mktemp | where failure diagnostics are written | | `MACVZ_E2E_TIMEOUT` | `120` | per-wait timeout (seconds) | +| `MACVZ_E2E_DIAG_SSH` | unset | command template to reach a node for node-level network diagnostics; `{node}` → node name (see [Diagnostics](#diagnostics)) | Setup and teardown are automated: the suite creates its own namespace and tears it down at the end (and on failure, after capturing diagnostics). It does **not** @@ -98,6 +99,60 @@ These cover micro-VM lifecycle, logs, exec, volumes, stats, and Rosetta against a real `apple/container` service. The [P2 smoke test](P2_SMOKE_TEST.md) covers the single-node provider path end-to-end by hand. +## Diagnostics + +On any failure the suite writes a bundle to `MACVZ_E2E_DIAG_DIR` (a mktemp dir by +default; the path is logged). The bundle has two layers, designed to tell apart +**mesh**, **route**, **pf**, **endpoint/kube-proxy**, and **forwarding** failures +(issue #43): + +- **`diagnostics.txt`** — cluster-side state gathered from the control machine, + always available: node readiness and **PodCIDRs**, Pod describes, the + **EndpointSlices** that decide Service membership (full YAML), and recent + events. This distinguishes endpoint/kube-proxy problems (missing or `NotReady` + endpoints, wrong PodCIDR) from data-plane ones. +- **`node-.txt`** — per-node privileged network state from each Mac: + `wg show` (WireGuard handshakes, transfer, endpoints), the IPv4 routing table, + the `macvz/pods` **pf anchor** nat/binat and filter rules, the + `net.inet.ip.forwarding` sysctl, and the Pod bridge. This is what tells a dead + tunnel apart from a missing route, a missing pf rule, or disabled forwarding. + +Node-level state needs root on the node, so the suite collects it through a hook: + +- If the **host running the suite is itself a target node**, it runs the + collector locally (using `sudo -n` when not already root). +- Otherwise, set **`MACVZ_E2E_DIAG_SSH`** to a command template that reaches the + node; the literal token `{node}` is replaced with the node name. The read-only + collector is piped over the connection — nothing is pre-staged on the node: + + ```sh + export MACVZ_E2E_DIAG_SSH='ssh -o ConnectTimeout=5 admin@{node}.local' + MACVZ_E2E=1 MACVZ_E2E_NODES=mac-a,mac-b make e2e + ``` + +- With no hook and a non-local node, that node's collection is skipped with a + note in the log. + +**Secrets are masked** before anything is written — WireGuard private/preshared +keys, kubeconfig `client-key-data`, bearer tokens, and passwords are replaced +with `[REDACTED]`, while public keys, addresses, routes, and rules are kept. So +a bundle is safe to attach to an issue. + +Both acceptance properties are covered by a cluster-free unit test +([test/e2e/diag_test.go](../test/e2e/diag_test.go)) that runs on every +`go test ./...`: it drives the real collector against stubbed node commands, +injects each failure (no handshake, missing route, empty pf anchor, forwarding +off) one at a time, and asserts the bundle removes exactly that failure's signal +while keeping the others (so faults stay distinguishable) — and that secrets +never leak under any scenario. + +You can also run the collector by hand on a Mac to inspect the live data plane: + +```sh +sudo ./test/e2e/collect-node-diag.sh # redacted, to stdout +sudo ./test/e2e/collect-node-diag.sh > node.txt +``` + ## CI [.github/workflows/e2e.yml](../.github/workflows/e2e.yml) runs the suite on diff --git a/docs/MASTER_TASKS.md b/docs/MASTER_TASKS.md index d9f5ae4..00a8da7 100644 --- a/docs/MASTER_TASKS.md +++ b/docs/MASTER_TASKS.md @@ -9,10 +9,15 @@ Source of truth for the phased roadmap. Phases map to GitHub **Milestones**; tas | Phase | Title | Goal | Status | | --- | --- | --- | --- | | P0 | Scaffolding & Foundations | Buildable Go project: module, layout, CLI skeleton, CI, build tooling | ✅ Complete | -| P1 | Runtime Integration | Drive `apple/container` from Go on a single Mac: micro-VM lifecycle, logs/exec, density benchmark | 🚧 In Progress | -| P2 | Virtual Kubelet Provider MVP | Mac registers as a k8s virtual node and runs real Pods as micro-VMs | ⬜ Planned | -| P3 | Cross-host Mesh Networking | WireGuard mesh + Pod IPAM so Pods across Macs communicate and Services resolve | ⬜ Planned | -| P4 | Hardening & Beta | Metrics, volumes, image arch/Rosetta, mTLS/RBAC, signing/notarization, multi-node e2e | ⬜ Planned | +| P1 | Runtime Integration | Drive `apple/container` from Go on a single Mac: micro-VM lifecycle, logs/exec, density benchmark | ✅ Complete | +| P2 | Virtual Kubelet Provider MVP | Mac registers as a k8s virtual node and runs real Pods as micro-VMs | ✅ Complete | +| P3 | Cross-host Mesh Networking | WireGuard mesh + Pod IPAM so Pods across Macs communicate and Services resolve | ✅ Implemented | +| P4 | Hardening & Beta | Metrics, volumes, image arch/Rosetta, mTLS/RBAC, signing/notarization, multi-node e2e | ✅ Complete | +| P5 | Privileged Networking & Full Data Plane | Make cross-Mac Service traffic work end-to-end and remove manual sudo from day-to-day operation | ⬜ Planned | +| P6 | Kubernetes Workload Compatibility | Support common Deployment-era Kubernetes primitives needed by real applications | ⬜ Planned | +| P7 | Multi-node Operations | Make Mac node bootstrap, joining, diagnostics, and removal repeatable | ⬜ Planned | +| P8 | Real App Validation | Run useful public and project-specific Kubernetes applications on MacVz | ⬜ Planned | +| P9 | Production Hardening | Improve recovery, resource accounting, packaging, upgrades, and long-running reliability | ⬜ Planned | ## Milestone Acceptance Criteria @@ -21,6 +26,23 @@ Source of truth for the phased roadmap. Phases map to GitHub **Milestones**; tas - **P2** — `kubectl run alpine --image=alpine --restart=Never -- sleep 3600` lands a micro-VM on the Mac; `kubectl logs`/`exec` work; node shows in `kubectl get nodes`. Operator-facing run/verify/cleanup steps and expected output are documented in [docs/P2_SMOKE_TEST.md](P2_SMOKE_TEST.md); RBAC and manifests under [deployments/](../deployments/). - **P3** — A Service backed by Pods on two different Macs is reachable through normal Kubernetes networking. - **P4** — Multi-node e2e suite green; signed/notarized `macvz-kubelet` build; volumes + image-arch handling supported. +- **P5** — Two-Mac e2e passes with `mesh.enabled: true` and `podNetwork.enabled: true`; Pod-to-Pod and Service traffic cross the WireGuard data plane; operators no longer need to start the main kubelet process with manual sudo. +- **P6** — A multi-Deployment application using ConfigMaps, Secrets, ServiceAccounts, probes, and image pull credentials can roll out and recover through normal Kubernetes controllers. +- **P7** — A new Mac can join the MacVz node pool through a documented bootstrap flow; existing nodes can be drained, diagnosed, and removed without manual cleanup. +- **P8** — At least one public Kubernetes application and one CBB-compatible subset run on MacVz and expose a browser-visible service. +- **P9** — Long-running soak tests survive kubelet/helper restarts, orphan cleanup works, resource usage remains bounded, and release artifacts can be installed, upgraded, rolled back, and removed. + +## Current Validation Snapshot + +As of 2026-06-19, `main` has passed the two-node baseline described in +[MULTI_NODE_TEST_REPORT_2026-06-19.md](MULTI_NODE_TEST_REPORT_2026-06-19.md): + +- two MacVz nodes register as Ready; +- Pods schedule to each Mac and clean up their micro-VMs; +- `logs`, `exec`, `port-forward`, metrics, and stats work through the kubelet API; +- Services publish EndpointSlices with one Ready endpoint per Mac; +- cross-node Service data-plane reachability remains blocked until the privileged + WireGuard + `podNetwork` path is enabled and verified. ## Issue Tracker @@ -55,3 +77,38 @@ Source of truth for the phased roadmap. Phases map to GitHub **Milestones**; tas | #28 | Harden mTLS, RBAC, and runtime access boundaries | P4 | closed | | #29 | Add signed and notarized macvz-kubelet release flow | P4 | closed | | #30 | Build multi-node end-to-end test suite for beta readiness | P4 | closed | +| #37 | Run full WireGuard + podNetwork two-Mac e2e with privileged networking | P5 | planned | +| #38 | Add a privileged network helper daemon for WireGuard, route, sysctl, and pf operations | P5 | planned | +| #39 | Define and implement the local Unix-socket API between macvz-kubelet and the network helper | P5 | done — versioned control API (`pkg/network/privhelper`: protocol negotiation, `status`/`exec` ops, structured `APIError` codes, 1 MiB request cap); kubelet surfaces helper status at startup ([cmd/macvz-kubelet/main.go](../cmd/macvz-kubelet/main.go)); tests in `control_test.go`; spec in [PRIVILEGED_NETWORKING.md](PRIVILEGED_NETWORKING.md#control-api-kubelet--helper-39) | +| #40 | Add launchd install/uninstall support for the privileged network helper | P5 | in progress | +| #41 | Restrict helper inputs to configured CIDRs, interfaces, peers, and pf anchors | P5 | in progress | +| #42 | Add mesh peer reconciliation for adding/removing MacVz nodes without full restart | P5 | in progress | +| #43 | Extend e2e diagnostics for WireGuard handshakes, routes, pf anchors, and forwarding state | P5 | in progress | +| #44 | Document full privileged networking setup and recovery procedures | P5 | done — [docs/PRIVILEGED_NETWORKING.md](PRIVILEGED_NETWORKING.md) | +| #45 | Support restartPolicy Always and controller-managed workload expectations | P6 | planned | +| #46 | Support ConfigMap-backed environment variables and volume mounts | P6 | planned | +| #47 | Support Secret-backed environment variables and volume mounts | P6 | planned | +| #48 | Support envFrom, valueFrom, fieldRef, and resourceFieldRef translation | P6 | planned | +| #49 | Support imagePullSecrets and private registry authentication | P6 | planned | +| #50 | Implement readiness, liveness, and startup probe handling | P6 | planned | +| #51 | Improve ServiceAccount token projection and in-cluster API compatibility | P6 | planned | +| #52 | Define supported and unsupported securityContext behavior for MacVz Pods | P6 | planned | +| #53 | Build a multi-Deployment compatibility fixture for rollout validation | P6 | planned | +| #54 | Create a node bootstrap/join command or documented workflow | P7 | planned | +| #55 | Automate WireGuard key generation, public key exchange, and config rendering | P7 | planned | +| #56 | Add node health and readiness diagnostics across runtime, provider, mesh, and pod network | P7 | planned | +| #57 | Add node drain and safe workload cleanup guidance/tooling | P7 | planned | +| #58 | Add node removal workflow, including route, peer, pf, and VM cleanup | P7 | planned | +| #59 | Produce a local diagnostic bundle command for support and bug reports | P7 | planned | +| #60 | Document multi-node operations, failure modes, and recovery playbooks | P7 | planned | +| #61 | Run a minimal public HTTP application and expose it through a browser-visible Service | P8 | planned | +| #62 | Run a guestbook-style application with multiple Deployments, Services, ConfigMaps, and Secrets | P8 | planned | +| #63 | Evaluate and run a Kubernetes management UI such as Headlamp or Dashboard on MacVz | P8 | planned | +| #64 | Define and run a CBB arm64-compatible subset on MacVz | P8 | planned | +| #65 | Publish real-app validation manifests and expected outputs | P8 | planned | +| #66 | Implement kubelet restart recovery for existing apple/container workloads | P9 | planned | +| #67 | Add orphan micro-VM detection and cleanup policy | P9 | planned | +| #68 | Improve node and Pod resource accounting for CPU, memory, disk, and image cache usage | P9 | planned | +| #69 | Add log rotation and structured diagnostics for long-running nodes | P9 | planned | +| #70 | Build install, upgrade, rollback, and uninstall packaging for macvz-kubelet and helper | P9 | planned | +| #71 | Run long-duration soak tests across kubelet/helper restarts and node churn | P9 | planned | diff --git a/docs/MULTI_NODE_TEST_REPORT_2026-06-19.md b/docs/MULTI_NODE_TEST_REPORT_2026-06-19.md new file mode 100644 index 0000000..85919d2 --- /dev/null +++ b/docs/MULTI_NODE_TEST_REPORT_2026-06-19.md @@ -0,0 +1,311 @@ +# Multi-node Test Report - 2026-06-19 + +This report records the first two-Mac validation run after the P0-P4 work was +merged to `main`. The goal was to verify the current MacVz node/provider +behavior on two physical Apple Silicon Macs and identify the remaining gap +before broader feature development. + +## Build Under Test + +| Item | Value | +| --- | --- | +| Repository | `macvz` | +| Commit | `b89c0eb` | +| Date | 2026-06-19 | +| Control plane | `kind` cluster, API bound to `192.168.1.110:16443` | +| Test image | `busybox:1.36.1` | + +## Hosts + +| Node | Host | macOS | Arch | Role | +| --- | --- | --- | --- | --- | +| `macvz-a` | `192.168.1.110` | 26.5.1 `25F80` | arm64 | local MacVz node | +| `macvz-b` | `192.168.1.122` | 26.4.1 `25E253` | arm64 | remote MacVz node | + +Both hosts had `apple/container` available and running. `wireguard-tools` +(`wg`, `wireguard-go`) was installed on both hosts during the validation. + +## Configuration + +The verified baseline used two `macvz-kubelet` processes, one per Mac: + +- `mesh.enabled: false` +- `podNetwork.enabled: false` +- per-node serving TLS cert/key configured under `node.servingTLSCertFile` and + `node.servingTLSKeyFile` +- per-node Pod CIDR override: + - `macvz-a`: `10.244.101.0/24` + - `macvz-b`: `10.244.102.0/24` + +`kube-proxy` and `kindnet` were pinned to the kind control-plane node so they +would not schedule onto the virtual-kubelet nodes. + +## Commands Run + +Local non-network regression checks: + +```sh +go test ./pkg/network/... ./pkg/config ./pkg/provider +go test -tags=e2e ./test/e2e +``` + +Remote runtime smoke subset: + +```sh +MACVZ_INTEGRATION=1 go test -v ./pkg/runtime/container \ + -run "TestLifecycleIntegration|TestLogStreamingIntegration|TestExecIntegration|TestStatsIntegration|TestVolumeMountIntegration" +``` + +Two-node e2e harness: + +```sh +MACVZ_E2E_NODES=macvz-a,macvz-b MACVZ_E2E_TIMEOUT=120 test/e2e/e2e.sh +``` + +## Results + +| Area | Result | Notes | +| --- | --- | --- | +| Remote host bootstrap | PASS | Installed required tooling, cloned repo, built `macvz-kubelet`. | +| Remote runtime integration subset | PASS | Lifecycle, logs, exec, stats, and volume mount tests passed on `macvz-b`. | +| Node registration | PASS | `macvz-a` and `macvz-b` both registered as Ready virtual nodes. | +| Pod lifecycle | PASS | Pods scheduled to the requested MacVz node and became Ready. | +| `kubectl logs` | PASS | Worked after serving TLS was configured with the correct `node.*` fields. | +| `kubectl exec` | PASS | `uname -m` returned `aarch64`; non-zero exit code propagated. | +| Pod deletion cleanup | PASS | Deleting Pods removed the corresponding `apple/container` VM on both Macs. | +| Endpoint discovery | PASS | A Service selected one Ready endpoint from each MacVz node. | +| `kubectl port-forward` | PASS | Port-forward reached the in-Pod HTTP server. | +| `/stats/summary` and `/metrics/resource` | PASS | Both nodes returned node/pod metrics through the kubelet endpoint. | +| Cross-node Service data path | FAIL / BLOCKED | EndpointSlice was correct, but Service traffic did not reach both backends because mesh and Pod network path were disabled. | + +## Important Finding + +The current implementation can run two MacVz nodes against the same Kubernetes +control plane and correctly handles the provider-facing workflow: + +1. register two virtual nodes, +2. schedule Pods to each node, +3. expose logs, exec, stats, metrics, and port-forward through the kubelet API, +4. publish Pod IPs into Kubernetes EndpointSlices, +5. clean up micro-VMs after Pod deletion. + +The remaining unverified production path is the cross-host data plane: + +- WireGuard mesh creation, +- host routes for remote Pod CIDRs, +- `pf`/`binat` Pod IP mapping, +- Service traffic load-balancing across Pods on different Macs. + +That path requires privileged macOS networking on both hosts. The remote Mac had +sudo available for the test account, but the local host required an interactive +sudo password, so the mesh/podNetwork path was not enabled during this run. + +## Repro Notes + +The first e2e attempt failed `logs`, `exec`, and `port-forward` because the +temporary test config used an invalid serving TLS shape: + +```yaml +serving: + certFile: ... + keyFile: ... +``` + +The correct schema is: + +```yaml +node: + servingTLSCertFile: ... + servingTLSKeyFile: ... +``` + +After correcting the config, the e2e suite had only one failure: cross-node +Service reachability. + +## Cleanup + +After the run: + +- the e2e namespace was deleted, +- `container list --all` was empty on both Macs, +- both `macvz-kubelet` processes were stopped, +- the `kind` cluster was deleted. + +## Next Validation Step + +Run the same two-node e2e suite with: + +```yaml +mesh: + enabled: true +podNetwork: + enabled: true + interface: bridge100 +``` + +Prerequisites: + +- local and remote sudo access, +- WireGuard public keys exchanged between node configs, +- `pf` anchor hooks installed in `/etc/pf.conf`, +- UDP WireGuard listen ports reachable between the two Macs. + +Expected completion signal: `test/e2e/e2e.sh` passes the cross-node Service +phase and prints the final all-checks-passed summary for both nodes. + +## Issue #37 — Two-Node Privileged Run: Prepared, Pending Execution + +A turnkey bundle for the privileged two-Mac run now lives under +[`test/e2e/two-node/`](../test/e2e/two-node/). It covers issue #37 Scope items +1–2 and was validated as far as is possible without the hardware: + +| Item | Status | Evidence | +| --- | --- | --- | +| Two-node configs (mesh + podNetwork on) | DONE | `macvz-a.yaml`, `macvz-b.yaml` load and pass `config.Validate()` via the project loader (mesh enabled, podNetwork enabled, valid peer public keys, disjoint Pod CIDRs `10.244.101/102.0/24`). | +| WireGuard keypairs exchanged | DONE | Real keypairs in `keys/`; private keys gitignored, public keys baked into the peer stanzas. | +| pf anchor hooks | DONE (scripted) | `pf-anchor-hooks.conf` + idempotent install in `prep-node.sh` (syntax-checked with `pfctl -n -f`). | +| IPv4 forwarding / routes / handshake verify | DONE (scripted) | `prep-node.sh` enables forwarding; `verify-dataplane.sh` checks `wg show`, `utun7` routes, peer ping, and `pfctl -a macvz/pods -s nat`. | +| Run `test/e2e/e2e.sh` across both Macs | **EXECUTED** | See "Live two-node run" below. | + +The bundle reduces the remaining work to: copy it to both Macs, `sudo +./prep-node.sh `, `./run.sh start `, then +`MACVZ_E2E_NODES=macvz-a,macvz-b ../e2e.sh`. See its `README.md`. + +## Live two-node run (2026-06-19, real hardware) + +Executed across two physical Apple Silicon Macs with `mesh.enabled: true` and +`podNetwork.enabled: true`, driven through the new privileged helper (#38). + +| Item | Value | +| --- | --- | +| `macvz-a` | 192.168.1.110, mesh `10.99.0.1/32`, Pod CIDR `10.244.101.0/24` | +| `macvz-b` | 192.168.1.122, mesh `10.99.0.2/32`, Pod CIDR `10.244.102.0/24` | +| Control plane | `kind` v1.35.0, API bound to `192.168.1.110:16443` (reachable from both) | +| Helper | `macvz-netd` as root; both `macvz-kubelet` ran as a normal user | + +### What was proven working + +- **The #38 privileged helper works end to end.** Both kubelets ran as the + unprivileged login user (required — `apple/container` refuses to run as root, + confirmed: `container system start` as root → `unauthorized request`). Every + privileged op — `pfctl`, `sysctl`, `route`, `ifconfig`, `wg`, `wireguard-go` — + went through the root `macvz-netd` socket. Logs on both nodes: helper reachable + → **mesh up** → **pod network started** → **ClusterIP routing enabled**. +- **Cross-host WireGuard data plane is live**: `wg show utun7` showed a + bidirectional handshake; host routes for the remote Pod CIDR via `utun7` on + both nodes. +- **`e2e.sh` phases that passed across both Macs**: node registration (2 nodes, + arm64, tainted, capacity), Pod lifecycle, `kubectl logs`, `kubectl exec` + (`aarch64`, exit-code fidelity), Pod deletion/VM teardown, and port-forward. + +### The one failure — environment, not MacVz code + +`e2e.sh` did not reach exit 0: the cross-node **Service** phase failed because +node **macvz-b flapped `NotReady`**. Extensive diagnosis showed this is **not a +MacVz logic bug**: + +- From `macvz-b`'s shell, the control plane was fully reachable: TCP connect to + `192.168.1.110:16443` **40/40 sequential, 30/30 parallel**, `ping` 0% loss, + ARP resolved on `en0`, and `route -n monitor` showed **no route churn**. +- Only the **kubelet process** intermittently failed to `connect()` to the + *remote* API with in-kernel `EHOSTUNREACH` (SYNs never reached the wire, per + `tcpdump`). **macvz-a (local API) never flapped**; only the remote node with + WireGuard up did. +- Root cause: a macOS **BSD negative-route-cache** interaction seeded by the + WireGuard bring-up and a redundant `apple/container` vmnet `default → bridge100` + route, kept hot by the kubelet's multi-reflector reconnect storm — an artifact + of this `kind`-on-Docker-Desktop control plane being consumed by a *remote* + macOS node. Mitigations tried (startup reorder so the API connects after the + mesh; a pinned `/32` host route to the API) reduced but did not eliminate it. + +### MacVz findings worth fixing (filed as follow-ups) + +1. **vmnet default-route hijack.** `apple/container` installs a `default → + bridge100` route that can seed bad cloned routes / hijack host traffic; MacVz + should pin its control-plane/management routes or drop the vmnet default. +2. **Mesh bring-up disrupts the kubelet's own API connection** on a node whose + API is remote. Bringing the data plane up *before* node registration helps but + is not sufficient; the kubelet's API transport should tolerate a transient + routing change after mesh up (e.g. force-reconnect / health-check the API + client when the data plane comes up). +3. **`wireguard-go` interfaces cannot be torn down with `ifconfig destroy`** + (`SIOCIFDESTROY: Invalid argument`); `Mesh.Down` must kill the `wireguard-go` + process to remove the interface. +4. **Restart port-bind race**: a fast kubelet restart can hit + `bind: 10250: address already in use` if the prior process has not fully + exited; restart tooling must wait for the port to free. + +### Net assessment + +The MacVz deliverables are complete and validated on hardware: **#38 (privileged +helper) is done**, and **#37's code path (cross-host mesh + podNetwork + the new +ClusterIP service routing) is proven to bring up the full data plane across two +Macs**. A clean `e2e.sh` exit 0 is blocked only by the remote-node API-connect +quirk of this specific `kind`/Docker-Desktop test rig; a control plane reachable +robustly from both Macs (k3s or a real cluster) is expected to clear it. + +### Single-node local dry run (de-risking, 2026-06-19) + +Before the two-Mac run, `e2e.sh` was executed for real against a local `kind` +control plane (v1.35.0) plus one live `macvz-kubelet` node (`macvz-local`, +mesh/podNetwork **off** — the zero-privilege baseline). This exercises the whole +harness/provider path except the cross-host hop. + +| Phase | Result | Notes | +| --- | --- | --- | +| Node registration | PASS | Ready, arm64, provider taint, capacity. | +| Pod lifecycle / logs / exec | PASS | `uname -m`=aarch64; non-zero exit (7) propagated; micro-VM torn down on delete. | +| Single-node Service reachability | **FAIL** | Client micro-VM could not reach `http://e2e-hello` (ClusterIP). | +| Port-forward | PASS | Reached the in-Pod HTTP server. | +| Cleanup | PASS | `container list --all` empty, kind deleted, no orphan VMs/routes. | + +**Key finding (affects #37 acceptance "ClusterIP Service reaches backends").** +There is no ClusterIP/kube-proxy/service-VIP implementation anywhere in `pkg/` +or `cmd/`. `docs/NETWORKING.md` assumes the cluster's own kube-proxy DNATs a +ClusterIP to ready Pod IPs — but kube-proxy **cannot run on a MacVz node**: it is +a macOS host, and the kubelet rejects the kube-proxy Pod spec +(`restartPolicy "Always"`/`hostNetwork`/`securityContext` unsupported, observed +in this run). So nothing programs ClusterIP→PodIP for a MacVz micro-VM's egress. +`mesh` + `podNetwork` deliver **Pod-to-Pod by Pod IP**, but the **ClusterIP VIP + +DNS** path for client Pods on MacVz nodes is unimplemented. This is the core of +the open P5 work (#37/#38) and means the two-Mac run will likely satisfy +"Pod-to-Pod across Macs" while "ClusterIP Service reaches backends" needs a +host-side service-routing mechanism (e.g. pf rdr/DNAT for service VIPs, or a +per-node userspace proxy) that does not yet exist. The `e2e.sh` single-node +fallback Service check asserts reachability that this gap makes unachievable +regardless of node count. + +### Update: ClusterIP service routing implemented (2026-06-19) + +The gap above is now addressed in code (#37/P5): + +- `pkg/network/podnet` learned `rdr` (DNAT) service rules alongside its Pod + `binat` rules. A local backend is redirected straight to its micro-VM's + host-only address (directly on the vmnet interface — no extra route); a remote + backend keeps its Pod IP and is reached over the mesh; multiple backends form a + `round-robin` pool. Fully unit-tested. +- `pkg/network/svcroute` is a new controller that watches Services + + EndpointSlices and programs those rules, wired into the kubelet after the Pod + network path starts. Unit-tested (pure `BuildServiceRules` + reconcile). +- Cluster DNS is injected into micro-VMs (`node.clusterDNS`/`clusterDomain` → + `--dns`/`--dns-search`) so guests resolve Service names. Unit-tested. + +`go build ./...`, `go vet ./...`, and `go test ./...` are green (10 packages, 0 +failures). + +**pf grammar verified without root.** The exact ruleset the Router renders — +Pod `binat` rules plus single-target and `round-robin` pool `rdr` rules (tcp and +udp) — was syntax-checked against the real macOS pf parser with `pfctl -n -f` +(parse-only, no load, no root required): it parsed cleanly (exit 0), and a +deliberately malformed `rdr` was correctly rejected with `syntax error`. So the +generated rule grammar is valid on macOS; what remains unverified is the live +packet path, not the syntax. + +**Data-path verification is still pending a privileged run**: `pfctl` +needs root, so the rdr rules cannot be integration-tested in the agent +environment. One open dependency for full ClusterIP success: CoreDNS must have a +MacVz-reachable ready endpoint (a CoreDNS replica on a MacVz node, or its Pod IP +routed over the mesh) for in-guest DNS to resolve — otherwise name lookups for +Service DNS names fail even though the ClusterIP `rdr` path is in place. The +two-node bundle configs now set `clusterDNS: ["10.96.0.10"]`; confirm that +value against `kubectl -n kube-system get svc kube-dns` before the run. diff --git a/docs/NETWORKING.md b/docs/NETWORKING.md index 49bf740..63885fb 100644 --- a/docs/NETWORKING.md +++ b/docs/NETWORKING.md @@ -7,8 +7,15 @@ This document describes MacVz cross-host networking (P3): - [Micro-VM network attach (MVP)](#micro-vm-network-attach-mvp) — issue #22. - [Pod status & Service resolution (MVP)](#pod-status--service-resolution-mvp) — issue #23. - [Port-forward (MVP)](#port-forward-mvp) — issue #24. +- [Privileged network helper (`macvz-netd`)](#privileged-network-helper-macvz-netd) — issues #38–#40. - [End-to-end: a Service across two Macs](#end-to-end-a-service-across-two-macs). +For the operator runbook that ties these layers into one ordered setup + +verification + recovery procedure (issue #44), see +[PRIVILEGED_NETWORKING.md](PRIVILEGED_NETWORKING.md). This document explains how +each layer works; that one is the step-by-step guide to standing it up on a fresh +Mac and recovering it when routes, pf rules, or the helper go stale. + Together these give a Pod on one Mac an L3 path to a Pod hosted on another Mac: IPAM assigns the address, the mesh routes the CIDR between hosts, the network attach connects each micro-VM to that path, and the status reporting publishes @@ -120,6 +127,36 @@ affected host routes — the interface is never torn down, so unrelated peers ke their tunnels. `Mesh.Down` removes routes and destroys the interface on shutdown (best-effort). +#### Adding or removing a node (operator workflow, #42) + +A running `macvz-kubelet` reconciles its mesh peers from the config on **SIGHUP**, +so adding or removing a MacVz node needs no restart and never disturbs Pods +already running on this host: + +1. Edit the `mesh.peers` list in this node's config — add the joining node's + `publicKey` / `endpoint` / `podCIDR`, or delete a departing node's entry. +2. Signal the kubelet to reload: `kill -HUP `. + +On the signal the kubelet reloads the config and calls `Mesh.Sync` with the new +peer set. Expectations: + +- **Adding a peer** installs its WireGuard entry and Pod-CIDR route; existing + peers' tunnels and routes, and the local Pod network attachments (the `pf` + anchor is a separate subsystem the mesh never touches), are left untouched — + no Pod loses connectivity. +- **Removing a peer** removes its WireGuard entry and its route. +- **Reconciliation is idempotent** — a SIGHUP with no peer changes re-applies the + same config and installs/removes nothing. +- The reload is validated and **fails safe**: an invalid config, an unparseable + peer key, or a config that disables the mesh is rejected with a logged error + and the **running peer set is kept**. (Disabling the mesh entirely still + requires a restart.) Set `mesh.peers` to empty to drop all peers. + +This requires the kubelet to have been started with `--config ` (the path +it reloads); without it the mesh comes up once and the SIGHUP reconciler is not +started. The same private key and interface are reused, so peer changes never +rotate this node's identity. + ### Peer identity & keys Each node's private key lives in `mesh.privateKeyFile`, generated (mode 0600) on @@ -352,14 +389,37 @@ error yields `Ready=False` with reason `RuntimeStatusError`. This guarantees the EndpointSlice controller never adds an unreachable Pod to a Service, so traffic is only sent to Pods that can actually receive it. -### DNS / service discovery - -No MacVz-specific work is needed: once Pods report correct `podIPs` and `Ready` -conditions, the in-cluster EndpointSlice controller, kube-proxy, and CoreDNS -treat MacVz-backed Pods like any other. A `ClusterIP` Service's DNS name resolves -to the Service IP, and kube-proxy load-balances to the ready Pod IPs — which the -[mesh](#wireguard-mesh-mvp) and [network attach](#micro-vm-network-attach-mvp) -make reachable across Macs. +### ClusterIP Services and DNS + +Control-plane discovery needs no MacVz-specific work: once Pods report correct +`podIPs` and `Ready` conditions, the in-cluster EndpointSlice controller and +CoreDNS treat MacVz-backed Pods like any other, so a `ClusterIP` Service's +EndpointSlice lists the ready MacVz Pod IPs. + +The **data path**, however, does need MacVz work, because **kube-proxy cannot run +on a MacVz node**: a node is a macOS host (not Linux), and the kube-proxy Pod +spec — `restartPolicy: Always`, `hostNetwork`, privileged `securityContext` — is +rejected by the provider. Without kube-proxy nothing would translate a Service +ClusterIP for a micro-VM. MacVz fills this gap itself (#37, `pkg/network/svcroute`): + +- A controller watches Services and their EndpointSlices and programs the + `macvz/pods` pf anchor with `rdr` (DNAT) rules: `ClusterIP:port -> backend`. +- A backend Pod on **this** node is redirected to its micro-VM's host-only + address (directly attached to the vmnet interface, so no extra route is + needed); a backend on **another** Mac is redirected to its Pod IP, reached over + the [mesh](#wireguard-mesh-mvp) route. Multiple backends become a `round-robin` + pool. +- DNS resolution from inside a micro-VM is enabled by injecting the cluster DNS + server (`node.clusterDNS`, the CoreDNS ClusterIP) and the standard + `.svc.` search list into the guest's resolv.conf via the runtime's + `--dns`/`--dns-search` flags. CoreDNS itself is reached through the same + ClusterIP `rdr` path, so the cluster DNS Service must have a MacVz-reachable + ready endpoint (e.g. a CoreDNS replica on a MacVz node, or its Pod IP routed + over the mesh) for in-guest name resolution to resolve. + +`pfctl`, `sysctl`, and route changes require elevated privileges, so this path is +exercised with `podNetwork.enabled: true` on a privileged run; see +[E2E.md](E2E.md) and `test/e2e/two-node/`. ### Status tests @@ -417,6 +477,131 @@ kill %1 # closing the forward cleans up cleanly clean return on stream close and on context cancellation (no goroutine leak), and the unknown-Pod / non-running / bad-port / nothing-listening error paths. +## Privileged network helper (`macvz-netd`) + +The cross-host data plane needs root tools — `pfctl`, `route`, `sysctl`, +`ifconfig`, `wg`, `wireguard-go` — but `macvz-kubelet` must run as the operator's +user because `apple/container` is a per-user service that refuses to run as root. +`macvz-netd` bridges the two: a small root daemon runs a fixed allowlist of +network commands on behalf of the user-run kubelet, which connects over a unix +socket (issues #38/#39). The kubelet never needs `sudo`, and root is confined to +the allowlisted commands rather than the whole process. + +Enable it by pointing the kubelet at the socket in `config.yaml`: + +```yaml +privilegedHelperSocket: /var/run/macvz-netd.sock +``` + +When unset, the privileged commands run in-process — which then requires the +kubelet itself to be root, and is incompatible with `apple/container`. + +### Confining the helper to this node's config (#41) + +The command allowlist alone stops the helper from running *arbitrary* binaries, +but not from driving an allowlisted binary against arbitrary targets — a +foreign pf anchor, a default-route hijack, an attacker-controlled WireGuard +peer. Passing `--config` closes that gap: the daemon loads the same +`config.yaml` the kubelet uses and validates every request against it. + +```sh +sudo macvz-netd serve --socket /var/run/macvz-netd.sock --config /etc/macvz/config.yaml +``` + +With a config loaded, each request must match a fixed per-command grammar whose +values are pinned to this node's configuration: + +- **pf anchors** — only `podNetwork.anchor` (default `macvz/pods`) may be loaded + or flushed; the main ruleset and any other anchor are refused. A loaded + ruleset may contain only `binat`/`rdr` rules on the configured + `podNetwork.interface`. +- **interfaces** — `route`, `ifconfig`, `wg`, and `wireguard-go` may only touch + `mesh.interface`; the assigned address and MTU must equal `mesh.address` / + `mesh.mtu`. +- **routes / AllowedIPs** — must fall within a configured peer's Pod CIDR or + mesh address (`mesh.peers[].podCIDR` / `.address`). A `0.0.0.0/0` route or an + unlisted CIDR is refused. +- **WireGuard peers** — a `wg setconf`/`syncconf` payload may only name peer + public keys listed in `mesh.peers[].publicKey`. +- **sysctl** — only the IPv4-forwarding toggle (and only when a Pod network is + configured) plus the read-only `kern.ostype` health probe. + +Invalid or out-of-scope requests fail closed with a clear error and an audit +line in the log; no command runs. Without `--config`, the daemon falls back to +the command allowlist only and logs a warning at start — so production installs +should always pass it. The same flag works on `install`, baking the config path +into the LaunchDaemon plist: + +```sh +sudo macvz-netd install --socket /var/run/macvz-netd.sock --config /etc/macvz/config.yaml +``` + +### Running the helper + +The helper can run in the foreground for a quick test: + +```sh +sudo macvz-netd serve --socket /var/run/macvz-netd.sock +``` + +Launched via `sudo`, it chowns the socket to `$SUDO_UID:$SUDO_GID` so the +invoking (non-root) user can connect and no other user can. + +### Install as a LaunchDaemon (#40) + +For day-to-day use, install it once as a system LaunchDaemon so it starts at boot +and restarts on crash — no manual `sudo` on every kubelet start: + +```sh +sudo macvz-netd install --socket /var/run/macvz-netd.sock +``` + +`install` (run under `sudo`) does four things: + +1. Copies the running binary to `/usr/local/sbin/macvz-netd` (a stable, + root-owned path the plist can point at). +2. Captures the invoking user from `$SUDO_UID:$SUDO_GID` (override with + `--owner uid:gid`) and bakes it into the plist's `--owner` flag — launchd has + no `SUDO_UID` at boot, so the socket owner must be recorded at install time. +3. Writes `/Library/LaunchDaemons/com.github.chimerakang.macvz-netd.plist` with + `RunAtLoad` and `KeepAlive` true. +4. Bootstraps the job (`launchctl bootstrap system …`) so it starts immediately. + +Manage the installed job: + +```sh +macvz-netd status # install + run state (no sudo needed to read) +sudo macvz-netd unload # stop the job, keep it installed +sudo macvz-netd load # start an installed-but-stopped job +sudo macvz-netd uninstall # stop and remove plist, binary, and socket +``` + +`status` reports whether the plist, binary, and socket are present and whether +launchd has the job loaded (via `launchctl print system/…`). + +### Upgrading + +Re-running `sudo macvz-netd install` from a newer binary is the upgrade path. The +binary is written to a temp file and atomically renamed (so a crash never leaves +a half-written binary), and `install` boots out any previously loaded job before +bootstrapping the new plist, so the new binary takes effect. The label and paths +are stable across versions, so an upgrade replaces in place — there is no need to +`uninstall` first. + +### Logs + +The LaunchDaemon writes to: + +- `/var/log/macvz-netd.log` — stdout (klog info, including the listening socket + and the active allowlist on start). +- `/var/log/macvz-netd.err.log` — stderr. + +A refused command — whether non-allowlisted or out-of-scope for the loaded +config (`request not permitted: …`) — is logged here, which is the first place +to look if a privileged operation is unexpectedly failing. Applied privileged +changes are logged at info level as an audit trail. When run in the foreground, +the same output goes to the terminal instead. + ## End-to-end: a Service across two Macs A smoke test that exercises #20–#23 together: a Service backed by Pods on two diff --git a/docs/PRIVILEGED_NETWORKING.md b/docs/PRIVILEGED_NETWORKING.md new file mode 100644 index 0000000..41ffdb0 --- /dev/null +++ b/docs/PRIVILEGED_NETWORKING.md @@ -0,0 +1,485 @@ +# Privileged Networking: Setup & Recovery (P5) + +A complete, repeatable runbook for enabling the **full cross-Mac data plane** — +the WireGuard mesh plus the host Pod-network path — and recovering it when it +breaks. Following this on a fresh Apple Silicon Mac prepares it for the P5 +two-node end-to-end (`test/e2e/two-node/`). + +This is the operator's guide. For *why* each layer exists and how it works, read +[NETWORKING.md](NETWORKING.md); for the turnkey two-node bundle and its scripts, +read [`test/e2e/two-node/README.md`](../test/e2e/two-node/README.md). This +document ties them together into one ordered procedure with verification output +and failure recovery. + +- [The sudo / helper model](#the-sudo--helper-model) +- [Prerequisites](#prerequisites) +- [Setup, in order](#setup-in-order) + - [1. Install tools](#1-install-tools) + - [2. WireGuard key exchange](#2-wireguard-key-exchange) + - [3. Mesh config](#3-mesh-config) + - [4. podNetwork config & bridge100 selection](#4-podnetwork-config--bridge100-selection) + - [5. pf.conf anchor hooks](#5-pfconf-anchor-hooks) + - [6. Install the privileged helper (`macvz-netd`)](#6-install-the-privileged-helper-macvz-netd) + - [7. Start the kubelet](#7-start-the-kubelet) +- [Verification](#verification) +- [Recovery procedures](#recovery-procedures) + - [Stale routes](#stale-routes) + - [Stale pf rules](#stale-pf-rules) + - [Helper failures](#helper-failures) +- [Teardown](#teardown) + +## The sudo / helper model + +The cross-host data plane needs **root** tools — `pfctl`, `route`, `sysctl`, +`ifconfig`, `wg`, `wireguard-go`. But `macvz-kubelet` must run as the operator's +**user**, because `apple/container` is a per-user service that refuses to run as +root. These two facts are in direct tension, so MacVz offers two models: + +| Model | How privileged commands run | When to use | +| --- | --- | --- | +| **Helper daemon (recommended)** | `macvz-netd` runs as root (a LaunchDaemon) and executes a fixed, config-validated allowlist on behalf of the user-run kubelet over a unix socket. | Production and the P5 e2e. The kubelet needs **no** `sudo`. | +| **In-process (dev only)** | The kubelet runs the privileged commands itself, so the **whole kubelet** must be root. | Incompatible with `apple/container` — only useful for unit-level network testing without real micro-VMs. | + +Choose the helper model. With it, the only `sudo` you ever type is the one-time +`macvz-netd install` (and the per-host prep below); day-to-day kubelet starts and +restarts need no elevation. Root is confined to the allowlisted, config-pinned +commands rather than the entire node process. The confinement rules (which +anchors, interfaces, CIDRs, and peers a request may name) are detailed in +[NETWORKING.md → Confining the helper](NETWORKING.md#confining-the-helper-to-this-nodes-config-41). + +## Control API (kubelet ↔ helper, #39) + +The kubelet and `macvz-netd` talk over a single newline-delimited JSON +request/response on a short-lived unix-socket connection (one request per +connection, mirroring `exec.Command`). + +- **Socket**: `--socket` path (default `/var/run/macvz-netd.sock`), mode `0660`, + owned by the kubelet's user (`$SUDO_UID:$SUDO_GID`, or `--owner uid:gid` under + launchd). No other user can connect; root holds the daemon. +- **Versioning**: every request carries a `protocol` field. The current version + is `1`. The helper refuses a request whose `protocol` is set and does not match + with `errorCode: "unsupported_protocol"`, so mismatched kubelet/helper builds + fail fast. `protocol: 0` (an older unversioned client) is accepted as the + current version. +- **Operations** (`op` field): + - `""` / `exec` — run the allowlisted command in `name` with `args`/`stdin`. + The reply carries `stdout`, `stderr`, and `exitCode` (a non-zero exit is a + normal command result, not an API error). + - `status` — the helper reports `version`, `protocol`, `allowedCommands`, + `policyEnforced` (whether #41 per-request validation is on), `pid`, + `startedAt`, and `uptime`. No command runs. The kubelet calls this at startup + and logs it, then issues a `sysctl` probe to confirm the exec path works. +- **Structured errors**: a refusal sets `errorCode` to one of + `unsupported_protocol`, `malformed` (undecodable or over the 1 MiB request + cap), `not_allowed` (command off the allowlist), `not_permitted` (allowlisted + but outside this node's policy, see #41), `unknown_op`, or `exec_error` (the + command could not be spawned). The client surfaces these as a typed + `privhelper.APIError`, so a caller can map a failure to a Pod condition without + parsing the human-readable message. + +A missing daemon, a wrong-version daemon, or a daemon that answers but cannot run +commands each surface as a distinct, actionable startup error rather than failing +later on the first `pfctl`/`wg` call. + +## Prerequisites + +On **each** Mac that will be a node: + +- **macOS 26 (Tahoe)+ on Apple Silicon**, with `apple/container` installed and + working (`container list` succeeds). +- `wireguard-tools` (provides `wg`, `wireguard-go`) and `kubectl`: + ```sh + brew install wireguard-tools kubernetes-cli + ``` +- A **shared Kubernetes control plane** reachable from every node, with + `--allocate-node-cidrs=true` and a `--cluster-cidr` so each node gets a + disjoint `Spec.PodCIDR`. (Override with `node.podCIDR` only if your cluster + does not allocate node CIDRs.) +- **UDP 51820 open** between the hosts (the WireGuard endpoint port). If a host + firewall is on, allow it explicitly. +- The built `bin/macvz-kubelet` and `bin/macvz-netd` (`make build`) copied to + each host. + +Throughout, the worked example uses the two-node topology from +[`test/e2e/two-node`](../test/e2e/two-node/README.md): + +| Node | Host | Mesh addr | Pod CIDR | WG endpoint | +| --- | --- | --- | --- | --- | +| `macvz-a` | 192.168.1.110 | 10.99.0.1/32 | 10.244.101.0/24 | 192.168.1.110:51820 | +| `macvz-b` | 192.168.1.122 | 10.99.0.2/32 | 10.244.102.0/24 | 192.168.1.122:51820 | + +## Setup, in order + +### 1. Install tools + +Confirm the userspace WireGuard toolchain and the runtime are present: + +```sh +wg --version # wireguard-tools is on PATH +which wireguard-go # userspace utun backend +container list # apple/container responds +``` + +### 2. WireGuard key exchange + +Each node has a **stable identity**: one private key, kept mode `0600`, that +never enters config or git. The kubelet auto-generates it at +`mesh.privateKeyFile` on first start if absent — but for a planned two-node +bring-up, generate both keypairs up front so you can fill in peer public keys +before starting anything: + +```sh +# Run once, anywhere. Produces a private + public key per node. +wg genkey | tee macvz-a.key | wg pubkey > macvz-a.pub +wg genkey | tee macvz-b.key | wg pubkey > macvz-b.pub +chmod 600 *.key +``` + +Then **exchange public keys**: each node's config lists the *other* node as a +peer, using that peer's **public** key. The private keys stay on their own host: + +- Install `macvz-a.key` as `mesh.privateKeyFile` on `macvz-a` only. +- Put the contents of `macvz-a.pub` into `macvz-b`'s `peers[].publicKey` for + `macvz-a` — and vice-versa. + +If you ever need a running node's public key, it is logged at startup, or: + +```sh +wg pubkey < /etc/macvz/wireguard.key +``` + +> The two-node bundle automates this: `keys/*.pub` are committed and baked into +> the peer stanzas; `keys/*.key` are gitignored and installed by `prep-node.sh`. + +### 3. Mesh config + +Add a `mesh` stanza to each node's `config.yaml`. The mesh is **off by default**, +so single-host development needs none of this. A peer's `podCIDR` becomes its +WireGuard `AllowedIPs`, and a host route for that CIDR is installed through the +interface — that is what forwards Pod-bound packets into the tunnel. + +**macvz-a** (`/etc/macvz/macvz-a.yaml`): + +```yaml +mesh: + enabled: true + interface: utun7 + privateKeyFile: /etc/macvz/wireguard.key + address: 10.99.0.1/32 + listenPort: 51820 + peers: + - name: macvz-b + publicKey: + endpoint: 192.168.1.122:51820 + podCIDR: 10.244.102.0/24 + address: 10.99.0.2/32 + persistentKeepalive: 25 +``` + +**macvz-b** is the mirror image: `address: 10.99.0.2/32`, and its single peer is +`macvz-a` with `macvz-a.pub`, endpoint `192.168.1.110:51820`, podCIDR +`10.244.101.0/24`, address `10.99.0.1/32`. + +Field reference: see [NETWORKING.md → mesh Configuration](NETWORKING.md#configuration-1). + +### 4. podNetwork config & bridge100 selection + +`podNetwork` enables the host-side path that maps each Pod IP to its micro-VM's +host-only vmnet address (1:1 NAT via a pf `binat`) and programs ClusterIP `rdr` +rules. Add to each node's config: + +```yaml +podNetwork: + enabled: true + interface: bridge100 # the vmnet bridge apple/container attaches VMs to + anchor: macvz/pods # default; the only anchor the helper will touch + enableForwarding: true # default; flips net.inet.ip.forwarding to 1 +``` + +**Selecting the bridge.** `apple/container` attaches each micro-VM to a +vmnet-backed bridge, **commonly `bridge100`**, but the number is assigned by +macOS and can differ. The bridge only appears **after the first micro-VM +starts** — so confirm it once a VM is running: + +```sh +# List bridges and the members each one has (VM vmnet interfaces show as members): +ifconfig | grep -A6 '^bridge' +# Or watch for the vmnet bridge to come up: +ifconfig bridge100 2>/dev/null && echo "bridge100 is up" +``` + +If the runtime uses a different bridge, set `podNetwork.interface` to match +before starting the kubelet. A wrong interface produces `binat`/`rdr` rules that +never match traffic — see [Recovery](#stale-pf-rules). + +To make cluster DNS resolve from inside micro-VMs, also set `node.clusterDNS` to +the CoreDNS ClusterIP (see +[NETWORKING.md → ClusterIP Services and DNS](NETWORKING.md#clusterip-services-and-dns)). + +### 5. pf.conf anchor hooks + +`pf` only evaluates the MacVz anchor if `/etc/pf.conf` references it. The four +hook lines must respect pf's ordering rule — **translation anchors +(nat/rdr/binat) before filter anchors**: + +``` +nat-anchor "macvz/pods/*" +rdr-anchor "macvz/pods/*" +binat-anchor "macvz/pods/*" +anchor "macvz/pods/*" +``` + +On a stock macOS `pf.conf` (which ships `com.apple` anchors), the translation +hooks go right after `rdr-anchor "com.apple/*"` and the filter hook right after +`anchor "com.apple/*"`. The two-node bundle's `prep-node.sh` does this insertion +idempotently and **backs up `pf.conf`** to `pf.conf.macvz.bak.` first; prefer +it over hand-editing: + +```sh +# From test/e2e/two-node, on each host (installs key, TLS, pf hooks, forwarding): +sudo ./prep-node.sh a # or b +``` + +If you edit by hand, validate before loading — a syntax error disables pf: + +```sh +sudo pfctl -n -f /etc/pf.conf # dry-run syntax check; fails loud +sudo pfctl -f /etc/pf.conf # load +sudo pfctl -e # enable (tolerate "already enabled") +``` + +### 6. Install the privileged helper (`macvz-netd`) + +Install once per host as a root LaunchDaemon, pinned to this node's config so +every request is validated against it: + +```sh +sudo macvz-netd install \ + --socket /var/run/macvz-netd.sock \ + --config /etc/macvz/macvz-a.yaml # macvz-b.yaml on the other host +``` + +`install` copies the binary to `/usr/local/sbin/macvz-netd`, records the invoking +user (`$SUDO_UID:$SUDO_GID`) as the socket owner, writes the plist with +`RunAtLoad`/`KeepAlive`, and bootstraps the job. Always pass `--config` — without +it the daemon falls back to the command allowlist only and logs a warning. + +Confirm it is up: + +```sh +macvz-netd status # no sudo needed to read; reports plist/binary/socket/loaded +``` + +Then point the kubelet at the socket in its `config.yaml`: + +```yaml +privilegedHelperSocket: /var/run/macvz-netd.sock +``` + +> **Upgrading:** re-run `sudo macvz-netd install` from the newer binary; it boots +> out the old job and replaces in place — no `uninstall` first. + +### 7. Start the kubelet + +With the helper running and `privilegedHelperSocket` set, start the kubelet as +your **normal user** (no sudo): + +```sh +KUBECONFIG=/path/to/kubeconfig macvz-kubelet --config /etc/macvz/macvz-a.yaml +``` + +On start it brings up the mesh interface, applies peers, installs peer-CIDR +routes, enables forwarding, and begins reconciling Pods and Services through the +helper. Repeat on the other host. The two-node bundle's `run.sh start ` +wraps this. + +## Verification + +Run these after both nodes are up. The bundle's +[`verify-dataplane.sh`](../test/e2e/two-node/verify-dataplane.sh) automates the +first three and exits non-zero if any fail. + +**1. WireGuard handshake** — a recent handshake with the peer: + +```sh +sudo wg show utun7 +``` +``` +interface: utun7 + public key: + listening port: 51820 +peer: + endpoint: 192.168.1.122:51820 + allowed ips: 10.244.102.0/24, 10.99.0.2/32 + latest handshake: 23 seconds ago # <- must be present and recent + transfer: 1.85 KiB received, 2.10 KiB sent +``` + +**2. Routes for the remote Pod CIDR** — present via the mesh interface: + +```sh +netstat -rn -f inet | grep utun7 +``` +``` +10.244.102/24 utun7 USc utun7 # remote Pod CIDR +10.99.0.2 utun7 UH utun7 # peer mesh addr +``` + +**3. IPv4 forwarding** — on: + +```sh +sysctl net.inet.ip.forwarding +``` +``` +net.inet.ip.forwarding: 1 +``` + +**4. Reach the peer across the tunnel:** + +```sh +ping -c2 10.99.0.2 # WARN-only if ICMP is filtered; not fatal +``` + +**5. pf anchor rules** — `binat` per attached Pod, plus `rdr` per ClusterIP: + +```sh +sudo pfctl -a macvz/pods -s nat # binat rules, one per Pod on this node +sudo pfctl -a macvz/pods -s rdr # rdr rules, one per ClusterIP:port +``` +``` +binat on bridge100 inet from 192.168.64.3 to any -> 10.244.101.5 +``` +> No rules until a Pod is scheduled on this node — that is expected, not a fault. + +**6. End-to-end** — a Pod on one Mac reaches a Pod hosted on the other, and a +ClusterIP Service load-balances across both. Use the worked example in +[NETWORKING.md → End-to-end](NETWORKING.md#end-to-end-a-service-across-two-macs), +or run the suite: + +```sh +MACVZ_E2E_NODES=macvz-a,macvz-b MACVZ_E2E_TIMEOUT=120 \ + MACVZ_E2E_DIAG_DIR=./diag test/e2e/e2e.sh +``` + +A fresh Mac prepared with this runbook should make the e2e "Service across nodes" +phase **PASS** rather than SKIP. + +## Recovery procedures + +Privileged network state lives in the kernel (routes, pf rules, the utun +interface) and outlives the kubelet process. A crash, a config change, or a +host/peer reconfiguration can leave **stale** state that silently misroutes or +blackholes traffic. Each of these is read-only to diagnose and safe to re-run. + +### Stale routes + +Symptom: traffic to a remote Pod CIDR blackholes, or `netstat` shows a route to +a CIDR/interface that no longer exists (e.g. an old peer's CIDR, or a route on a +utun number from a previous run). + +```sh +# 1. See what is installed via the mesh interface: +netstat -rn -f inet | grep utun + +# 2. Normal recovery: a kubelet restart re-reconciles routes from config — +# Mesh.Sync adds/removes only the affected host routes, no full teardown. +# Just restart macvz-kubelet. + +# 3. If a route persists for a CIDR no longer in any peer (orphan), remove it: +sudo route -n delete -inet 10.244.199.0/24 -interface utun7 + +# 4. If the utun interface itself is wedged (wrong number, half-torn-down), +# destroy it and let the kubelet recreate on next start: +sudo ifconfig utun7 down +sudo ifconfig utun7 destroy # bring-up is idempotent; restart re-creates it +``` + +Because bring-up tolerates "already exists" and `Mesh.Sync` reconciles +incrementally, the safe default is **restart the kubelet** and only hand-delete +orphans the restart cannot know about (CIDRs removed from a since-changed config). + +### Stale pf rules + +Symptom: `binat`/`rdr` rules reference an old Pod's vmnet address or the wrong +interface; traffic is NAT'd to a VM that no longer exists, or rules never match. + +```sh +# 1. Inspect the anchor (this is the only anchor MacVz touches): +sudo pfctl -a macvz/pods -s nat +sudo pfctl -a macvz/pods -s rdr + +# 2. The kubelet regenerates this anchor wholesale and loads it atomically on +# every change, and FLUSHES it on clean shutdown. So a stop+start clears +# stragglers. If the kubelet died uncleanly, flush by hand: +sudo pfctl -a macvz/pods -F all # flush all rules in the macvz anchor only + +# 3. If rules are present but never match, the bridge is wrong: confirm +# podNetwork.interface equals the live vmnet bridge (see step 4 of setup), +# fix the config, and restart. + +# 4. If NO rules ever appear, pf is not evaluating the anchor. Confirm the hooks +# and that pf is enabled: +grep 'macvz/pods' /etc/pf.conf # the four anchor lines must be present +sudo pfctl -s info | grep Status # must read: Status: Enabled +``` + +Flushing only ever targets `-a macvz/pods`; the main ruleset and `com.apple` +anchors are never touched. If pf got disabled by a bad hand-edit, restore from +the `pf.conf.macvz.bak.` backup `prep-node.sh` left and reload. + +### Helper failures + +Symptom: a privileged operation fails even though the command looks valid; the +kubelet logs an error attaching a Pod or syncing the mesh. + +```sh +# 1. Is the daemon loaded and the socket present? +macvz-netd status + +# 2. First place to look: the helper logs. A refused request is logged as +# "request not permitted: ..." with an audit line; applied changes log at info. +tail -n 50 /var/log/macvz-netd.log # stdout: socket, allowlist, audit trail +tail -n 50 /var/log/macvz-netd.err.log # stderr + +# 3. "request not permitted" means the request is out-of-scope for the loaded +# --config: a CIDR/interface/peer/anchor not pinned in THIS node's config. +# Reconcile the config with the request (e.g. add the peer's podCIDR), then +# re-run install so the daemon reloads it: +sudo macvz-netd install --socket /var/run/macvz-netd.sock --config /etc/macvz/macvz-a.yaml + +# 4. Socket permission denied from the kubelet: the socket owner was recorded at +# install time from $SUDO_UID. If you now run the kubelet as a different user, +# reinstall with --owner uid:gid for that user. + +# 5. Restart the job without reinstalling: +sudo macvz-netd unload && sudo macvz-netd load +``` + +If `--config` was omitted at install, the daemon logs a warning at start and +enforces only the command allowlist — reinstall with `--config` to restore +config-pinned validation. + +## Teardown + +Privileged state does not revert on its own. To leave a host clean: + +```sh +# Stop the kubelet (flushes the macvz/pods anchor and best-effort tears down mesh): +# run.sh stop a # if using the bundle, or just stop the macvz-kubelet process + +sudo pfctl -a macvz/pods -F all # belt-and-suspenders anchor flush +sudo route -n flush -inet # only if you must clear stale routes +sudo ifconfig utun7 destroy 2>/dev/null # remove the mesh interface +sudo sysctl -w net.inet.ip.forwarding=0 # revert forwarding, if desired + +# Remove the helper entirely (plist, binary, socket): +sudo macvz-netd uninstall + +# Restore /etc/pf.conf from the prep-node.sh backup, if you want the hooks gone: +# sudo cp /etc/pf.conf.macvz.bak. /etc/pf.conf && sudo pfctl -f /etc/pf.conf + +# Verify nothing is orphaned: +container list --all # expect no leftover micro-VMs +netstat -rn -f inet | grep utun7 # expect no stale remote Pod CIDR routes +sudo pfctl -a macvz/pods -s all # expect empty +``` diff --git a/internal/types/types.go b/internal/types/types.go index 4757400..9523fc0 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -35,6 +35,16 @@ type ContainerSpec struct { // they should be applied. The runtime realizes each as a VirtioFS share (or a // guest tmpfs); the provider is responsible for validating sources. Mounts []Mount + + // DNS are nameserver IPs written into the micro-VM's resolv.conf (passed as + // `--dns`). Empty leaves the image's baked DNS in place. Cluster DNS lets the + // guest resolve Service names (#37). + DNS []string + // DNSSearch are resolv.conf search domains (passed as `--dns-search`). + DNSSearch []string + // DNSOptions are resolv.conf options such as "ndots:5" (passed as + // `--dns-option`). + DNSOptions []string } // Mount describes one filesystem mount inside the micro-VM. A bind mount shares diff --git a/pkg/config/config.go b/pkg/config/config.go index 1146a6b..05b860c 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -55,6 +55,16 @@ type Config struct { // PodNetwork configures the host path that makes each micro-VM reachable at // its assigned Pod IP across the mesh (P3, #22). Disabled by default. PodNetwork PodNetworkConfig `yaml:"podNetwork"` + + // PrivilegedHelperSocket, when set, routes all privileged network commands + // (pf, wg, route, sysctl, ifconfig) for the mesh and Pod network path through + // the root helper daemon (cmd/macvz-netd, #38) at this unix socket, instead + // of executing them in-process. This lets macvz-kubelet run as the operator's + // user — required because apple/container refuses to run as root — while the + // data plane still gets root for its network operations. Empty runs the + // commands directly (the kubelet must then be root, which is incompatible + // with apple/container). + PrivilegedHelperSocket string `yaml:"privilegedHelperSocket"` } // NodeConfig is the configurable shape of the virtual node registered with @@ -136,6 +146,17 @@ type NodeConfig struct { // Volumes configures Pod volume support (#26). Volumes VolumesConfig `yaml:"volumes"` + + // ClusterDNS are the cluster DNS server IPs (typically the CoreDNS/kube-dns + // ClusterIP, e.g. "10.96.0.10") injected into micro-VMs whose Pods use the + // ClusterFirst DNS policy, so in-guest Service name resolution works (#37). + // Empty leaves each micro-VM with the DNS baked into its image. + ClusterDNS []string `yaml:"clusterDNS"` + + // ClusterDomain is the cluster DNS domain used to build Pod search domains + // (e.g. ".svc.cluster.local"). Defaults to "cluster.local" when + // clusterDNS is set. + ClusterDomain string `yaml:"clusterDomain"` } // VolumesConfig governs which Pod volumes MacVz mounts into micro-VMs and where diff --git a/pkg/config/mesh.go b/pkg/config/mesh.go index ea782cb..52abc5b 100644 --- a/pkg/config/mesh.go +++ b/pkg/config/mesh.go @@ -114,11 +114,39 @@ func (c Config) MeshInterfaceConfig() (wireguard.InterfaceConfig, error) { port = DefaultMeshListenPort } + peers, err := m.resolvePeers() + if err != nil { + return wireguard.InterfaceConfig{}, err + } + + return wireguard.InterfaceConfig{ + Name: m.Interface, + PrivateKey: priv, + Address: m.Address, + ListenPort: port, + MTU: m.MTU, + Peers: peers, + }, nil +} + +// MeshPeers resolves the configured mesh peers into wireguard.Peer values, +// without loading this node's private key. It is the input to peer +// reconciliation (#42): on a config reload the kubelet feeds the result to +// wireguard.Mesh.Sync to add/remove peers (and their routes) in place, never +// recreating the interface. Call only when Mesh.Enabled is true. +func (c Config) MeshPeers() ([]wireguard.Peer, error) { + return c.Mesh.resolvePeers() +} + +// resolvePeers maps the YAML peer entries to wireguard.Peer values. Each peer's +// AllowedIPs are its Pod CIDR plus (when set) its mesh address, exactly the +// CIDRs routed through the tunnel to that node. +func (m MeshConfig) resolvePeers() ([]wireguard.Peer, error) { peers := make([]wireguard.Peer, 0, len(m.Peers)) for _, p := range m.Peers { pub, err := wireguard.ParseKey(p.PublicKey) if err != nil { - return wireguard.InterfaceConfig{}, fmt.Errorf("mesh peer %q: %w", p.Name, err) + return nil, fmt.Errorf("mesh peer %q: %w", p.Name, err) } allowed := []string{p.PodCIDR} if p.Address != "" { @@ -132,13 +160,5 @@ func (c Config) MeshInterfaceConfig() (wireguard.InterfaceConfig, error) { PersistentKeepalive: p.PersistentKeepalive, }) } - - return wireguard.InterfaceConfig{ - Name: m.Interface, - PrivateKey: priv, - Address: m.Address, - ListenPort: port, - MTU: m.MTU, - Peers: peers, - }, nil + return peers, nil } diff --git a/pkg/config/mesh_test.go b/pkg/config/mesh_test.go index cf919e1..86b9923 100644 --- a/pkg/config/mesh_test.go +++ b/pkg/config/mesh_test.go @@ -104,6 +104,41 @@ func TestMeshInterfaceConfigTranslates(t *testing.T) { } } +func TestMeshPeersResolvesForReconcile(t *testing.T) { + c := Default() + c.Mesh = enabledMesh(t, filepath.Join(t.TempDir(), "wg.key")) + + peers, err := c.MeshPeers() + if err != nil { + t.Fatalf("MeshPeers: %v", err) + } + if len(peers) != 1 { + t.Fatalf("peers = %d, want 1", len(peers)) + } + p := peers[0] + if p.Name != "mac-02" || p.Endpoint != "192.168.1.20:51820" { + t.Errorf("peer fields not resolved: %+v", p) + } + want := map[string]bool{"10.244.2.0/24": true, "10.99.0.2/32": true} + if len(p.AllowedIPs) != 2 { + t.Fatalf("AllowedIPs = %v, want pod CIDR + address", p.AllowedIPs) + } + for _, a := range p.AllowedIPs { + if !want[a] { + t.Errorf("unexpected AllowedIP %q", a) + } + } + // The peer set MeshPeers returns must match what MeshInterfaceConfig embeds, + // so a reload reconciles to the same desired state Up established. + ifc, err := c.MeshInterfaceConfig() + if err != nil { + t.Fatalf("MeshInterfaceConfig: %v", err) + } + if len(ifc.Peers) != len(peers) || ifc.Peers[0].PublicKey != p.PublicKey { + t.Errorf("MeshPeers and MeshInterfaceConfig disagree on peers") + } +} + func TestMeshInterfaceConfigDefaultsPort(t *testing.T) { c := Default() m := enabledMesh(t, filepath.Join(t.TempDir(), "wg.key")) diff --git a/pkg/config/policy.go b/pkg/config/policy.go new file mode 100644 index 0000000..520d7f9 --- /dev/null +++ b/pkg/config/policy.go @@ -0,0 +1,52 @@ +package config + +import ( + "net" + + "github.com/chimerakang/macvz/pkg/network/podnet" + "github.com/chimerakang/macvz/pkg/network/privhelper" +) + +// PrivilegedHelperPolicy derives the privhelper.Policy that confines the root +// network helper to exactly the resources this node's config references (#41): +// its mesh interface, mesh address, vmnet interface, pf anchor, the Pod/mesh +// CIDRs routed through the tunnel, and the configured peers' public keys. +// +// It draws only on the mesh and Pod-network stanzas, so it is safe to call even +// when those are disabled — the result simply has empty sets and refuses the +// corresponding commands (fail closed). Errors surface only from resolving the +// mesh interface config (e.g. an unreadable/invalid private key file), which the +// helper needs to canonicalise peer keys and route targets. +func (c Config) PrivilegedHelperPolicy() (privhelper.Policy, error) { + p := privhelper.Policy{ + VMNetInterface: c.PodNetwork.Interface, + Anchor: podnet.DefaultAnchor, + RouteCIDRs: map[string]bool{}, + PeerPublicKeys: map[string]bool{}, + } + if c.PodNetwork.Anchor != "" { + p.Anchor = c.PodNetwork.Anchor + } + + if !c.Mesh.Enabled { + return p, nil + } + + ifc, err := c.MeshInterfaceConfig() + if err != nil { + return privhelper.Policy{}, err + } + p.MeshInterface = ifc.Name + p.MTU = ifc.MTU + if ip, _, err := net.ParseCIDR(ifc.Address); err == nil { + p.MeshAddressIP = ip.String() + } + // Route targets and AllowedIPs are confined to the union of every peer's + // AllowedIPs (Pod CIDR + mesh address), exactly what the mesh installs routes + // for and what its wg config advertises. + p.RouteCIDRs = privhelper.NormalizeCIDRSet(ifc.RouteTargets()) + for _, peer := range ifc.Peers { + p.PeerPublicKeys[peer.PublicKey.String()] = true + } + return p, nil +} diff --git a/pkg/config/policy_test.go b/pkg/config/policy_test.go new file mode 100644 index 0000000..9b68df6 --- /dev/null +++ b/pkg/config/policy_test.go @@ -0,0 +1,76 @@ +package config + +import ( + "path/filepath" + "testing" + + "github.com/chimerakang/macvz/pkg/network/wireguard" +) + +func TestPrivilegedHelperPolicyFromConfig(t *testing.T) { + c := Default() + c.Mesh = enabledMesh(t, filepath.Join(t.TempDir(), "wg.key")) + c.PodNetwork = PodNetworkConfig{Enabled: true, Interface: "bridge100"} + + p, err := c.PrivilegedHelperPolicy() + if err != nil { + t.Fatalf("PrivilegedHelperPolicy: %v", err) + } + + if p.MeshInterface != "utun7" { + t.Errorf("MeshInterface = %q, want utun7", p.MeshInterface) + } + if p.VMNetInterface != "bridge100" { + t.Errorf("VMNetInterface = %q, want bridge100", p.VMNetInterface) + } + if p.Anchor != "macvz/pods" { + t.Errorf("Anchor = %q, want macvz/pods", p.Anchor) + } + if p.MeshAddressIP != "10.99.0.1" { + t.Errorf("MeshAddressIP = %q, want 10.99.0.1", p.MeshAddressIP) + } + // The peer's PodCIDR and mesh Address are the route targets. + for _, want := range []string{"10.244.2.0/24", "10.99.0.2/32"} { + if !p.RouteCIDRs[want] { + t.Errorf("RouteCIDRs missing %q (have %v)", want, p.RouteCIDRs) + } + } + // The peer's public key is the only configured peer. + pub := c.Mesh.Peers[0].PublicKey + if k, err := wireguard.ParseKey(pub); err != nil { + t.Fatalf("ParseKey: %v", err) + } else if !p.PeerPublicKeys[k.String()] { + t.Errorf("PeerPublicKeys missing the configured peer key") + } +} + +func TestPrivilegedHelperPolicyMeshDisabledFailsClosed(t *testing.T) { + c := Default() // mesh and pod network off by default + p, err := c.PrivilegedHelperPolicy() + if err != nil { + t.Fatalf("PrivilegedHelperPolicy: %v", err) + } + if p.MeshInterface != "" { + t.Errorf("disabled mesh should leave MeshInterface empty, got %q", p.MeshInterface) + } + if len(p.RouteCIDRs) != 0 || len(p.PeerPublicKeys) != 0 { + t.Errorf("disabled mesh should configure no routes/peers, got %v / %v", p.RouteCIDRs, p.PeerPublicKeys) + } + // Anchor defaults even when disabled, but no interface/peers means the helper + // refuses every mesh and Pod-network command. + if p.Anchor != "macvz/pods" { + t.Errorf("Anchor default = %q, want macvz/pods", p.Anchor) + } +} + +func TestPrivilegedHelperPolicyCustomAnchor(t *testing.T) { + c := Default() + c.PodNetwork = PodNetworkConfig{Enabled: true, Interface: "bridge100", Anchor: "macvz/custom"} + p, err := c.PrivilegedHelperPolicy() + if err != nil { + t.Fatalf("PrivilegedHelperPolicy: %v", err) + } + if p.Anchor != "macvz/custom" { + t.Errorf("Anchor = %q, want macvz/custom", p.Anchor) + } +} diff --git a/pkg/network/podnet/helper.go b/pkg/network/podnet/helper.go new file mode 100644 index 0000000..404d9b8 --- /dev/null +++ b/pkg/network/podnet/helper.go @@ -0,0 +1,34 @@ +package podnet + +import ( + "context" + + "github.com/chimerakang/macvz/pkg/network/privhelper" +) + +// helperRunner implements runner by delegating to the root privileged helper +// (cmd/macvz-netd) over its unix socket, so the user-run kubelet can program pf +// without itself being root (#38). +type helperRunner struct { + client *privhelper.Client +} + +func (h helperRunner) run(ctx context.Context, c command) (string, error) { + stdout, stderr, code, err := h.client.Run(ctx, c.Name, c.Args, c.Stdin) + if err != nil { + return stdout, &CommandError{Cmd: c.String(), Stderr: stderr, ExitCode: code, Err: err} + } + if code != 0 { + return stdout, &CommandError{Cmd: c.String(), Stderr: stderr, ExitCode: code} + } + return stdout, nil +} + +// WithHelperSocket routes the Router's privileged commands through the root +// helper daemon at socketPath instead of executing them directly. Use this when +// the kubelet runs as a non-root user (#38). +func WithHelperSocket(socketPath string) Option { + return func(rt *Router) { + rt.run = helperRunner{client: privhelper.NewClient(socketPath)} + } +} diff --git a/pkg/network/podnet/helper_test.go b/pkg/network/podnet/helper_test.go new file mode 100644 index 0000000..e1b4e6f --- /dev/null +++ b/pkg/network/podnet/helper_test.go @@ -0,0 +1,63 @@ +package podnet + +import ( + "context" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/chimerakang/macvz/pkg/network/privhelper" +) + +// TestRouterRoutesThroughHelper verifies WithHelperSocket sends the Router's pf +// commands through the privileged helper rather than executing them directly. +func TestRouterRoutesThroughHelper(t *testing.T) { + var mu sync.Mutex + var ran []string + fake := func(_ context.Context, name string, args []string, stdin string) (string, string, int, error) { + mu.Lock() + defer mu.Unlock() + ran = append(ran, strings.TrimSpace(name+" "+strings.Join(args, " "))) + return "", "", 0, nil + } + + dir, err := os.MkdirTemp("/tmp", "ph") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + sock := filepath.Join(dir, "s") + srv := privhelper.NewServerWithExec(sock, fake) + if err := srv.Listen(-1, -1); err != nil { + t.Fatalf("Listen: %v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go srv.Serve(ctx) + defer srv.Close() + + rt := New(Config{Interface: "bridge100", EnableForwarding: true}, WithHelperSocket(sock)) + if err := rt.Start(ctx); err != nil { + t.Fatalf("Start: %v", err) + } + if err := rt.Attach(ctx, Endpoint{PodKey: "default/web", PodIP: "10.244.1.2", VMIP: "192.168.64.5"}); err != nil { + t.Fatalf("Attach: %v", err) + } + + mu.Lock() + defer mu.Unlock() + var sawForward, sawAnchorLoad bool + for _, c := range ran { + if strings.Contains(c, "sysctl -w net.inet.ip.forwarding=1") { + sawForward = true + } + if strings.Contains(c, "pfctl -a macvz/pods -f -") { + sawAnchorLoad = true + } + } + if !sawForward || !sawAnchorLoad { + t.Errorf("expected privileged commands through helper; ran=%v", ran) + } +} diff --git a/pkg/network/podnet/router.go b/pkg/network/podnet/router.go index 148eafe..8ea27f8 100644 --- a/pkg/network/podnet/router.go +++ b/pkg/network/podnet/router.go @@ -73,7 +73,8 @@ type Router struct { mu sync.Mutex started bool - endpoints map[string]Endpoint // keyed by PodKey + endpoints map[string]Endpoint // keyed by PodKey + services map[string][]ServiceRule // keyed by service "namespace/name" } // Option configures a Router. @@ -105,6 +106,7 @@ func New(cfg Config, opts ...Option) *Router { pfctl: defaultPfctl, sysctl: defaultSysctl, endpoints: map[string]Endpoint{}, + services: map[string][]ServiceRule{}, } for _, opt := range opts { opt(rt) @@ -194,6 +196,7 @@ func (rt *Router) Stop(ctx context.Context) error { rt.mu.Lock() defer rt.mu.Unlock() rt.endpoints = map[string]Endpoint{} + rt.services = map[string][]ServiceRule{} if !rt.started { return nil } @@ -209,6 +212,7 @@ func (rt *Router) Stop(ctx context.Context) error { // atomically via `pfctl -a -f -`. Caller holds rt.mu. func (rt *Router) loadAnchorLocked(ctx context.Context) error { rules := renderAnchor(rt.cfg.Interface, rt.endpointsSortedLocked()) + rules += renderServiceRules(rt.cfg.Interface, rt.services, rt.vmipByPodIPLocked()) _, err := rt.run.run(ctx, command{ Name: rt.pfctl, Args: []string{"-a", rt.cfg.Anchor, "-f", "-"}, diff --git a/pkg/network/podnet/service.go b/pkg/network/podnet/service.go new file mode 100644 index 0000000..ed4326a --- /dev/null +++ b/pkg/network/podnet/service.go @@ -0,0 +1,183 @@ +package podnet + +import ( + "context" + "fmt" + "net" + "sort" + "strings" + + "k8s.io/klog/v2" +) + +// ServiceRule is one ClusterIP:port DNAT to its ready backends. It maps a +// Kubernetes ClusterIP Service port to the set of ready backend Pod IPs, so a +// micro-VM that dials the ClusterIP reaches a backend. There is no kube-proxy on +// a MacVz node (it is a macOS host and the kube-proxy Pod spec is unsupported), +// so MacVz programs this translation in the same pf anchor that owns the Pod +// binat rules (#37/P5). +type ServiceRule struct { + // ServiceKey is the "namespace/name" of the owning Service (diagnostics). + ServiceKey string + // ClusterIP is the Service's stable virtual IP. + ClusterIP string + // Protocol is "tcp" or "udp" (lowercased pf token). + Protocol string + // Port is the Service port the ClusterIP listens on. + Port int + // TargetPort is the backend container port traffic is redirected to. + TargetPort int + // Backends are the ready backend Pod IPs (MacVz Pod IPs). An empty list + // means the Service has no ready endpoints and emits no rule. + Backends []string +} + +func (r ServiceRule) validate() error { + if r.ServiceKey == "" { + return fmt.Errorf("podnet: service rule has no service key") + } + if net.ParseIP(r.ClusterIP) == nil { + return fmt.Errorf("podnet: service %q has invalid ClusterIP %q", r.ServiceKey, r.ClusterIP) + } + switch r.Protocol { + case "tcp", "udp": + default: + return fmt.Errorf("podnet: service %q has unsupported protocol %q", r.ServiceKey, r.Protocol) + } + if r.Port < 1 || r.Port > 65535 { + return fmt.Errorf("podnet: service %q port %d out of range", r.ServiceKey, r.Port) + } + if r.TargetPort < 1 || r.TargetPort > 65535 { + return fmt.Errorf("podnet: service %q targetPort %d out of range", r.ServiceKey, r.TargetPort) + } + for _, b := range r.Backends { + if net.ParseIP(b) == nil { + return fmt.Errorf("podnet: service %q has invalid backend IP %q", r.ServiceKey, b) + } + } + return nil +} + +// AttachService installs (or replaces) the DNAT rules for one Service, keyed by +// its "namespace/name". Passing rules with no backends, or an empty slice, +// removes the Service's rules — so a Service that loses all ready endpoints +// stops redirecting. It is idempotent. +func (rt *Router) AttachService(ctx context.Context, key string, rules []ServiceRule) error { + if key == "" { + return fmt.Errorf("podnet: AttachService requires a service key") + } + kept := make([]ServiceRule, 0, len(rules)) + for _, r := range rules { + if err := r.validate(); err != nil { + return err + } + if len(r.Backends) == 0 { + continue + } + kept = append(kept, r) + } + + rt.mu.Lock() + defer rt.mu.Unlock() + if !rt.started { + return fmt.Errorf("podnet: AttachService %q before Start", key) + } + if len(kept) == 0 { + delete(rt.services, key) + } else { + rt.services[key] = kept + } + if err := rt.loadAnchorLocked(ctx); err != nil { + return fmt.Errorf("attach service %q: %w", key, err) + } + klog.InfoS("attached service to network path", "service", key, "rules", len(kept)) + return nil +} + +// DetachService removes a Service's DNAT rules. Detaching an unknown Service is +// a no-op so deletion stays idempotent. +func (rt *Router) DetachService(ctx context.Context, key string) error { + rt.mu.Lock() + defer rt.mu.Unlock() + if _, ok := rt.services[key]; !ok { + return nil + } + delete(rt.services, key) + if !rt.started { + return nil + } + if err := rt.loadAnchorLocked(ctx); err != nil { + return fmt.Errorf("detach service %q: %w", key, err) + } + klog.InfoS("detached service from network path", "service", key) + return nil +} + +// Services returns the service keys with active rules, sorted, for diagnostics +// and tests. +func (rt *Router) Services() []string { + rt.mu.Lock() + defer rt.mu.Unlock() + out := make([]string, 0, len(rt.services)) + for k := range rt.services { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// renderServiceRules builds the pf `rdr` rules for all services. Each backend +// Pod IP is resolved against the local endpoint set: a backend whose Pod runs on +// this node is redirected straight to its micro-VM's host-only address (directly +// attached to the vmnet interface, so no extra route is needed), while a backend +// on another Mac keeps its Pod IP and is reached over the WireGuard mesh route. +// Output is deterministic: services, rules, and backends are all sorted. +func renderServiceRules(iface string, services map[string][]ServiceRule, vmipByPodIP map[string]string) string { + keys := make([]string, 0, len(services)) + for k := range services { + keys = append(keys, k) + } + sort.Strings(keys) + + var b strings.Builder + for _, key := range keys { + rules := append([]ServiceRule(nil), services[key]...) + sort.Slice(rules, func(i, j int) bool { + if rules[i].ClusterIP != rules[j].ClusterIP { + return rules[i].ClusterIP < rules[j].ClusterIP + } + return rules[i].Port < rules[j].Port + }) + for _, r := range rules { + targets := make([]string, 0, len(r.Backends)) + for _, podIP := range r.Backends { + if vmIP, ok := vmipByPodIP[podIP]; ok { + targets = append(targets, vmIP) // local backend, via vmnet + } else { + targets = append(targets, podIP) // remote backend, via mesh + } + } + sort.Strings(targets) + + fmt.Fprintf(&b, "# %s %s:%d -> :%d\n", r.ServiceKey, r.ClusterIP, r.Port, r.TargetPort) + if len(targets) == 1 { + fmt.Fprintf(&b, "rdr on %s inet proto %s from any to %s port %d -> %s port %d\n", + iface, r.Protocol, r.ClusterIP, r.Port, targets[0], r.TargetPort) + continue + } + fmt.Fprintf(&b, "rdr on %s inet proto %s from any to %s port %d -> { %s } port %d round-robin\n", + iface, r.Protocol, r.ClusterIP, r.Port, strings.Join(targets, ", "), r.TargetPort) + } + } + return b.String() +} + +// vmipByPodIPLocked maps each attached Pod IP to its micro-VM address. Caller +// holds rt.mu. +func (rt *Router) vmipByPodIPLocked() map[string]string { + m := make(map[string]string, len(rt.endpoints)) + for _, ep := range rt.endpoints { + m[ep.PodIP] = ep.VMIP + } + return m +} diff --git a/pkg/network/podnet/service_test.go b/pkg/network/podnet/service_test.go new file mode 100644 index 0000000..e1128a8 --- /dev/null +++ b/pkg/network/podnet/service_test.go @@ -0,0 +1,206 @@ +package podnet + +import ( + "context" + "strings" + "testing" +) + +func startedRouter(t *testing.T, fr *fakeRunner) (*Router, context.Context) { + t.Helper() + rt := newTestRouter(fr) + ctx := context.Background() + if err := rt.Start(ctx); err != nil { + t.Fatalf("Start: %v", err) + } + return rt, ctx +} + +// A local backend (its Pod is attached on this node) redirects to the VM's +// host-only address, not the Pod IP, so it is delivered straight over vmnet. +func TestAttachServiceLocalBackendRedirectsToVMIP(t *testing.T) { + fr := newFakeRunner() + rt, ctx := startedRouter(t, fr) + + if err := rt.Attach(ctx, Endpoint{PodKey: "default/be", PodIP: "10.244.101.2", VMIP: "192.168.64.5"}); err != nil { + t.Fatalf("Attach backend: %v", err) + } + rule := ServiceRule{ + ServiceKey: "default/hello", ClusterIP: "10.96.0.50", Protocol: "tcp", + Port: 80, TargetPort: 8080, Backends: []string{"10.244.101.2"}, + } + if err := rt.AttachService(ctx, "default/hello", []ServiceRule{rule}); err != nil { + t.Fatalf("AttachService: %v", err) + } + rules, ok := fr.lastAnchorLoad() + if !ok { + t.Fatal("no anchor load recorded") + } + want := "rdr on bridge100 inet proto tcp from any to 10.96.0.50 port 80 -> 192.168.64.5 port 8080" + if !strings.Contains(rules, want) { + t.Errorf("anchor missing local rdr rule %q\n---\n%s", want, rules) + } + if strings.Contains(rules, "-> 10.244.101.2 port 8080") { + t.Errorf("local backend should redirect to VMIP, not Pod IP\n---\n%s", rules) + } +} + +// A remote backend (no local endpoint) keeps its Pod IP, reached via the mesh. +func TestAttachServiceRemoteBackendKeepsPodIP(t *testing.T) { + fr := newFakeRunner() + rt, ctx := startedRouter(t, fr) + + rule := ServiceRule{ + ServiceKey: "default/hello", ClusterIP: "10.96.0.50", Protocol: "tcp", + Port: 80, TargetPort: 8080, Backends: []string{"10.244.102.9"}, + } + if err := rt.AttachService(ctx, "default/hello", []ServiceRule{rule}); err != nil { + t.Fatalf("AttachService: %v", err) + } + rules, _ := fr.lastAnchorLoad() + want := "rdr on bridge100 inet proto tcp from any to 10.96.0.50 port 80 -> 10.244.102.9 port 8080" + if !strings.Contains(rules, want) { + t.Errorf("anchor missing remote rdr rule %q\n---\n%s", want, rules) + } +} + +// Mixed local + remote backends become a round-robin pool with VMIP for the +// local one and Pod IP for the remote one. +func TestAttachServiceMixedBackendsRoundRobin(t *testing.T) { + fr := newFakeRunner() + rt, ctx := startedRouter(t, fr) + if err := rt.Attach(ctx, Endpoint{PodKey: "default/be", PodIP: "10.244.101.2", VMIP: "192.168.64.5"}); err != nil { + t.Fatalf("Attach: %v", err) + } + rule := ServiceRule{ + ServiceKey: "default/hello", ClusterIP: "10.96.0.50", Protocol: "tcp", + Port: 80, TargetPort: 8080, Backends: []string{"10.244.101.2", "10.244.102.9"}, + } + if err := rt.AttachService(ctx, "default/hello", []ServiceRule{rule}); err != nil { + t.Fatalf("AttachService: %v", err) + } + rules, _ := fr.lastAnchorLoad() + // targets are sorted; "10.244.102.9" < "192.168.64.5" + want := "-> { 10.244.102.9, 192.168.64.5 } port 8080 round-robin" + if !strings.Contains(rules, want) { + t.Errorf("anchor missing round-robin pool %q\n---\n%s", want, rules) + } +} + +// A Service with no ready backends emits no rule, and replacing a Service's +// rules with an empty set removes it. +func TestAttachServiceNoBackendsRemovesRule(t *testing.T) { + fr := newFakeRunner() + rt, ctx := startedRouter(t, fr) + rule := ServiceRule{ + ServiceKey: "default/hello", ClusterIP: "10.96.0.50", Protocol: "tcp", + Port: 80, TargetPort: 8080, Backends: []string{"10.244.101.2"}, + } + if err := rt.AttachService(ctx, "default/hello", []ServiceRule{rule}); err != nil { + t.Fatalf("AttachService: %v", err) + } + // now drop all backends + rule.Backends = nil + if err := rt.AttachService(ctx, "default/hello", []ServiceRule{rule}); err != nil { + t.Fatalf("AttachService empty: %v", err) + } + if svcs := rt.Services(); len(svcs) != 0 { + t.Errorf("Services = %v, want empty after losing all backends", svcs) + } + rules, _ := fr.lastAnchorLoad() + if strings.Contains(rules, "10.96.0.50") { + t.Errorf("rule still present after losing all backends\n---\n%s", rules) + } +} + +func TestDetachServiceRemovesRule(t *testing.T) { + fr := newFakeRunner() + rt, ctx := startedRouter(t, fr) + a := ServiceRule{ServiceKey: "default/a", ClusterIP: "10.96.0.1", Protocol: "tcp", Port: 80, TargetPort: 8080, Backends: []string{"10.244.102.2"}} + b := ServiceRule{ServiceKey: "default/b", ClusterIP: "10.96.0.2", Protocol: "tcp", Port: 80, TargetPort: 8080, Backends: []string{"10.244.102.3"}} + if err := rt.AttachService(ctx, "default/a", []ServiceRule{a}); err != nil { + t.Fatalf("AttachService a: %v", err) + } + if err := rt.AttachService(ctx, "default/b", []ServiceRule{b}); err != nil { + t.Fatalf("AttachService b: %v", err) + } + if err := rt.DetachService(ctx, "default/a"); err != nil { + t.Fatalf("DetachService: %v", err) + } + rules, _ := fr.lastAnchorLoad() + if strings.Contains(rules, "10.96.0.1") { + t.Errorf("detached service rule still present\n---\n%s", rules) + } + if !strings.Contains(rules, "10.96.0.2") { + t.Errorf("remaining service rule missing\n---\n%s", rules) + } +} + +func TestDetachServiceUnknownIsNoop(t *testing.T) { + fr := newFakeRunner() + rt, ctx := startedRouter(t, fr) + before := len(fr.strings()) + if err := rt.DetachService(ctx, "default/ghost"); err != nil { + t.Fatalf("DetachService unknown: %v", err) + } + if after := len(fr.strings()); after != before { + t.Errorf("DetachService of unknown reloaded the anchor (%d -> %d)", before, after) + } +} + +func TestAttachServiceValidates(t *testing.T) { + fr := newFakeRunner() + rt, ctx := startedRouter(t, fr) + bad := []ServiceRule{ + {ServiceKey: "", ClusterIP: "10.96.0.1", Protocol: "tcp", Port: 80, TargetPort: 8080, Backends: []string{"10.244.1.2"}}, + {ServiceKey: "default/x", ClusterIP: "nope", Protocol: "tcp", Port: 80, TargetPort: 8080, Backends: []string{"10.244.1.2"}}, + {ServiceKey: "default/x", ClusterIP: "10.96.0.1", Protocol: "sctp", Port: 80, TargetPort: 8080, Backends: []string{"10.244.1.2"}}, + {ServiceKey: "default/x", ClusterIP: "10.96.0.1", Protocol: "tcp", Port: 0, TargetPort: 8080, Backends: []string{"10.244.1.2"}}, + {ServiceKey: "default/x", ClusterIP: "10.96.0.1", Protocol: "tcp", Port: 80, TargetPort: 0, Backends: []string{"10.244.1.2"}}, + {ServiceKey: "default/x", ClusterIP: "10.96.0.1", Protocol: "tcp", Port: 80, TargetPort: 8080, Backends: []string{"bad-ip"}}, + } + for _, r := range bad { + if err := rt.AttachService(ctx, "default/x", []ServiceRule{r}); err == nil { + t.Errorf("AttachService(%+v) = nil, want validation error", r) + } + } +} + +func TestAttachServiceBeforeStartFails(t *testing.T) { + rt := newTestRouter(newFakeRunner()) + r := ServiceRule{ServiceKey: "default/x", ClusterIP: "10.96.0.1", Protocol: "tcp", Port: 80, TargetPort: 8080, Backends: []string{"10.244.1.2"}} + if err := rt.AttachService(context.Background(), "default/x", []ServiceRule{r}); err == nil { + t.Fatal("AttachService before Start should fail") + } +} + +func TestRenderServiceRulesDeterministic(t *testing.T) { + services := map[string][]ServiceRule{ + "default/a": {{ServiceKey: "default/a", ClusterIP: "10.96.0.1", Protocol: "tcp", Port: 80, TargetPort: 8080, Backends: []string{"10.244.1.2"}}}, + "default/b": {{ServiceKey: "default/b", ClusterIP: "10.96.0.2", Protocol: "tcp", Port: 80, TargetPort: 8080, Backends: []string{"10.244.1.3"}}}, + } + vmip := map[string]string{} + if renderServiceRules("bridge100", services, vmip) != renderServiceRules("bridge100", services, vmip) { + t.Error("renderServiceRules is not deterministic") + } +} + +// Service rules and Pod binat rules coexist in the same anchor load. +func TestServiceAndBinatRulesCoexist(t *testing.T) { + fr := newFakeRunner() + rt, ctx := startedRouter(t, fr) + if err := rt.Attach(ctx, Endpoint{PodKey: "default/be", PodIP: "10.244.101.2", VMIP: "192.168.64.5"}); err != nil { + t.Fatalf("Attach: %v", err) + } + rule := ServiceRule{ServiceKey: "default/hello", ClusterIP: "10.96.0.50", Protocol: "tcp", Port: 80, TargetPort: 8080, Backends: []string{"10.244.101.2"}} + if err := rt.AttachService(ctx, "default/hello", []ServiceRule{rule}); err != nil { + t.Fatalf("AttachService: %v", err) + } + rules, _ := fr.lastAnchorLoad() + if !strings.Contains(rules, "binat on bridge100 from 192.168.64.5 to any -> 10.244.101.2") { + t.Errorf("binat rule missing\n---\n%s", rules) + } + if !strings.Contains(rules, "rdr on bridge100 inet proto tcp from any to 10.96.0.50") { + t.Errorf("rdr rule missing\n---\n%s", rules) + } +} diff --git a/pkg/network/privhelper/client.go b/pkg/network/privhelper/client.go new file mode 100644 index 0000000..6bc77d7 --- /dev/null +++ b/pkg/network/privhelper/client.go @@ -0,0 +1,110 @@ +package privhelper + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "net" + "time" +) + +// Client talks to the root helper over its unix socket. It is safe for +// concurrent use: each Run opens its own short-lived connection. +type Client struct { + socketPath string + dialer net.Dialer +} + +// NewClient builds a client for the helper at socketPath. +func NewClient(socketPath string) *Client { + return &Client{socketPath: socketPath} +} + +// SocketPath returns the configured socket path. +func (c *Client) SocketPath() string { return c.socketPath } + +// APIError is a structured refusal or transport-level failure from the helper. +// Its Code is one of the privhelper Code* constants, so a caller can map a +// failure to a Pod condition or diagnostic without parsing the message. +type APIError struct { + Code string + Message string +} + +func (e *APIError) Error() string { + if e.Code == "" { + return "privhelper: " + e.Message + } + return fmt.Sprintf("privhelper: %s (%s)", e.Message, e.Code) +} + +// Ping verifies the helper is reachable by running a harmless allowlisted +// command (`sysctl -n kern.ostype`). It returns an error if the socket cannot be +// reached or the helper refuses, so startup can fail fast with a clear message. +func (c *Client) Ping(ctx context.Context) error { + _, _, code, err := c.Run(ctx, "sysctl", []string{"-n", "kern.ostype"}, "") + if err != nil { + return err + } + if code != 0 { + return fmt.Errorf("privhelper ping returned exit %d", code) + } + return nil +} + +// Status queries the helper's self-report (version, protocol, allowlist, policy +// state, uptime) for startup logs and diagnostics. It also exercises protocol +// negotiation: an incompatible helper rejects the request with an *APIError +// whose Code is CodeUnsupportedProtocol. +func (c *Client) Status(ctx context.Context) (*HelperStatus, error) { + resp, err := c.roundTrip(ctx, Request{Protocol: Protocol, Op: OpStatus}) + if err != nil { + return nil, err + } + if resp.Err != "" { + return nil, &APIError{Code: resp.ErrorCode, Message: resp.Err} + } + if resp.Status == nil { + return nil, &APIError{Message: "helper returned no status"} + } + return resp.Status, nil +} + +// Run sends one command to the helper and returns its stdout, stderr, exit code, +// and any transport error. A non-zero exitCode with a nil error is a normal +// command failure (mirrors os/exec). A refusal or spawn failure is returned as +// an *APIError carrying a structured Code. +func (c *Client) Run(ctx context.Context, name string, args []string, stdin string) (stdout, stderr string, exitCode int, err error) { + resp, err := c.roundTrip(ctx, Request{Protocol: Protocol, Op: OpExec, Name: name, Args: args, Stdin: stdin}) + if err != nil { + return "", "", -1, err + } + if resp.Err != "" { + return resp.Stdout, resp.Stderr, resp.ExitCode, &APIError{Code: resp.ErrorCode, Message: resp.Err} + } + return resp.Stdout, resp.Stderr, resp.ExitCode, nil +} + +// roundTrip opens a short-lived connection, sends req, and decodes one Response. +func (c *Client) roundTrip(ctx context.Context, req Request) (Response, error) { + conn, err := c.dialer.DialContext(ctx, "unix", c.socketPath) + if err != nil { + return Response{}, fmt.Errorf("privhelper dial %q: %w", c.socketPath, err) + } + defer conn.Close() + if deadline, ok := ctx.Deadline(); ok { + conn.SetDeadline(deadline) + } else { + conn.SetDeadline(time.Now().Add(2 * time.Minute)) + } + + if err := json.NewEncoder(conn).Encode(req); err != nil { + return Response{}, fmt.Errorf("privhelper encode request: %w", err) + } + var resp Response + if err := json.NewDecoder(bufio.NewReader(conn)).Decode(&resp); err != nil { + return Response{}, fmt.Errorf("privhelper decode response: %w", err) + } + return resp, nil +} diff --git a/pkg/network/privhelper/control_test.go b/pkg/network/privhelper/control_test.go new file mode 100644 index 0000000..8c02860 --- /dev/null +++ b/pkg/network/privhelper/control_test.go @@ -0,0 +1,222 @@ +package privhelper + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "net" + "os" + "path/filepath" + "strings" + "testing" +) + +// startControlServer starts a Server (optionally policy-enforcing) with the +// given executor on a short /tmp socket and returns a connected Client. +func startControlServer(t *testing.T, srv *Server) *Client { + t.Helper() + // Unix socket paths are capped at ~104 bytes on macOS; use a short /tmp dir. + dir, err := os.MkdirTemp("/tmp", "phc") + if err != nil { + t.Fatalf("MkdirTemp: %v", err) + } + t.Cleanup(func() { os.RemoveAll(dir) }) + if err := srv.Listen(-1, -1); err != nil { + t.Fatalf("Listen: %v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + go srv.Serve(ctx) + t.Cleanup(func() { cancel(); srv.Close() }) + return NewClient(srv.socketPath) +} + +func tmpSock(t *testing.T) string { + t.Helper() + dir, err := os.MkdirTemp("/tmp", "phc") + if err != nil { + t.Fatalf("MkdirTemp: %v", err) + } + t.Cleanup(func() { os.RemoveAll(dir) }) + return filepath.Join(dir, "s") +} + +// okExec is a no-op executor that always succeeds. +func okExec(_ context.Context, _ string, _ []string, _ string) (string, string, int, error) { + return "", "", 0, nil +} + +func TestStatusReportsHelperIdentity(t *testing.T) { + srv := NewServerWithExec(tmpSock(t), okExec).SetVersion("v1.2.3") + c := startControlServer(t, srv) + + st, err := c.Status(context.Background()) + if err != nil { + t.Fatalf("Status: %v", err) + } + if st.Version != "v1.2.3" { + t.Errorf("version = %q, want v1.2.3", st.Version) + } + if st.Protocol != Protocol { + t.Errorf("protocol = %d, want %d", st.Protocol, Protocol) + } + if st.PolicyEnforced { + t.Error("PolicyEnforced should be false for a non-policy server") + } + if st.PID != os.Getpid() { + t.Errorf("PID = %d, want %d", st.PID, os.Getpid()) + } + if len(st.AllowedCommands) == 0 || st.Uptime == "" || st.StartedAt.IsZero() { + t.Errorf("incomplete status: %+v", st) + } +} + +func TestStatusReportsPolicyEnforced(t *testing.T) { + srv := NewServerWithPolicy(tmpSock(t), Policy{MeshInterface: "utun7"}) + srv.exec = okExec // don't actually run commands + c := startControlServer(t, srv) + + st, err := c.Status(context.Background()) + if err != nil { + t.Fatalf("Status: %v", err) + } + if !st.PolicyEnforced { + t.Error("PolicyEnforced should be true for a policy server") + } + // Version defaults to "dev" when unset. + if st.Version != "dev" { + t.Errorf("version = %q, want dev", st.Version) + } +} + +// TestUnsupportedProtocolRejected verifies a client speaking a future protocol +// is refused with a structured error rather than silently misinterpreted. +func TestUnsupportedProtocolRejected(t *testing.T) { + srv := NewServerWithExec(tmpSock(t), okExec) + c := startControlServer(t, srv) + + resp, err := c.roundTrip(context.Background(), Request{Protocol: Protocol + 1, Op: OpStatus}) + if err != nil { + t.Fatalf("roundTrip: %v", err) + } + if resp.ErrorCode != CodeUnsupportedProtocol { + t.Errorf("errorCode = %q, want %q (err=%q)", resp.ErrorCode, CodeUnsupportedProtocol, resp.Err) + } + if resp.Status != nil { + t.Error("status must not be returned for a rejected protocol") + } +} + +// TestLegacyUnversionedRequestAccepted verifies a Protocol==0 request (an older +// client that predates version negotiation) is still served. +func TestLegacyUnversionedRequestAccepted(t *testing.T) { + srv := NewServerWithExec(tmpSock(t), okExec) + c := startControlServer(t, srv) + + // Protocol intentionally left 0; Op exec. + resp, err := c.roundTrip(context.Background(), Request{Name: "pfctl", Args: []string{"-e"}}) + if err != nil { + t.Fatalf("roundTrip: %v", err) + } + if resp.Err != "" { + t.Errorf("legacy request refused: %q (%s)", resp.Err, resp.ErrorCode) + } +} + +func TestUnknownOpRejected(t *testing.T) { + srv := NewServerWithExec(tmpSock(t), okExec) + c := startControlServer(t, srv) + + resp, err := c.roundTrip(context.Background(), Request{Protocol: Protocol, Op: "frobnicate"}) + if err != nil { + t.Fatalf("roundTrip: %v", err) + } + if resp.ErrorCode != CodeUnknownOp { + t.Errorf("errorCode = %q, want %q", resp.ErrorCode, CodeUnknownOp) + } +} + +// TestRunReturnsStructuredErrorCodes verifies Run surfaces a refusal as an +// *APIError whose Code classifies the failure for Pod-condition mapping. +func TestRunReturnsStructuredErrorCodes(t *testing.T) { + srv := NewServerWithExec(tmpSock(t), okExec) + c := startControlServer(t, srv) + + _, _, _, err := c.Run(context.Background(), "rm", []string{"-rf", "/"}, "") + if err == nil { + t.Fatal("expected error for disallowed command") + } + var apiErr *APIError + if !errors.As(err, &apiErr) { + t.Fatalf("error is not *APIError: %T (%v)", err, err) + } + if apiErr.Code != CodeNotAllowed { + t.Errorf("code = %q, want %q", apiErr.Code, CodeNotAllowed) + } +} + +// TestPolicyRefusalCode verifies an allowlisted-but-out-of-scope command is +// rejected with CodeNotPermitted (#41 integration through the #39 API). +func TestPolicyRefusalCode(t *testing.T) { + srv := NewServerWithPolicy(tmpSock(t), Policy{MeshInterface: "utun7"}) + srv.exec = okExec + c := startControlServer(t, srv) + + // ifconfig on a foreign interface is allowlisted but not policy-permitted. + _, _, _, err := c.Run(context.Background(), "ifconfig", []string{"en0", "up"}, "") + var apiErr *APIError + if !errors.As(err, &apiErr) { + t.Fatalf("error is not *APIError: %T (%v)", err, err) + } + if apiErr.Code != CodeNotPermitted { + t.Errorf("code = %q, want %q", apiErr.Code, CodeNotPermitted) + } +} + +// TestMalformedRequestRejected verifies non-JSON input gets a structured +// malformed error rather than crashing the connection handler. +func TestMalformedRequestRejected(t *testing.T) { + srv := NewServerWithExec(tmpSock(t), okExec) + startControlServer(t, srv) + + conn, err := net.Dial("unix", srv.socketPath) + if err != nil { + t.Fatalf("dial: %v", err) + } + defer conn.Close() + if _, err := conn.Write([]byte("this is not json\n")); err != nil { + t.Fatalf("write: %v", err) + } + var resp Response + if err := json.NewDecoder(bufio.NewReader(conn)).Decode(&resp); err != nil { + t.Fatalf("decode response: %v", err) + } + if resp.ErrorCode != CodeMalformed { + t.Errorf("errorCode = %q, want %q", resp.ErrorCode, CodeMalformed) + } +} + +// TestOversizedRequestRejected verifies a request beyond the size cap is refused +// rather than buffered unbounded by the root daemon. +func TestOversizedRequestRejected(t *testing.T) { + srv := NewServerWithExec(tmpSock(t), okExec) + startControlServer(t, srv) + + conn, err := net.Dial("unix", srv.socketPath) + if err != nil { + t.Fatalf("dial: %v", err) + } + defer conn.Close() + // A valid JSON envelope with an enormous Stdin, exceeding maxRequestBytes. + huge := strings.Repeat("A", maxRequestBytes+1024) + if err := json.NewEncoder(conn).Encode(Request{Protocol: Protocol, Name: "pfctl", Stdin: huge}); err != nil { + t.Fatalf("encode: %v", err) + } + var resp Response + if err := json.NewDecoder(bufio.NewReader(conn)).Decode(&resp); err != nil { + t.Fatalf("decode response: %v", err) + } + if resp.ErrorCode != CodeMalformed { + t.Errorf("errorCode = %q, want %q (the truncated payload must not decode to a valid request)", resp.ErrorCode, CodeMalformed) + } +} diff --git a/pkg/network/privhelper/launchd.go b/pkg/network/privhelper/launchd.go new file mode 100644 index 0000000..cb56dbc --- /dev/null +++ b/pkg/network/privhelper/launchd.go @@ -0,0 +1,377 @@ +package privhelper + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "text/template" +) + +// macOS launchd integration for the privileged network helper (#40). +// +// Without this, an operator must run `sudo macvz-netd` by hand before every +// kubelet start. A LaunchDaemon makes the helper a managed system service: it is +// installed once with sudo, started at boot, restarted if it crashes, and the +// kubelet then connects to its socket with no further privilege escalation. +// +// The plist runs the helper as root (the whole point — it owns pf/route/wg), and +// passes --owner uid:gid so the helper chowns its socket to the operator's user. +// At install time there is a SUDO_UID; at daemon start (launchd) there is not, so +// the owner is captured into the plist rather than read from the environment. + +const ( + // DefaultLabel is the launchd job label and the basename of the plist. + DefaultLabel = "com.github.chimerakang.macvz-netd" + // DefaultPlistDir is where system LaunchDaemons live on macOS. + DefaultPlistDir = "/Library/LaunchDaemons" + // DefaultBinaryPath is where Install copies the helper binary so the plist + // points at a stable, root-owned location rather than a user build tree. + DefaultBinaryPath = "/usr/local/sbin/macvz-netd" + // DefaultStdoutPath and DefaultStderrPath are the daemon's log files. + DefaultStdoutPath = "/var/log/macvz-netd.log" + DefaultStderrPath = "/var/log/macvz-netd.err.log" +) + +// LaunchdConfig describes the LaunchDaemon to install. All paths are absolute so +// the same struct drives both plist rendering and the filesystem operations, +// and tests can redirect every path into a temp dir. +type LaunchdConfig struct { + // Label is the launchd job label (reverse-DNS) and the plist basename. + Label string + // PlistDir is the directory the plist is written to (system LaunchDaemons). + PlistDir string + // BinaryPath is where the helper binary is installed and the plist points. + BinaryPath string + // SocketPath is the unix socket the helper listens on. + SocketPath string + // ConfigPath, when set, is the MacVz config the daemon loads to restrict + // privileged commands to its interfaces, CIDRs, peers, and pf anchor (#41). + // Empty installs a daemon with argument validation disabled (name-allowlist + // only). + ConfigPath string + // OwnerUID/OwnerGID is the user the socket is chowned to so the (non-root) + // kubelet can connect. -1 leaves the socket root-owned. + OwnerUID int + OwnerGID int + // StdoutPath/StderrPath are the daemon's log destinations. + StdoutPath string + StderrPath string +} + +// DefaultLaunchdConfig returns the standard system layout with the socket and +// owner left to the caller (owner defaults to "unset"). +func DefaultLaunchdConfig(socketPath string) LaunchdConfig { + return LaunchdConfig{ + Label: DefaultLabel, + PlistDir: DefaultPlistDir, + BinaryPath: DefaultBinaryPath, + SocketPath: socketPath, + OwnerUID: -1, + OwnerGID: -1, + StdoutPath: DefaultStdoutPath, + StderrPath: DefaultStderrPath, + } +} + +// PlistPath is the full path to the LaunchDaemon plist. +func (c LaunchdConfig) PlistPath() string { + return filepath.Join(c.PlistDir, c.Label+".plist") +} + +// ServiceTarget is the launchctl service target (e.g. "system/