diff --git a/cmd/macvz-kubelet/main.go b/cmd/macvz-kubelet/main.go index c1d37cc..c303f69 100644 --- a/cmd/macvz-kubelet/main.go +++ b/cmd/macvz-kubelet/main.go @@ -102,9 +102,14 @@ func run(ctx context.Context, configPath, runtimeBinary string) error { return fmt.Errorf("build kubernetes client: %w", err) } - // Construct the provider over the runtime driver. Pod lifecycle methods are - // stubbed until #16; this issue only wires the node controller loop. - p := provider.New(cfg.NodeName, driver) + internalIP := cfg.Node.InternalIP + if internalIP == "" { + internalIP = detectInternalIP() + } + + // Construct the provider over the runtime driver. The node's reachable + // address is reported as each Pod's HostIP so Services/`-o wide` resolve it. + p := provider.New(cfg.NodeName, driver, provider.WithHostIP(internalIP)) // Resolve the configured node shape (capacity/taints validated at load). capacity, err := cfg.Capacity() @@ -115,10 +120,6 @@ func run(ctx context.Context, configPath, runtimeBinary string) error { if err != nil { return fmt.Errorf("node taints: %w", err) } - internalIP := cfg.Node.InternalIP - if internalIP == "" { - internalIP = detectInternalIP() - } nodeSpec := provider.NodeSpec{ KubeletVersion: version.Version, OS: cfg.Node.OS, @@ -197,6 +198,28 @@ func run(ctx context.Context, configPath, runtimeBinary string) error { return nil } + // Enable coordinated Pod IPAM now that Kubernetes has assigned this node a + // Pod CIDR, recovering any existing allocations before Pods are reconciled. + if err := setupIPAM(ctx, cfg, clientset, p); err != nil { + return fmt.Errorf("setup pod IPAM: %w", err) + } + + // Bring up the cross-host WireGuard mesh (if enabled) so remote Pod CIDRs are + // routed through encrypted tunnels before Pods start landing on this node. + stopMesh, err := setupMesh(ctx, cfg) + if err != nil { + return fmt.Errorf("setup mesh: %w", err) + } + defer stopMesh() + + // Start the host Pod network path so each micro-VM is reachable at its Pod IP + // across the mesh, and attach it to the provider before Pods are reconciled. + stopPodNet, err := setupPodNetwork(ctx, cfg, p) + if err != nil { + return fmt.Errorf("setup pod network: %w", err) + } + defer stopPodNet() + // Start the Pod lifecycle controller so scheduled Pods become micro-VMs. stopPods, err := startPodController(ctx, cfg, clientset, p, runtime.NumCPU()) if err != nil { diff --git a/cmd/macvz-kubelet/serve.go b/cmd/macvz-kubelet/serve.go index 8c12df4..a56605d 100644 --- a/cmd/macvz-kubelet/serve.go +++ b/cmd/macvz-kubelet/serve.go @@ -9,11 +9,16 @@ import ( "time" "github.com/chimerakang/macvz/pkg/config" + "github.com/chimerakang/macvz/pkg/network" + "github.com/chimerakang/macvz/pkg/network/podnet" + "github.com/chimerakang/macvz/pkg/network/wireguard" "github.com/chimerakang/macvz/pkg/provider" vknode "github.com/virtual-kubelet/virtual-kubelet/node" vkapi "github.com/virtual-kubelet/virtual-kubelet/node/api" "github.com/virtual-kubelet/virtual-kubelet/node/nodeutil" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" @@ -25,6 +30,75 @@ import ( // informerResync is how often the shared informers do a full relist. const informerResync = time.Minute +// podCIDRWaitTimeout bounds how long startup waits for Kubernetes to assign this +// node a Pod CIDR before continuing without coordinated IPAM. +const podCIDRWaitTimeout = 30 * time.Second + +// setupIPAM enables coordinated Pod IPAM for this node. The address range is the +// node's Kubernetes-assigned Spec.PodCIDR (or cfg.Node.PodCIDR when set as an +// override). It then recovers existing allocations from the API server so a +// restart neither leaks addresses nor reassigns a live Pod's IP. +// +// IPAM is best-effort: on a cluster that assigns no Pod CIDR (and with no +// override configured), it logs and returns nil so Pods still run with the Pod +// IP derived from the runtime-reported address. +func setupIPAM(ctx context.Context, cfg config.Config, clientset *kubernetes.Clientset, p *provider.Provider) error { + cidr := cfg.Node.PodCIDR + if cidr == "" { + var err error + cidr, err = waitForPodCIDR(ctx, clientset, cfg.NodeName) + if err != nil { + klog.ErrorS(err, "coordinated Pod IPAM disabled; Pod IPs will come from the runtime", + "hint", "run kube-controller-manager with --allocate-node-cidrs or set node.podCIDR") + return nil + } + } + + ipam, err := network.NewPodIPAM(cidr) + if err != nil { + return fmt.Errorf("build pod IPAM for %q: %w", cidr, err) + } + p.SetIPAM(ipam) + klog.InfoS("coordinated Pod IPAM enabled", "node", cfg.NodeName, "podCIDR", ipam.CIDR()) + + // Rebuild allocations from Kubernetes state before the Pod controller runs. + podList, err := clientset.CoreV1().Pods(corev1.NamespaceAll).List(ctx, metav1.ListOptions{ + FieldSelector: fields.OneTermEqualSelector("spec.nodeName", cfg.NodeName).String(), + }) + if err != nil { + // A failed recovery list is non-fatal: the allocator starts empty and + // re-derives IPs as Pods are (re)created. + klog.ErrorS(err, "could not list existing Pods for IPAM recovery", "node", cfg.NodeName) + return nil + } + pods := make([]*corev1.Pod, 0, len(podList.Items)) + for i := range podList.Items { + pods = append(pods, &podList.Items[i]) + } + p.RecoverAllocations(pods) + return nil +} + +// waitForPodCIDR polls the node until Kubernetes assigns its Spec.PodCIDR, which +// happens shortly after registration on clusters with node-CIDR allocation. +func waitForPodCIDR(ctx context.Context, clientset *kubernetes.Clientset, nodeName string) (string, error) { + deadlineCtx, cancel := context.WithTimeout(ctx, podCIDRWaitTimeout) + defer cancel() + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + for { + node, err := clientset.CoreV1().Nodes().Get(deadlineCtx, nodeName, metav1.GetOptions{}) + if err == nil && node.Spec.PodCIDR != "" { + return node.Spec.PodCIDR, nil + } + select { + case <-deadlineCtx.Done(): + return "", fmt.Errorf("node %q has no Spec.PodCIDR after %s", nodeName, podCIDRWaitTimeout) + case <-ticker.C: + } + } +} + // startPodController wires the Virtual Kubelet pod controller: pod/configmap/ // secret/service informers (pods filtered to this node), an event recorder, and // the controller itself driving the provider's Pod lifecycle. It returns a @@ -77,6 +151,64 @@ func startPodController(ctx context.Context, cfg config.Config, clientset *kuber return eb.Shutdown, nil } +// setupMesh brings up this node's WireGuard mesh when enabled, returning a +// cleanup func that tears it back down. The mesh encrypts and routes cross-host +// Pod traffic (issue #21); each peer's Pod CIDR becomes a route through the +// tunnel. When the mesh is disabled it is a no-op returning a no-op cleanup. +func setupMesh(ctx context.Context, cfg config.Config) (func(), error) { + if !cfg.Mesh.Enabled { + klog.InfoS("WireGuard mesh disabled; Pods are reachable only on their local node") + return func() {}, nil + } + + ifc, err := cfg.MeshInterfaceConfig() + if err != nil { + return nil, fmt.Errorf("resolve mesh config: %w", err) + } + mesh := wireguard.New(ifc) + if err := mesh.Up(ctx); err != nil { + return nil, fmt.Errorf("bring up mesh interface %q: %w", ifc.Name, err) + } + klog.InfoS("WireGuard mesh up", + "interface", mesh.InterfaceName(), + "publicKey", ifc.PrivateKey.PublicKey().String(), + "peers", mesh.Peers(), + "routes", mesh.InstalledRoutes(), + ) + + return func() { + // Tear down with a fresh context: the root ctx is already cancelled by + // the time cleanup runs during shutdown. + if err := mesh.Down(context.Background()); err != nil { + klog.ErrorS(err, "failed to tear down mesh", "interface", ifc.Name) + } + }, nil +} + +// setupPodNetwork starts the host Pod network path (when enabled) and attaches +// it to the provider so each micro-VM is reachable at its Pod IP across the mesh +// (#22). It returns a cleanup func that flushes the host rules. When disabled it +// is a no-op returning a no-op cleanup. +func setupPodNetwork(ctx context.Context, cfg config.Config, p *provider.Provider) (func(), error) { + if !cfg.PodNetwork.Enabled { + klog.InfoS("Pod network path disabled; Pods keep the runtime host-only address") + return func() {}, nil + } + + router := podnet.New(cfg.PodNetworkRouterConfig()) + if err := router.Start(ctx); err != nil { + return nil, fmt.Errorf("start pod network path: %w", err) + } + p.SetPodNetwork(router) + klog.InfoS("Pod network path started", "interface", cfg.PodNetwork.Interface) + + return func() { + if err := router.Stop(context.Background()); err != nil { + klog.ErrorS(err, "failed to stop pod network path") + } + }, nil +} + // startKubeletServer starts the HTTPS kubelet API used by `kubectl logs`/`exec`, // routing to the provider. It is a no-op (returning a no-op cleanup) when no // serving certificate is configured, mirroring upstream Virtual Kubelet: Pods @@ -95,6 +227,7 @@ func startKubeletServer(ctx context.Context, cfg config.Config, p *provider.Prov handler := vkapi.PodHandler(vkapi.PodHandlerConfig{ RunInContainer: p.RunInContainer, GetContainerLogs: p.GetContainerLogs, + PortForward: p.PortForward, GetPods: p.GetPods, GetPodsFromKubernetes: p.GetPods, }, false) diff --git a/config.example.yaml b/config.example.yaml index 652772a..b2ca854 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -35,6 +35,11 @@ node: # Node InternalIP. Leave empty to auto-detect the host's primary outbound IPv4. internalIP: "" + # Pod CIDR override for coordinated Pod IPAM. Leave empty to use the + # Kubernetes-assigned Node.Spec.PodCIDR. Only set this on clusters that do not + # allocate node CIDRs automatically, and give every MacVz node a disjoint CIDR. + podCIDR: "" + # Extra labels/annotations merged onto the node (override built-ins on clash). labels: {} annotations: {} @@ -61,3 +66,28 @@ node: kubeletPort: 10250 servingTLSCertFile: "" # e.g. /etc/macvz/pki/kubelet.crt servingTLSKeyFile: "" # e.g. /etc/macvz/pki/kubelet.key + +# Cross-host WireGuard mesh. Disabled by default; see docs/NETWORKING.md for a +# complete two-node setup and pf anchor requirements. +mesh: + enabled: false + interface: utun7 + privateKeyFile: /etc/macvz/wireguard.key + address: 10.99.0.1/32 + listenPort: 51820 + mtu: 0 + peers: + - name: mac-mini-02 + publicKey: "" + endpoint: 192.168.1.20:51820 + podCIDR: 10.244.2.0/24 + address: 10.99.0.2/32 + persistentKeepalive: 25 + +# Host Pod network path. Disabled by default; enable with mesh once the +# apple/container vmnet interface is known on the host. +podNetwork: + enabled: false + interface: bridge100 + anchor: macvz/pods + enableForwarding: true diff --git a/docs/MASTER_TASKS.md b/docs/MASTER_TASKS.md index cf4a62b..785d924 100644 --- a/docs/MASTER_TASKS.md +++ b/docs/MASTER_TASKS.md @@ -44,11 +44,11 @@ Source of truth for the phased roadmap. Phases map to GitHub **Milestones**; tas | #17 | Translate Kubernetes Pod specs into runtime workload specs | P2 | in progress | | #18 | Wire kubectl logs and exec through the runtime | P2 | in progress | | #19 | Add RBAC, manifests, and P2 MVP smoke test docs | P2 | in progress | -| #20 | Implement Kubernetes-coordinated Pod IPAM | P3 | open | -| #21 | Bring up WireGuard mesh between MacVz nodes | P3 | open | -| #22 | Connect micro-VM networking to the controllable Pod network path | P3 | open | -| #23 | Report Pod IPs and readiness so Services resolve across MacVz nodes | P3 | open | -| #24 | Implement kubectl port-forward for MacVz-backed Pods | P3 | open | +| #20 | Implement Kubernetes-coordinated Pod IPAM | P3 | closed | +| #21 | Bring up WireGuard mesh between MacVz nodes | P3 | closed | +| #22 | Connect micro-VM networking to the controllable Pod network path | P3 | closed | +| #23 | Report Pod IPs and readiness so Services resolve across MacVz nodes | P3 | closed | +| #24 | Implement kubectl port-forward for MacVz-backed Pods | P3 | closed | | #25 | Implement node and pod metrics reporting | P4 | open | | #26 | Support VirtioFS-backed volumes for MacVz Pods | P4 | open | | #27 | Handle image architecture and Rosetta-for-Linux behavior | P4 | open | diff --git a/docs/NETWORKING.md b/docs/NETWORKING.md new file mode 100644 index 0000000..49bf740 --- /dev/null +++ b/docs/NETWORKING.md @@ -0,0 +1,502 @@ +# MacVz Networking + +This document describes MacVz cross-host networking (P3): + +- [Pod IPAM (MVP)](#pod-ipam-mvp) — issue #20. +- [WireGuard mesh (MVP)](#wireguard-mesh-mvp) — issue #21. +- [Micro-VM network attach (MVP)](#micro-vm-network-attach-mvp) — issue #22. +- [Pod status & Service resolution (MVP)](#pod-status--service-resolution-mvp) — issue #23. +- [Port-forward (MVP)](#port-forward-mvp) — issue #24. +- [End-to-end: a Service across two Macs](#end-to-end-a-service-across-two-macs). + +Together these give a Pod on one Mac an L3 path to a Pod hosted on another Mac: +IPAM assigns the address, the mesh routes the CIDR between hosts, the network +attach connects each micro-VM to that path, and the status reporting publishes +the address and readiness so Kubernetes Services resolve normally. + +## Pod IPAM (MVP) + +This section describes the MVP Pod IP address management (IPAM) behavior for +MacVz, delivered in issue #20. + +## Allocation model + +MacVz does **not** invent a new cluster-wide IPAM coordinator. It reuses state +Kubernetes already owns: + +- When `kube-controller-manager` runs with `--allocate-node-cidrs=true` and a + `--cluster-cidr`, Kubernetes assigns every node a **disjoint** `Spec.PodCIDR` + (e.g. `10.244.1.0/24` on one Mac, `10.244.2.0/24` on another). +- Each `macvz-kubelet` allocates Pod IPs **only** from its own node's PodCIDR. + +Because the per-node CIDRs are disjoint and Kubernetes-owned, two MacVz nodes can +never produce the same Pod IP for different Pods. Collision avoidance is a +property of the CIDR partitioning, not of any peer-to-peer negotiation. + +### Within a node + +`pkg/network.PodIPAM` hands out host addresses from the node's CIDR in order: + +- The **network address** (offset 0) is reserved. +- The **first host address** (offset 1) is reserved as the gateway, matching + typical CNI bridge conventions; the data path using it is wired in #22. +- For IPv4, the **broadcast address** (last) is reserved. +- Remaining addresses are allocated to Pods. IPv6 CIDRs are supported. + +Allocation is keyed by `namespace/name` and is **idempotent**: Virtual Kubelet +may re-issue `CreatePod`, and the same Pod always gets the same IP. Released +addresses return to the pool and are reused. + +## Lifecycle + +- **Create** — `Provider.CreatePod` allocates an IP and records it as the Pod's + `status.PodIP`. If the Pod fails to start (or is a terminal failure such as an + architecture mismatch), the IP is released so it is not leaked. +- **Delete** — `Provider.DeletePod` releases the IP back to the pool. +- **Restart recovery** — on startup, `macvz-kubelet` lists the Pods assigned to + this node and calls `Provider.RecoverAllocations`, which **reserves** the + `status.PodIP` already recorded in Kubernetes. A restarted provider therefore + neither leaks addresses nor reassigns a live Pod's IP. Kubernetes is the + durable record; the in-memory allocator is rebuilt from it. + +## Configuration + +| Field | Default | Meaning | +| --- | --- | --- | +| `node.podCIDR` | _(empty)_ | Override for the Pod CIDR. When empty, MacVz waits for and uses the Kubernetes-assigned `Node.Spec.PodCIDR`. | + +Set `node.podCIDR` only on clusters that do **not** allocate node CIDRs; you must +then give each node a disjoint range manually to preserve collision avoidance. + +### Graceful degradation + +If no Pod CIDR is available (node-CIDR allocation is off and no override is set), +coordinated IPAM is disabled with a log message and Pods still run — the +`status.PodIP` is then derived from the runtime-reported address. This keeps +single-host development working without cluster-wide IPAM. + +### Pod IPAM tests + +- `pkg/network/ipam_test.go` — allocation, release/reuse, idempotency, reserve + (restart recovery), exhaustion, out-of-range rejection, IPv6, concurrency, and + the cross-node collision-avoidance property. +- `pkg/provider/ipam_test.go` — Pod IP assignment on create, release on delete, + release on failure, recovery from existing Kubernetes Pods, and the no-IPAM + fallback to the runtime-reported address. + +## WireGuard mesh (MVP) + +This section describes the encrypted host-to-host mesh that carries Pod traffic +between Macs, delivered in issue #21 (`pkg/network/wireguard`). + +### Model + +Each Mac runs **one WireGuard interface**. Every other MacVz node is a *peer*, +and a peer's `AllowedIPs` is its **Pod CIDR** (from [Pod IPAM](#pod-ipam-mvp)) +plus its mesh address. Traffic for a remote Pod is therefore encrypted and +tunnelled to the Mac that hosts it, and a **host route** for that CIDR is +installed through the interface so the kernel forwards Pod-bound packets into the +tunnel. + +macOS has no in-kernel WireGuard, so MacVz drives the userspace toolchain from +Homebrew's `wireguard-tools` (`wg`, `wireguard-go`) plus the system `route` and +`ifconfig`. Bring-up runs, in order: + +1. `wireguard-go ` — create the userspace `utun` interface. +2. `wg setconf /dev/stdin` — apply keys and peers (config via stdin, no + file on disk). +3. `ifconfig inet alias` + `ifconfig up` — assign + the mesh address and bring the link up. +4. `route -q -n add -inet -interface ` — one route per peer CIDR. + +Interface creation and route installation tolerate "already exists" / "File +exists", so bring-up is **idempotent** and safe to re-run. + +### Reconcile without restart + +When nodes join or leave, `Mesh.Sync` re-applies the peer set with +`wg syncconf` (which adds/removes peers atomically) and adds/removes only the +affected host routes — the interface is never torn down, so unrelated peers keep +their tunnels. `Mesh.Down` removes routes and destroys the interface on shutdown +(best-effort). + +### Peer identity & keys + +Each node's private key lives in `mesh.privateKeyFile`, generated (mode 0600) on +first start if absent, giving the node a **stable identity across restarts** +without keys ever appearing in config or git. Share each node's **public key** +(logged at startup, or `wg pubkey < key`) with its peers. + +### Prerequisites + +```sh +brew install wireguard-tools # provides wg, wg-quick, wireguard-go +``` + +`wireguard-go`, `ifconfig`, and `route` require elevated privileges to create +interfaces and edit the routing table; run `macvz-kubelet` accordingly. + +### Configuration + +Add a `mesh` stanza to each node's config. The mesh is **disabled by default** +so single-host development needs no setup. + +| Field | Meaning | +| --- | --- | +| `mesh.enabled` | Turn the mesh on. | +| `mesh.interface` | Interface name to manage (e.g. `utun7`). | +| `mesh.privateKeyFile` | Path to this node's private key (auto-generated if absent). | +| `mesh.address` | This node's mesh address, CIDR (e.g. `10.99.0.1/32`). | +| `mesh.listenPort` | WireGuard UDP port (default `51820`). | +| `mesh.mtu` | Optional interface MTU. | +| `mesh.peers[]` | `name`, `publicKey`, `endpoint` (host:port), `podCIDR`, `address`, `persistentKeepalive`. | + +### Two-node example + +**mac-01** (`10.99.0.1`, Pod CIDR `10.244.1.0/24`): + +```yaml +mesh: + enabled: true + interface: utun7 + privateKeyFile: /var/lib/macvz/wg.key + address: 10.99.0.1/32 + listenPort: 51820 + peers: + - name: mac-02 + publicKey: + endpoint: 192.168.1.20:51820 + podCIDR: 10.244.2.0/24 + address: 10.99.0.2/32 + persistentKeepalive: 25 +``` + +**mac-02** (`10.99.0.2`, Pod CIDR `10.244.2.0/24`): + +```yaml +mesh: + enabled: true + interface: utun7 + privateKeyFile: /var/lib/macvz/wg.key + address: 10.99.0.2/32 + listenPort: 51820 + peers: + - name: mac-01 + publicKey: + endpoint: 192.168.1.10:51820 + podCIDR: 10.244.1.0/24 + address: 10.99.0.1/32 + persistentKeepalive: 25 +``` + +### Verifying + +```sh +wg show utun7 # handshake established with the peer +netstat -rn -f inet | grep utun7 # routes for remote Pod CIDRs are present +ping 10.99.0.2 # reach the peer across the tunnel +``` + +### Mesh tests + +- `pkg/network/wireguard/key_test.go` — key generation/clamping, base64 round + trip, public-key derivation, and persistent load-or-create. +- `pkg/network/wireguard/config_test.go` — config validation, `wg-quick` vs `wg` + rendering, deterministic output, and route-target de-duplication. +- `pkg/network/wireguard/mesh_test.go` — bring-up command sequence, idempotent + tolerance of benign errors, fatal-error propagation, peer/route reconcile via + `Sync`, and best-effort `Down`. +- `pkg/config/mesh_test.go` — mesh config validation and translation into the + `wireguard` package's interface config. + +## Micro-VM network attach (MVP) + +This section describes how each `apple/container` micro-VM is connected to the +Pod network path, delivered in issue #22 (`pkg/network/podnet`). It is the piece +that makes a Pod reachable at its [assigned Pod IP](#pod-ipam-mvp) and routable +across the [mesh](#wireguard-mesh-mvp). + +### The apple/container constraint + +`apple/container` attaches each micro-VM to a **vmnet-backed** network and gives +it a **host-only address** (e.g. `192.168.64.x`) over DHCP. The CLI does **not** +let MacVz push an arbitrary guest IP, so a micro-VM cannot natively own its +Kubernetes Pod IP. MacVz bridges this gap on the host rather than inside the +guest. + +### Chosen path: host-side 1:1 NAT (pf `binat`) + +For each Pod, MacVz programs a packet-filter `binat` (bidirectional NAT) rule in +a dedicated anchor (`macvz/pods`) that maps the Pod's assigned Pod IP to the +micro-VM's host-only address: + +``` +binat on from to any -> +``` + +- **Inbound** — packets the mesh delivers for the Pod IP (the node's Pod CIDR is + routed here, see the mesh section) are DNAT'd to the VM address and forwarded + onto the vmnet interface. +- **Outbound** — packets the VM sends are SNAT'd to appear to originate from the + Pod IP; the route for a remote Pod CIDR (installed by the mesh) carries them + into the WireGuard tunnel. + +IPv4 forwarding is enabled so the host routes between the mesh interface and the +vmnet interface. The result satisfies the acceptance criteria: each Pod is +addressed by its **MacVz Pod IP** (not an opaque host-only address), and a Pod +on one Mac reaches a Pod hosted on another Mac at **L3**. + +The provider observes the VM's host-only address from the runtime once the guest +has acquired DHCP, then attaches the mapping; it detaches on Pod deletion. The +anchor ruleset is regenerated wholesale and loaded atomically (`pfctl -a +macvz/pods -f -`) on every change. + +### Alternative considered: gvisor-tap-vsock + +A fully userspace path — attaching the guest over `vsock`/a file handle and +terminating its traffic in a userspace network stack (gvisor-tap-vsock style) — +would let the Pod own its IP directly and avoid `pf`/root. It requires a guest +agent and a userspace gateway process, so it is **deferred past the MVP**; the +host-NAT path above needs no guest changes and works with stock +`apple/container` images. + +### Prerequisites + +`pf` must evaluate the MacVz anchor. Reference it from `/etc/pf.conf` once: + +``` +nat-anchor "macvz/pods/*" +rdr-anchor "macvz/pods/*" +binat-anchor "macvz/pods/*" +anchor "macvz/pods/*" +``` + +`pfctl` and `sysctl` require elevated privileges. Identify the vmnet interface +backing the micro-VMs (commonly `bridge100`) with `ifconfig` and set it as +`podNetwork.interface`. + +### Configuration + +| Field | Meaning | +| --- | --- | +| `podNetwork.enabled` | Turn the host Pod network path on. | +| `podNetwork.interface` | vmnet interface the micro-VMs attach to (e.g. `bridge100`). | +| `podNetwork.anchor` | pf anchor to manage (default `macvz/pods`). | +| `podNetwork.enableForwarding` | Enable IPv4 forwarding (default `true`). | + +```yaml +podNetwork: + enabled: true + interface: bridge100 +``` + +### Verifying + +```sh +sudo pfctl -a macvz/pods -s nat # binat rules, one per Pod +sysctl net.inet.ip.forwarding # = 1 +# From a Pod on mac-01, reach a Pod hosted on mac-02 by its Pod IP: +kubectl exec -- ping -c1 +``` + +### Failure modes + +- **No binat rules appear** — the anchor is not referenced from `pf.conf`, or pf + is disabled. Add the anchor hooks above and confirm `pfctl -s info` shows + `Status: Enabled`. +- **`CreatePod` retries with "micro-VM address not available yet"** — the guest + has not finished DHCP. The provider polls briefly and the next reconcile + succeeds once the VM reports an address; persistent failure points at vmnet/DHCP + on the host. +- **Inbound works, outbound does not (or vice-versa)** — IP forwarding is off + (`sysctl net.inet.ip.forwarding` should be `1`) or the remote Pod CIDR has no + mesh route (`netstat -rn` should list it via the WireGuard interface). +- **Pod IP unreachable from another Mac** — verify the mesh tunnel first + (`wg show`), then that the destination node's Pod CIDR routes to it. + +### Pod network tests + +- `pkg/network/podnet/router_test.go` — Start (forwarding + pf enable, tolerating + "already enabled"), `binat` rule rendering, attach/detach anchor reloads, + endpoint validation, deterministic rendering, and anchor flush on Stop. +- `pkg/provider/podnet_test.go` — attach on create, detach on delete, rollback + + IP release when attach fails, failure when the VM IP never appears, and the + disabled-path no-op. + +## Pod status & Service resolution (MVP) + +This section describes how MacVz reports Pod status to Kubernetes so Endpoints / +EndpointSlices and Services work normally, delivered in issue #23 +(`pkg/provider`). + +### What the provider publishes + +`Provider.reconcileStatus` builds the `PodStatus` that Virtual Kubelet pushes to +the API server: + +- **`status.podIP` and `status.podIPs`** — both are set from the + [assigned Pod IP](#pod-ipam-mvp). The EndpointSlice controller reads + `podIPs`, so populating it is what makes a MacVz-backed Pod appear as a usable + Service endpoint. (Without an IPAM allocation, the runtime-reported address is + used as a fallback.) +- **`status.hostIP` / `status.hostIPs`** — the node's reachable address, so + `kubectl get pod -o wide` and topology-aware routing resolve the hosting Mac. +- **Conditions** — `Initialized`, `ContainersReady`, and `Ready`. + +### Readiness gates endpoint membership + +A Pod is reported `Ready` only when it is **both running and addressable** — the +phase is `Running` *and* a Pod IP is present. A running Pod without an address is +held `Ready=False` with reason `PodNetworkNotReady`, and a transient runtime +error yields `Ready=False` with reason `RuntimeStatusError`. This guarantees the +EndpointSlice controller never adds an unreachable Pod to a Service, so traffic +is only sent to Pods that can actually receive it. + +### DNS / service discovery + +No MacVz-specific work is needed: once Pods report correct `podIPs` and `Ready` +conditions, the in-cluster EndpointSlice controller, kube-proxy, and CoreDNS +treat MacVz-backed Pods like any other. A `ClusterIP` Service's DNS name resolves +to the Service IP, and kube-proxy load-balances to the ready Pod IPs — which the +[mesh](#wireguard-mesh-mvp) and [network attach](#micro-vm-network-attach-mvp) +make reachable across Macs. + +### Status tests + +- `pkg/provider/status_test.go` — `podIP`/`podIPs`/`hostIP` population, readiness + True when running with an IP, `Ready=False`/`PodNetworkNotReady` when running + without an IP, and `Ready=False`/`RuntimeStatusError` on a runtime error. + +## Port-forward (MVP) + +This section describes `kubectl port-forward` for MacVz-backed Pods, delivered in +issue #24 (`pkg/provider`). + +### How it works + +`kubectl port-forward` opens a stream to the node's kubelet API server (the same +HTTPS endpoint that serves `logs`/`exec`), which Virtual Kubelet routes to +`Provider.PortForward`. Because the kubelet runs on the **same Mac** as the Pod's +micro-VM, it dials the VM's address directly — the host can always reach the +guest's vmnet address, so port-forward works **with or without** the cross-host +[Pod network path](#micro-vm-network-attach-mvp) or [mesh](#wireguard-mesh-mvp). +Bytes are copied bidirectionally between the Kubernetes stream and the TCP +connection until either side closes or the request is cancelled; both copy +goroutines and the connection are always reaped, so closing the forward leaks +nothing. + +The target address is the live runtime-reported micro-VM address, falling back +to the address observed when the Pod was attached to the network path. + +### Error behavior + +- **Unknown Pod** → `NotFound` (kubectl reports the Pod does not exist). +- **Non-running Pod** → a clear "container is not running" error, not `NotFound`. +- **Nothing listening on the port** → the dial is refused and the error surfaces + to kubectl, matching normal Kubernetes behavior. +- **Out-of-range port** → rejected before dialing. + +### Requirements + +Port-forward uses the kubelet API server, so it needs the serving TLS cert/key +(`node.servingTLSCertFile`/`servingTLSKeyFile`) just like `logs`/`exec`; without +them the server is not started and these subcommands are unavailable. + +### Smoke test + +```sh +# A Pod with a process listening on 8080 inside the micro-VM: +kubectl port-forward pod/ 18080:8080 & +curl -s localhost:18080 # reaches the in-Pod process +kill %1 # closing the forward cleans up cleanly +``` + +### Port-forward tests + +- `pkg/provider/portforward_test.go` — byte proxying through a loopback target, + clean return on stream close and on context cancellation (no goroutine leak), + and the unknown-Pod / non-running / bad-port / nothing-listening error paths. + +## End-to-end: a Service across two Macs + +A smoke test that exercises #20–#23 together: a Service backed by Pods on two +different Macs, reached from a client Pod. + +### Prerequisites + +- Two `macvz-kubelet` nodes registered and `Ready` (`kubectl get nodes`), each + with a Kubernetes-assigned `Spec.PodCIDR`. +- The [WireGuard mesh](#wireguard-mesh-mvp) up between them (`wg show` shows a + handshake), and the [Pod network path](#micro-vm-network-attach-mvp) enabled on + both (`podNetwork.enabled: true`). + +### 1. Deploy Pods on both Macs + +Schedule one replica to each node (the example pins by hostname; adjust to your +node names). MacVz Pods must tolerate the provider taint. + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: { name: hello } +spec: + replicas: 2 + selector: { matchLabels: { app: hello } } + template: + metadata: { labels: { app: hello } } + spec: + tolerations: + - key: virtual-kubelet.io/provider + operator: Exists + effect: NoSchedule + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: { matchLabels: { app: hello } } + containers: + - name: hello + image: + ports: [{ containerPort: 8080 }] +--- +apiVersion: v1 +kind: Service +metadata: { name: hello } +spec: + selector: { app: hello } + ports: [{ port: 80, targetPort: 8080 }] +``` + +### 2. Verify Pod IPs and endpoints + +```sh +# (#20/#23) Pods show MacVz-assigned Pod IPs from each node's CIDR, on both Macs: +kubectl get pods -l app=hello -o wide + +# (#23) The Service has an EndpointSlice listing both ready Pod IPs: +kubectl get endpointslices -l kubernetes.io/service-name=hello -o wide +``` + +Expect two `Ready` endpoints, one Pod IP from each node's Pod CIDR. + +### 3. Reach the Service across the mesh + +```sh +# From a client Pod (on either Mac), the Service resolves and load-balances to +# both backends — including the one hosted on the *other* Mac (traffic crosses +# the WireGuard tunnel, #21/#22): +kubectl run client --rm -it --restart=Never \ + --overrides='{"spec":{"tolerations":[{"key":"virtual-kubelet.io/provider","operator":"Exists"}]}}' \ + --image= -- sh -c 'for i in $(seq 10); do curl -s hello; done' +``` + +Hitting the Service repeatedly should reach Pods on both Macs. + +### Troubleshooting + +- **An endpoint is missing / `NotReady`** — check the Pod's conditions + (`kubectl describe pod`): `PodNetworkNotReady` means no Pod IP yet (IPAM/ + network attach), `RuntimeStatusError` means the runtime probe failed. +- **Endpoint present but unreachable cross-host** — this is a data-path issue, + not status: verify the [mesh](#wireguard-mesh-mvp) tunnel and the + [network attach](#micro-vm-network-attach-mvp) failure modes. diff --git a/go.mod b/go.mod index 97c3b39..0e2526d 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.25.8 require ( github.com/virtual-kubelet/virtual-kubelet v1.12.0 + golang.org/x/crypto v0.53.0 gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.35.0 k8s.io/apimachinery v0.35.0 @@ -82,14 +83,13 @@ require ( go.uber.org/zap v1.27.0 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/crypto v0.51.0 // indirect golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect golang.org/x/net v0.55.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/sync v0.20.0 // indirect - golang.org/x/sys v0.45.0 // indirect - golang.org/x/term v0.43.0 // indirect - golang.org/x/text v0.37.0 // indirect + golang.org/x/sync v0.21.0 // indirect + golang.org/x/sys v0.46.0 // indirect + golang.org/x/term v0.44.0 // indirect + golang.org/x/text v0.38.0 // indirect golang.org/x/time v0.14.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260610212136-7ab31c22f7ad // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260610212136-7ab31c22f7ad // indirect diff --git a/go.sum b/go.sum index 4a6d1e6..9286080 100644 --- a/go.sum +++ b/go.sum @@ -258,8 +258,8 @@ go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= -golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= +golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= +golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= @@ -268,8 +268,8 @@ golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvx golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= -golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= +golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= +golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -289,20 +289,20 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= -golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= -golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= +golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= -golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= +golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -313,8 +313,8 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= -golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= +golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= +golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pkg/config/config.go b/pkg/config/config.go index fae1698..a11be37 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -4,6 +4,7 @@ package config import ( "fmt" + "net" "os" "time" @@ -39,6 +40,14 @@ type Config struct { // Node describes the shape this Mac advertises as a Kubernetes node: // capacity, addresses, labels, and scheduling taints. Node NodeConfig `yaml:"node"` + + // Mesh configures the cross-host WireGuard mesh (P3). Disabled by default so + // single-host development needs no networking setup. + Mesh MeshConfig `yaml:"mesh"` + + // PodNetwork configures the host path that makes each micro-VM reachable at + // its assigned Pod IP across the mesh (P3, #22). Disabled by default. + PodNetwork PodNetworkConfig `yaml:"podNetwork"` } // NodeConfig is the configurable shape of the virtual node registered with @@ -63,6 +72,12 @@ type NodeConfig struct { // detects the host's primary outbound IPv4 at startup. InternalIP string `yaml:"internalIP"` + // PodCIDR overrides the address range used for coordinated Pod IPAM. When + // empty, the kubelet uses the Pod CIDR that Kubernetes assigns to this node + // (Node.Spec.PodCIDR). Set it only on clusters that do not allocate node + // CIDRs, where each node must be given a disjoint range manually. + PodCIDR string `yaml:"podCIDR"` + // Labels and Annotations are merged onto the built-in node metadata. User // entries override built-ins on key collision. Labels map[string]string `yaml:"labels"` @@ -294,5 +309,16 @@ func (c Config) Validate() error { if (c.Node.ServingTLSCertFile == "") != (c.Node.ServingTLSKeyFile == "") { return fmt.Errorf("node.servingTLSCertFile and node.servingTLSKeyFile must be set together") } + if c.Node.PodCIDR != "" { + if _, _, err := net.ParseCIDR(c.Node.PodCIDR); err != nil { + return fmt.Errorf("node.podCIDR %q is not a valid CIDR: %w", c.Node.PodCIDR, err) + } + } + if err := c.validateMesh(); err != nil { + return err + } + if err := c.validatePodNetwork(); err != nil { + return err + } return nil } diff --git a/pkg/config/mesh.go b/pkg/config/mesh.go new file mode 100644 index 0000000..ea782cb --- /dev/null +++ b/pkg/config/mesh.go @@ -0,0 +1,144 @@ +package config + +import ( + "fmt" + "net" + + "github.com/chimerakang/macvz/pkg/network/wireguard" +) + +// MeshConfig configures this node's WireGuard mesh interface and its peers. The +// mesh is opt-in: when Enabled is false the kubelet skips all networking setup, +// keeping single-host development zero-config. +type MeshConfig struct { + // Enabled turns the cross-host WireGuard mesh on. + Enabled bool `yaml:"enabled"` + + // Interface is the WireGuard interface name to manage (e.g. "utun7"). + Interface string `yaml:"interface"` + + // PrivateKeyFile holds this node's WireGuard private key (base64). It is + // generated on first start if absent, giving the node a stable identity + // without keys ever living in this config or in git. + PrivateKeyFile string `yaml:"privateKeyFile"` + + // Address is this node's address on the mesh network in CIDR form + // (e.g. "10.99.0.1/32"). + Address string `yaml:"address"` + + // ListenPort is the UDP port WireGuard listens on. + ListenPort int `yaml:"listenPort"` + + // MTU optionally overrides the interface MTU (0 keeps the default). + MTU int `yaml:"mtu"` + + // Peers are the other MacVz nodes in the mesh. + Peers []MeshPeerConfig `yaml:"peers"` +} + +// MeshPeerConfig describes one remote node in the mesh. +type MeshPeerConfig struct { + // Name is the peer's node name (for diagnostics). + Name string `yaml:"name"` + // PublicKey is the peer's WireGuard public key (base64). + PublicKey string `yaml:"publicKey"` + // Endpoint is the peer's reachable "host:port". May be empty for a peer that + // initiates the connection from behind NAT. + Endpoint string `yaml:"endpoint"` + // PodCIDR is the peer's Kubernetes-assigned Pod CIDR, routed through the + // tunnel so traffic to its Pods is encrypted and delivered to that node. + PodCIDR string `yaml:"podCIDR"` + // Address is the peer's mesh address in CIDR form (e.g. "10.99.0.2/32"), + // also routed through the tunnel. + Address string `yaml:"address"` + // PersistentKeepalive, in seconds, keeps NAT mappings alive (0 disables it). + PersistentKeepalive int `yaml:"persistentKeepalive"` +} + +// DefaultMeshListenPort is the standard WireGuard UDP port, used when a mesh is +// enabled without an explicit listenPort. +const DefaultMeshListenPort = 51820 + +// validateMesh checks mesh fields when the mesh is enabled. Disabled meshes are +// not validated so an incomplete stanza never blocks single-host startup. +func (c Config) validateMesh() error { + m := c.Mesh + if !m.Enabled { + return nil + } + if m.Interface == "" { + return fmt.Errorf("mesh.interface is required when the mesh is enabled") + } + if m.PrivateKeyFile == "" { + return fmt.Errorf("mesh.privateKeyFile is required when the mesh is enabled") + } + if _, _, err := net.ParseCIDR(m.Address); err != nil { + return fmt.Errorf("mesh.address %q is not a CIDR: %w", m.Address, err) + } + if m.ListenPort < 0 || m.ListenPort > 65535 { + return fmt.Errorf("mesh.listenPort %d out of range", m.ListenPort) + } + for i, p := range m.Peers { + if _, err := wireguard.ParseKey(p.PublicKey); err != nil { + return fmt.Errorf("mesh.peers[%d] (%q) publicKey is invalid: %w", i, p.Name, err) + } + if _, _, err := net.ParseCIDR(p.PodCIDR); err != nil { + return fmt.Errorf("mesh.peers[%d] (%q) podCIDR %q is not a CIDR: %w", i, p.Name, p.PodCIDR, err) + } + if p.Address != "" { + if _, _, err := net.ParseCIDR(p.Address); err != nil { + return fmt.Errorf("mesh.peers[%d] (%q) address %q is not a CIDR: %w", i, p.Name, p.Address, err) + } + } + if p.Endpoint != "" { + if _, _, err := net.SplitHostPort(p.Endpoint); err != nil { + return fmt.Errorf("mesh.peers[%d] (%q) endpoint %q is not host:port: %w", i, p.Name, p.Endpoint, err) + } + } + } + return nil +} + +// MeshInterfaceConfig resolves the YAML mesh config into a +// wireguard.InterfaceConfig, loading (or generating) the node's private key. +// Call only when Mesh.Enabled is true. +func (c Config) MeshInterfaceConfig() (wireguard.InterfaceConfig, error) { + m := c.Mesh + priv, err := wireguard.LoadOrCreateKey(m.PrivateKeyFile) + if err != nil { + return wireguard.InterfaceConfig{}, err + } + + port := m.ListenPort + if port == 0 { + port = DefaultMeshListenPort + } + + peers := make([]wireguard.Peer, 0, len(m.Peers)) + for _, p := range m.Peers { + pub, err := wireguard.ParseKey(p.PublicKey) + if err != nil { + return wireguard.InterfaceConfig{}, fmt.Errorf("mesh peer %q: %w", p.Name, err) + } + allowed := []string{p.PodCIDR} + if p.Address != "" { + allowed = append(allowed, p.Address) + } + peers = append(peers, wireguard.Peer{ + Name: p.Name, + PublicKey: pub, + Endpoint: p.Endpoint, + AllowedIPs: allowed, + PersistentKeepalive: p.PersistentKeepalive, + }) + } + + return wireguard.InterfaceConfig{ + Name: m.Interface, + PrivateKey: priv, + Address: m.Address, + ListenPort: port, + MTU: m.MTU, + Peers: peers, + }, nil +} diff --git a/pkg/config/mesh_test.go b/pkg/config/mesh_test.go new file mode 100644 index 0000000..cf919e1 --- /dev/null +++ b/pkg/config/mesh_test.go @@ -0,0 +1,119 @@ +package config + +import ( + "path/filepath" + "testing" + + "github.com/chimerakang/macvz/pkg/network/wireguard" +) + +func enabledMesh(t *testing.T, keyPath string) MeshConfig { + t.Helper() + peerKey, err := wireguard.GenerateKey() + if err != nil { + t.Fatalf("GenerateKey: %v", err) + } + return MeshConfig{ + Enabled: true, + Interface: "utun7", + PrivateKeyFile: keyPath, + Address: "10.99.0.1/32", + ListenPort: 51820, + Peers: []MeshPeerConfig{{ + Name: "mac-02", + PublicKey: peerKey.PublicKey().String(), + Endpoint: "192.168.1.20:51820", + PodCIDR: "10.244.2.0/24", + Address: "10.99.0.2/32", + }}, + } +} + +func TestValidateMeshDisabledSkipsChecks(t *testing.T) { + c := Default() + c.Mesh = MeshConfig{Enabled: false, Address: "garbage"} // invalid, but disabled + if err := c.Validate(); err != nil { + t.Errorf("disabled mesh should not be validated, got %v", err) + } +} + +func TestValidateMeshEnabled(t *testing.T) { + c := Default() + c.Mesh = enabledMesh(t, filepath.Join(t.TempDir(), "wg.key")) + if err := c.Validate(); err != nil { + t.Errorf("valid mesh rejected: %v", err) + } +} + +func TestValidateMeshRejectsBadFields(t *testing.T) { + base := enabledMesh(t, filepath.Join(t.TempDir(), "wg.key")) + cases := map[string]func(m *MeshConfig){ + "no interface": func(m *MeshConfig) { m.Interface = "" }, + "no key file": func(m *MeshConfig) { m.PrivateKeyFile = "" }, + "bad address": func(m *MeshConfig) { m.Address = "10.99.0.1" }, + "bad port": func(m *MeshConfig) { m.ListenPort = 70000 }, + "peer bad key": func(m *MeshConfig) { m.Peers[0].PublicKey = "nope" }, + "peer bad cidr": func(m *MeshConfig) { m.Peers[0].PodCIDR = "nope" }, + "peer bad endp": func(m *MeshConfig) { m.Peers[0].Endpoint = "noport" }, + } + for name, mutate := range cases { + t.Run(name, func(t *testing.T) { + c := Default() + m := base + m.Peers = append([]MeshPeerConfig(nil), base.Peers...) + mutate(&m) + c.Mesh = m + if err := c.Validate(); err == nil { + t.Errorf("Validate() = nil, want error for %s", name) + } + }) + } +} + +func TestMeshInterfaceConfigTranslates(t *testing.T) { + keyPath := filepath.Join(t.TempDir(), "wg.key") + c := Default() + c.Mesh = enabledMesh(t, keyPath) + + ifc, err := c.MeshInterfaceConfig() + if err != nil { + t.Fatalf("MeshInterfaceConfig: %v", err) + } + if ifc.Name != "utun7" || ifc.Address != "10.99.0.1/32" || ifc.ListenPort != 51820 { + t.Errorf("interface fields not translated: %+v", ifc) + } + if ifc.PrivateKey.IsZero() { + t.Error("private key was not loaded/generated") + } + if len(ifc.Peers) != 1 { + t.Fatalf("peers = %d, want 1", len(ifc.Peers)) + } + // AllowedIPs must combine the peer's Pod CIDR and mesh address. + want := map[string]bool{"10.244.2.0/24": true, "10.99.0.2/32": true} + for _, a := range ifc.Peers[0].AllowedIPs { + if !want[a] { + t.Errorf("unexpected AllowedIP %q", a) + } + } + if len(ifc.Peers[0].AllowedIPs) != 2 { + t.Errorf("AllowedIPs = %v, want pod CIDR + address", ifc.Peers[0].AllowedIPs) + } + // The resolved config must satisfy the wireguard package's own validation. + if err := ifc.Validate(); err != nil { + t.Errorf("translated config fails wireguard validation: %v", err) + } +} + +func TestMeshInterfaceConfigDefaultsPort(t *testing.T) { + c := Default() + m := enabledMesh(t, filepath.Join(t.TempDir(), "wg.key")) + m.ListenPort = 0 + c.Mesh = m + ifc, err := c.MeshInterfaceConfig() + if err != nil { + t.Fatalf("MeshInterfaceConfig: %v", err) + } + if ifc.ListenPort != DefaultMeshListenPort { + t.Errorf("ListenPort = %d, want default %d", ifc.ListenPort, DefaultMeshListenPort) + } +} diff --git a/pkg/config/podnetwork.go b/pkg/config/podnetwork.go new file mode 100644 index 0000000..17c1d27 --- /dev/null +++ b/pkg/config/podnetwork.go @@ -0,0 +1,57 @@ +package config + +import ( + "fmt" + + "github.com/chimerakang/macvz/pkg/network/podnet" +) + +// PodNetworkConfig configures the host Pod network path that makes each +// micro-VM reachable at its assigned Pod IP across the mesh (#22). It is opt-in: +// when disabled, Pods keep apple/container's host-only address and cross-host +// routing is unavailable. +type PodNetworkConfig struct { + // Enabled turns the host Pod network path on. It is only meaningful together + // with an enabled mesh and Kubernetes-assigned Pod CIDR. + Enabled bool `yaml:"enabled"` + + // Interface is the host vmnet interface apple/container micro-VMs attach to + // (e.g. "bridge100"). The binat rules are scoped to it. + Interface string `yaml:"interface"` + + // Anchor is the pf anchor MacVz manages. Defaults to podnet.DefaultAnchor. + // The operator must reference this anchor from the main pf.conf. + Anchor string `yaml:"anchor"` + + // EnableForwarding turns on IPv4 forwarding so the host routes between the + // mesh interface and the vmnet interface. Defaults to true when enabled; + // set false only when forwarding is managed externally. + EnableForwarding *bool `yaml:"enableForwarding"` +} + +// validatePodNetwork checks the Pod network fields when enabled. +func (c Config) validatePodNetwork() error { + pn := c.PodNetwork + if !pn.Enabled { + return nil + } + if pn.Interface == "" { + return fmt.Errorf("podNetwork.interface is required when the Pod network is enabled") + } + return nil +} + +// PodNetworkRouterConfig resolves the YAML Pod network config into a +// podnet.Config. Call only when PodNetwork.Enabled is true. +func (c Config) PodNetworkRouterConfig() podnet.Config { + pn := c.PodNetwork + forwarding := true + if pn.EnableForwarding != nil { + forwarding = *pn.EnableForwarding + } + return podnet.Config{ + Interface: pn.Interface, + Anchor: pn.Anchor, + EnableForwarding: forwarding, + } +} diff --git a/pkg/config/podnetwork_test.go b/pkg/config/podnetwork_test.go new file mode 100644 index 0000000..5f812fe --- /dev/null +++ b/pkg/config/podnetwork_test.go @@ -0,0 +1,40 @@ +package config + +import "testing" + +func TestValidatePodNetworkDisabledSkipsChecks(t *testing.T) { + c := Default() + c.PodNetwork = PodNetworkConfig{Enabled: false} // no interface, but disabled + if err := c.Validate(); err != nil { + t.Errorf("disabled Pod network should not be validated, got %v", err) + } +} + +func TestValidatePodNetworkRequiresInterface(t *testing.T) { + c := Default() + c.PodNetwork = PodNetworkConfig{Enabled: true} + if err := c.Validate(); err == nil { + t.Error("enabled Pod network without an interface should fail validation") + } +} + +func TestPodNetworkRouterConfigDefaultsForwardingTrue(t *testing.T) { + c := Default() + c.PodNetwork = PodNetworkConfig{Enabled: true, Interface: "bridge100"} + rc := c.PodNetworkRouterConfig() + if rc.Interface != "bridge100" { + t.Errorf("Interface = %q, want bridge100", rc.Interface) + } + if !rc.EnableForwarding { + t.Error("EnableForwarding should default to true") + } +} + +func TestPodNetworkRouterConfigForwardingOverride(t *testing.T) { + off := false + c := Default() + c.PodNetwork = PodNetworkConfig{Enabled: true, Interface: "bridge100", EnableForwarding: &off} + if c.PodNetworkRouterConfig().EnableForwarding { + t.Error("EnableForwarding override to false was not honored") + } +} diff --git a/pkg/network/doc.go b/pkg/network/doc.go index 80153f3..5619d8a 100644 --- a/pkg/network/doc.go +++ b/pkg/network/doc.go @@ -1,4 +1,10 @@ -// Package network will provide cross-host Pod connectivity: a WireGuard mesh -// between Macs plus Pod IPAM coordinated through Kubernetes, so Pods scheduled -// on different nodes reach each other directly. Implemented in milestone P3. +// Package network provides cross-host Pod connectivity for MacVz: a WireGuard +// mesh between Macs (P3, #21) plus Pod IPAM coordinated through Kubernetes, so +// Pods scheduled on different nodes reach each other directly. +// +// The Pod IPAM piece (#20) is implemented here as PodIPAM. Each macvz-kubelet +// allocates Pod IPs only from its own node's Kubernetes-assigned Spec.PodCIDR; +// because Kubernetes hands every node a disjoint CIDR, two nodes can never +// allocate the same address for different Pods. See PodIPAM for the full MVP +// allocation model, and docs/NETWORKING.md for operational behavior. package network diff --git a/pkg/network/ipam.go b/pkg/network/ipam.go new file mode 100644 index 0000000..8a934ed --- /dev/null +++ b/pkg/network/ipam.go @@ -0,0 +1,240 @@ +package network + +import ( + "errors" + "fmt" + "math/big" + "net" + "sync" +) + +// Pod IPAM — MVP allocation model +// +// Cross-host clusters must not let two Macs hand out the same Pod IP. Rather than +// inventing a new coordination layer, MacVz leans on state Kubernetes already +// owns: when kube-controller-manager runs with --allocate-node-cidrs, every Node +// is assigned a disjoint Spec.PodCIDR. Each macvz-kubelet allocates Pod IPs only +// from its own node's PodCIDR, so two nodes can never produce the same address +// for different Pods — collision avoidance is a property of the (Kubernetes-owned) +// CIDR partitioning, not of any peer-to-peer agreement. +// +// Within a single node, PodIPAM hands out host addresses from the CIDR in order, +// skipping the network address (and, for IPv4, the broadcast address). The first +// usable host address is treated as the gateway and reserved, mirroring typical +// CNI bridge conventions; the cross-host data path that uses it is wired in #22. +// +// Allocation is keyed by "namespace/name" and is idempotent: allocating the same +// key twice returns the same IP, so Virtual Kubelet's at-least-once CreatePod is +// safe. Released addresses are returned to the pool and reused. Durability lives +// in Kubernetes: a restarted provider rebuilds its in-memory allocations by +// reserving the PodIPs already recorded on the API server (see Reserve), so a +// restart neither leaks addresses nor reassigns a live Pod's IP. +// +// PodIPAM is safe for concurrent use. + +// ErrExhausted is returned when the CIDR has no free host address left. +var ErrExhausted = errors.New("network: pod CIDR exhausted") + +// ErrOutOfRange is returned by Reserve when the IP is not a usable host address +// within the managed CIDR. +var ErrOutOfRange = errors.New("network: address outside managed CIDR") + +// PodIPAM allocates Pod IPs from a single node's Kubernetes-assigned Pod CIDR. +type PodIPAM struct { + ipnet *net.IPNet + bytes int // address width in bytes (4 for IPv4, 16 for IPv6) + + base *big.Int // network address as an integer + first *big.Int // first usable host offset (inclusive), relative to base + last *big.Int // last usable host offset (inclusive), relative to base + + mu sync.Mutex + byKey map[string]string // key -> ip + byIP map[string]string // ip -> key + cursor *big.Int // next offset to try (relative to base) +} + +// NewPodIPAM builds an allocator over the given CIDR (e.g. a node's +// Spec.PodCIDR, "10.244.1.0/24"). It reserves the network address, the IPv4 +// broadcast address, and the first usable host address (the gateway). +func NewPodIPAM(cidr string) (*PodIPAM, error) { + ip, ipnet, err := net.ParseCIDR(cidr) + if err != nil { + return nil, fmt.Errorf("network: parse pod CIDR %q: %w", cidr, err) + } + // Normalize to the canonical address width: IPv4 CIDRs parse to a 4-byte + // mask, IPv6 to 16. Use the mask length as the source of truth. + width := len(ipnet.Mask) + isIPv4 := ip.To4() != nil && width == net.IPv4len + + ones, bits := ipnet.Mask.Size() + hostBits := bits - ones + size := new(big.Int).Lsh(big.NewInt(1), uint(hostBits)) // 2^hostBits + + base := ipToBig(ipnet.IP) + + // Usable host range: skip the network address (offset 0). Reserve the first + // host (offset 1) as the gateway. For IPv4 also skip the broadcast (last). + first := big.NewInt(2) + last := new(big.Int).Sub(size, big.NewInt(1)) // size-1 (last offset) + if isIPv4 { + last.Sub(last, big.NewInt(1)) // skip broadcast -> size-2 + } + + a := &PodIPAM{ + ipnet: ipnet, + bytes: width, + base: base, + first: first, + last: last, + byKey: map[string]string{}, + byIP: map[string]string{}, + cursor: new(big.Int).Set(first), + } + return a, nil +} + +// CIDR returns the managed CIDR in canonical form. +func (a *PodIPAM) CIDR() string { return a.ipnet.String() } + +// Allocate returns the Pod IP for key, assigning a fresh one if needed. It is +// idempotent: repeated calls for the same key return the same IP. ErrExhausted +// is returned when no host address is free. +func (a *PodIPAM) Allocate(key string) (string, error) { + a.mu.Lock() + defer a.mu.Unlock() + + if ip, ok := a.byKey[key]; ok { + return ip, nil + } + if a.first.Cmp(a.last) > 0 { + return "", ErrExhausted + } + + span := new(big.Int).Sub(a.last, a.first) + span.Add(span, big.NewInt(1)) // number of usable offsets + off := new(big.Int).Set(a.cursor) + for i := new(big.Int).SetInt64(0); i.Cmp(span) < 0; i.Add(i, big.NewInt(1)) { + if off.Cmp(a.last) > 0 { + off.Set(a.first) // wrap + } + ip := a.offsetIP(off) + if _, taken := a.byIP[ip]; !taken { + a.byKey[key] = ip + a.byIP[ip] = key + a.cursor.Add(off, big.NewInt(1)) // advance past the one we took + return ip, nil + } + off.Add(off, big.NewInt(1)) + } + return "", ErrExhausted +} + +// Reserve binds key to a specific IP, used to rebuild allocator state from +// Kubernetes on restart. It is idempotent when (key, ip) already match. It +// returns ErrOutOfRange if ip is not a usable host address in the CIDR, or an +// error if the IP is already held by a different key. +func (a *PodIPAM) Reserve(key, ip string) error { + parsed := net.ParseIP(ip) + if parsed == nil { + return fmt.Errorf("network: reserve %q: invalid IP %q", key, ip) + } + canon := canonIP(parsed, a.bytes) + if canon == "" || !a.usable(parsed) { + return fmt.Errorf("network: reserve %q at %s: %w", key, ip, ErrOutOfRange) + } + + a.mu.Lock() + defer a.mu.Unlock() + + if owner, ok := a.byIP[canon]; ok { + if owner == key { + return nil + } + return fmt.Errorf("network: reserve %s for %q: already held by %q", canon, key, owner) + } + if existing, ok := a.byKey[key]; ok && existing != canon { + return fmt.Errorf("network: reserve %q at %s: key already holds %s", key, canon, existing) + } + a.byKey[key] = canon + a.byIP[canon] = key + return nil +} + +// Release frees the IP held by key, returning it to the pool. Releasing an +// unknown key is a no-op, so deletion is idempotent. +func (a *PodIPAM) Release(key string) { + a.mu.Lock() + defer a.mu.Unlock() + ip, ok := a.byKey[key] + if !ok { + return + } + delete(a.byKey, key) + delete(a.byIP, ip) +} + +// IP returns the IP currently held by key, or "" if none. +func (a *PodIPAM) IP(key string) string { + a.mu.Lock() + defer a.mu.Unlock() + return a.byKey[key] +} + +// Allocations returns a snapshot copy of the current key->IP assignments. +func (a *PodIPAM) Allocations() map[string]string { + a.mu.Lock() + defer a.mu.Unlock() + out := make(map[string]string, len(a.byKey)) + for k, v := range a.byKey { + out[k] = v + } + return out +} + +// offsetIP renders the host address at base+off as a canonical string. +func (a *PodIPAM) offsetIP(off *big.Int) string { + addr := new(big.Int).Add(a.base, off) + return bigToIP(addr, a.bytes).String() +} + +// usable reports whether ip falls within the managed CIDR's usable host range. +func (a *PodIPAM) usable(ip net.IP) bool { + if !a.ipnet.Contains(ip) { + return false + } + off := new(big.Int).Sub(ipToBig(ip), a.base) + return off.Cmp(a.first) >= 0 && off.Cmp(a.last) <= 0 +} + +// canonIP renders ip at the allocator's width, or "" if it does not fit. +func canonIP(ip net.IP, width int) string { + switch width { + case net.IPv4len: + v4 := ip.To4() + if v4 == nil { + return "" + } + return v4.String() + default: + return ip.String() + } +} + +// ipToBig converts an IP to a big.Int using its canonical byte width. +func ipToBig(ip net.IP) *big.Int { + if v4 := ip.To4(); v4 != nil { + return new(big.Int).SetBytes(v4) + } + return new(big.Int).SetBytes(ip.To16()) +} + +// bigToIP converts an integer back to an IP of the given byte width. +func bigToIP(n *big.Int, width int) net.IP { + b := n.Bytes() + if len(b) < width { + pad := make([]byte, width-len(b)) + b = append(pad, b...) + } + return net.IP(b[len(b)-width:]) +} diff --git a/pkg/network/ipam_test.go b/pkg/network/ipam_test.go new file mode 100644 index 0000000..c8d06b6 --- /dev/null +++ b/pkg/network/ipam_test.go @@ -0,0 +1,229 @@ +package network + +import ( + "errors" + "fmt" + "net" + "sync" + "testing" +) + +func mustIPAM(t *testing.T, cidr string) *PodIPAM { + t.Helper() + a, err := NewPodIPAM(cidr) + if err != nil { + t.Fatalf("NewPodIPAM(%q): %v", cidr, err) + } + return a +} + +func TestNewPodIPAMRejectsBadCIDR(t *testing.T) { + if _, err := NewPodIPAM("not-a-cidr"); err == nil { + t.Fatal("expected error for invalid CIDR") + } +} + +func TestAllocateSkipsNetworkGatewayAndBroadcast(t *testing.T) { + a := mustIPAM(t, "10.244.1.0/24") + ip, err := a.Allocate("default/p1") + if err != nil { + t.Fatalf("Allocate: %v", err) + } + // .0 network and .1 gateway are reserved, so the first handout is .2. + if ip != "10.244.1.2" { + t.Errorf("first allocation = %q, want 10.244.1.2", ip) + } + if !a.usable(net.ParseIP(ip)) { + t.Errorf("%s should be within the usable range", ip) + } + // Network, gateway, and broadcast must never be in range. + for _, bad := range []string{"10.244.1.0", "10.244.1.1", "10.244.1.255"} { + if a.usable(net.ParseIP(bad)) { + t.Errorf("%s should not be usable", bad) + } + } +} + +func TestAllocateIsIdempotentPerKey(t *testing.T) { + a := mustIPAM(t, "10.244.1.0/24") + first, err := a.Allocate("default/p1") + if err != nil { + t.Fatalf("Allocate: %v", err) + } + again, err := a.Allocate("default/p1") + if err != nil { + t.Fatalf("Allocate (retry): %v", err) + } + if first != again { + t.Errorf("idempotent Allocate returned %q then %q", first, again) + } + if got := len(a.Allocations()); got != 1 { + t.Errorf("idempotent Allocate produced %d allocations, want 1", got) + } +} + +func TestAllocateHandsOutDistinctIPs(t *testing.T) { + a := mustIPAM(t, "10.244.1.0/24") + seen := map[string]string{} + for i := 0; i < 50; i++ { + key := fmt.Sprintf("default/p%d", i) + ip, err := a.Allocate(key) + if err != nil { + t.Fatalf("Allocate %s: %v", key, err) + } + if other, dup := seen[ip]; dup { + t.Fatalf("IP %s handed to both %s and %s", ip, other, key) + } + seen[ip] = key + } +} + +func TestReleaseReturnsIPToPool(t *testing.T) { + // /29 yields exactly 5 usable hosts (offsets 2..6), so once the pool is full + // the only address a new allocation can get is the one we just released. + a := mustIPAM(t, "10.0.0.0/29") + const usable = 5 + keys := make([]string, usable) + for i := range keys { + keys[i] = fmt.Sprintf("default/p%d", i) + if _, err := a.Allocate(keys[i]); err != nil { + t.Fatalf("Allocate %s: %v", keys[i], err) + } + } + freed := a.IP(keys[2]) + a.Release(keys[2]) + if a.IP(keys[2]) != "" { + t.Error("Release did not clear the key") + } + reused, err := a.Allocate("default/new") + if err != nil { + t.Fatalf("Allocate after release: %v", err) + } + if reused != freed { + t.Errorf("freed IP %s not reused (got %s)", freed, reused) + } +} + +func TestReleaseUnknownKeyIsNoop(t *testing.T) { + a := mustIPAM(t, "10.244.1.0/24") + a.Release("default/ghost") // must not panic +} + +func TestExhaustion(t *testing.T) { + // /30 has one usable host after reserving network/gateway/broadcast. + a := mustIPAM(t, "10.0.0.0/30") + if _, err := a.Allocate("default/p1"); err != nil { + t.Fatalf("first Allocate: %v", err) + } + _, err := a.Allocate("default/p2") + if !errors.Is(err, ErrExhausted) { + t.Fatalf("expected ErrExhausted, got %v", err) + } +} + +func TestReserveRecoversState(t *testing.T) { + a := mustIPAM(t, "10.244.1.0/24") + // Simulate restart recovery: a Pod already recorded at .5 in Kubernetes. + if err := a.Reserve("default/p1", "10.244.1.5"); err != nil { + t.Fatalf("Reserve: %v", err) + } + if a.IP("default/p1") != "10.244.1.5" { + t.Errorf("Reserve did not bind key, got %q", a.IP("default/p1")) + } + // A fresh allocation must not collide with the reserved address. + for i := 0; i < 100; i++ { + ip, err := a.Allocate(fmt.Sprintf("default/x%d", i)) + if err != nil { + t.Fatalf("Allocate: %v", err) + } + if ip == "10.244.1.5" { + t.Fatalf("allocator reissued reserved IP 10.244.1.5") + } + } +} + +func TestReserveIdempotentAndConflict(t *testing.T) { + a := mustIPAM(t, "10.244.1.0/24") + if err := a.Reserve("default/p1", "10.244.1.5"); err != nil { + t.Fatalf("Reserve: %v", err) + } + // Same (key, ip) again is fine. + if err := a.Reserve("default/p1", "10.244.1.5"); err != nil { + t.Fatalf("idempotent Reserve: %v", err) + } + // Different key, same IP -> conflict. + if err := a.Reserve("default/p2", "10.244.1.5"); err == nil { + t.Error("expected conflict reserving a held IP for a different key") + } +} + +func TestReserveRejectsOutOfRange(t *testing.T) { + a := mustIPAM(t, "10.244.1.0/24") + for _, ip := range []string{"10.244.2.5", "10.244.1.1", "10.244.1.0"} { + if err := a.Reserve("default/p1", ip); !errors.Is(err, ErrOutOfRange) { + t.Errorf("Reserve(%s): expected ErrOutOfRange, got %v", ip, err) + } + } +} + +// TestCrossNodeCollisionAvoidance is the core P3 acceptance property: two nodes +// with disjoint Kubernetes-assigned PodCIDRs can never produce the same Pod IP. +func TestCrossNodeCollisionAvoidance(t *testing.T) { + nodeA := mustIPAM(t, "10.244.1.0/24") + nodeB := mustIPAM(t, "10.244.2.0/24") + + seen := map[string]string{} + for i := 0; i < 100; i++ { + for name, a := range map[string]*PodIPAM{"A": nodeA, "B": nodeB} { + ip, err := a.Allocate(fmt.Sprintf("ns/p-%s-%d", name, i)) + if err != nil { + t.Fatalf("node %s Allocate: %v", name, err) + } + if owner, dup := seen[ip]; dup { + t.Fatalf("IP %s allocated on both node %s and node %s", ip, owner, name) + } + seen[ip] = name + } + } +} + +func TestIPv6Allocation(t *testing.T) { + a := mustIPAM(t, "fd00:10:244:1::/64") + ip, err := a.Allocate("default/p1") + if err != nil { + t.Fatalf("Allocate: %v", err) + } + if net.ParseIP(ip) == nil || net.ParseIP(ip).To4() != nil { + t.Errorf("expected an IPv6 address, got %q", ip) + } + if !a.usable(net.ParseIP(ip)) { + t.Errorf("%s should be usable", ip) + } +} + +func TestConcurrentAllocateDistinct(t *testing.T) { + a := mustIPAM(t, "10.244.1.0/24") // ~253 usable + const n = 200 + var wg sync.WaitGroup + ips := make([]string, n) + errs := make([]error, n) + for i := 0; i < n; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + ips[i], errs[i] = a.Allocate(fmt.Sprintf("default/p%d", i)) + }(i) + } + wg.Wait() + + seen := map[string]bool{} + for i := 0; i < n; i++ { + if errs[i] != nil { + t.Fatalf("Allocate p%d: %v", i, errs[i]) + } + if seen[ips[i]] { + t.Fatalf("duplicate IP %s under concurrency", ips[i]) + } + seen[ips[i]] = true + } +} diff --git a/pkg/network/podnet/doc.go b/pkg/network/podnet/doc.go new file mode 100644 index 0000000..526f7cc --- /dev/null +++ b/pkg/network/podnet/doc.go @@ -0,0 +1,41 @@ +// Package podnet connects each apple/container micro-VM to the MacVz-controlled +// Pod network path (issue #22, milestone P3), so a Pod is reachable at its +// assigned MacVz Pod IP (pkg/network IPAM, #20) and can route across the +// WireGuard mesh (pkg/network/wireguard, #21). +// +// # The apple/container constraint +// +// apple/container attaches each micro-VM to a vmnet-backed network and assigns +// it a host-only address (e.g. 192.168.64.x) over DHCP. The CLI does not let us +// push an arbitrary guest IP, so the micro-VM cannot natively own its Kubernetes +// Pod IP. MacVz therefore bridges the gap on the host. +// +// # Chosen MVP path: host-side 1:1 NAT (pf binat) +// +// For each Pod, MacVz installs a packet-filter binat rule that maps the Pod's +// assigned Pod IP to the micro-VM's host-only address bidirectionally: +// +// binat on from to any -> +// +// - Inbound: packets arriving for the Pod IP (delivered to this Mac by the +// mesh, which routes the node's Pod CIDR here) are DNAT'd to the VM address +// and forwarded to the vmnet interface. +// - Outbound: packets the VM sends are SNAT'd so they appear to originate from +// the Pod IP; the route for a remote Pod CIDR (installed by the mesh) sends +// them into the WireGuard tunnel. +// +// IP forwarding is enabled so the host routes between the mesh interface and the +// vmnet interface. The result: a Pod on one Mac reaches a Pod IP hosted on +// another Mac at L3, and every Pod is addressed by its MacVz Pod IP rather than +// an opaque host-only address. +// +// The Router owns a single pf anchor whose ruleset is regenerated wholesale on +// every Attach/Detach (pf anchors load atomically). All command execution goes +// through the runner interface, so the rule generation and lifecycle are +// unit-tested against a fake without touching the host's packet filter. +// +// An alternative fully-userspace path (gvisor-tap-vsock over a vsock/file-handle +// attachment, terminating the guest's traffic in a userspace network stack) is +// evaluated in docs/NETWORKING.md; it avoids pf and root but requires a guest +// agent and is deferred past the MVP. +package podnet diff --git a/pkg/network/podnet/router.go b/pkg/network/podnet/router.go new file mode 100644 index 0000000..148eafe --- /dev/null +++ b/pkg/network/podnet/router.go @@ -0,0 +1,265 @@ +package podnet + +import ( + "context" + "errors" + "fmt" + "net" + "sort" + "strings" + "sync" + + "k8s.io/klog/v2" +) + +// Default tool names, resolved through PATH (never hardcoded absolute paths). +const ( + defaultPfctl = "pfctl" + defaultSysctl = "sysctl" +) + +// DefaultAnchor is the pf anchor MacVz owns. The operator must reference it from +// the main pf.conf (see docs/NETWORKING.md) so the kernel evaluates these rules. +const DefaultAnchor = "macvz/pods" + +// ipForwardingSysctl toggles IPv4 forwarding so the host routes between the mesh +// interface and the vmnet interface that backs the micro-VMs. +const ipForwardingSysctl = "net.inet.ip.forwarding" + +// Endpoint binds a Pod's assigned MacVz Pod IP to the host-only address that +// apple/container gave its micro-VM. +type Endpoint struct { + // PodKey is the "namespace/name" identifying the Pod. + PodKey string + // PodIP is the MacVz-assigned Pod IP (from IPAM, #20). + PodIP string + // VMIP is the micro-VM's apple/container host-only address. + VMIP string +} + +func (e Endpoint) validate() error { + if e.PodKey == "" { + return fmt.Errorf("podnet: endpoint has no pod key") + } + if net.ParseIP(e.PodIP) == nil { + return fmt.Errorf("podnet: endpoint %q has invalid PodIP %q", e.PodKey, e.PodIP) + } + if net.ParseIP(e.VMIP) == nil { + return fmt.Errorf("podnet: endpoint %q has invalid VMIP %q", e.PodKey, e.VMIP) + } + return nil +} + +// Config configures the Router. +type Config struct { + // Interface is the host vmnet interface apple/container micro-VMs attach to + // (e.g. "bridge100"). It scopes the binat rules. + Interface string + // Anchor is the pf anchor to manage. Defaults to DefaultAnchor. + Anchor string + // EnableForwarding turns on IPv4 forwarding during Start. Disable only when + // forwarding is managed externally. + EnableForwarding bool +} + +// Router programs the host packet filter so each micro-VM is reachable at its +// MacVz Pod IP. It owns one pf anchor and regenerates it wholesale on every +// change. It is safe for concurrent use. +type Router struct { + run runner + cfg Config + pfctl string + sysctl string + + mu sync.Mutex + started bool + endpoints map[string]Endpoint // keyed by PodKey +} + +// Option configures a Router. +type Option func(*Router) + +// WithRunner injects a command runner (used by tests). +func WithRunner(r runner) Option { return func(rt *Router) { rt.run = r } } + +// WithTools overrides external binary names. Empty values keep the default. +func WithTools(pfctl, sysctl string) Option { + return func(rt *Router) { + if pfctl != "" { + rt.pfctl = pfctl + } + if sysctl != "" { + rt.sysctl = sysctl + } + } +} + +// New builds a Router for the given configuration. +func New(cfg Config, opts ...Option) *Router { + if cfg.Anchor == "" { + cfg.Anchor = DefaultAnchor + } + rt := &Router{ + run: cliRunner{}, + cfg: cfg, + pfctl: defaultPfctl, + sysctl: defaultSysctl, + endpoints: map[string]Endpoint{}, + } + for _, opt := range opts { + opt(rt) + } + return rt +} + +// Start prepares the host: it enables IPv4 forwarding (when configured), ensures +// pf is enabled, and loads an empty anchor so the ruleset has a known baseline. +func (rt *Router) Start(ctx context.Context) error { + rt.mu.Lock() + defer rt.mu.Unlock() + + if rt.cfg.Interface == "" { + return fmt.Errorf("podnet: interface is required") + } + if rt.cfg.EnableForwarding { + if _, err := rt.run.run(ctx, command{Name: rt.sysctl, Args: []string{"-w", ipForwardingSysctl + "=1"}}); err != nil { + return fmt.Errorf("enable ip forwarding: %w", err) + } + } + // Enabling pf when it is already on is reported as an error we tolerate. + if _, err := rt.runTolerating(ctx, command{Name: rt.pfctl, Args: []string{"-e"}}, pfAlreadyEnabled); err != nil { + return fmt.Errorf("enable pf: %w", err) + } + rt.started = true + if err := rt.loadAnchorLocked(ctx); err != nil { + return err + } + klog.InfoS("pod network started", "interface", rt.cfg.Interface, "anchor", rt.cfg.Anchor, "forwarding", rt.cfg.EnableForwarding) + return nil +} + +// Attach maps a Pod's IP to its micro-VM's address. It is idempotent: attaching +// the same endpoint twice reloads the same ruleset. +func (rt *Router) Attach(ctx context.Context, ep Endpoint) error { + if err := ep.validate(); err != nil { + return err + } + rt.mu.Lock() + defer rt.mu.Unlock() + if !rt.started { + return fmt.Errorf("podnet: Attach %q before Start", ep.PodKey) + } + rt.endpoints[ep.PodKey] = ep + if err := rt.loadAnchorLocked(ctx); err != nil { + return fmt.Errorf("attach %q (%s -> %s): %w", ep.PodKey, ep.PodIP, ep.VMIP, err) + } + klog.InfoS("attached pod to network path", "pod", ep.PodKey, "podIP", ep.PodIP, "vmIP", ep.VMIP) + return nil +} + +// Detach removes a Pod's mapping. Detaching an unknown Pod is a no-op, so +// deletion stays idempotent. +func (rt *Router) Detach(ctx context.Context, podKey string) error { + rt.mu.Lock() + defer rt.mu.Unlock() + if _, ok := rt.endpoints[podKey]; !ok { + return nil + } + delete(rt.endpoints, podKey) + if !rt.started { + return nil + } + if err := rt.loadAnchorLocked(ctx); err != nil { + return fmt.Errorf("detach %q: %w", podKey, err) + } + klog.InfoS("detached pod from network path", "pod", podKey) + return nil +} + +// Endpoints returns a snapshot of the currently attached endpoints, sorted by +// Pod key, so callers and tests can confirm the active mappings. +func (rt *Router) Endpoints() []Endpoint { + rt.mu.Lock() + defer rt.mu.Unlock() + out := make([]Endpoint, 0, len(rt.endpoints)) + for _, ep := range rt.endpoints { + out = append(out, ep) + } + sort.Slice(out, func(i, j int) bool { return out[i].PodKey < out[j].PodKey }) + return out +} + +// Stop flushes the anchor, removing all MacVz rules. Best-effort. +func (rt *Router) Stop(ctx context.Context) error { + rt.mu.Lock() + defer rt.mu.Unlock() + rt.endpoints = map[string]Endpoint{} + if !rt.started { + return nil + } + if _, err := rt.run.run(ctx, command{Name: rt.pfctl, Args: []string{"-a", rt.cfg.Anchor, "-F", "all"}}); err != nil { + klog.ErrorS(err, "failed to flush pod network anchor", "anchor", rt.cfg.Anchor) + } + rt.started = false + klog.InfoS("pod network stopped", "anchor", rt.cfg.Anchor) + return nil +} + +// loadAnchorLocked renders the current ruleset and loads it into the anchor +// atomically via `pfctl -a -f -`. Caller holds rt.mu. +func (rt *Router) loadAnchorLocked(ctx context.Context) error { + rules := renderAnchor(rt.cfg.Interface, rt.endpointsSortedLocked()) + _, err := rt.run.run(ctx, command{ + Name: rt.pfctl, + Args: []string{"-a", rt.cfg.Anchor, "-f", "-"}, + Stdin: rules, + }) + if err != nil { + return fmt.Errorf("load pf anchor %q: %w", rt.cfg.Anchor, err) + } + return nil +} + +func (rt *Router) endpointsSortedLocked() []Endpoint { + out := make([]Endpoint, 0, len(rt.endpoints)) + for _, ep := range rt.endpoints { + out = append(out, ep) + } + sort.Slice(out, func(i, j int) bool { return out[i].PodKey < out[j].PodKey }) + return out +} + +// renderAnchor builds the pf anchor body: one bidirectional NAT rule per Pod, +// mapping the VM's host-only address to its Pod IP. Rendering is deterministic +// (endpoints are pre-sorted) so identical state yields identical rules. +func renderAnchor(iface string, endpoints []Endpoint) string { + var b strings.Builder + b.WriteString("# Managed by macvz (issue #22). Do not edit by hand.\n") + for _, ep := range endpoints { + fmt.Fprintf(&b, "# %s\n", ep.PodKey) + fmt.Fprintf(&b, "binat on %s from %s to any -> %s\n", iface, ep.VMIP, ep.PodIP) + } + return b.String() +} + +// runTolerating runs a command, treating an error whose stderr contains any of +// the benign substrings as success (e.g. "pf already enabled"). +func (rt *Router) runTolerating(ctx context.Context, c command, benign ...string) (string, error) { + out, err := rt.run.run(ctx, c) + if err == nil { + return out, nil + } + var ce *CommandError + if errors.As(err, &ce) { + msg := strings.ToLower(ce.Stderr) + for _, b := range benign { + if b != "" && strings.Contains(msg, b) { + return out, nil + } + } + } + return out, err +} + +// pfAlreadyEnabled is the stderr pfctl prints when pf is already on. +const pfAlreadyEnabled = "pf already enabled" diff --git a/pkg/network/podnet/router_test.go b/pkg/network/podnet/router_test.go new file mode 100644 index 0000000..7bc895a --- /dev/null +++ b/pkg/network/podnet/router_test.go @@ -0,0 +1,227 @@ +package podnet + +import ( + "context" + "strings" + "sync" + "testing" +) + +// fakeRunner records commands and can inject failures. +type fakeRunner struct { + mu sync.Mutex + cmds []command + failOn map[string]string // substring of command string -> stderr to fail with +} + +func newFakeRunner() *fakeRunner { return &fakeRunner{failOn: map[string]string{}} } + +func (f *fakeRunner) run(_ context.Context, c command) (string, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.cmds = append(f.cmds, c) + for sub, stderr := range f.failOn { + if strings.Contains(c.String(), sub) { + return "", &CommandError{Cmd: c.String(), Stderr: stderr, ExitCode: 1} + } + } + return "", nil +} + +func (f *fakeRunner) strings() []string { + f.mu.Lock() + defer f.mu.Unlock() + out := make([]string, len(f.cmds)) + for i, c := range f.cmds { + out[i] = c.String() + } + return out +} + +// lastAnchorLoad returns the stdin of the most recent `pfctl -a ... -f -` call. +func (f *fakeRunner) lastAnchorLoad() (string, bool) { + f.mu.Lock() + defer f.mu.Unlock() + for i := len(f.cmds) - 1; i >= 0; i-- { + if strings.Contains(f.cmds[i].String(), "-f -") { + return f.cmds[i].Stdin, true + } + } + return "", false +} + +func contains(haystack []string, sub string) bool { + for _, s := range haystack { + if strings.Contains(s, sub) { + return true + } + } + return false +} + +func newTestRouter(fr *fakeRunner) *Router { + return New(Config{Interface: "bridge100", EnableForwarding: true}, WithRunner(fr)) +} + +func TestStartEnablesForwardingAndPF(t *testing.T) { + fr := newFakeRunner() + rt := newTestRouter(fr) + if err := rt.Start(context.Background()); err != nil { + t.Fatalf("Start: %v", err) + } + cmds := fr.strings() + if !contains(cmds, "sysctl -w net.inet.ip.forwarding=1") { + t.Errorf("Start did not enable forwarding\nran: %v", cmds) + } + if !contains(cmds, "pfctl -e") { + t.Errorf("Start did not enable pf\nran: %v", cmds) + } + if !contains(cmds, "pfctl -a macvz/pods -f -") { + t.Errorf("Start did not load a baseline anchor\nran: %v", cmds) + } +} + +func TestStartToleratesPFAlreadyEnabled(t *testing.T) { + fr := newFakeRunner() + fr.failOn["pfctl -e"] = "pfctl: pf already enabled" + rt := newTestRouter(fr) + if err := rt.Start(context.Background()); err != nil { + t.Fatalf("Start should tolerate 'pf already enabled', got: %v", err) + } +} + +func TestStartRequiresInterface(t *testing.T) { + rt := New(Config{}, WithRunner(newFakeRunner())) + if err := rt.Start(context.Background()); err == nil { + t.Fatal("Start should require an interface") + } +} + +func TestAttachInstallsBinatRule(t *testing.T) { + fr := newFakeRunner() + rt := newTestRouter(fr) + ctx := context.Background() + if err := rt.Start(ctx); err != nil { + t.Fatalf("Start: %v", err) + } + ep := Endpoint{PodKey: "default/web", PodIP: "10.244.1.2", VMIP: "192.168.64.5"} + if err := rt.Attach(ctx, ep); err != nil { + t.Fatalf("Attach: %v", err) + } + rules, ok := fr.lastAnchorLoad() + if !ok { + t.Fatal("no anchor load recorded") + } + want := "binat on bridge100 from 192.168.64.5 to any -> 10.244.1.2" + if !strings.Contains(rules, want) { + t.Errorf("anchor missing rule %q\n---\n%s", want, rules) + } + if eps := rt.Endpoints(); len(eps) != 1 || eps[0].PodKey != "default/web" { + t.Errorf("Endpoints = %v, want [default/web]", eps) + } +} + +func TestAttachValidatesEndpoint(t *testing.T) { + fr := newFakeRunner() + rt := newTestRouter(fr) + if err := rt.Start(context.Background()); err != nil { + t.Fatalf("Start: %v", err) + } + bad := []Endpoint{ + {PodKey: "", PodIP: "10.244.1.2", VMIP: "192.168.64.5"}, + {PodKey: "default/x", PodIP: "nope", VMIP: "192.168.64.5"}, + {PodKey: "default/x", PodIP: "10.244.1.2", VMIP: "nope"}, + } + for _, ep := range bad { + if err := rt.Attach(context.Background(), ep); err == nil { + t.Errorf("Attach(%+v) = nil, want validation error", ep) + } + } +} + +func TestAttachBeforeStartFails(t *testing.T) { + rt := newTestRouter(newFakeRunner()) + ep := Endpoint{PodKey: "default/web", PodIP: "10.244.1.2", VMIP: "192.168.64.5"} + if err := rt.Attach(context.Background(), ep); err == nil { + t.Fatal("Attach before Start should fail") + } +} + +func TestDetachRemovesRule(t *testing.T) { + fr := newFakeRunner() + rt := newTestRouter(fr) + ctx := context.Background() + if err := rt.Start(ctx); err != nil { + t.Fatalf("Start: %v", err) + } + a := Endpoint{PodKey: "default/a", PodIP: "10.244.1.2", VMIP: "192.168.64.5"} + b := Endpoint{PodKey: "default/b", PodIP: "10.244.1.3", VMIP: "192.168.64.6"} + if err := rt.Attach(ctx, a); err != nil { + t.Fatalf("Attach a: %v", err) + } + if err := rt.Attach(ctx, b); err != nil { + t.Fatalf("Attach b: %v", err) + } + if err := rt.Detach(ctx, "default/a"); err != nil { + t.Fatalf("Detach: %v", err) + } + rules, _ := fr.lastAnchorLoad() + if strings.Contains(rules, "10.244.1.2") { + t.Errorf("detached pod's rule still present\n---\n%s", rules) + } + if !strings.Contains(rules, "10.244.1.3") { + t.Errorf("remaining pod's rule missing\n---\n%s", rules) + } + if eps := rt.Endpoints(); len(eps) != 1 || eps[0].PodKey != "default/b" { + t.Errorf("Endpoints = %v, want [default/b]", eps) + } +} + +func TestDetachUnknownIsNoop(t *testing.T) { + fr := newFakeRunner() + rt := newTestRouter(fr) + ctx := context.Background() + if err := rt.Start(ctx); err != nil { + t.Fatalf("Start: %v", err) + } + before := len(fr.strings()) + if err := rt.Detach(ctx, "default/ghost"); err != nil { + t.Fatalf("Detach unknown: %v", err) + } + if after := len(fr.strings()); after != before { + t.Errorf("Detach of unknown pod reloaded the anchor (%d -> %d cmds)", before, after) + } +} + +func TestRenderAnchorDeterministic(t *testing.T) { + eps := []Endpoint{ + {PodKey: "default/a", PodIP: "10.244.1.2", VMIP: "192.168.64.5"}, + {PodKey: "default/b", PodIP: "10.244.1.3", VMIP: "192.168.64.6"}, + } + first := renderAnchor("bridge100", eps) + second := renderAnchor("bridge100", eps) + if first != second { + t.Error("renderAnchor is not deterministic") + } +} + +func TestStopFlushesAnchor(t *testing.T) { + fr := newFakeRunner() + rt := newTestRouter(fr) + ctx := context.Background() + if err := rt.Start(ctx); err != nil { + t.Fatalf("Start: %v", err) + } + if err := rt.Attach(ctx, Endpoint{PodKey: "default/a", PodIP: "10.244.1.2", VMIP: "192.168.64.5"}); err != nil { + t.Fatalf("Attach: %v", err) + } + if err := rt.Stop(ctx); err != nil { + t.Fatalf("Stop: %v", err) + } + if !contains(fr.strings(), "pfctl -a macvz/pods -F all") { + t.Errorf("Stop did not flush the anchor\nran: %v", fr.strings()) + } + if eps := rt.Endpoints(); len(eps) != 0 { + t.Errorf("endpoints remain after Stop: %v", eps) + } +} diff --git a/pkg/network/podnet/runner.go b/pkg/network/podnet/runner.go new file mode 100644 index 0000000..7baa604 --- /dev/null +++ b/pkg/network/podnet/runner.go @@ -0,0 +1,76 @@ +package podnet + +import ( + "bytes" + "context" + "errors" + "fmt" + "os/exec" + "strings" +) + +// command is a single external command invocation. +type command struct { + // Name is the binary to run, resolved through PATH (e.g. "pfctl", "sysctl"). + Name string + // Args are the command arguments. + Args []string + // Stdin, when non-empty, is fed to the command's standard input. The router + // uses it to load a pf anchor ruleset via `pfctl ... -f -`, avoiding temp + // files on disk. + Stdin string +} + +func (c command) String() string { + return strings.TrimSpace(c.Name + " " + strings.Join(c.Args, " ")) +} + +// runner executes external commands. It is an interface so the Router can be +// unit-tested against a fake without touching the host packet filter, mirroring +// the runner in pkg/network/wireguard and pkg/runtime/container. +type runner interface { + run(ctx context.Context, c command) (string, error) +} + +// CommandError describes a failed Pod-network command invocation. +type CommandError struct { + Cmd string + ExitCode int + Stderr string + Err error +} + +func (e *CommandError) Error() string { + msg := strings.TrimSpace(e.Stderr) + if msg == "" && e.Err != nil { + msg = e.Err.Error() + } + if msg == "" { + msg = "command failed" + } + return fmt.Sprintf("%s: %s", e.Cmd, msg) +} + +func (e *CommandError) Unwrap() error { return e.Err } + +// cliRunner is the production runner backed by the host's networking tools. +type cliRunner struct{} + +func (cliRunner) run(ctx context.Context, c command) (string, error) { + var stdout, stderr bytes.Buffer + cmd := exec.CommandContext(ctx, c.Name, c.Args...) + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if c.Stdin != "" { + cmd.Stdin = strings.NewReader(c.Stdin) + } + if err := cmd.Run(); err != nil { + ce := &CommandError{Cmd: c.String(), Stderr: stderr.String(), Err: err, ExitCode: -1} + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + ce.ExitCode = exitErr.ExitCode() + } + return stdout.String(), ce + } + return stdout.String(), nil +} diff --git a/pkg/network/wireguard/config.go b/pkg/network/wireguard/config.go new file mode 100644 index 0000000..23d71f1 --- /dev/null +++ b/pkg/network/wireguard/config.go @@ -0,0 +1,173 @@ +package wireguard + +import ( + "fmt" + "net" + "sort" + "strconv" + "strings" +) + +// Peer is a remote MacVz node in the mesh. +type Peer struct { + // Name is the node name. It is emitted as a comment for human-readable + // `wg show` / config diffs and is not part of the WireGuard protocol. + Name string + // PublicKey identifies the peer cryptographically. + PublicKey Key + // Endpoint is the peer's reachable "host:port". It may be empty for a peer + // behind NAT that initiates the connection itself. + Endpoint string + // AllowedIPs are the CIDRs routed to this peer — its Pod CIDR plus its + // address on the mesh network. + AllowedIPs []string + // PersistentKeepalive, in seconds, keeps NAT mappings alive (0 disables it). + PersistentKeepalive int +} + +// InterfaceConfig is the desired state of this node's WireGuard interface. +type InterfaceConfig struct { + // Name is the interface name (e.g. "utun7"). On macOS wg-quick assigns a + // utun device; this is the name the Mesh manages and routes through. + Name string + // PrivateKey is this node's WireGuard private key. + PrivateKey Key + // Address is this node's address on the mesh network in CIDR form + // (e.g. "10.99.0.1/32"). + Address string + // ListenPort is the UDP port WireGuard listens on. + ListenPort int + // MTU optionally overrides the interface MTU (0 leaves the default). + MTU int + // Peers are the other nodes in the mesh. + Peers []Peer +} + +// Validate checks the interface config is internally consistent before it is +// rendered or applied. +func (c InterfaceConfig) Validate() error { + if c.Name == "" { + return fmt.Errorf("wireguard: interface name is required") + } + if c.PrivateKey.IsZero() { + return fmt.Errorf("wireguard: interface %q has no private key", c.Name) + } + if c.Address == "" { + return fmt.Errorf("wireguard: interface %q has no mesh address", c.Name) + } + if _, _, err := net.ParseCIDR(c.Address); err != nil { + return fmt.Errorf("wireguard: interface %q address %q is not a CIDR: %w", c.Name, c.Address, err) + } + if c.ListenPort < 1 || c.ListenPort > 65535 { + return fmt.Errorf("wireguard: interface %q listenPort %d out of range", c.Name, c.ListenPort) + } + if c.MTU < 0 { + return fmt.Errorf("wireguard: interface %q mtu is negative", c.Name) + } + seen := map[Key]string{} + for _, p := range c.Peers { + if err := p.validate(); err != nil { + return err + } + if other, dup := seen[p.PublicKey]; dup { + return fmt.Errorf("wireguard: peers %q and %q share a public key", other, p.Name) + } + seen[p.PublicKey] = p.Name + } + return nil +} + +func (p Peer) validate() error { + if p.PublicKey.IsZero() { + return fmt.Errorf("wireguard: peer %q has no public key", p.Name) + } + if len(p.AllowedIPs) == 0 { + return fmt.Errorf("wireguard: peer %q has no allowedIPs", p.Name) + } + for _, cidr := range p.AllowedIPs { + if _, _, err := net.ParseCIDR(cidr); err != nil { + return fmt.Errorf("wireguard: peer %q allowedIP %q is not a CIDR: %w", p.Name, cidr, err) + } + } + if p.Endpoint != "" { + if _, _, err := net.SplitHostPort(p.Endpoint); err != nil { + return fmt.Errorf("wireguard: peer %q endpoint %q is not host:port: %w", p.Name, p.Endpoint, err) + } + } + if p.PersistentKeepalive < 0 { + return fmt.Errorf("wireguard: peer %q persistentKeepalive is negative", p.Name) + } + return nil +} + +// QuickConfig renders a wg-quick(8) configuration, including the Address/MTU +// extensions wg-quick understands. Used to bring the interface up. +func (c InterfaceConfig) QuickConfig() string { + return c.render(true) +} + +// SyncConfig renders a wg(8) configuration (no wg-quick Address/MTU keys), as +// consumed by `wg setconf` / `wg syncconf` to reconcile peers in place. +func (c InterfaceConfig) SyncConfig() string { + return c.render(false) +} + +// render builds the textual config. When quick is true the wg-quick-only keys +// (Address, MTU) are included. Peers are emitted in a stable order so repeated +// renders are byte-identical and diffs are meaningful. +func (c InterfaceConfig) render(quick bool) string { + var b strings.Builder + b.WriteString("[Interface]\n") + b.WriteString("PrivateKey = " + c.PrivateKey.String() + "\n") + if quick { + b.WriteString("Address = " + c.Address + "\n") + if c.MTU > 0 { + b.WriteString("MTU = " + strconv.Itoa(c.MTU) + "\n") + } + } + b.WriteString("ListenPort = " + strconv.Itoa(c.ListenPort) + "\n") + + for _, p := range sortedPeers(c.Peers) { + b.WriteString("\n[Peer]\n") + if p.Name != "" { + b.WriteString("# " + p.Name + "\n") + } + b.WriteString("PublicKey = " + p.PublicKey.String() + "\n") + if p.Endpoint != "" { + b.WriteString("Endpoint = " + p.Endpoint + "\n") + } + b.WriteString("AllowedIPs = " + strings.Join(p.AllowedIPs, ", ") + "\n") + if p.PersistentKeepalive > 0 { + b.WriteString("PersistentKeepalive = " + strconv.Itoa(p.PersistentKeepalive) + "\n") + } + } + return b.String() +} + +// sortedPeers returns peers ordered by public key for deterministic rendering. +func sortedPeers(peers []Peer) []Peer { + out := append([]Peer(nil), peers...) + sort.Slice(out, func(i, j int) bool { + return out[i].PublicKey.String() < out[j].PublicKey.String() + }) + return out +} + +// RouteTargets returns the de-duplicated set of CIDRs that must be routed +// through the interface — the union of every peer's AllowedIPs. These are the +// host routes installed so the kernel forwards remote-Pod traffic into the +// tunnel. +func (c InterfaceConfig) RouteTargets() []string { + seen := map[string]bool{} + var out []string + for _, p := range c.Peers { + for _, cidr := range p.AllowedIPs { + if !seen[cidr] { + seen[cidr] = true + out = append(out, cidr) + } + } + } + sort.Strings(out) + return out +} diff --git a/pkg/network/wireguard/config_test.go b/pkg/network/wireguard/config_test.go new file mode 100644 index 0000000..b4357d9 --- /dev/null +++ b/pkg/network/wireguard/config_test.go @@ -0,0 +1,129 @@ +package wireguard + +import ( + "strings" + "testing" +) + +func testKey(t *testing.T) Key { + t.Helper() + k, err := GenerateKey() + if err != nil { + t.Fatalf("GenerateKey: %v", err) + } + return k +} + +func testConfig(t *testing.T) InterfaceConfig { + t.Helper() + return InterfaceConfig{ + Name: "utun7", + PrivateKey: testKey(t), + Address: "10.99.0.1/32", + ListenPort: 51820, + Peers: []Peer{ + { + Name: "mac-02", + PublicKey: testKey(t), + Endpoint: "192.168.1.20:51820", + AllowedIPs: []string{"10.244.2.0/24", "10.99.0.2/32"}, + PersistentKeepalive: 25, + }, + }, + } +} + +func TestValidateRejectsBadConfig(t *testing.T) { + good := testConfig(t) + cases := map[string]func(c *InterfaceConfig){ + "no name": func(c *InterfaceConfig) { c.Name = "" }, + "no key": func(c *InterfaceConfig) { c.PrivateKey = Key{} }, + "bad address": func(c *InterfaceConfig) { c.Address = "10.99.0.1" }, + "bad port": func(c *InterfaceConfig) { c.ListenPort = 0 }, + "negative mtu": func(c *InterfaceConfig) { c.MTU = -1 }, + "peer no key": func(c *InterfaceConfig) { c.Peers[0].PublicKey = Key{} }, + "peer no allow": func(c *InterfaceConfig) { c.Peers[0].AllowedIPs = nil }, + "peer bad cidr": func(c *InterfaceConfig) { c.Peers[0].AllowedIPs = []string{"nope"} }, + "peer bad endpt": func(c *InterfaceConfig) { c.Peers[0].Endpoint = "noport" }, + } + for name, mutate := range cases { + t.Run(name, func(t *testing.T) { + c := good + c.Peers = append([]Peer(nil), good.Peers...) + mutate(&c) + if err := c.Validate(); err == nil { + t.Errorf("Validate() = nil, want error for %s", name) + } + }) + } +} + +func TestValidateRejectsDuplicatePeerKeys(t *testing.T) { + c := testConfig(t) + dup := c.Peers[0] + dup.Name = "mac-03" + c.Peers = append(c.Peers, dup) + if err := c.Validate(); err == nil { + t.Error("expected error for duplicate peer public keys") + } +} + +func TestQuickConfigIncludesAddressAndMTU(t *testing.T) { + c := testConfig(t) + c.MTU = 1380 + out := c.QuickConfig() + for _, want := range []string{"[Interface]", "Address = 10.99.0.1/32", "MTU = 1380", "ListenPort = 51820", "[Peer]", "# mac-02", "Endpoint = 192.168.1.20:51820", "AllowedIPs = 10.244.2.0/24, 10.99.0.2/32", "PersistentKeepalive = 25"} { + if !strings.Contains(out, want) { + t.Errorf("QuickConfig missing %q\n---\n%s", want, out) + } + } +} + +func TestSyncConfigOmitsQuickExtensions(t *testing.T) { + c := testConfig(t) + c.MTU = 1380 + out := c.SyncConfig() + if strings.Contains(out, "Address =") || strings.Contains(out, "MTU =") { + t.Errorf("SyncConfig must not contain wg-quick keys\n---\n%s", out) + } + if !strings.Contains(out, "PublicKey =") || !strings.Contains(out, "ListenPort = 51820") { + t.Errorf("SyncConfig missing required wg keys\n---\n%s", out) + } +} + +func TestRenderIsDeterministic(t *testing.T) { + c := testConfig(t) + // Add peers out of key order; render must be stable across calls. + p2 := Peer{Name: "z", PublicKey: testKey(t), AllowedIPs: []string{"10.244.3.0/24"}} + p3 := Peer{Name: "a", PublicKey: testKey(t), AllowedIPs: []string{"10.244.4.0/24"}} + c.Peers = append(c.Peers, p2, p3) + first := c.SyncConfig() + second := c.SyncConfig() + if first != second { + t.Error("SyncConfig is not deterministic") + } + // A reordered peer slice must render identically (peers sort by key). + c.Peers[1], c.Peers[2] = c.Peers[2], c.Peers[1] + if reordered := c.SyncConfig(); reordered != first { + t.Error("SyncConfig depends on peer slice order") + } +} + +func TestRouteTargetsDedupesAndSorts(t *testing.T) { + c := testConfig(t) + c.Peers = append(c.Peers, Peer{ + Name: "mac-03", + PublicKey: testKey(t), + AllowedIPs: []string{"10.244.3.0/24", "10.244.2.0/24"}, // 2.0 overlaps mac-02 + }) + got := c.RouteTargets() + want := []string{"10.244.2.0/24", "10.244.3.0/24", "10.99.0.2/32"} + if len(got) != len(want) { + t.Fatalf("RouteTargets = %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Errorf("RouteTargets[%d] = %q, want %q", i, got[i], want[i]) + } + } +} diff --git a/pkg/network/wireguard/doc.go b/pkg/network/wireguard/doc.go new file mode 100644 index 0000000..a96311c --- /dev/null +++ b/pkg/network/wireguard/doc.go @@ -0,0 +1,20 @@ +// Package wireguard brings up the encrypted host-to-host mesh that carries Pod +// traffic between MacVz nodes (issue #21, milestone P3). +// +// Each Apple Silicon Mac runs one WireGuard interface. Peers are other MacVz +// nodes; a peer's AllowedIPs is its Kubernetes-assigned Pod CIDR (see +// pkg/network IPAM, #20), so traffic destined for a remote Pod is encrypted and +// tunnelled to the Mac that hosts it. Host routes for those CIDRs are installed +// through the interface so the kernel forwards Pod-bound packets into the tunnel. +// +// macOS has no in-kernel WireGuard, so the Mesh drives the userspace toolchain +// shipped by Homebrew's wireguard-tools (`wg`, `wg-quick`) plus `route`, exactly +// as pkg/runtime/container drives the apple/container CLI. All command execution +// goes through the runner interface, so the orchestration logic is unit-tested +// against a fake without touching the host network. +// +// The MVP takes peer identity, endpoints, and Pod CIDRs from configuration; +// reconciliation (`wg syncconf`) applies peer additions and removals without +// tearing the interface down. Dynamic peer discovery via Kubernetes Node +// metadata is layered on in later P3 work. +package wireguard diff --git a/pkg/network/wireguard/key.go b/pkg/network/wireguard/key.go new file mode 100644 index 0000000..8d70714 --- /dev/null +++ b/pkg/network/wireguard/key.go @@ -0,0 +1,103 @@ +package wireguard + +import ( + "crypto/rand" + "encoding/base64" + "fmt" + "os" + "path/filepath" + "strings" + + "golang.org/x/crypto/curve25519" +) + +// KeyLen is the length of a WireGuard key in bytes (Curve25519). +const KeyLen = 32 + +// Key is a WireGuard Curve25519 key (private or public). Its string form is the +// standard base64 encoding used by the `wg` tool and config files. +type Key [KeyLen]byte + +// GenerateKey returns a new, clamped WireGuard private key. +func GenerateKey() (Key, error) { + var k Key + if _, err := rand.Read(k[:]); err != nil { + return Key{}, fmt.Errorf("wireguard: read random bytes: %w", err) + } + clamp(&k) + return k, nil +} + +// clamp applies the Curve25519 private-key clamping WireGuard expects, so a key +// generated here behaves identically to one produced by `wg genkey`. +func clamp(k *Key) { + k[0] &= 248 + k[31] &= 127 + k[31] |= 64 +} + +// PublicKey derives the public key for a private key. +func (k Key) PublicKey() Key { + var pub, priv [KeyLen]byte + copy(priv[:], k[:]) + // curve25519.ScalarBaseMult computes pub = priv * basepoint. + curve25519.ScalarBaseMult(&pub, &priv) + return Key(pub) +} + +// String returns the base64 encoding of the key, as written to config and shown +// by `wg`. +func (k Key) String() string { + return base64.StdEncoding.EncodeToString(k[:]) +} + +// IsZero reports whether the key is the all-zero value (i.e. unset). +func (k Key) IsZero() bool { + return k == Key{} +} + +// ParseKey decodes a base64-encoded WireGuard key. +func ParseKey(s string) (Key, error) { + b, err := base64.StdEncoding.DecodeString(strings.TrimSpace(s)) + if err != nil { + return Key{}, fmt.Errorf("wireguard: decode key: %w", err) + } + if len(b) != KeyLen { + return Key{}, fmt.Errorf("wireguard: key is %d bytes, want %d", len(b), KeyLen) + } + var k Key + copy(k[:], b) + return k, nil +} + +// LoadOrCreateKey reads a base64 private key from path, generating and +// persisting a new one (mode 0600) if the file does not exist. This gives each +// node a stable identity across restarts without hardcoding keys in config. +func LoadOrCreateKey(path string) (Key, error) { + data, err := os.ReadFile(path) + switch { + case err == nil: + k, perr := ParseKey(string(data)) + if perr != nil { + return Key{}, fmt.Errorf("wireguard: parse key file %q: %w", path, perr) + } + return k, nil + case !os.IsNotExist(err): + return Key{}, fmt.Errorf("wireguard: read key file %q: %w", path, err) + } + + // Not present: generate and persist a new private key. + k, err := GenerateKey() + if err != nil { + return Key{}, err + } + if dir := filepath.Dir(path); dir != "" && dir != "." { + if err := os.MkdirAll(dir, 0o700); err != nil { + return Key{}, fmt.Errorf("wireguard: create key dir %q: %w", dir, err) + } + } + if err := os.WriteFile(path, []byte(k.String()+"\n"), 0o600); err != nil { + return Key{}, fmt.Errorf("wireguard: write key file %q: %w", path, err) + } + return k, nil +} diff --git a/pkg/network/wireguard/key_test.go b/pkg/network/wireguard/key_test.go new file mode 100644 index 0000000..5dec276 --- /dev/null +++ b/pkg/network/wireguard/key_test.go @@ -0,0 +1,108 @@ +package wireguard + +import ( + "os" + "path/filepath" + "testing" +) + +func TestGenerateKeyIsClampedAndUnique(t *testing.T) { + k1, err := GenerateKey() + if err != nil { + t.Fatalf("GenerateKey: %v", err) + } + k2, err := GenerateKey() + if err != nil { + t.Fatalf("GenerateKey: %v", err) + } + if k1 == k2 { + t.Fatal("two generated keys are identical") + } + // Clamping invariants from the Curve25519 spec. + if k1[0]&7 != 0 { + t.Errorf("low 3 bits of byte 0 not cleared: %08b", k1[0]) + } + if k1[31]&0x80 != 0 { + t.Errorf("high bit of byte 31 not cleared: %08b", k1[31]) + } + if k1[31]&0x40 == 0 { + t.Errorf("bit 6 of byte 31 not set: %08b", k1[31]) + } +} + +func TestKeyStringRoundTrip(t *testing.T) { + k, err := GenerateKey() + if err != nil { + t.Fatalf("GenerateKey: %v", err) + } + parsed, err := ParseKey(k.String()) + if err != nil { + t.Fatalf("ParseKey: %v", err) + } + if parsed != k { + t.Errorf("round trip mismatch: %s != %s", parsed, k) + } +} + +func TestPublicKeyIsStableAndPublic(t *testing.T) { + k, err := GenerateKey() + if err != nil { + t.Fatalf("GenerateKey: %v", err) + } + pub1 := k.PublicKey() + pub2 := k.PublicKey() + if pub1 != pub2 { + t.Error("PublicKey is not deterministic") + } + if pub1 == k { + t.Error("public key equals private key") + } + if pub1.IsZero() { + t.Error("derived public key is zero") + } +} + +func TestParseKeyRejectsBadInput(t *testing.T) { + for _, s := range []string{"", "not-base64!!", "c2hvcnQ="} { // last decodes to 5 bytes + if _, err := ParseKey(s); err == nil { + t.Errorf("ParseKey(%q) = nil error, want failure", s) + } + } +} + +func TestLoadOrCreateKeyPersists(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "keys", "wg.key") + + created, err := LoadOrCreateKey(path) + if err != nil { + t.Fatalf("LoadOrCreateKey (create): %v", err) + } + // File must exist with 0600 perms. + info, err := os.Stat(path) + if err != nil { + t.Fatalf("stat key file: %v", err) + } + if perm := info.Mode().Perm(); perm != 0o600 { + t.Errorf("key file perm = %o, want 600", perm) + } + + // Second call returns the same key (stable identity across restarts). + loaded, err := LoadOrCreateKey(path) + if err != nil { + t.Fatalf("LoadOrCreateKey (load): %v", err) + } + if loaded != created { + t.Errorf("reloaded key differs: %s != %s", loaded, created) + } +} + +func TestLoadOrCreateKeyRejectsCorruptFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "wg.key") + if err := os.WriteFile(path, []byte("garbage"), 0o600); err != nil { + t.Fatalf("seed file: %v", err) + } + if _, err := LoadOrCreateKey(path); err == nil { + t.Fatal("expected error for corrupt key file") + } +} diff --git a/pkg/network/wireguard/mesh.go b/pkg/network/wireguard/mesh.go new file mode 100644 index 0000000..4da07d4 --- /dev/null +++ b/pkg/network/wireguard/mesh.go @@ -0,0 +1,299 @@ +package wireguard + +import ( + "context" + "errors" + "fmt" + "net" + "sort" + "strconv" + "strings" + "sync" + + "k8s.io/klog/v2" +) + +// Default tool names. They are resolved through PATH (Homebrew's wireguard-tools +// and the system route/ifconfig), never hardcoded absolute paths. +const ( + defaultWG = "wg" + defaultWireGuardGo = "wireguard-go" + defaultIfconfig = "ifconfig" + defaultRoute = "route" +) + +// tools names the external binaries the Mesh drives. +type tools struct { + wg string + wireguardGo string + ifconfig string + route string +} + +// Mesh manages this node's WireGuard interface and the host routes that steer +// remote-Pod traffic into the tunnel. It is safe for concurrent use. +// +// The lifecycle is: Up (create interface, apply config, assign address, install +// routes), Sync (reconcile peers and routes in place as nodes join/leave), and +// Down (tear everything back down). Sync never recreates the interface, so the +// data path is not interrupted for unrelated peers. +type Mesh struct { + run runner + tools tools + + mu sync.Mutex + cfg InterfaceConfig + routes map[string]bool // route targets currently installed + up bool +} + +// Option configures a Mesh. +type Option func(*Mesh) + +// WithRunner injects a command runner (used by tests). +func WithRunner(r runner) Option { return func(m *Mesh) { m.run = r } } + +// WithTools overrides the external binary names. Empty values keep the default. +func WithTools(wg, wireguardGo, ifconfig, route string) Option { + return func(m *Mesh) { + if wg != "" { + m.tools.wg = wg + } + if wireguardGo != "" { + m.tools.wireguardGo = wireguardGo + } + if ifconfig != "" { + m.tools.ifconfig = ifconfig + } + if route != "" { + m.tools.route = route + } + } +} + +// New builds a Mesh for the given interface configuration. +func New(cfg InterfaceConfig, opts ...Option) *Mesh { + m := &Mesh{ + run: cliRunner{}, + tools: tools{wg: defaultWG, wireguardGo: defaultWireGuardGo, ifconfig: defaultIfconfig, route: defaultRoute}, + cfg: cfg, + routes: map[string]bool{}, + } + for _, opt := range opts { + opt(m) + } + return m +} + +// InterfaceName returns the managed interface name. +func (m *Mesh) InterfaceName() string { + m.mu.Lock() + defer m.mu.Unlock() + return m.cfg.Name +} + +// Peers returns the names of the peers currently configured. +func (m *Mesh) Peers() []string { + m.mu.Lock() + defer m.mu.Unlock() + names := make([]string, 0, len(m.cfg.Peers)) + for _, p := range m.cfg.Peers { + names = append(names, p.Name) + } + sort.Strings(names) + return names +} + +// Up creates the WireGuard interface, applies the configuration, assigns the +// node's mesh address, and installs host routes for every peer's AllowedIPs. +func (m *Mesh) Up(ctx context.Context) error { + m.mu.Lock() + defer m.mu.Unlock() + + if err := m.cfg.Validate(); err != nil { + return err + } + ip, err := addressIP(m.cfg.Address) + if err != nil { + return err + } + + // Create the userspace interface. Tolerate "already exists" so Up is safe to + // re-run after a partial start. + if _, err := m.runTolerating(ctx, command{Name: m.tools.wireguardGo, Args: []string{m.cfg.Name}}, errInterfaceExists); err != nil { + return fmt.Errorf("create interface %q: %w", m.cfg.Name, err) + } + // Apply WireGuard peers/keys via stdin (no config file on disk). + if _, err := m.run.run(ctx, command{Name: m.tools.wg, Args: []string{"setconf", m.cfg.Name, "/dev/stdin"}, Stdin: m.cfg.SyncConfig()}); err != nil { + return fmt.Errorf("apply wireguard config to %q: %w", m.cfg.Name, err) + } + // Assign the mesh address (macOS point-to-point alias) and bring the link up. + if _, err := m.runTolerating(ctx, command{Name: m.tools.ifconfig, Args: []string{m.cfg.Name, "inet", ip, ip, "alias"}}, errAddressExists); err != nil { + return fmt.Errorf("assign address %s to %q: %w", ip, m.cfg.Name, err) + } + if m.cfg.MTU > 0 { + if _, err := m.run.run(ctx, command{Name: m.tools.ifconfig, Args: []string{m.cfg.Name, "mtu", strconv.Itoa(m.cfg.MTU)}}); err != nil { + return fmt.Errorf("set mtu %d on %q: %w", m.cfg.MTU, m.cfg.Name, err) + } + } + if _, err := m.run.run(ctx, command{Name: m.tools.ifconfig, Args: []string{m.cfg.Name, "up"}}); err != nil { + return fmt.Errorf("bring up %q: %w", m.cfg.Name, err) + } + + m.up = true + if err := m.reconcileRoutesLocked(ctx, m.cfg.RouteTargets()); err != nil { + return err + } + klog.InfoS("wireguard mesh up", "interface", m.cfg.Name, "address", m.cfg.Address, "peers", len(m.cfg.Peers), "routes", len(m.routes)) + return nil +} + +// Sync reconciles the mesh to a new peer set without recreating the interface: +// it re-applies the WireGuard config with `wg syncconf` (which adds and removes +// peers atomically) and adds/removes host routes to match. It is the path used +// when nodes join or leave the cluster. +func (m *Mesh) Sync(ctx context.Context, peers []Peer) error { + m.mu.Lock() + defer m.mu.Unlock() + + next := m.cfg + next.Peers = peers + if err := next.Validate(); err != nil { + return err + } + if !m.up { + return fmt.Errorf("wireguard: Sync called before Up on %q", m.cfg.Name) + } + + if _, err := m.run.run(ctx, command{Name: m.tools.wg, Args: []string{"syncconf", m.cfg.Name, "/dev/stdin"}, Stdin: next.SyncConfig()}); err != nil { + return fmt.Errorf("sync wireguard config on %q: %w", m.cfg.Name, err) + } + m.cfg = next + if err := m.reconcileRoutesLocked(ctx, next.RouteTargets()); err != nil { + return err + } + klog.InfoS("wireguard mesh synced", "interface", m.cfg.Name, "peers", len(peers), "routes", len(m.routes)) + return nil +} + +// Down removes installed routes and tears the interface down. It is best-effort: +// it logs and continues past individual failures so a partially-applied mesh can +// always be cleaned up. +func (m *Mesh) Down(ctx context.Context) error { + m.mu.Lock() + defer m.mu.Unlock() + + for target := range m.routes { + if _, err := m.runTolerating(ctx, m.routeCmd("delete", target), errRouteMissing); err != nil { + klog.ErrorS(err, "wireguard: failed to delete route", "target", target, "interface", m.cfg.Name) + } + } + m.routes = map[string]bool{} + + if _, err := m.runTolerating(ctx, command{Name: m.tools.ifconfig, Args: []string{m.cfg.Name, "down"}}, errInterfaceMissing); err != nil { + klog.ErrorS(err, "wireguard: failed to bring interface down", "interface", m.cfg.Name) + } + // wireguard-go installs the utun; remove it so the name is reusable. + if _, err := m.runTolerating(ctx, command{Name: m.tools.ifconfig, Args: []string{m.cfg.Name, "destroy"}}, errInterfaceMissing); err != nil { + klog.ErrorS(err, "wireguard: failed to destroy interface", "interface", m.cfg.Name) + } + m.up = false + klog.InfoS("wireguard mesh down", "interface", m.cfg.Name) + return nil +} + +// InstalledRoutes returns the route targets currently installed, sorted. It +// lets operators (and tests) confirm remote Pod CIDRs are routed through the +// tunnel, satisfying the "routes installed and visible" acceptance criterion. +func (m *Mesh) InstalledRoutes() []string { + m.mu.Lock() + defer m.mu.Unlock() + out := make([]string, 0, len(m.routes)) + for r := range m.routes { + out = append(out, r) + } + sort.Strings(out) + return out +} + +// reconcileRoutesLocked installs routes present in desired but missing, and +// removes routes installed but no longer desired. Caller holds m.mu. +func (m *Mesh) reconcileRoutesLocked(ctx context.Context, desired []string) error { + want := make(map[string]bool, len(desired)) + for _, t := range desired { + want[t] = true + } + // Remove stale routes first so a CIDR that moved to another peer is not + // briefly routed to two interfaces. + for target := range m.routes { + if want[target] { + continue + } + if _, err := m.runTolerating(ctx, m.routeCmd("delete", target), errRouteMissing); err != nil { + return fmt.Errorf("delete route %s: %w", target, err) + } + delete(m.routes, target) + } + for _, target := range desired { + if m.routes[target] { + continue + } + if _, err := m.runTolerating(ctx, m.routeCmd("add", target), errRouteExists); err != nil { + return fmt.Errorf("add route %s: %w", target, err) + } + m.routes[target] = true + } + return nil +} + +// routeCmd builds a macOS `route` command for add/delete of a CIDR through the +// managed interface. +func (m *Mesh) routeCmd(op, target string) command { + return command{Name: m.tools.route, Args: []string{"-q", "-n", op, routeFamily(target), target, "-interface", m.cfg.Name}} +} + +func routeFamily(target string) string { + ip, _, err := net.ParseCIDR(target) + if err != nil || ip.To4() != nil { + return "-inet" + } + return "-inet6" +} + +// runTolerating runs a command, treating an error whose stderr contains any of +// the given benign substrings as success. This makes interface/route operations +// idempotent (e.g. "route already in table", "interface already exists"). +func (m *Mesh) runTolerating(ctx context.Context, c command, benign ...string) (string, error) { + out, err := m.run.run(ctx, c) + if err == nil { + return out, nil + } + var ce *CommandError + if errors.As(err, &ce) { + msg := strings.ToLower(ce.Stderr) + for _, b := range benign { + if b != "" && strings.Contains(msg, b) { + return out, nil + } + } + } + return out, err +} + +// Benign stderr fragments that make operations idempotent across re-runs. +const ( + errInterfaceExists = "already exists" + errInterfaceMissing = "does not exist" + errAddressExists = "already in list" + errRouteExists = "file exists" + errRouteMissing = "not in table" +) + +// addressIP extracts the bare IP from a CIDR mesh address. +func addressIP(cidr string) (string, error) { + ip, _, err := net.ParseCIDR(cidr) + if err != nil { + return "", fmt.Errorf("wireguard: parse mesh address %q: %w", cidr, err) + } + return ip.String(), nil +} diff --git a/pkg/network/wireguard/mesh_test.go b/pkg/network/wireguard/mesh_test.go new file mode 100644 index 0000000..060724f --- /dev/null +++ b/pkg/network/wireguard/mesh_test.go @@ -0,0 +1,225 @@ +package wireguard + +import ( + "context" + "strings" + "sync" + "testing" +) + +// fakeRunner records every command and can inject failures. +type fakeRunner struct { + mu sync.Mutex + cmds []command + // failOn maps a substring of command.String() to the stderr it should fail + // with, so tests can exercise tolerated and fatal error paths. + failOn map[string]string +} + +func newFakeRunner() *fakeRunner { return &fakeRunner{failOn: map[string]string{}} } + +func (f *fakeRunner) run(_ context.Context, c command) (string, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.cmds = append(f.cmds, c) + for sub, stderr := range f.failOn { + if strings.Contains(c.String(), sub) { + return "", &CommandError{Cmd: c.String(), Stderr: stderr, ExitCode: 1} + } + } + return "", nil +} + +func (f *fakeRunner) strings() []string { + f.mu.Lock() + defer f.mu.Unlock() + out := make([]string, len(f.cmds)) + for i, c := range f.cmds { + out[i] = c.String() + } + return out +} + +func (f *fakeRunner) find(sub string) *command { + f.mu.Lock() + defer f.mu.Unlock() + for i := range f.cmds { + if strings.Contains(f.cmds[i].String(), sub) { + return &f.cmds[i] + } + } + return nil +} + +func contains(haystack []string, sub string) bool { + for _, s := range haystack { + if strings.Contains(s, sub) { + return true + } + } + return false +} + +func TestUpBringsInterfaceUpConfiguresAndRoutes(t *testing.T) { + fr := newFakeRunner() + cfg := testConfig(t) + cfg.MTU = 1380 + m := New(cfg, WithRunner(fr)) + if err := m.Up(context.Background()); err != nil { + t.Fatalf("Up: %v", err) + } + cmds := fr.strings() + + for _, want := range []string{ + "wireguard-go utun7", + "wg setconf utun7 /dev/stdin", + "ifconfig utun7 inet 10.99.0.1 10.99.0.1 alias", + "ifconfig utun7 mtu 1380", + "ifconfig utun7 up", + "route -q -n add -inet 10.244.2.0/24 -interface utun7", + "route -q -n add -inet 10.99.0.2/32 -interface utun7", + } { + if !contains(cmds, want) { + t.Errorf("Up did not run %q\nran: %v", want, cmds) + } + } + + // The applied config must be passed via stdin, not a temp file. + setconf := fr.find("wg setconf") + if setconf == nil || !strings.Contains(setconf.Stdin, "PublicKey =") { + t.Errorf("wg setconf did not receive config on stdin") + } + + routes := m.InstalledRoutes() + if len(routes) != 2 { + t.Errorf("InstalledRoutes = %v, want 2", routes) + } +} + +func TestRouteCmdUsesIPv6Family(t *testing.T) { + m := New(testConfig(t), WithRunner(newFakeRunner())) + cmd := m.routeCmd("add", "fd00:10:244:2::/64") + if got := cmd.String(); !strings.Contains(got, "add -inet6 fd00:10:244:2::/64") { + t.Errorf("IPv6 route command = %q, want -inet6", got) + } +} + +func TestUpToleratesExistingInterfaceAndRoutes(t *testing.T) { + fr := newFakeRunner() + fr.failOn["wireguard-go utun7"] = "interface already exists" + fr.failOn["route -q -n add -inet 10.244.2.0/24"] = "add net 10.244.2.0: gateway utun7: File exists" + m := New(testConfig(t), WithRunner(fr)) + if err := m.Up(context.Background()); err != nil { + t.Fatalf("Up should tolerate benign 'already exists'/'File exists', got: %v", err) + } + if got := len(m.InstalledRoutes()); got != 2 { + t.Errorf("InstalledRoutes = %d, want 2", got) + } +} + +func TestUpFailsOnFatalError(t *testing.T) { + fr := newFakeRunner() + fr.failOn["wg setconf"] = "permission denied" + m := New(testConfig(t), WithRunner(fr)) + if err := m.Up(context.Background()); err == nil { + t.Fatal("Up should fail when wg setconf fails fatally") + } +} + +func TestUpValidatesConfig(t *testing.T) { + bad := testConfig(t) + bad.Address = "" // invalid + m := New(bad, WithRunner(newFakeRunner())) + if err := m.Up(context.Background()); err == nil { + t.Fatal("Up should reject an invalid config before running commands") + } +} + +func TestSyncAddsAndRemovesPeersAndRoutes(t *testing.T) { + fr := newFakeRunner() + m := New(testConfig(t), WithRunner(fr)) + ctx := context.Background() + if err := m.Up(ctx); err != nil { + t.Fatalf("Up: %v", err) + } + + // Replace the single peer (mac-02, CIDR 10.244.2.0/24) with a new peer + // (mac-09, CIDR 10.244.9.0/24). The old route must go, the new one appear. + newPeer := Peer{ + Name: "mac-09", + PublicKey: testKey(t), + Endpoint: "192.168.1.90:51820", + AllowedIPs: []string{"10.244.9.0/24", "10.99.0.9/32"}, + } + if err := m.Sync(ctx, []Peer{newPeer}); err != nil { + t.Fatalf("Sync: %v", err) + } + + cmds := fr.strings() + if !contains(cmds, "wg syncconf utun7 /dev/stdin") { + t.Errorf("Sync did not run wg syncconf\nran: %v", cmds) + } + if !contains(cmds, "route -q -n delete -inet 10.244.2.0/24 -interface utun7") { + t.Errorf("Sync did not remove the departed peer's route\nran: %v", cmds) + } + if !contains(cmds, "route -q -n add -inet 10.244.9.0/24 -interface utun7") { + t.Errorf("Sync did not add the new peer's route\nran: %v", cmds) + } + + got := m.InstalledRoutes() + want := map[string]bool{"10.244.9.0/24": true, "10.99.0.9/32": true} + if len(got) != len(want) { + t.Fatalf("InstalledRoutes = %v, want %v", got, want) + } + for _, r := range got { + if !want[r] { + t.Errorf("unexpected route %q still installed", r) + } + } + if names := m.Peers(); len(names) != 1 || names[0] != "mac-09" { + t.Errorf("Peers = %v, want [mac-09]", names) + } +} + +func TestSyncBeforeUpFails(t *testing.T) { + m := New(testConfig(t), WithRunner(newFakeRunner())) + if err := m.Sync(context.Background(), nil); err == nil { + t.Fatal("Sync before Up should fail") + } +} + +func TestDownRemovesRoutesAndInterface(t *testing.T) { + fr := newFakeRunner() + m := New(testConfig(t), WithRunner(fr)) + ctx := context.Background() + if err := m.Up(ctx); err != nil { + t.Fatalf("Up: %v", err) + } + if err := m.Down(ctx); err != nil { + t.Fatalf("Down: %v", err) + } + cmds := fr.strings() + if !contains(cmds, "route -q -n delete -inet 10.244.2.0/24 -interface utun7") { + t.Errorf("Down did not delete routes\nran: %v", cmds) + } + if !contains(cmds, "ifconfig utun7 destroy") { + t.Errorf("Down did not destroy the interface\nran: %v", cmds) + } + if got := m.InstalledRoutes(); len(got) != 0 { + t.Errorf("routes remain after Down: %v", got) + } +} + +func TestDownIsBestEffort(t *testing.T) { + fr := newFakeRunner() + m := New(testConfig(t), WithRunner(fr)) + ctx := context.Background() + if err := m.Up(ctx); err != nil { + t.Fatalf("Up: %v", err) + } + // Even if teardown commands fail, Down must not error (cleanup is best-effort). + fr.failOn["ifconfig utun7 destroy"] = "some unexpected failure" + if err := m.Down(ctx); err != nil { + t.Fatalf("Down should be best-effort, got: %v", err) + } +} diff --git a/pkg/network/wireguard/runner.go b/pkg/network/wireguard/runner.go new file mode 100644 index 0000000..630b5e3 --- /dev/null +++ b/pkg/network/wireguard/runner.go @@ -0,0 +1,79 @@ +package wireguard + +import ( + "bytes" + "context" + "errors" + "fmt" + "os/exec" + "strings" +) + +// command is a single external command invocation. +type command struct { + // Name is the binary to run, resolved through PATH (e.g. "wg", "wg-quick", + // "route"). It is never a hardcoded absolute path. + Name string + // Args are the command arguments. + Args []string + // Stdin, when non-empty, is fed to the command's standard input. The mesh + // uses it to pass generated WireGuard config to `wg setconf`/`syncconf` via + // /dev/stdin, avoiding temp files on disk. + Stdin string +} + +func (c command) String() string { + return strings.TrimSpace(c.Name + " " + strings.Join(c.Args, " ")) +} + +// runner executes external commands. It is an interface so the Mesh can be +// unit-tested against a fake without touching the host network, mirroring the +// apple/container driver in pkg/runtime/container. +type runner interface { + // run executes the command to completion and returns its combined stdout. + // On a non-zero exit it returns a *CommandError carrying stderr and the code. + run(ctx context.Context, c command) (string, error) +} + +// CommandError describes a failed mesh command invocation. +type CommandError struct { + Cmd string + ExitCode int + Stderr string + Err error +} + +func (e *CommandError) Error() string { + msg := strings.TrimSpace(e.Stderr) + if msg == "" && e.Err != nil { + msg = e.Err.Error() + } + if msg == "" { + msg = "command failed" + } + return fmt.Sprintf("%s: %s", e.Cmd, msg) +} + +func (e *CommandError) Unwrap() error { return e.Err } + +// cliRunner is the production runner backed by the host's WireGuard toolchain. +type cliRunner struct{} + +func (cliRunner) run(ctx context.Context, c command) (string, error) { + var stdout, stderr bytes.Buffer + cmd := exec.CommandContext(ctx, c.Name, c.Args...) + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if c.Stdin != "" { + cmd.Stdin = strings.NewReader(c.Stdin) + } + if err := cmd.Run(); err != nil { + ce := &CommandError{Cmd: c.String(), Stderr: stderr.String(), Err: err, ExitCode: -1} + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + ce.ExitCode = exitErr.ExitCode() + } + return stdout.String(), ce + } + return stdout.String(), nil +} diff --git a/pkg/provider/ipam.go b/pkg/provider/ipam.go new file mode 100644 index 0000000..0bf222a --- /dev/null +++ b/pkg/provider/ipam.go @@ -0,0 +1,65 @@ +package provider + +import ( + "github.com/chimerakang/macvz/pkg/network" + corev1 "k8s.io/api/core/v1" + "k8s.io/klog/v2" +) + +// ipamRef returns the configured allocator, or nil when coordinated IPAM is off. +func (p *Provider) ipamRef() *network.PodIPAM { + p.mu.RLock() + defer p.mu.RUnlock() + return p.ipam +} + +// allocateIP assigns a stable Pod IP for key from the node's Pod CIDR. When IPAM +// is disabled it returns "" with no error, leaving the Pod IP to be derived from +// the runtime-reported address. +func (p *Provider) allocateIP(key string) (string, error) { + ipam := p.ipamRef() + if ipam == nil { + return "", nil + } + return ipam.Allocate(key) +} + +// releaseIP returns key's Pod IP to the pool. It is a no-op when IPAM is +// disabled or key holds no address, so deletion stays idempotent. +func (p *Provider) releaseIP(key string) { + if ipam := p.ipamRef(); ipam != nil { + ipam.Release(key) + } +} + +// RecoverAllocations rebuilds the allocator from Kubernetes state so a restarted +// provider neither leaks addresses nor reassigns a live Pod's IP. It reserves +// the PodIP already recorded for each Pod that this node owns. It is a no-op when +// IPAM is disabled. +// +// Call it once at startup, after SetIPAM and before the Pod controller runs. +func (p *Provider) RecoverAllocations(pods []*corev1.Pod) { + ipam := p.ipamRef() + if ipam == nil { + return + } + recovered := 0 + for _, pod := range pods { + ip := pod.Status.PodIP + if ip == "" { + continue + } + key := podKey(pod.Namespace, pod.Name) + if err := ipam.Reserve(key, ip); err != nil { + // An out-of-range or conflicting address predates this CIDR (e.g. a + // CIDR change). Skip it rather than fail startup; the Pod will be + // reassigned on its next create. + klog.ErrorS(err, "skipping IPAM recovery for Pod", "pod", key, "ip", ip) + continue + } + recovered++ + } + if recovered > 0 { + klog.InfoS("recovered Pod IP allocations from Kubernetes", "count", recovered, "cidr", ipam.CIDR()) + } +} diff --git a/pkg/provider/ipam_test.go b/pkg/provider/ipam_test.go new file mode 100644 index 0000000..df8614b --- /dev/null +++ b/pkg/provider/ipam_test.go @@ -0,0 +1,120 @@ +package provider + +import ( + "context" + "fmt" + "testing" + + "github.com/chimerakang/macvz/pkg/network" + "github.com/chimerakang/macvz/pkg/runtime" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func newIPAMProvider(t *testing.T, cidr string) (*Provider, *recordingRuntime, *network.PodIPAM) { + t.Helper() + ipam, err := network.NewPodIPAM(cidr) + if err != nil { + t.Fatalf("NewPodIPAM: %v", err) + } + rt := newRecordingRuntime() + return New("mac-01", rt, WithIPAM(ipam)), rt, ipam +} + +func TestCreatePodAssignsStablePodIP(t *testing.T) { + p, _, _ := newIPAMProvider(t, "10.244.1.0/24") + ctx := context.Background() + if err := p.CreatePod(ctx, testPod("web")); err != nil { + t.Fatalf("CreatePod: %v", err) + } + got, err := p.GetPod(ctx, "default", "p1") + if err != nil { + t.Fatalf("GetPod: %v", err) + } + if got.Status.PodIP != "10.244.1.2" { + t.Errorf("PodIP = %q, want 10.244.1.2", got.Status.PodIP) + } + // The IP must survive status reconciliation (it is not overwritten by the + // runtime-reported address, which the fake leaves empty). + st, err := p.GetPodStatus(ctx, "default", "p1") + if err != nil { + t.Fatalf("GetPodStatus: %v", err) + } + if st.PodIP != "10.244.1.2" { + t.Errorf("reconciled PodIP = %q, want 10.244.1.2", st.PodIP) + } +} + +func TestDeletePodReleasesPodIP(t *testing.T) { + p, _, ipam := newIPAMProvider(t, "10.244.1.0/24") + ctx := context.Background() + if err := p.CreatePod(ctx, testPod("web")); err != nil { + t.Fatalf("CreatePod: %v", err) + } + if got := len(ipam.Allocations()); got != 1 { + t.Fatalf("allocations after create = %d, want 1", got) + } + if err := p.DeletePod(ctx, testPod("web")); err != nil { + t.Fatalf("DeletePod: %v", err) + } + if got := len(ipam.Allocations()); got != 0 { + t.Errorf("allocations after delete = %d, want 0 (IP leaked)", got) + } +} + +func TestCreatePodReleasesIPOnFailure(t *testing.T) { + p, rt, ipam := newIPAMProvider(t, "10.244.1.0/24") + rt.startErr = fmt.Errorf("boom") + if err := p.CreatePod(context.Background(), testPod("web")); err == nil { + t.Fatal("expected CreatePod to fail when Start fails") + } + if got := len(ipam.Allocations()); got != 0 { + t.Errorf("failed CreatePod leaked %d IP allocations, want 0", got) + } +} + +func TestRecoverAllocationsReservesExistingPodIPs(t *testing.T) { + p, _, ipam := newIPAMProvider(t, "10.244.1.0/24") + existing := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "survivor"}, + Status: corev1.PodStatus{PodIP: "10.244.1.5"}, + } + noIP := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "pending"}, + } + p.RecoverAllocations([]*corev1.Pod{existing, noIP}) + + if got := ipam.IP("default/survivor"); got != "10.244.1.5" { + t.Errorf("recovered IP = %q, want 10.244.1.5", got) + } + if got := len(ipam.Allocations()); got != 1 { + t.Errorf("allocations after recovery = %d, want 1", got) + } + + // A new Pod created after recovery must not reuse the reserved IP. + if err := p.CreatePod(context.Background(), testPod("web")); err != nil { + t.Fatalf("CreatePod: %v", err) + } + got, _ := p.GetPod(context.Background(), "default", "p1") + if got.Status.PodIP == "10.244.1.5" { + t.Errorf("new Pod reused recovered IP 10.244.1.5") + } +} + +// TestNoIPAMLeavesPodIPToRuntime confirms the provider still works without +// coordinated IPAM: the PodIP is then derived from the runtime-reported address. +func TestNoIPAMLeavesPodIPToRuntime(t *testing.T) { + p, rt := newTestProvider() // no WithIPAM + ctx := context.Background() + if err := p.CreatePod(ctx, testPod("web")); err != nil { + t.Fatalf("CreatePod: %v", err) + } + rt.setStatus("wl-1", runtime.Status{ID: "wl-1", Phase: runtime.PhaseRunning, IP: "172.16.0.9"}) + st, err := p.GetPodStatus(ctx, "default", "p1") + if err != nil { + t.Fatalf("GetPodStatus: %v", err) + } + if st.PodIP != "172.16.0.9" { + t.Errorf("PodIP = %q, want runtime-reported 172.16.0.9", st.PodIP) + } +} diff --git a/pkg/provider/pod.go b/pkg/provider/pod.go index 19dfa70..f3501b5 100644 --- a/pkg/provider/pod.go +++ b/pkg/provider/pod.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" + "github.com/chimerakang/macvz/pkg/network/podnet" "github.com/chimerakang/macvz/pkg/runtime" "github.com/virtual-kubelet/virtual-kubelet/errdefs" corev1 "k8s.io/api/core/v1" @@ -21,6 +22,9 @@ import ( func (p *Provider) CreatePod(ctx context.Context, pod *corev1.Pod) error { key := podKey(pod.Namespace, pod.Name) + p.createMu.Lock() + defer p.createMu.Unlock() + p.mu.RLock() _, exists := p.pods[key] p.mu.RUnlock() @@ -37,9 +41,26 @@ func (p *Provider) CreatePod(ctx context.Context, pod *corev1.Pod) error { } c := pod.Spec.Containers[0] + // Allocate a stable Pod IP from this node's Pod CIDR. Until the Pod is + // committed to the store, any early return must release it so a retry (or a + // terminal failure) does not leak the address. + podIP, err := p.allocateIP(key) + if err != nil { + return fmt.Errorf("allocate pod IP for %q: %w", key, err) + } + committed := false + if podIP != "" { + defer func() { + if !committed { + p.releaseIP(key) + } + }() + } + st := &podState{pod: pod.DeepCopy()} now := metav1.Now() st.pod.Status.StartTime = &now + st.pod.Status.PodIP = podIP if err := p.rt.Pull(ctx, spec.Image); err != nil { // An image with no arm64 variant can never run here (P1 surfaces this @@ -62,11 +83,30 @@ func (p *Provider) CreatePod(ctx context.Context, pod *corev1.Pod) error { return fmt.Errorf("start container %q: %w", c.Name, err) } + // Wire the micro-VM into the Pod network path so it is reachable at its Pod + // IP across the mesh (#22). This needs both the assigned Pod IP and the VM's + // host-only address; if either is unavailable, roll back so the retry starts + // clean (a missing VM IP is transient — the guest is still acquiring DHCP). + if pn := p.podNetRef(); pn != nil && podIP != "" { + vmIP := p.observeVMIP(ctx, st) + if vmIP == "" { + p.rollback(ctx, st) + return fmt.Errorf("pod %q: micro-VM address not available yet for network attach", key) + } + if err := pn.Attach(ctx, podnet.Endpoint{PodKey: key, PodIP: podIP, VMIP: vmIP}); err != nil { + p.rollback(ctx, st) + return fmt.Errorf("attach pod %q network path (%s -> %s): %w", key, podIP, vmIP, err) + } + st.vmIP = vmIP + st.attached = true + } + st.pod.Status = p.reconcileStatus(ctx, st) p.mu.Lock() p.pods[key] = st p.mu.Unlock() - klog.InfoS("created Pod", "pod", key, "workloadID", spec.Name) + committed = true // the allocated Pod IP is now owned by the stored Pod. + klog.InfoS("created Pod", "pod", key, "workloadID", spec.Name, "podIP", podIP) return nil } @@ -138,6 +178,11 @@ func (p *Provider) DeletePod(ctx context.Context, pod *corev1.Pod) error { return errdefs.NotFoundf("pod %q is not known to this node", key) } + // Tear down the network path mapping, then return the Pod's IP to the pool so + // it can be reused by a future Pod. + p.detachPodNetwork(ctx, st, key) + p.releaseIP(key) + timeout := stopTimeout(pod) for _, w := range st.workloads { if err := p.rt.Stop(ctx, w.id, timeout); err != nil && !errors.Is(err, runtime.ErrNotFound) { diff --git a/pkg/provider/pod_test.go b/pkg/provider/pod_test.go index a38a7f1..3b473b6 100644 --- a/pkg/provider/pod_test.go +++ b/pkg/provider/pod_test.go @@ -33,6 +33,10 @@ type recordingRuntime struct { pullErr, createErr, startErr, stopErr, destroyErr, statusErr error + // runningIP, when set, is reported as the workload's host-only address once + // it is started, so tests can exercise the Pod network attach path. + runningIP string + // log/exec behavior logData string lastLogOpts runtime.LogOptions @@ -78,7 +82,7 @@ func (r *recordingRuntime) Start(_ context.Context, id string) error { return r.startErr } r.startedIDs = append(r.startedIDs, id) - r.statuses[id] = runtime.Status{ID: id, Phase: runtime.PhaseRunning, StartedAt: time.Now()} + r.statuses[id] = runtime.Status{ID: id, Phase: runtime.PhaseRunning, StartedAt: time.Now(), IP: r.runningIP} return nil } @@ -224,6 +228,32 @@ func TestCreatePodIdempotent(t *testing.T) { } } +func TestCreatePodConcurrentIdempotent(t *testing.T) { + p, rt := newTestProvider() + ctx := context.Background() + const attempts = 10 + errs := make(chan error, attempts) + var wg sync.WaitGroup + for i := 0; i < attempts; i++ { + wg.Add(1) + go func() { + defer wg.Done() + errs <- p.CreatePod(ctx, testPod("web")) + }() + } + wg.Wait() + close(errs) + for err := range errs { + if err != nil { + t.Fatalf("CreatePod: %v", err) + } + } + _, creates, _, _, _ := rt.counts() + if creates != 1 { + t.Errorf("concurrent CreatePod created %d workloads, want 1", creates) + } +} + func TestCreatePodRollsBackOnStartError(t *testing.T) { p, rt := newTestProvider() rt.startErr = fmt.Errorf("boom") diff --git a/pkg/provider/podnet.go b/pkg/provider/podnet.go new file mode 100644 index 0000000..5f8503f --- /dev/null +++ b/pkg/provider/podnet.go @@ -0,0 +1,70 @@ +package provider + +import ( + "context" + "time" + + "github.com/chimerakang/macvz/pkg/network/podnet" + "k8s.io/klog/v2" +) + +// PodNetwork wires a Pod's micro-VM into the MacVz Pod network path so the Pod +// is reachable at its assigned Pod IP across the WireGuard mesh (#22). It is +// satisfied by *podnet.Router and is optional: when nil, Pods keep the runtime's +// host-only address and cross-host routing is unavailable. +type PodNetwork interface { + Attach(ctx context.Context, ep podnet.Endpoint) error + Detach(ctx context.Context, podKey string) error +} + +// VM-IP polling: apple/container assigns the micro-VM its host-only address over +// DHCP shortly after boot, so the address may not be readable the instant Start +// returns. CreatePod polls briefly for it before attaching the network path. +const ( + vmIPPollAttempts = 20 + vmIPPollInterval = 500 * time.Millisecond +) + +// podNetRef returns the configured PodNetwork, or nil when it is disabled. +func (p *Provider) podNetRef() PodNetwork { + p.mu.RLock() + defer p.mu.RUnlock() + return p.podNet +} + +// observeVMIP polls the runtime for the micro-VM's host-only address backing the +// Pod's (single, MVP) workload. It returns "" if the address never appears +// within the poll budget or the context is cancelled. +func (p *Provider) observeVMIP(ctx context.Context, st *podState) string { + if len(st.workloads) == 0 { + return "" + } + id := st.workloads[0].id + for attempt := 0; attempt < vmIPPollAttempts; attempt++ { + if rs, err := p.rt.Status(ctx, id); err == nil && rs.IP != "" { + return rs.IP + } + if attempt == vmIPPollAttempts-1 { + break + } + select { + case <-ctx.Done(): + return "" + case <-time.After(vmIPPollInterval): + } + } + return "" +} + +// detachPodNetwork removes a Pod's network mapping. It is a no-op when the +// network path is disabled or the Pod was never attached. +func (p *Provider) detachPodNetwork(ctx context.Context, st *podState, key string) { + if !st.attached { + return + } + if pn := p.podNetRef(); pn != nil { + if err := pn.Detach(ctx, key); err != nil { + klog.ErrorS(err, "failed to detach pod from network path", "pod", key) + } + } +} diff --git a/pkg/provider/podnet_test.go b/pkg/provider/podnet_test.go new file mode 100644 index 0000000..108766b --- /dev/null +++ b/pkg/provider/podnet_test.go @@ -0,0 +1,142 @@ +package provider + +import ( + "context" + "fmt" + "sync" + "testing" + + "github.com/chimerakang/macvz/pkg/network" + "github.com/chimerakang/macvz/pkg/network/podnet" + corev1 "k8s.io/api/core/v1" +) + +// fakePodNet records attach/detach calls and can fail attaches. +type fakePodNet struct { + mu sync.Mutex + attached map[string]podnet.Endpoint + detached []string + attachErr error + attachCnt int + detachCnt int +} + +func newFakePodNet() *fakePodNet { + return &fakePodNet{attached: map[string]podnet.Endpoint{}} +} + +func (f *fakePodNet) Attach(_ context.Context, ep podnet.Endpoint) error { + f.mu.Lock() + defer f.mu.Unlock() + f.attachCnt++ + if f.attachErr != nil { + return f.attachErr + } + f.attached[ep.PodKey] = ep + return nil +} + +func (f *fakePodNet) Detach(_ context.Context, podKey string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.detachCnt++ + delete(f.attached, podKey) + f.detached = append(f.detached, podKey) + return nil +} + +func (f *fakePodNet) get(key string) (podnet.Endpoint, bool) { + f.mu.Lock() + defer f.mu.Unlock() + ep, ok := f.attached[key] + return ep, ok +} + +func newPodNetProvider(t *testing.T) (*Provider, *recordingRuntime, *fakePodNet) { + t.Helper() + ipam, err := network.NewPodIPAM("10.244.1.0/24") + if err != nil { + t.Fatalf("NewPodIPAM: %v", err) + } + rt := newRecordingRuntime() + rt.runningIP = "192.168.64.5" // the VM's host-only address + pn := newFakePodNet() + p := New("mac-01", rt, WithIPAM(ipam), WithPodNetwork(pn)) + return p, rt, pn +} + +func TestCreatePodAttachesNetworkPath(t *testing.T) { + p, _, pn := newPodNetProvider(t) + ctx := context.Background() + if err := p.CreatePod(ctx, testPod("web")); err != nil { + t.Fatalf("CreatePod: %v", err) + } + ep, ok := pn.get("default/p1") + if !ok { + t.Fatal("Pod was not attached to the network path") + } + if ep.PodIP != "10.244.1.2" || ep.VMIP != "192.168.64.5" { + t.Errorf("attached endpoint = %+v, want podIP 10.244.1.2 / vmIP 192.168.64.5", ep) + } +} + +func TestDeletePodDetachesNetworkPath(t *testing.T) { + p, _, pn := newPodNetProvider(t) + ctx := context.Background() + if err := p.CreatePod(ctx, testPod("web")); err != nil { + t.Fatalf("CreatePod: %v", err) + } + if err := p.DeletePod(ctx, testPod("web")); err != nil { + t.Fatalf("DeletePod: %v", err) + } + if _, ok := pn.get("default/p1"); ok { + t.Error("Pod was not detached on delete") + } + if pn.detachCnt != 1 { + t.Errorf("detach called %d times, want 1", pn.detachCnt) + } +} + +func TestCreatePodRollsBackAndReleasesIPWhenAttachFails(t *testing.T) { + p, rt, pn := newPodNetProvider(t) + pn.attachErr = fmt.Errorf("pfctl boom") + if err := p.CreatePod(context.Background(), testPod("web")); err == nil { + t.Fatal("expected CreatePod to fail when network attach fails") + } + // The workload must be rolled back and the IP released so a retry is clean. + _, creates, _, _, destroys := rt.counts() + if creates != 1 || destroys != 1 { + t.Errorf("expected 1 create and 1 rollback destroy, got creates=%d destroys=%d", creates, destroys) + } + if p.ipam != nil { + if got := len(p.ipam.Allocations()); got != 0 { + t.Errorf("failed attach leaked %d IP allocations", got) + } + } +} + +func TestCreatePodFailsWhenVMIPNeverAppears(t *testing.T) { + p, rt, pn := newPodNetProvider(t) + rt.runningIP = "" // VM never reports an address + // Keep the poll fast for the test by cancelling once the first probe is done. + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := p.CreatePod(ctx, testPod("web")); err == nil { + t.Fatal("expected CreatePod to fail when the VM IP never appears") + } + if pn.attachCnt != 0 { + t.Errorf("attach should not be called without a VM IP, got %d calls", pn.attachCnt) + } +} + +func TestCreatePodWithoutPodNetworkSkipsAttach(t *testing.T) { + // No WithPodNetwork: Pods still run, just without the network path. + p, _ := newTestProvider() + if err := p.CreatePod(context.Background(), testPod("web")); err != nil { + t.Fatalf("CreatePod: %v", err) + } + got, _ := p.GetPod(context.Background(), "default", "p1") + if got.Status.Phase != corev1.PodRunning { + t.Errorf("phase = %q, want Running", got.Status.Phase) + } +} diff --git a/pkg/provider/portforward.go b/pkg/provider/portforward.go new file mode 100644 index 0000000..530e7ac --- /dev/null +++ b/pkg/provider/portforward.go @@ -0,0 +1,122 @@ +package provider + +import ( + "context" + "fmt" + "io" + "net" + "strconv" + "time" + + "github.com/chimerakang/macvz/pkg/runtime" + "github.com/virtual-kubelet/virtual-kubelet/errdefs" + vkapi "github.com/virtual-kubelet/virtual-kubelet/node/api" + "k8s.io/klog/v2" +) + +// Compile-time assertion that PortForward satisfies the Virtual Kubelet handler +// wired into the kubelet API server. +var _ vkapi.PortForwardHandlerFunc = (*Provider)(nil).PortForward + +// portForwardDialTimeout bounds the connection attempt to the micro-VM. +const portForwardDialTimeout = 10 * time.Second + +// PortForward proxies a single `kubectl port-forward` stream to a port inside a +// MacVz-backed Pod. +// +// The kubelet runs on the same Mac as the Pod's micro-VM, so it dials the VM's +// address directly (the host can always reach the guest's vmnet address, with or +// without the cross-host Pod network path). Bytes are copied bidirectionally +// between the Kubernetes stream and the TCP connection until either side closes +// or the context is cancelled; both copy goroutines and the connection are +// always reaped, so closing the forward leaks nothing. +func (p *Provider) PortForward(ctx context.Context, namespace, name string, port int32, stream io.ReadWriteCloser) error { + key := podKey(namespace, name) + if port < 1 || port > 65535 { + return fmt.Errorf("port-forward to pod %q: port %d is out of range", key, port) + } + + p.mu.RLock() + st, ok := p.pods[key] + p.mu.RUnlock() + if !ok { + return errdefs.NotFoundf("pod %q is not known to this node", key) + } + + host, err := p.portForwardTarget(ctx, st, key) + if err != nil { + return err + } + + addr := net.JoinHostPort(host, strconv.Itoa(int(port))) + dialer := net.Dialer{Timeout: portForwardDialTimeout} + conn, err := dialer.DialContext(ctx, "tcp", addr) + if err != nil { + // Nothing listening on that port inside the Pod, or the guest is + // unreachable: surface a clear error to kubectl. + return fmt.Errorf("port-forward to pod %q: dial %s: %w", key, addr, err) + } + klog.InfoS("port-forward established", "pod", key, "port", port, "target", addr) + + err = proxyStream(ctx, stream, conn) + klog.InfoS("port-forward closed", "pod", key, "port", port) + return err +} + +// portForwardTarget resolves the address to dial for a Pod: the live +// runtime-reported micro-VM address, falling back to the last observed VM IP. It +// returns a clear error when the Pod is not running or has no address yet. +func (p *Provider) portForwardTarget(ctx context.Context, st *podState, key string) (string, error) { + if len(st.workloads) == 0 { + return "", fmt.Errorf("port-forward to pod %q: pod has no running container", key) + } + id := st.workloads[0].id + + rs, err := p.rt.Status(ctx, id) + if err != nil { + // Fall back to the address observed at attach time if the runtime probe + // fails transiently; otherwise the Pod is effectively unreachable. + if st.vmIP != "" { + return st.vmIP, nil + } + return "", fmt.Errorf("port-forward to pod %q: %w", key, err) + } + if rs.Phase != runtime.PhaseRunning { + return "", fmt.Errorf("port-forward to pod %q: container is not running (%s)", key, rs.Phase) + } + if rs.IP != "" { + return rs.IP, nil + } + if st.vmIP != "" { + return st.vmIP, nil + } + return "", fmt.Errorf("port-forward to pod %q: micro-VM has no address yet", key) +} + +// proxyStream copies bytes both ways between the port-forward stream and the +// micro-VM connection. It returns when either direction ends or ctx is +// cancelled, then closes both endpoints and waits for both copies to finish so +// no goroutine or connection is leaked. +func proxyStream(ctx context.Context, stream io.ReadWriteCloser, conn net.Conn) error { + done := make(chan error, 2) + cp := func(dst io.Writer, src io.Reader) { _, err := io.Copy(dst, src); done <- err } + go cp(conn, stream) + go cp(stream, conn) + + var first error + select { + case <-ctx.Done(): + first = ctx.Err() + case first = <-done: + } + + // Unblock and reap the other direction. + _ = conn.Close() + _ = stream.Close() + <-done + + if first == io.EOF { + return nil + } + return first +} diff --git a/pkg/provider/portforward_test.go b/pkg/provider/portforward_test.go new file mode 100644 index 0000000..25ee6a3 --- /dev/null +++ b/pkg/provider/portforward_test.go @@ -0,0 +1,152 @@ +package provider + +import ( + "context" + "io" + "net" + "testing" + "time" + + "github.com/chimerakang/macvz/pkg/runtime" + "github.com/virtual-kubelet/virtual-kubelet/errdefs" +) + +// echoListener starts a loopback TCP echo server standing in for a process +// listening inside a micro-VM. It returns the listening port and a stop func. +func echoListener(t *testing.T) (int, func()) { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + go func() { _, _ = io.Copy(c, c); _ = c.Close() }() + } + }() + return ln.Addr().(*net.TCPAddr).Port, func() { _ = ln.Close() } +} + +// runningPodProvider creates a Provider with one running Pod whose micro-VM +// address is loopback, so PortForward dials a local test listener. +func runningPodProvider(t *testing.T) *Provider { + t.Helper() + rt := newRecordingRuntime() + rt.runningIP = "127.0.0.1" + p := New("mac-01", rt) + if err := p.CreatePod(context.Background(), testPod("web")); err != nil { + t.Fatalf("CreatePod: %v", err) + } + return p +} + +func TestPortForwardProxiesBytes(t *testing.T) { + port, stop := echoListener(t) + defer stop() + p := runningPodProvider(t) + + client, server := net.Pipe() // server end is handed to PortForward + pfErr := make(chan error, 1) + go func() { + pfErr <- p.PortForward(context.Background(), "default", "p1", int32(port), server) + }() + + // Write through the forward and read the echoed bytes back. + go func() { _, _ = client.Write([]byte("ping")) }() + buf := make([]byte, 4) + _ = client.SetReadDeadline(time.Now().Add(3 * time.Second)) + if _, err := io.ReadFull(client, buf); err != nil { + t.Fatalf("read echoed bytes: %v", err) + } + if string(buf) != "ping" { + t.Errorf("echoed %q, want ping", buf) + } + + // Closing the client side must end the forward without leaking goroutines. + _ = client.Close() + select { + case err := <-pfErr: + if err != nil { + t.Errorf("PortForward returned %v, want nil after clean close", err) + } + case <-time.After(3 * time.Second): + t.Fatal("PortForward did not return after stream close (leak)") + } +} + +func TestPortForwardCancellationReturns(t *testing.T) { + port, stop := echoListener(t) + defer stop() + p := runningPodProvider(t) + + ctx, cancel := context.WithCancel(context.Background()) + _, server := net.Pipe() + pfErr := make(chan error, 1) + go func() { pfErr <- p.PortForward(ctx, "default", "p1", int32(port), server) }() + + cancel() // tearing down the forward must unblock the proxy + select { + case <-pfErr: + case <-time.After(3 * time.Second): + t.Fatal("PortForward did not return after context cancellation (leak)") + } +} + +func TestPortForwardUnknownPod(t *testing.T) { + p, _ := newTestProvider() + _, server := net.Pipe() + err := p.PortForward(context.Background(), "default", "missing", 80, server) + if !errdefs.IsNotFound(err) { + t.Errorf("PortForward to unknown pod = %v, want NotFound", err) + } +} + +func TestPortForwardNotRunningPod(t *testing.T) { + p := runningPodProvider(t) + // Flip the workload to stopped: port-forward must fail clearly, not hang. + rtErrToStopped(p) + _, server := net.Pipe() + err := p.PortForward(context.Background(), "default", "p1", 8080, server) + if err == nil { + t.Fatal("expected error port-forwarding to a non-running pod") + } + if errdefs.IsNotFound(err) { + t.Errorf("non-running pod should not be reported as NotFound: %v", err) + } +} + +func TestPortForwardRejectsBadPort(t *testing.T) { + p := runningPodProvider(t) + _, server := net.Pipe() + for _, port := range []int32{0, -1, 70000} { + if err := p.PortForward(context.Background(), "default", "p1", port, server); err == nil { + t.Errorf("PortForward(port=%d) = nil, want range error", port) + } + } +} + +func TestPortForwardNothingListening(t *testing.T) { + // Reserve a port then release it, so the dial is refused. + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + port := ln.Addr().(*net.TCPAddr).Port + _ = ln.Close() + + p := runningPodProvider(t) + _, server := net.Pipe() + if err := p.PortForward(context.Background(), "default", "p1", int32(port), server); err == nil { + t.Fatal("expected dial error when nothing is listening on the port") + } +} + +// rtErrToStopped flips the provider's single workload to a stopped status. +func rtErrToStopped(p *Provider) { + rt := p.rt.(*recordingRuntime) + rt.setStatus("wl-1", runtime.Status{ID: "wl-1", Phase: runtime.PhaseStopped}) +} diff --git a/pkg/provider/provider.go b/pkg/provider/provider.go index 0aaefac..0161a0d 100644 --- a/pkg/provider/provider.go +++ b/pkg/provider/provider.go @@ -12,6 +12,7 @@ import ( "sync" "time" + "github.com/chimerakang/macvz/pkg/network" "github.com/chimerakang/macvz/pkg/runtime" "github.com/virtual-kubelet/virtual-kubelet/node" corev1 "k8s.io/api/core/v1" @@ -27,6 +28,45 @@ type Provider struct { mu sync.RWMutex pods map[string]*podState + + // createMu serializes CreatePod. Virtual Kubelet can retry or race the same + // Pod key; serializing keeps the idempotency check and runtime side effects + // together so duplicate workloads are not leaked. + createMu sync.Mutex + + // ipam, when set, allocates each Pod a stable IP from this node's + // Kubernetes-assigned Pod CIDR. It is nil on clusters without coordinated + // IPAM, in which case the Pod IP falls back to the runtime-reported address. + ipam *network.PodIPAM + + // podNet, when set, wires each Pod's micro-VM into the Pod network path so it + // is reachable at its assigned Pod IP across the mesh (#22). Nil disables it. + podNet PodNetwork + + // hostIP is this node's reachable address, reported as each Pod's HostIP so + // `kubectl get pod -o wide` and topology-aware routing resolve the host. Set + // once at startup before the Pod controller runs; treated as immutable after. + hostIP string +} + +// Option configures a Provider at construction time. +type Option func(*Provider) + +// WithIPAM attaches a Pod IP allocator so Pods receive stable, collision-free +// addresses from this node's Kubernetes-assigned Pod CIDR. +func WithIPAM(ipam *network.PodIPAM) Option { + return func(p *Provider) { p.ipam = ipam } +} + +// WithPodNetwork attaches the Pod network path that makes each micro-VM +// reachable at its Pod IP across the mesh (#22). +func WithPodNetwork(pn PodNetwork) Option { + return func(p *Provider) { p.podNet = pn } +} + +// WithHostIP sets the node address reported as each Pod's HostIP. +func WithHostIP(ip string) Option { + return func(p *Provider) { p.hostIP = ip } } // podState tracks one Pod and the runtime workloads backing its containers. @@ -40,6 +80,12 @@ type podState struct { // unsupported spec, or an image with no arm64 variant), so they surface a // clear, stable Failed status instead of being re-derived as Pending. terminalStatus *corev1.PodStatus + // vmIP is the micro-VM's apple/container host-only address, observed once the + // VM has booted. It is mapped to the Pod IP by the Pod network path (#22). + vmIP string + // attached records whether the Pod's micro-VM has been wired into the Pod + // network path, so DeletePod knows to tear the mapping down. + attached bool } // workload binds a Pod container to a runtime workload ID. @@ -49,12 +95,34 @@ type workload struct { } // New constructs a Provider bound to a node name and runtime driver. -func New(nodeName string, rt runtime.Runtime) *Provider { - return &Provider{ +func New(nodeName string, rt runtime.Runtime, opts ...Option) *Provider { + p := &Provider{ nodeName: nodeName, rt: rt, pods: make(map[string]*podState), } + for _, opt := range opts { + opt(p) + } + return p +} + +// SetIPAM attaches a Pod IP allocator after construction. The node's Pod CIDR is +// only known once Kubernetes assigns it (after node registration), so the +// allocator is wired in then, before the Pod controller starts. It must not be +// called once Pods are being reconciled. +func (p *Provider) SetIPAM(ipam *network.PodIPAM) { + p.mu.Lock() + defer p.mu.Unlock() + p.ipam = ipam +} + +// SetPodNetwork attaches the Pod network path after construction, before the Pod +// controller starts. It must not be called once Pods are being reconciled. +func (p *Provider) SetPodNetwork(pn PodNetwork) { + p.mu.Lock() + defer p.mu.Unlock() + p.podNet = pn } // Compile-time assertion that Provider satisfies the Virtual Kubelet contract. diff --git a/pkg/provider/status.go b/pkg/provider/status.go index c80ebca..66feb56 100644 --- a/pkg/provider/status.go +++ b/pkg/provider/status.go @@ -61,9 +61,30 @@ func (p *Provider) reconcileStatus(ctx context.Context, st *podState) corev1.Pod status.ContainerStatuses = statuses status.Phase = aggregatePhase(statuses) - status.Conditions = podConditions(status.Phase) - if status.Message != "" { - status.Conditions = podConditionsWithRuntimeError(status.Conditions, status.Message) + + // Publish the Pod's address in both the singular and plural fields. The + // EndpointSlice controller reads PodIPs, so populating it is what makes a + // MacVz-backed Pod show up as a usable Service endpoint. + if status.PodIP != "" { + status.PodIPs = []corev1.PodIP{{IP: status.PodIP}} + } + // HostIP lets `kubectl get pod -o wide` and topology-aware routing resolve the + // node hosting the Pod. + if p.hostIP != "" { + status.HostIP = p.hostIP + status.HostIPs = []corev1.HostIP{{IP: p.hostIP}} + } + + // Readiness drives EndpointSlice membership: a Pod is Ready only when it is + // running AND has an address, so endpoints never point at an unreachable Pod. + ready := status.Phase == corev1.PodRunning && status.PodIP != "" + status.Conditions = podConditions(status.Phase, ready) + switch { + case status.Message != "": + status.Conditions = withConditionReason(status.Conditions, "RuntimeStatusError", status.Message) + case status.Phase == corev1.PodRunning && status.PodIP == "": + status.Conditions = withConditionReason(status.Conditions, "PodNetworkNotReady", + "waiting for Pod IP allocation and network attach") } return status } @@ -171,9 +192,10 @@ func aggregatePhase(statuses []corev1.ContainerStatus) corev1.PodPhase { } } -// podConditions returns the standard Pod conditions for a phase. -func podConditions(phase corev1.PodPhase) []corev1.PodCondition { - ready := phase == corev1.PodRunning +// podConditions returns the standard Pod conditions for a phase. Readiness is +// passed in explicitly because a Pod is only Ready when it is both running and +// addressable (see reconcileStatus), not merely running. +func podConditions(phase corev1.PodPhase, ready bool) []corev1.PodCondition { cond := func(t corev1.PodConditionType, status bool) corev1.PodCondition { s := corev1.ConditionFalse if status { @@ -182,7 +204,7 @@ func podConditions(phase corev1.PodPhase) []corev1.PodCondition { return corev1.PodCondition{Type: t, Status: s} } // Initialized stays true once the Pod has been started; ContainersReady and - // Ready track the running state. + // Ready track the live ready state. initialized := phase != corev1.PodPending return []corev1.PodCondition{ cond(corev1.PodInitialized, initialized), @@ -191,12 +213,15 @@ func podConditions(phase corev1.PodPhase) []corev1.PodCondition { } } -func podConditionsWithRuntimeError(conditions []corev1.PodCondition, msg string) []corev1.PodCondition { +// withConditionReason forces the readiness conditions False and annotates them +// with a reason/message, used when a running Pod is not yet a valid endpoint +// (transient runtime error, or no Pod IP yet). +func withConditionReason(conditions []corev1.PodCondition, reason, msg string) []corev1.PodCondition { out := append([]corev1.PodCondition(nil), conditions...) for i := range out { if out[i].Type == corev1.PodReady || out[i].Type == corev1.ContainersReady { out[i].Status = corev1.ConditionFalse - out[i].Reason = "RuntimeStatusError" + out[i].Reason = reason out[i].Message = msg } } diff --git a/pkg/provider/status_test.go b/pkg/provider/status_test.go new file mode 100644 index 0000000..78f5936 --- /dev/null +++ b/pkg/provider/status_test.go @@ -0,0 +1,116 @@ +package provider + +import ( + "context" + "testing" + + "github.com/chimerakang/macvz/pkg/network" + "github.com/chimerakang/macvz/pkg/runtime" + corev1 "k8s.io/api/core/v1" +) + +func condition(st *corev1.PodStatus, t corev1.PodConditionType) (corev1.PodCondition, bool) { + for _, c := range st.Conditions { + if c.Type == t { + return c, true + } + } + return corev1.PodCondition{}, false +} + +func newReadyProvider(t *testing.T) (*Provider, *recordingRuntime) { + t.Helper() + ipam, err := network.NewPodIPAM("10.244.1.0/24") + if err != nil { + t.Fatalf("NewPodIPAM: %v", err) + } + rt := newRecordingRuntime() + return New("mac-01", rt, WithIPAM(ipam), WithHostIP("192.168.1.10")), rt +} + +func TestStatusPopulatesPodIPsAndHostIP(t *testing.T) { + p, _ := newReadyProvider(t) + ctx := context.Background() + if err := p.CreatePod(ctx, testPod("web")); err != nil { + t.Fatalf("CreatePod: %v", err) + } + st, err := p.GetPodStatus(ctx, "default", "p1") + if err != nil { + t.Fatalf("GetPodStatus: %v", err) + } + if st.PodIP != "10.244.1.2" { + t.Errorf("PodIP = %q, want 10.244.1.2", st.PodIP) + } + // PodIPs is what the EndpointSlice controller reads. + if len(st.PodIPs) != 1 || st.PodIPs[0].IP != "10.244.1.2" { + t.Errorf("PodIPs = %v, want [10.244.1.2]", st.PodIPs) + } + if st.HostIP != "192.168.1.10" { + t.Errorf("HostIP = %q, want 192.168.1.10", st.HostIP) + } + if len(st.HostIPs) != 1 || st.HostIPs[0].IP != "192.168.1.10" { + t.Errorf("HostIPs = %v, want [192.168.1.10]", st.HostIPs) + } +} + +func TestRunningPodWithIPIsReady(t *testing.T) { + p, _ := newReadyProvider(t) + ctx := context.Background() + if err := p.CreatePod(ctx, testPod("web")); err != nil { + t.Fatalf("CreatePod: %v", err) + } + st, err := p.GetPodStatus(ctx, "default", "p1") + if err != nil { + t.Fatalf("GetPodStatus: %v", err) + } + for _, ct := range []corev1.PodConditionType{corev1.PodReady, corev1.ContainersReady} { + c, ok := condition(st, ct) + if !ok || c.Status != corev1.ConditionTrue { + t.Errorf("%s = %v (ok=%v), want True", ct, c.Status, ok) + } + } +} + +func TestRunningPodWithoutIPIsNotReady(t *testing.T) { + // No IPAM and the fake runtime reports no address: a running Pod must NOT be + // Ready, so EndpointSlices never include an unreachable Pod. + p, _ := newTestProvider() + ctx := context.Background() + if err := p.CreatePod(ctx, testPod("web")); err != nil { + t.Fatalf("CreatePod: %v", err) + } + st, err := p.GetPodStatus(ctx, "default", "p1") + if err != nil { + t.Fatalf("GetPodStatus: %v", err) + } + if st.Phase != corev1.PodRunning { + t.Fatalf("phase = %q, want Running", st.Phase) + } + if len(st.PodIPs) != 0 { + t.Errorf("PodIPs = %v, want empty without an address", st.PodIPs) + } + ready, _ := condition(st, corev1.PodReady) + if ready.Status != corev1.ConditionFalse { + t.Errorf("Ready = %v, want False without a Pod IP", ready.Status) + } + if ready.Reason != "PodNetworkNotReady" { + t.Errorf("Ready reason = %q, want PodNetworkNotReady", ready.Reason) + } +} + +func TestRuntimeErrorKeepsPodNotReady(t *testing.T) { + p, rt := newReadyProvider(t) + ctx := context.Background() + if err := p.CreatePod(ctx, testPod("web")); err != nil { + t.Fatalf("CreatePod: %v", err) + } + rt.statusErr = runtime.ErrNotReady + st, err := p.GetPodStatus(ctx, "default", "p1") + if err != nil { + t.Fatalf("GetPodStatus: %v", err) + } + ready, _ := condition(st, corev1.PodReady) + if ready.Status != corev1.ConditionFalse || ready.Reason != "RuntimeStatusError" { + t.Errorf("Ready = %v reason=%q, want False/RuntimeStatusError", ready.Status, ready.Reason) + } +}