Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ macvz/
- Register the Mac as a virtual node; advertise CPU/RAM capacity so the standard scheduler can place Pods.
- Translate Pod spec → `pkg/runtime` calls (image, command/args, env, resource limits).
- Wire up `kubectl logs` and `kubectl exec` through the provider.
- **Acceptance:** `kubectl run alpine --image=alpine -- sleep 3600` lands a micro-VM on the Mac; `kubectl logs`/`exec` work.
- **Acceptance:** `kubectl run alpine --image=alpine --restart=Never -- sleep 3600` lands a micro-VM on the Mac; `kubectl logs`/`exec` work.

### Phase 3 — Cross-host mesh networking

Expand Down
2 changes: 1 addition & 1 deletion README.zh-TW.md
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ macvz/
- 將 Mac 註冊為虛擬節點;回報 CPU/RAM 容量,讓標準調度器能派 Pod。
- 將 Pod spec 翻譯成 `pkg/runtime` 呼叫(映像、command/args、env、資源限制)。
- 打通 `kubectl logs` 與 `kubectl exec`。
- **驗收:** `kubectl run alpine --image=alpine -- sleep 3600` 能在 Mac 上落地一個 micro-VM,且 `kubectl logs`/`exec` 正常。
- **驗收:** `kubectl run alpine --image=alpine --restart=Never -- sleep 3600` 能在 Mac 上落地一個 micro-VM,且 `kubectl logs`/`exec` 正常。

### 階段三:跨主機網格網路

Expand Down
166 changes: 159 additions & 7 deletions cmd/macvz-kubelet/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,19 @@ import (
"context"
"flag"
"fmt"
"net"
"os"
"os/signal"
"runtime"
"syscall"

"github.com/chimerakang/macvz/internal/version"
"github.com/chimerakang/macvz/pkg/config"
"github.com/chimerakang/macvz/pkg/provider"
"github.com/chimerakang/macvz/pkg/runtime/container"
vknode "github.com/virtual-kubelet/virtual-kubelet/node"
corev1 "k8s.io/api/core/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/klog/v2"
)

Expand All @@ -40,15 +48,19 @@ func main() {
return
}

if err := run(configPath, runtimeBinary); err != nil {
// Cancel the root context on SIGINT/SIGTERM for graceful shutdown.
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()

if err := run(ctx, configPath, runtimeBinary); err != nil {
klog.ErrorS(err, "macvz-kubelet exited with error")
klog.Flush()
os.Exit(1)
}
klog.Flush()
}

func run(configPath, runtimeBinary string) error {
func run(ctx context.Context, configPath, runtimeBinary string) error {
cfg, err := config.Load(configPath)
if err != nil {
return fmt.Errorf("load config: %w", err)
Expand All @@ -73,16 +85,156 @@ func run(configPath, runtimeBinary string) error {

// Build the apple/container driver and report runtime readiness (P1).
driver := container.New(container.Config{Binary: bin})
if err := driver.Ready(context.Background()); err != nil {
if err := driver.Ready(ctx); err != nil {
klog.ErrorS(err, "apple/container runtime is not ready",
"hint", "ensure the binary is installed and `container system start` has been run")
} else {
klog.InfoS("apple/container runtime is ready")
}
_ = driver // handed to the provider in P2

// TODO(P2): construct the provider from driver and start the Virtual Kubelet
// node controller here.
klog.InfoS("node registration not yet implemented (lands in P2); exiting cleanly")
// Build the Kubernetes client from kubeconfig / in-cluster config.
restCfg, err := cfg.RestConfig()
if err != nil {
return fmt.Errorf("kubernetes client config: %w", err)
}
clientset, err := kubernetes.NewForConfig(restCfg)
if err != nil {
return fmt.Errorf("build kubernetes client: %w", err)
}

// Construct the provider over the runtime driver. Pod lifecycle methods are
// stubbed until #16; this issue only wires the node controller loop.
p := provider.New(cfg.NodeName, driver)

// Resolve the configured node shape (capacity/taints validated at load).
capacity, err := cfg.Capacity()
if err != nil {
return fmt.Errorf("node capacity: %w", err)
}
taints, err := cfg.Taints()
if err != nil {
return fmt.Errorf("node taints: %w", err)
}
internalIP := cfg.Node.InternalIP
if internalIP == "" {
internalIP = detectInternalIP()
}
nodeSpec := provider.NodeSpec{
KubeletVersion: version.Version,
OS: cfg.Node.OS,
Arch: cfg.Node.Arch,
InternalIP: internalIP,
Capacity: capacity,
Labels: cfg.Node.Labels,
Annotations: cfg.Node.Annotations,
Taints: taints,
}
if cfg.Node.ServingTLSCertFile != "" && cfg.Node.ServingTLSKeyFile != "" {
nodeSpec.KubeletPort = cfg.Node.KubeletPort
}
node := p.BuildNode(ctx, nodeSpec)

pingInterval, statusInterval, err := cfg.HeartbeatIntervals()
if err != nil {
return fmt.Errorf("heartbeat intervals: %w", err)
}

klog.InfoS("registering virtual node",
"node", cfg.NodeName,
"cpu", cfg.Node.CPU, "memory", cfg.Node.Memory, "pods", cfg.Node.Pods,
"internalIP", internalIP, "taints", len(taints),
"enableLease", cfg.Node.EnableLease, "pingInterval", pingInterval, "statusInterval", statusInterval,
)

// Node provider re-probes runtime readiness on the status interval and
// surfaces changes as node conditions.
nodeProvider := p.NewNodeStatusProvider(nodeSpec, statusInterval)

// Log transient status-update errors and keep going: the control loop retries
// on the next interval, and client-go applies its own backoff. Returning nil
// signals "handled, retry possible" to Virtual Kubelet.
statusErrHandler := func(_ context.Context, err error) error {
klog.ErrorS(err, "transient node status update error; will retry", "node", cfg.NodeName)
return nil
}

opts := []vknode.NodeControllerOpt{
vknode.WithNodePingInterval(pingInterval),
vknode.WithNodeStatusUpdateInterval(statusInterval),
vknode.WithNodeStatusUpdateErrorHandler(statusErrHandler),
}
if cfg.Node.EnableLease {
// The Lease in kube-node-lease is the modern node heartbeat; Kubernetes
// marks the node NotReady if it is not renewed within the lease duration.
leaseClient := clientset.CoordinationV1().Leases(corev1.NamespaceNodeLease)
opts = append(opts, vknode.WithNodeEnableLeaseV1(leaseClient, cfg.Node.LeaseDurationSeconds))
}

nc, err := vknode.NewNodeController(
nodeProvider,
node,
clientset.CoreV1().Nodes(),
opts...,
)
if err != nil {
return fmt.Errorf("create node controller: %w", err)
}

klog.InfoS("starting virtual-kubelet node controller", "node", cfg.NodeName)
go func() {
if err := nc.Run(ctx); err != nil && ctx.Err() == nil {
klog.ErrorS(err, "node controller stopped unexpectedly")
}
}()

select {
case <-nc.Ready():
klog.InfoS("virtual node registered and ready", "node", cfg.NodeName)
case <-nc.Done():
return fmt.Errorf("node controller exited before becoming ready: %w", nc.Err())
case <-ctx.Done():
klog.InfoS("shutdown requested before node became ready")
return nil
}

// Start the Pod lifecycle controller so scheduled Pods become micro-VMs.
stopPods, err := startPodController(ctx, cfg, clientset, p, runtime.NumCPU())
if err != nil {
return fmt.Errorf("start pod controller: %w", err)
}
defer stopPods()

// Start the kubelet API server for kubectl logs/exec (no-op without certs).
stopServer, err := startKubeletServer(ctx, cfg, p)
if err != nil {
return fmt.Errorf("start kubelet server: %w", err)
}
defer stopServer()

// Block until shutdown is requested, then let the cancelled context unwind
// the controller run loops.
<-ctx.Done()
klog.InfoS("shutdown signal received; stopping controllers", "node", cfg.NodeName)
<-nc.Done()
if err := nc.Err(); err != nil {
return fmt.Errorf("node controller shutdown: %w", err)
}
klog.InfoS("macvz-kubelet stopped cleanly")
return nil
}

// detectInternalIP returns the host's primary outbound IPv4 address, used as
// the node's InternalIP when not set in config. It opens no connection (UDP
// dial just selects a route), and returns "" if detection fails — the node then
// registers without an InternalIP.
func detectInternalIP() string {
conn, err := net.Dial("udp", "8.8.8.8:80")
if err != nil {
return ""
}
defer func() { _ = conn.Close() }()
if addr, ok := conn.LocalAddr().(*net.UDPAddr); ok {
return addr.IP.String()
}
return ""
}
123 changes: 123 additions & 0 deletions cmd/macvz-kubelet/serve.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
package main

import (
"context"
"crypto/tls"
"fmt"
"net/http"
"path"
"time"

"github.com/chimerakang/macvz/pkg/config"
"github.com/chimerakang/macvz/pkg/provider"
vknode "github.com/virtual-kubelet/virtual-kubelet/node"
vkapi "github.com/virtual-kubelet/virtual-kubelet/node/api"
"github.com/virtual-kubelet/virtual-kubelet/node/nodeutil"
corev1 "k8s.io/api/core/v1"
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/scheme"
corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
"k8s.io/client-go/tools/record"
"k8s.io/klog/v2"
)

// informerResync is how often the shared informers do a full relist.
const informerResync = time.Minute

// startPodController wires the Virtual Kubelet pod controller: pod/configmap/
// secret/service informers (pods filtered to this node), an event recorder, and
// the controller itself driving the provider's Pod lifecycle. It returns a
// cleanup func to release the event broadcaster.
func startPodController(ctx context.Context, cfg config.Config, clientset *kubernetes.Clientset, p *provider.Provider, workers int) (func(), error) {
podFactory := informers.NewSharedInformerFactoryWithOptions(clientset, informerResync, nodeutil.PodInformerFilter(cfg.NodeName))
scmFactory := informers.NewSharedInformerFactoryWithOptions(clientset, informerResync)

podInformer := podFactory.Core().V1().Pods()
secretInformer := scmFactory.Core().V1().Secrets()
configMapInformer := scmFactory.Core().V1().ConfigMaps()
serviceInformer := scmFactory.Core().V1().Services()

eb := record.NewBroadcaster()
eb.StartRecordingToSink(&corev1client.EventSinkImpl{Interface: clientset.CoreV1().Events(corev1.NamespaceAll)})
recorder := eb.NewRecorder(scheme.Scheme, corev1.EventSource{Component: path.Join(cfg.NodeName, "pod-controller")})

pc, err := vknode.NewPodController(vknode.PodControllerConfig{
PodClient: clientset.CoreV1(),
EventRecorder: recorder,
Provider: p,
PodInformer: podInformer,
SecretInformer: secretInformer,
ConfigMapInformer: configMapInformer,
ServiceInformer: serviceInformer,
})
if err != nil {
eb.Shutdown()
return nil, fmt.Errorf("create pod controller: %w", err)
}

podFactory.Start(ctx.Done())
scmFactory.Start(ctx.Done())

go func() {
if err := pc.Run(ctx, workers); err != nil && ctx.Err() == nil {
klog.ErrorS(err, "pod controller stopped unexpectedly")
}
}()

select {
case <-pc.Ready():
klog.InfoS("pod controller ready", "node", cfg.NodeName, "workers", workers)
case <-pc.Done():
eb.Shutdown()
return nil, fmt.Errorf("pod controller exited before becoming ready: %w", pc.Err())
case <-ctx.Done():
}

return eb.Shutdown, nil
}

// startKubeletServer starts the HTTPS kubelet API used by `kubectl logs`/`exec`,
// routing to the provider. It is a no-op (returning a no-op cleanup) when no
// serving certificate is configured, mirroring upstream Virtual Kubelet: Pods
// still run, but logs/exec are unavailable until certs are provided.
func startKubeletServer(ctx context.Context, cfg config.Config, p *provider.Provider) (func(), error) {
if cfg.Node.ServingTLSCertFile == "" || cfg.Node.ServingTLSKeyFile == "" {
klog.InfoS("kubelet TLS serving disabled (no servingTLSCertFile/KeyFile); kubectl logs/exec will be unavailable")
return func() {}, nil
}

cert, err := tls.LoadX509KeyPair(cfg.Node.ServingTLSCertFile, cfg.Node.ServingTLSKeyFile)
if err != nil {
return nil, fmt.Errorf("load serving TLS keypair: %w", err)
}

handler := vkapi.PodHandler(vkapi.PodHandlerConfig{
RunInContainer: p.RunInContainer,
GetContainerLogs: p.GetContainerLogs,
GetPods: p.GetPods,
GetPodsFromKubernetes: p.GetPods,
}, false)

addr := fmt.Sprintf(":%d", cfg.Node.KubeletPort)
listener, err := tls.Listen("tcp", addr, &tls.Config{
Certificates: []tls.Certificate{cert},
MinVersion: tls.VersionTLS12,
})
if err != nil {
return nil, fmt.Errorf("listen on %s: %w", addr, err)
}

srv := &http.Server{Handler: handler, ReadHeaderTimeout: 30 * time.Second}
go func() {
if err := srv.Serve(listener); err != nil && err != http.ErrServerClosed && ctx.Err() == nil {
klog.ErrorS(err, "kubelet API server stopped unexpectedly")
}
}()
klog.InfoS("kubelet API server listening", "addr", addr)

return func() {
_ = srv.Close()
_ = listener.Close()
}, nil
}
44 changes: 44 additions & 0 deletions config.example.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -17,3 +17,47 @@ runtimeBinary: container

# Log verbosity: "info" or "debug" (klog -v levels also honored via flags).
logLevel: info

# Node shape advertised to Kubernetes. All fields are optional.
node:
# Capacity / allocatable resources. Kept conservative until P1 density data is
# recorded; raise once per-VM RAM overhead is measured.
cpu: "2"
memory: 4Gi
pods: "20"

# Workload platform reported in node info and kubernetes.io/{os,arch} labels.
# MacVz runs Linux micro-VMs on Apple Silicon, so this is linux/arm64 — not
# the Darwin host.
os: linux
arch: arm64

# Node InternalIP. Leave empty to auto-detect the host's primary outbound IPv4.
internalIP: ""

# Extra labels/annotations merged onto the node (override built-ins on clash).
labels: {}
annotations: {}

# Scheduling taints. Defaults to the well-known Virtual Kubelet provider taint
# so only Pods that tolerate MacVz are scheduled here. Set to [] to disable.
taints:
- key: virtual-kubelet.io/provider
value: macvz
effect: NoSchedule

# Heartbeat / lease tuning. The coordination.k8s.io Lease is the node
# heartbeat: Kubernetes marks the node NotReady if it is not renewed within
# leaseDurationSeconds. Disable enableLease only on clusters without lease
# support (then node status updates double as the heartbeat).
enableLease: true
leaseDurationSeconds: 40
pingInterval: 10s # node liveness Ping cadence
statusUpdateInterval: 1m # node status push + runtime readiness re-probe

# Kubelet API (kubectl logs/exec). The Kubernetes API server connects to the
# node's InternalIP on kubeletPort over HTTPS. Provide a serving cert+key to
# enable logs/exec; leave them empty to run Pods without logs/exec support.
kubeletPort: 10250
servingTLSCertFile: "" # e.g. /etc/macvz/pki/kubelet.crt
servingTLSKeyFile: "" # e.g. /etc/macvz/pki/kubelet.key
Loading
Loading