diff --git a/cmd/controller/main.go b/cmd/controller/main.go index ab6ae2b2..4788bf82 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -84,6 +84,10 @@ func main() { var orgDefaultMaxSandboxes int var orgDefaultCPU string var orgDefaultMemory string + var orgPoolTemplateName string + var orgPoolTemplateNamespace string + var orgPoolName string + var orgPoolWarmMin int flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") @@ -119,6 +123,10 @@ func main() { flag.IntVar(&orgDefaultMaxSandboxes, "org-default-max-sandboxes", 50, "Default per-org sandbox/pod count ceiling applied to an Org's ResourceQuota when the Org sets no spec.quota override. The per-org abuse-control primitive: it bounds how much one tenant can schedule. Only used with --enable-org-tenancy.") flag.StringVar(&orgDefaultCPU, "org-default-cpu", "32", "Default per-org aggregate CPU limit ceiling (a Kubernetes quantity, for example 32 or 32000m) applied to an Org's ResourceQuota when the Org sets no spec.quota override. Only used with --enable-org-tenancy.") flag.StringVar(&orgDefaultMemory, "org-default-memory", "64Gi", "Default per-org aggregate memory limit ceiling (a Kubernetes quantity, for example 64Gi) applied to an Org's ResourceQuota when the Org sets no spec.quota override. Only used with --enable-org-tenancy.") + flag.StringVar(&orgPoolTemplateName, "org-pool-template-name", "", "Name of a reference SandboxPool (in --org-pool-template-namespace) whose full spec (image, resources, placement, snapshots, network) is cloned into every org namespace so a per-org create has a schedulable pool to fork from. Empty (the default) provisions NO per-org pool (bring-your-own-pool). Only used with --enable-org-tenancy.") + flag.StringVar(&orgPoolTemplateNamespace, "org-pool-template-namespace", "", "Namespace of the reference pool named by --org-pool-template-name. Empty defaults to the controller's own namespace. Only used with --enable-org-tenancy.") + flag.StringVar(&orgPoolName, "org-pool-name", "", "Name the cloned pool is created under in each org namespace; MUST match the pool name a create resolves (the SDK's image/pool argument). Empty defaults to --org-pool-template-name. Only used with --enable-org-tenancy.") + flag.IntVar(&orgPoolWarmMin, "org-pool-warm-min", 0, "warm.min override on the cloned per-org pool. Default 0: no idle warm husks, so N per-org pools cost nothing at rest; the first fork per org cold-starts and the pool warms on demand. Only used with --enable-org-tenancy.") flag.Parse() ctrl.SetLogger(zap.New(zap.UseDevMode(true))) @@ -390,12 +398,16 @@ func main() { os.Exit(1) } orgReconciler := &controller.OrgReconciler{ - Client: mgr.GetClient(), - PoolSecretsSubject: "mitos-controller", - PoolSecretsNamespace: poolControllerNamespace, - DefaultMaxSandboxes: int32(orgDefaultMaxSandboxes), //nolint:gosec // flag value, operator-controlled, bounded by ResourceQuota semantics - DefaultCPU: orgCPU, - DefaultMemory: orgMem, + Client: mgr.GetClient(), + PoolSecretsSubject: "mitos-controller", + PoolSecretsNamespace: poolControllerNamespace, + DefaultMaxSandboxes: int32(orgDefaultMaxSandboxes), //nolint:gosec // flag value, operator-controlled, bounded by ResourceQuota semantics + DefaultCPU: orgCPU, + DefaultMemory: orgMem, + PoolTemplateName: orgPoolTemplateName, + PoolTemplateNamespace: orgPoolTemplateNamespace, + OrgPoolName: orgPoolName, + OrgPoolWarmMin: int32(orgPoolWarmMin), //nolint:gosec // flag value, operator-controlled } if err := orgReconciler.SetupWithManager(mgr); err != nil { logger.Error(err, "unable to create controller", "controller", "Org") @@ -403,6 +415,12 @@ func main() { } logger.Info("org tenancy: ENABLED; provisioning per-org isolation namespaces (mitos-org-) with PSA privileged labels, a ResourceQuota ceiling, a LimitRange, a default-deny NetworkPolicy + DNS egress, and the mitos-pool-secrets RoleBinding. Cross-org isolation is the separate namespace + default-deny NetworkPolicy + ResourceQuota + the microVM; a NetworkPolicy-enforcing CNI is required", "default-max-sandboxes", orgDefaultMaxSandboxes, "default-cpu", orgDefaultCPU, "default-memory", orgDefaultMemory) + if orgPoolTemplateName != "" { + logger.Info("org tenancy: per-org pool ENABLED; cloning a reference pool spec into each org namespace with warm.min overridden so a per-org create has a schedulable pool to fork from", + "template-name", orgPoolTemplateName, "template-namespace", orgPoolTemplateNamespace, "org-pool-name", orgPoolName, "warm-min", orgPoolWarmMin) + } else { + logger.Info("org tenancy: per-org pool disabled (no --org-pool-template-name); each org namespace has NO pool until one is provisioned (bring-your-own-pool)") + } } else { logger.Info("org tenancy: disabled (default); self-host single-tenant. Pass --enable-org-tenancy for hosted multi-tenant per-org namespaces") } diff --git a/deploy/charts/mitos/templates/controller-deployment.yaml b/deploy/charts/mitos/templates/controller-deployment.yaml index 9e41ca21..7754b500 100644 --- a/deploy/charts/mitos/templates/controller-deployment.yaml +++ b/deploy/charts/mitos/templates/controller-deployment.yaml @@ -83,6 +83,16 @@ spec: - --org-default-max-sandboxes={{ .Values.controller.orgTenancy.defaultMaxSandboxes }} - --org-default-cpu={{ .Values.controller.orgTenancy.defaultCPU }} - --org-default-memory={{ .Values.controller.orgTenancy.defaultMemory }} + {{- if .Values.controller.orgTenancy.poolTemplate.name }} + - --org-pool-template-name={{ .Values.controller.orgTenancy.poolTemplate.name }} + {{- if .Values.controller.orgTenancy.poolTemplate.namespace }} + - --org-pool-template-namespace={{ .Values.controller.orgTenancy.poolTemplate.namespace }} + {{- end }} + {{- if .Values.controller.orgTenancy.poolTemplate.orgPoolName }} + - --org-pool-name={{ .Values.controller.orgTenancy.poolTemplate.orgPoolName }} + {{- end }} + - --org-pool-warm-min={{ .Values.controller.orgTenancy.poolTemplate.warmMin }} + {{- end }} {{- end }} {{- /* Usage metering (issue #602): the collector scrapes every forkd diff --git a/deploy/charts/mitos/values.schema.json b/deploy/charts/mitos/values.schema.json index 1f06266e..1220e4f5 100644 --- a/deploy/charts/mitos/values.schema.json +++ b/deploy/charts/mitos/values.schema.json @@ -240,6 +240,29 @@ "defaultMemory": { "description": "Default per-org aggregate memory limit ceiling, a Kubernetes quantity.", "type": "string" + }, + "poolTemplate": { + "description": "Per-org SandboxPool cloned from a reference pool: when name is set, clone that pool's full spec into every org namespace with warm.min overridden.", + "type": "object", + "additionalProperties": false, + "properties": { + "name": { + "description": "Reference SandboxPool to clone; empty disables per-org pools.", + "type": "string" + }, + "namespace": { + "description": "Namespace of the reference pool; empty defaults to the release namespace.", + "type": "string" + }, + "orgPoolName": { + "description": "Name the cloned pool takes per org; empty defaults to name.", + "type": "string" + }, + "warmMin": { + "description": "warm.min override on the cloned pool.", + "type": "integer" + } + } } } }, diff --git a/deploy/charts/mitos/values.yaml b/deploy/charts/mitos/values.yaml index 1d261569..c856137b 100644 --- a/deploy/charts/mitos/values.yaml +++ b/deploy/charts/mitos/values.yaml @@ -136,6 +136,25 @@ controller: defaultCPU: "32" # Default per-org aggregate memory limit ceiling (a Kubernetes quantity). defaultMemory: "64Gi" + # Per-org SandboxPool. When poolTemplate.name is set, the OrgReconciler clones + # that reference pool's FULL spec (image, resources, placement, snapshots, + # network) into every org namespace with warm.min overridden, so the per-org + # pool schedules exactly where the reference pool does (a bare image would not + # carry placement/resources). warmMin:0 means no idle warm husks, so N per-org + # pools cost nothing at rest (the snapshot is content-addressed and deduped per + # node); the first fork per org cold-starts and the pool warms on demand. + # orgPoolName MUST match the pool name a create resolves (the SDK image/pool + # argument). Empty name (the default) provisions no pool (bring-your-own-pool). + # Only used when enabled. + poolTemplate: + # Name of the reference SandboxPool to clone. Empty disables per-org pools. + name: "" + # Namespace of the reference pool. Empty defaults to the release namespace. + namespace: "" + # Name the cloned pool takes in each org namespace. Empty defaults to name. + orgPoolName: "" + # warm.min override on the cloned pool. + warmMin: 0 # Usage metering (issues #164/#211/#602): the live per-org metering scraper # and the bearer-gated internal usage API the hosted console reads. OFF by # default: a self-host install that does not want metering renders none of it. diff --git a/docs/threat-model.md b/docs/threat-model.md index d2b25a92..9684c3a5 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -1525,6 +1525,20 @@ deleted or drifted stack object enqueues the Org and is recreated; org deletion cascades through owner references (the cluster-scoped Org owns the cluster-scoped Namespace, a valid owner edge). +When `--org-pool-template-name` is set, the reconciler also clones a reference +SandboxPool's full spec (image, resources, placement, snapshots, network) into +each org namespace with `warm.min` overridden (default 0), named to match the +pool a create resolves. Cloning from one reference is what makes the per-org pool +schedule exactly where the reference does (a bare image would not carry the KVM +placement); `warm.min:0` keeps no idle warm husks, so N per-org pools cost nothing +at rest (the first fork per org cold-starts; the snapshot is content-addressed and +deduped per node). This does NOT move the isolation boundary: the pool and its +husks live inside the org's already-isolated namespace under the same default-deny +NetworkPolicy and ResourceQuota, so it grants no cross-tenant reach. It is the +enablement that lets the hosted deployment run per-org namespaces instead of one +shared namespace (which is what closes the shared-namespace referenced-object +class of issues in section 7b, the bare-name secret/workspace reference finding). + What isolates two orgs: **the separate namespace + the per-org default-deny NetworkPolicy + the per-org ResourceQuota ceiling + the microVM.** No single one of these is the whole boundary; the microVM remains the in-pod isolation diff --git a/internal/controller/org_builders_test.go b/internal/controller/org_builders_test.go index 5e9460ef..21562e8b 100644 --- a/internal/controller/org_builders_test.go +++ b/internal/controller/org_builders_test.go @@ -10,6 +10,53 @@ import ( "mitos.run/mitos/internal/tenant" ) +// TestBuildOrgPoolFromTemplate asserts the per-org pool clones the reference +// pool's full spec (image, resources, placement) and overrides ONLY warm.min to +// the configured floor, carries the org label and target name/namespace, so a +// per-org create in the org namespace has a schedulable pool to fork from (issue +// #288 multi-tenancy enablement). +func TestBuildOrgPoolFromTemplate(t *testing.T) { + tmpl := &v1.SandboxPool{ + Spec: v1.SandboxPoolSpec{ + Template: &v1.PoolTemplateSpec{ + Image: "ghcr.io/mitos-run/mitos-python:v1.13.0", + Resources: v1.SandboxResources{CPU: resource.MustParse("2"), Memory: resource.MustParse("512Mi")}, + }, + Warm: &v1.PoolWarm{Min: 8}, + Placement: &v1.PoolPlacement{ + NodeSelector: map[string]string{"mitos.run/kvm": "true"}, + }, + }, + } + org := &v1.Org{} + org.Name = "acme" + pool := buildOrgPoolFromTemplate(org, tenant.NamespaceForOrg("acme"), "python", tmpl, 0) + + if pool.Name != "python" { + t.Fatalf("name = %q, want python", pool.Name) + } + if pool.Namespace != "mitos-org-acme" { + t.Fatalf("namespace = %q, want mitos-org-acme", pool.Namespace) + } + if got := pool.Labels[tenant.OrgLabelKey]; got != "acme" { + t.Fatalf("org label = %q, want acme", got) + } + if pool.Spec.Template == nil || pool.Spec.Template.Image != "ghcr.io/mitos-run/mitos-python:v1.13.0" { + t.Fatalf("template image not cloned: %+v", pool.Spec.Template) + } + if pool.Spec.Placement == nil || pool.Spec.Placement.NodeSelector["mitos.run/kvm"] != "true" { + t.Fatalf("placement not cloned: %+v", pool.Spec.Placement) + } + if pool.Spec.Warm == nil || pool.Spec.Warm.Min != 0 { + t.Fatalf("warm.min = %+v, want a non-nil Warm with Min 0 (overridden)", pool.Spec.Warm) + } + // The clone must be independent: mutating it must not touch the template. + pool.Spec.Placement.NodeSelector["mitos.run/kvm"] = "mutated" + if tmpl.Spec.Placement.NodeSelector["mitos.run/kvm"] != "true" { + t.Fatalf("clone shares the template's placement map (not a deep copy)") + } +} + // TestBuildOrgDefaultDenyPolicy asserts the per-org NetworkPolicy is // default-deny in BOTH directions with a single DNS egress allow, applies to // every pod in the namespace, and carries the org label. diff --git a/internal/controller/org_controller.go b/internal/controller/org_controller.go index 64064f7a..f7ead423 100644 --- a/internal/controller/org_controller.go +++ b/internal/controller/org_controller.go @@ -65,6 +65,28 @@ type OrgReconciler struct { // DefaultMemory is the per-org aggregate memory limit ceiling applied when an // Org sets no Quota override. DefaultMemory resource.Quantity + + // PoolTemplateName is the name of a reference SandboxPool (in + // PoolTemplateNamespace) whose spec is cloned into every org namespace so a + // per-org create has a pool to fork from. Cloning the full spec (template, + // resources, placement, snapshots, network) from ONE source of truth is what + // makes the per-org pool schedulable on the same nodes as the reference pool, + // which a bare image cannot. Empty disables per-org pool provisioning (the + // self-host / bring-your-own-pool posture), so a single-tenant install is + // unaffected. + PoolTemplateName string + // PoolTemplateNamespace is where the reference pool lives. Empty defaults to + // the controller's own namespace (PoolSecretsNamespace, else "mitos"). + PoolTemplateNamespace string + // OrgPoolName is the name the cloned pool is created under in each org + // namespace. It MUST match the pool name a create resolves (the SDK passes it + // as the image/pool argument). Empty defaults to PoolTemplateName. + OrgPoolName string + // OrgPoolWarmMin overrides warm.min on the cloned per-org pool. Default 0: no + // dormant warm husks are held, so N per-org pools cost nothing at rest (the + // snapshot is content-addressed and deduped per node); the first fork per org + // cold-starts and the pool warms on demand. + OrgPoolWarmMin int32 } const ( @@ -128,9 +150,83 @@ func (r *OrgReconciler) ensureStack(ctx context.Context, org *v1.Org, ns string) if err := r.ensurePoolSecretsRoleBinding(ctx, org, ns); err != nil { return err } + if err := r.ensureOrgPool(ctx, org, ns); err != nil { + return err + } + return nil +} + +// ensureOrgPool clones the reference SandboxPool (PoolTemplateNamespace/ +// PoolTemplateName) into the org namespace when a template is configured. It is a +// no-op otherwise (the self-host / bring-your-own-pool posture), so a +// single-tenant install is unaffected. Cloning the full spec carries placement, +// resources, snapshots, and network from one source of truth, so the per-org +// pool schedules exactly where the reference pool does. The pool is created ONLY +// (never updated): once it exists the reconciler leaves its spec alone, so a +// warm.min bumped by the autoscaler or an operator persists. Owner-referenced to +// the Org for cascade deletion. +func (r *OrgReconciler) ensureOrgPool(ctx context.Context, org *v1.Org, ns string) error { + if r.PoolTemplateName == "" { + return nil + } + tns := r.PoolTemplateNamespace + if tns == "" { + tns = r.PoolSecretsNamespace + } + if tns == "" { + tns = defaultControllerNamespace + } + name := r.OrgPoolName + if name == "" { + name = r.PoolTemplateName + } + + // Create-once: if the per-org pool already exists, leave it alone. + existing := &v1.SandboxPool{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}} + if err := r.Get(ctx, client.ObjectKeyFromObject(existing), existing); err == nil { + return nil + } else if !apierrors.IsNotFound(err) { + return fmt.Errorf("get org pool %s/%s: %w", ns, name, err) + } + + var tmpl v1.SandboxPool + if err := r.Get(ctx, client.ObjectKey{Namespace: tns, Name: r.PoolTemplateName}, &tmpl); err != nil { + // A missing template is an operator misconfiguration, not a transient: fail + // the reconcile so it surfaces on the Org status with actionable context. + return fmt.Errorf("get org pool template %s/%s: %w", tns, r.PoolTemplateName, err) + } + + desired := buildOrgPoolFromTemplate(org, ns, name, &tmpl, r.OrgPoolWarmMin) + if err := r.setOwner(org, desired); err != nil { + return err + } + if err := r.Create(ctx, desired); err != nil && !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("create org pool %s/%s: %w", ns, name, err) + } return nil } +// buildOrgPoolFromTemplate builds a per-org SandboxPool by deep-copying the +// reference pool's spec and overriding only warm.min. Placement, resources, +// snapshots, template image, and network are carried verbatim so the per-org +// pool schedules exactly like the reference. warmMin defaults the pool to no +// idle warm husks (0) so N per-org pools cost nothing at rest. +func buildOrgPoolFromTemplate(org *v1.Org, ns, name string, tmpl *v1.SandboxPool, warmMin int32) *v1.SandboxPool { + spec := *tmpl.Spec.DeepCopy() + if spec.Warm == nil { + spec.Warm = &v1.PoolWarm{} + } + spec.Warm.Min = warmMin + return &v1.SandboxPool{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + Labels: tenant.OrgLabels(org.Name), + }, + Spec: spec, + } +} + // ensureNamespace creates or updates the org namespace with the org label and // the PSA privileged labels. // @@ -213,7 +309,7 @@ func buildOrgResourceQuota(org *v1.Org, ns string, defMaxSandboxes int32, defCPU } hard := corev1.ResourceList{ - corev1.ResourcePods: *resource.NewQuantity(int64(maxPods), resource.DecimalSI), + corev1.ResourcePods: *resource.NewQuantity(int64(maxPods), resource.DecimalSI), "count/sandboxes.mitos.run": *resource.NewQuantity(int64(maxSandboxes), resource.DecimalSI), corev1.ResourceLimitsCPU: cpu, corev1.ResourceLimitsMemory: mem, diff --git a/internal/controller/org_envtest_test.go b/internal/controller/org_envtest_test.go index 1e6f4a1b..b3e11c2a 100644 --- a/internal/controller/org_envtest_test.go +++ b/internal/controller/org_envtest_test.go @@ -8,6 +8,7 @@ import ( corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -157,6 +158,92 @@ func TestOrgReconcilerProvisionsIsolationStack(t *testing.T) { } } +// TestOrgReconcilerProvisionsPoolFromTemplate asserts that when a pool template +// is configured the reconciler clones it (full spec, warm.min overridden to 0) +// into the org namespace under OrgPoolName, owner-referenced to the Org, so a +// per-org create has a schedulable pool to fork from (issue #288). +func TestOrgReconcilerProvisionsPoolFromTemplate(t *testing.T) { + stamp := time.Now().UnixNano() + // Seed a reference pool in the controller namespace ("mitos"). Ensure the + // namespace exists first (envtest starts with none). + mitosNS := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "mitos"}} + if err := k8sClient.Create(ctx, mitosNS); err != nil && !apierrors.IsAlreadyExists(err) { + t.Fatalf("create mitos namespace: %v", err) + } + tmplName := fmt.Sprintf("python-%d", stamp) + tmpl := &v1.SandboxPool{ + ObjectMeta: metav1.ObjectMeta{Name: tmplName, Namespace: "mitos"}, + Spec: v1.SandboxPoolSpec{ + Template: &v1.PoolTemplateSpec{Image: "ghcr.io/mitos-run/mitos-python:v1.13.0"}, + Warm: &v1.PoolWarm{Min: 8}, + Placement: &v1.PoolPlacement{NodeSelector: map[string]string{"mitos.run/kvm": "true"}}, + }, + } + if err := k8sClient.Create(ctx, tmpl); err != nil { + t.Fatalf("create template pool: %v", err) + } + t.Cleanup(func() { _ = k8sClient.Delete(ctx, tmpl) }) + + orgID := fmt.Sprintf("pooled-%d", stamp) + org := &v1.Org{ObjectMeta: metav1.ObjectMeta{Name: orgID}} + if err := k8sClient.Create(ctx, org); err != nil { + t.Fatalf("create org: %v", err) + } + t.Cleanup(func() { _ = k8sClient.Delete(ctx, org) }) + + r := newOrgReconciler() + r.PoolTemplateName = tmplName + r.PoolTemplateNamespace = "mitos" + r.OrgPoolName = "python" + r.OrgPoolWarmMin = 0 + reconcileOrg(t, r, orgID) + + ns := tenant.NamespaceForOrg(orgID) + var pool v1.SandboxPool + if err := k8sClient.Get(ctx, types.NamespacedName{Name: "python", Namespace: ns}, &pool); err != nil { + t.Fatalf("get org pool: %v", err) + } + if pool.Spec.Template == nil || pool.Spec.Template.Image != "ghcr.io/mitos-run/mitos-python:v1.13.0" { + t.Errorf("pool template image = %+v, want cloned image", pool.Spec.Template) + } + if pool.Spec.Placement == nil || pool.Spec.Placement.NodeSelector["mitos.run/kvm"] != "true" { + t.Errorf("pool placement not cloned: %+v", pool.Spec.Placement) + } + if pool.Spec.Warm == nil || pool.Spec.Warm.Min != 0 { + t.Errorf("pool warm = %+v, want Min 0 (overridden)", pool.Spec.Warm) + } + if pool.Labels[tenant.OrgLabelKey] != orgID { + t.Errorf("pool org label = %q, want %q", pool.Labels[tenant.OrgLabelKey], orgID) + } + if !hasOrgOwner(pool.OwnerReferences, orgID) { + t.Errorf("pool missing Org owner reference: %+v", pool.OwnerReferences) + } +} + +// TestOrgReconcilerSkipsPoolWhenUnconfigured asserts that with no pool template +// (the self-host / bring-your-own-pool posture) NO pool is created, so a +// single-tenant install is unaffected. +func TestOrgReconcilerSkipsPoolWhenUnconfigured(t *testing.T) { + orgID := fmt.Sprintf("nopool-%d", time.Now().UnixNano()) + org := &v1.Org{ObjectMeta: metav1.ObjectMeta{Name: orgID}} + if err := k8sClient.Create(ctx, org); err != nil { + t.Fatalf("create org: %v", err) + } + t.Cleanup(func() { _ = k8sClient.Delete(ctx, org) }) + + r := newOrgReconciler() // PoolTemplateName empty + reconcileOrg(t, r, orgID) + + ns := tenant.NamespaceForOrg(orgID) + var list v1.SandboxPoolList + if err := k8sClient.List(ctx, &list, client.InNamespace(ns)); err != nil { + t.Fatalf("list pools: %v", err) + } + if len(list.Items) != 0 { + t.Fatalf("a pool was created despite no template: %+v", list.Items) + } +} + // TestOrgReconcilerTwoOrgsAreIsolated asserts two Orgs land in two DISTINCT // namespaces, each with its own full stack and ONLY its own org label. func TestOrgReconcilerTwoOrgsAreIsolated(t *testing.T) {