From 57a4eedb2b8dc0b82b27765752d0c0ac506b42c4 Mon Sep 17 00:00:00 2001 From: Jannes Stubbemann Date: Sun, 5 Jul 2026 23:35:46 +0200 Subject: [PATCH] feat(controller): clone a reference SandboxPool into each org namespace (#288) Per-org namespace tenancy provisioned everything EXCEPT a SandboxPool, so a create in an org namespace had no pool to fork from. This closes that gap so the hosted deployment can run per-org namespaces instead of one shared namespace (which is what removes the shared-namespace referenced-object class of findings, e.g. the bare-name secret/workspace reference). When --org-pool-template-name is set (and --enable-org-tenancy), the OrgReconciler CLONES that reference pool's full spec (image, resources, placement, snapshots, network) into every org namespace with warm.min overridden (default 0), named --org-pool-name. Cloning the full spec from one source of truth is what makes the per-org pool schedule exactly where the reference does; a bare image would not carry the KVM placement the hosted nodes require. warm.min:0 means no idle warm husks, so N per-org pools cost nothing at rest (the snapshot is content-addressed and deduped per node); the first fork per org cold-starts and the pool warms on demand. The pool is cloned once and never clobbered, so autoscaler or operator warm tuning persists. Empty template name (the default) provisions no pool, so a self-host / single-tenant install is unaffected. Chart: controller.orgTenancy.poolTemplate.{name,namespace,orgPoolName,warmMin}. Threat-model 7a notes the pool lives inside the already-isolated org namespace and grants no cross-tenant reach. TDD: builder unit test (clone + deep-copy independence) + envtest (clones from a seeded reference pool when configured, skips when not). Refs #288. Signed-off-by: Jannes Stubbemann --- cmd/controller/main.go | 30 +- .../templates/controller-deployment.yaml | 10 + deploy/charts/mitos/values.schema.json | 269 ++++++++++++++---- deploy/charts/mitos/values.yaml | 19 ++ docs/threat-model.md | 14 + internal/controller/org_builders_test.go | 47 +++ internal/controller/org_controller.go | 98 ++++++- internal/controller/org_envtest_test.go | 87 ++++++ 8 files changed, 509 insertions(+), 65 deletions(-) diff --git a/cmd/controller/main.go b/cmd/controller/main.go index 746e5c5a..8ec4b816 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -79,6 +79,10 @@ func main() { var orgDefaultMaxSandboxes int var orgDefaultCPU string var orgDefaultMemory string + var orgPoolTemplateName string + var orgPoolTemplateNamespace string + var orgPoolName string + var orgPoolWarmMin int flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") @@ -109,6 +113,10 @@ func main() { flag.IntVar(&orgDefaultMaxSandboxes, "org-default-max-sandboxes", 50, "Default per-org sandbox/pod count ceiling applied to an Org's ResourceQuota when the Org sets no spec.quota override. The per-org abuse-control primitive: it bounds how much one tenant can schedule. Only used with --enable-org-tenancy.") flag.StringVar(&orgDefaultCPU, "org-default-cpu", "32", "Default per-org aggregate CPU limit ceiling (a Kubernetes quantity, for example 32 or 32000m) applied to an Org's ResourceQuota when the Org sets no spec.quota override. Only used with --enable-org-tenancy.") flag.StringVar(&orgDefaultMemory, "org-default-memory", "64Gi", "Default per-org aggregate memory limit ceiling (a Kubernetes quantity, for example 64Gi) applied to an Org's ResourceQuota when the Org sets no spec.quota override. Only used with --enable-org-tenancy.") + flag.StringVar(&orgPoolTemplateName, "org-pool-template-name", "", "Name of a reference SandboxPool (in --org-pool-template-namespace) whose full spec (image, resources, placement, snapshots, network) is cloned into every org namespace so a per-org create has a schedulable pool to fork from. Empty (the default) provisions NO per-org pool (bring-your-own-pool). Only used with --enable-org-tenancy.") + flag.StringVar(&orgPoolTemplateNamespace, "org-pool-template-namespace", "", "Namespace of the reference pool named by --org-pool-template-name. Empty defaults to the controller's own namespace. Only used with --enable-org-tenancy.") + flag.StringVar(&orgPoolName, "org-pool-name", "", "Name the cloned pool is created under in each org namespace; MUST match the pool name a create resolves (the SDK's image/pool argument). Empty defaults to --org-pool-template-name. Only used with --enable-org-tenancy.") + flag.IntVar(&orgPoolWarmMin, "org-pool-warm-min", 0, "warm.min override on the cloned per-org pool. Default 0: no idle warm husks, so N per-org pools cost nothing at rest; the first fork per org cold-starts and the pool warms on demand. Only used with --enable-org-tenancy.") flag.Parse() ctrl.SetLogger(zap.New(zap.UseDevMode(true))) @@ -336,12 +344,16 @@ func main() { os.Exit(1) } orgReconciler := &controller.OrgReconciler{ - Client: mgr.GetClient(), - PoolSecretsSubject: "mitos-controller", - PoolSecretsNamespace: poolControllerNamespace, - DefaultMaxSandboxes: int32(orgDefaultMaxSandboxes), //nolint:gosec // flag value, operator-controlled, bounded by ResourceQuota semantics - DefaultCPU: orgCPU, - DefaultMemory: orgMem, + Client: mgr.GetClient(), + PoolSecretsSubject: "mitos-controller", + PoolSecretsNamespace: poolControllerNamespace, + DefaultMaxSandboxes: int32(orgDefaultMaxSandboxes), //nolint:gosec // flag value, operator-controlled, bounded by ResourceQuota semantics + DefaultCPU: orgCPU, + DefaultMemory: orgMem, + PoolTemplateName: orgPoolTemplateName, + PoolTemplateNamespace: orgPoolTemplateNamespace, + OrgPoolName: orgPoolName, + OrgPoolWarmMin: int32(orgPoolWarmMin), //nolint:gosec // flag value, operator-controlled } if err := orgReconciler.SetupWithManager(mgr); err != nil { logger.Error(err, "unable to create controller", "controller", "Org") @@ -349,6 +361,12 @@ func main() { } logger.Info("org tenancy: ENABLED; provisioning per-org isolation namespaces (mitos-org-) with PSA privileged labels, a ResourceQuota ceiling, a LimitRange, a default-deny NetworkPolicy + DNS egress, and the mitos-pool-secrets RoleBinding. Cross-org isolation is the separate namespace + default-deny NetworkPolicy + ResourceQuota + the microVM; a NetworkPolicy-enforcing CNI is required", "default-max-sandboxes", orgDefaultMaxSandboxes, "default-cpu", orgDefaultCPU, "default-memory", orgDefaultMemory) + if orgPoolTemplateName != "" { + logger.Info("org tenancy: per-org pool ENABLED; cloning a reference pool spec into each org namespace with warm.min overridden so a per-org create has a schedulable pool to fork from", + "template-name", orgPoolTemplateName, "template-namespace", orgPoolTemplateNamespace, "org-pool-name", orgPoolName, "warm-min", orgPoolWarmMin) + } else { + logger.Info("org tenancy: per-org pool disabled (no --org-pool-template-name); each org namespace has NO pool until one is provisioned (bring-your-own-pool)") + } } else { logger.Info("org tenancy: disabled (default); self-host single-tenant. Pass --enable-org-tenancy for hosted multi-tenant per-org namespaces") } diff --git a/deploy/charts/mitos/templates/controller-deployment.yaml b/deploy/charts/mitos/templates/controller-deployment.yaml index f52078c7..13e2fb2c 100644 --- a/deploy/charts/mitos/templates/controller-deployment.yaml +++ b/deploy/charts/mitos/templates/controller-deployment.yaml @@ -53,6 +53,16 @@ spec: - --org-default-max-sandboxes={{ .Values.controller.orgTenancy.defaultMaxSandboxes }} - --org-default-cpu={{ .Values.controller.orgTenancy.defaultCPU }} - --org-default-memory={{ .Values.controller.orgTenancy.defaultMemory }} + {{- if .Values.controller.orgTenancy.poolTemplate.name }} + - --org-pool-template-name={{ .Values.controller.orgTenancy.poolTemplate.name }} + {{- if .Values.controller.orgTenancy.poolTemplate.namespace }} + - --org-pool-template-namespace={{ .Values.controller.orgTenancy.poolTemplate.namespace }} + {{- end }} + {{- if .Values.controller.orgTenancy.poolTemplate.orgPoolName }} + - --org-pool-name={{ .Values.controller.orgTenancy.poolTemplate.orgPoolName }} + {{- end }} + - --org-pool-warm-min={{ .Values.controller.orgTenancy.poolTemplate.warmMin }} + {{- end }} {{- end }} {{- /* Usage metering (issue #602): the collector scrapes every forkd diff --git a/deploy/charts/mitos/values.schema.json b/deploy/charts/mitos/values.schema.json index 2b5c65bd..822ca250 100644 --- a/deploy/charts/mitos/values.schema.json +++ b/deploy/charts/mitos/values.schema.json @@ -21,7 +21,11 @@ "pullPolicy": { "description": "Image pull policy for the container.", "type": "string", - "enum": ["Always", "IfNotPresent", "Never"] + "enum": [ + "Always", + "IfNotPresent", + "Never" + ] } } }, @@ -55,7 +59,10 @@ }, "minAvailable": { "description": "minAvailable for the PDB; an integer count or a percentage string.", - "type": ["integer", "string"] + "type": [ + "integer", + "string" + ] } } }, @@ -94,7 +101,10 @@ "kind": { "description": "Kind of the issuer reference.", "type": "string", - "enum": ["ClusterIssuer", "Issuer"] + "enum": [ + "ClusterIssuer", + "Issuer" + ] } } }, @@ -149,13 +159,19 @@ "type": "object", "additionalProperties": false, "properties": { - "image": { "$ref": "#/definitions/componentImage" }, + "image": { + "$ref": "#/definitions/componentImage" + }, "replicas": { "description": "Number of controller replicas; multiple replicas run HA with leader election.", "type": "integer" }, - "podDisruptionBudget": { "$ref": "#/definitions/podDisruptionBudget" }, - "resources": { "$ref": "#/definitions/resources" }, + "podDisruptionBudget": { + "$ref": "#/definitions/podDisruptionBudget" + }, + "resources": { + "$ref": "#/definitions/resources" + }, "enableHuskPods": { "description": "Render --enable-husk-pods so each SandboxPool maintains a warm pool of pre-scheduled husk pods.", "type": "boolean" @@ -195,11 +211,38 @@ }, "defaultCPU": { "description": "Default per-org aggregate CPU limit ceiling, a Kubernetes quantity.", - "type": ["string", "integer", "number"] + "type": [ + "string", + "integer", + "number" + ] }, "defaultMemory": { "description": "Default per-org aggregate memory limit ceiling, a Kubernetes quantity.", "type": "string" + }, + "poolTemplate": { + "description": "Per-org SandboxPool cloned from a reference pool: when name is set, clone that pool's full spec into every org namespace with warm.min overridden.", + "type": "object", + "additionalProperties": false, + "properties": { + "name": { + "description": "Reference SandboxPool to clone; empty disables per-org pools.", + "type": "string" + }, + "namespace": { + "description": "Namespace of the reference pool; empty defaults to the release namespace.", + "type": "string" + }, + "orgPoolName": { + "description": "Name the cloned pool takes per org; empty defaults to name.", + "type": "string" + }, + "warmMin": { + "description": "warm.min override on the cloned pool.", + "type": "integer" + } + } } } }, @@ -220,16 +263,24 @@ "description": "Internal usage API listen address in :port form; the container and Service port derive from it.", "type": "string" }, - "tokenSecret": { "$ref": "#/definitions/secretRef" }, - "priceList": { "$ref": "#/definitions/rateMap" } + "tokenSecret": { + "$ref": "#/definitions/secretRef" + }, + "priceList": { + "$ref": "#/definitions/rateMap" + } } }, "extraArgs": { "description": "Extra args appended verbatim to the controller container args.", "type": "array", - "items": { "type": "string" } + "items": { + "type": "string" + } }, - "extraEnv": { "$ref": "#/definitions/extraEnv" } + "extraEnv": { + "$ref": "#/definitions/extraEnv" + } } }, "huskStub": { @@ -237,7 +288,9 @@ "type": "object", "additionalProperties": false, "properties": { - "image": { "$ref": "#/definitions/componentImage" } + "image": { + "$ref": "#/definitions/componentImage" + } } }, "forkd": { @@ -245,12 +298,16 @@ "type": "object", "additionalProperties": false, "properties": { - "image": { "$ref": "#/definitions/componentImage" }, + "image": { + "$ref": "#/definitions/componentImage" + }, "priorityClassName": { "description": "Priority class so forkd is never evicted under resource pressure; empty disables.", "type": "string" }, - "resources": { "$ref": "#/definitions/resources" }, + "resources": { + "$ref": "#/definitions/resources" + }, "dataDir": { "description": "Value for forkd --data-dir and the data hostPath mountPath.", "type": "string" @@ -267,7 +324,11 @@ "type": { "description": "Seccomp profile type.", "type": "string", - "enum": ["RuntimeDefault", "Unconfined", "Localhost"] + "enum": [ + "RuntimeDefault", + "Unconfined", + "Localhost" + ] }, "localhostProfile": { "description": "Path of the Localhost seccomp profile relative to the kubelet seccomp root.", @@ -278,17 +339,23 @@ "extraCapabilities": { "description": "Extra Linux capabilities added to forkd's hardened base set; the base set is not removable here.", "type": "array", - "items": { "type": "string" } + "items": { + "type": "string" + } }, "nodeSelector": { "description": "nodeSelector pinning forkd to KVM nodes.", "type": "object", - "additionalProperties": { "type": "string" } + "additionalProperties": { + "type": "string" + } }, "tolerations": { "description": "Tolerations so forkd lands on dedicated, tainted KVM workers; rendered verbatim.", "type": "array", - "items": { "type": "object" } + "items": { + "type": "object" + } } } }, @@ -301,7 +368,9 @@ "description": "Gate the device plugin DaemonSet.", "type": "boolean" }, - "image": { "$ref": "#/definitions/componentImage" } + "image": { + "$ref": "#/definitions/componentImage" + } } }, "kernelProvisioner": { @@ -339,7 +408,10 @@ "failurePolicy": { "description": "Webhook failure policy; Fail rejects a claim if the webhook is unreachable.", "type": "string", - "enum": ["Fail", "Ignore"] + "enum": [ + "Fail", + "Ignore" + ] } } }, @@ -370,7 +442,9 @@ "description": "Gate the facade Deployment plus its ServiceAccount and RBAC.", "type": "boolean" }, - "image": { "$ref": "#/definitions/componentImage" }, + "image": { + "$ref": "#/definitions/componentImage" + }, "defaultPool": { "description": "The mitos.run pool a Sandbox binds to when it carries no bridge annotation.", "type": "string" @@ -379,7 +453,9 @@ "description": "The cluster DNS domain the facade uses to build in-cluster URLs.", "type": "string" }, - "resources": { "$ref": "#/definitions/resources" } + "resources": { + "$ref": "#/definitions/resources" + } } }, "canary": { @@ -391,7 +467,9 @@ "description": "Gate the whole canary: Deployment, Service, ServiceMonitor, optional Secret.", "type": "boolean" }, - "image": { "$ref": "#/definitions/componentImage" }, + "image": { + "$ref": "#/definitions/componentImage" + }, "pool": { "description": "Template the canary forks each cycle; keep it small for a fast, cheap probe.", "type": "string" @@ -447,7 +525,9 @@ } } }, - "resources": { "$ref": "#/definitions/resources" } + "resources": { + "$ref": "#/definitions/resources" + } } }, "monitoring": { @@ -517,19 +597,25 @@ "imagePullSecrets": { "description": "Image pull secret names attached to every workload pod spec, rendered verbatim.", "type": "array", - "items": { "type": "object" } + "items": { + "type": "object" + } }, "commonLabels": { "description": "Extra labels merged onto every resource the chart renders.", "type": "object", - "additionalProperties": { "type": "string" } + "additionalProperties": { + "type": "string" + } }, "database": { "description": "Durable persistence for the front door: bring your own managed Postgres, referenced as a Secret.", "type": "object", "additionalProperties": false, "properties": { - "dsnSecretRef": { "$ref": "#/definitions/secretRef" } + "dsnSecretRef": { + "$ref": "#/definitions/secretRef" + } } }, "console": { @@ -544,15 +630,24 @@ "edition": { "description": "Runtime edition value; community for self-host, hosted for the SaaS.", "type": "string", - "enum": ["community", "hosted"] + "enum": [ + "community", + "hosted" + ] + }, + "image": { + "$ref": "#/definitions/componentImage" }, - "image": { "$ref": "#/definitions/componentImage" }, "replicas": { "description": "Console replicas; more than one requires a shared session store first.", "type": "integer" }, - "podDisruptionBudget": { "$ref": "#/definitions/podDisruptionBudget" }, - "resources": { "$ref": "#/definitions/resources" }, + "podDisruptionBudget": { + "$ref": "#/definitions/podDisruptionBudget" + }, + "resources": { + "$ref": "#/definitions/resources" + }, "signup": { "description": "Self-serve org signup; defaults off (waitlist mode), server-controlled per the #208 gate.", "type": "boolean" @@ -564,12 +659,16 @@ "autoAllowDomains": { "description": "Email domains whose signups bypass the manual allowlist approval, rendered comma-joined.", "type": "array", - "items": { "type": "string" } + "items": { + "type": "string" + } }, "authConnectors": { "description": "Social-login connectors advertised at GET /auth/connectors; known values are github and google.", "type": "array", - "items": { "type": "string" } + "items": { + "type": "string" + } }, "antiAbuse": { "description": "Signup anti-abuse knobs; everything defaults off or empty.", @@ -585,7 +684,9 @@ "description": "The public Friendly Captcha sitekey; not a secret.", "type": "string" }, - "secretRef": { "$ref": "#/definitions/secretRef" }, + "secretRef": { + "$ref": "#/definitions/secretRef" + }, "url": { "description": "Verification API base URL override; empty uses the binary's built-in default endpoint.", "type": "string" @@ -595,11 +696,16 @@ "disposableAllowDomains": { "description": "Email domains exempted from the disposable-email-domain blocklist, rendered comma-joined.", "type": "array", - "items": { "type": "string" } + "items": { + "type": "string" + } }, "signupIPLimit": { "description": "Per-IP signup velocity cap; empty renders no env, 0 disables the cap entirely.", - "type": ["string", "integer"] + "type": [ + "string", + "integer" + ] }, "signupIPWindow": { "description": "Per-IP signup velocity window, a Go duration string such as 30m.", @@ -641,7 +747,9 @@ } } }, - "extraEnv": { "$ref": "#/definitions/extraEnv" }, + "extraEnv": { + "$ref": "#/definitions/extraEnv" + }, "secrets": { "description": "Secret-store providers advertised in capabilities and wired into the binary.", "type": "object", @@ -674,7 +782,10 @@ "perOrg": { "description": "Per-tenant isolation mode: path or namespace.", "type": "string", - "enum": ["path", "namespace"] + "enum": [ + "path", + "namespace" + ] } } } @@ -694,8 +805,12 @@ "type": "object", "additionalProperties": false, "properties": { - "apiKeySecretRef": { "$ref": "#/definitions/secretRef" }, - "webhookSecretRef": { "$ref": "#/definitions/secretRef" }, + "apiKeySecretRef": { + "$ref": "#/definitions/secretRef" + }, + "webhookSecretRef": { + "$ref": "#/definitions/secretRef" + }, "baseURL": { "description": "Paddle API host; empty uses the binary's built-in live default.", "type": "string" @@ -710,7 +825,9 @@ } } }, - "rates": { "$ref": "#/definitions/rateMap" } + "rates": { + "$ref": "#/definitions/rateMap" + } } }, "onboarding": { @@ -751,7 +868,9 @@ } } }, - "ingress": { "$ref": "#/definitions/ingress" } + "ingress": { + "$ref": "#/definitions/ingress" + } } }, "gateway": { @@ -763,12 +882,16 @@ "description": "Render the gateway Deployment, Service, and RBAC.", "type": "boolean" }, - "image": { "$ref": "#/definitions/componentImage" }, + "image": { + "$ref": "#/definitions/componentImage" + }, "replicas": { "description": "Number of gateway replicas.", "type": "integer" }, - "podDisruptionBudget": { "$ref": "#/definitions/podDisruptionBudget" }, + "podDisruptionBudget": { + "$ref": "#/definitions/podDisruptionBudget" + }, "enforce": { "description": "Abuse-control enforcement: quotas, rate limits, and the kill-switch.", "type": "object", @@ -784,12 +907,16 @@ } } }, - "resources": { "$ref": "#/definitions/resources" }, + "resources": { + "$ref": "#/definitions/resources" + }, "singleTenantNamespace": { "description": "When non-empty, pins all sandbox operations to this fixed namespace instead of per-org namespaces.", "type": "string" }, - "ingress": { "$ref": "#/definitions/ingress" } + "ingress": { + "$ref": "#/definitions/ingress" + } } }, "telemetry": { @@ -809,8 +936,12 @@ "description": "The collector endpoint the network sink POSTs JSON to; required when enabled, empty fails closed.", "type": "string" }, - "saltSecretRef": { "$ref": "#/definitions/secretRef" }, - "tokenSecretRef": { "$ref": "#/definitions/secretRef" } + "saltSecretRef": { + "$ref": "#/definitions/secretRef" + }, + "tokenSecretRef": { + "$ref": "#/definitions/secretRef" + } } }, "expose": { @@ -822,7 +953,9 @@ "description": "Gate the expose-proxy Deployment, Service, and optional Ingress and Certificate.", "type": "boolean" }, - "image": { "$ref": "#/definitions/componentImage" }, + "image": { + "$ref": "#/definitions/componentImage" + }, "replicas": { "description": "Number of proxy replicas.", "type": "integer" @@ -839,7 +972,11 @@ "type": { "description": "Service type.", "type": "string", - "enum": ["ClusterIP", "NodePort", "LoadBalancer"] + "enum": [ + "ClusterIP", + "NodePort", + "LoadBalancer" + ] } } }, @@ -876,7 +1013,9 @@ "description": "Render a cert-manager Certificate for the wildcard domain.", "type": "boolean" }, - "issuerRef": { "$ref": "#/definitions/issuerRef" } + "issuerRef": { + "$ref": "#/definitions/issuerRef" + } } }, "secretName": { @@ -915,7 +1054,9 @@ } } }, - "resources": { "$ref": "#/definitions/resources" } + "resources": { + "$ref": "#/definitions/resources" + } } }, "dex": { @@ -973,7 +1114,9 @@ "description": "Number of frontdoor replicas.", "type": "integer" }, - "image": { "$ref": "#/definitions/componentImage" }, + "image": { + "$ref": "#/definitions/componentImage" + }, "marketingURL": { "description": "In-cluster URL of the marketing static-site Service; empty derives the default.", "type": "string" @@ -989,9 +1132,13 @@ "marketingPagesAddrs": { "description": "GitHub Pages anycast IP:port addresses for the marketing upstream; empty means in-cluster mode.", "type": "array", - "items": { "type": "string" } + "items": { + "type": "string" + } }, - "resources": { "$ref": "#/definitions/resources" } + "resources": { + "$ref": "#/definitions/resources" + } } }, "edge": { @@ -1020,7 +1167,9 @@ "description": "Name of the TLS Secret created by cert-manager and referenced by the Gateway listener.", "type": "string" }, - "issuerRef": { "$ref": "#/definitions/issuerRef" } + "issuerRef": { + "$ref": "#/definitions/issuerRef" + } } } } @@ -1046,7 +1195,9 @@ "description": "Full OCI image reference for the marketing static site; required when marketing.enabled is true.", "type": "string" }, - "resources": { "$ref": "#/definitions/resources" } + "resources": { + "$ref": "#/definitions/resources" + } } }, "networkPolicy": { @@ -1066,7 +1217,9 @@ "extraEgress": { "description": "Extra CiliumNetworkPolicy egress rule objects appended verbatim to the shared control-plane policy.", "type": "array", - "items": { "type": "object" } + "items": { + "type": "object" + } } } } diff --git a/deploy/charts/mitos/values.yaml b/deploy/charts/mitos/values.yaml index 840023a4..68de9901 100644 --- a/deploy/charts/mitos/values.yaml +++ b/deploy/charts/mitos/values.yaml @@ -93,6 +93,25 @@ controller: defaultCPU: "32" # Default per-org aggregate memory limit ceiling (a Kubernetes quantity). defaultMemory: "64Gi" + # Per-org SandboxPool. When poolTemplate.name is set, the OrgReconciler clones + # that reference pool's FULL spec (image, resources, placement, snapshots, + # network) into every org namespace with warm.min overridden, so the per-org + # pool schedules exactly where the reference pool does (a bare image would not + # carry placement/resources). warmMin:0 means no idle warm husks, so N per-org + # pools cost nothing at rest (the snapshot is content-addressed and deduped per + # node); the first fork per org cold-starts and the pool warms on demand. + # orgPoolName MUST match the pool name a create resolves (the SDK image/pool + # argument). Empty name (the default) provisions no pool (bring-your-own-pool). + # Only used when enabled. + poolTemplate: + # Name of the reference SandboxPool to clone. Empty disables per-org pools. + name: "" + # Namespace of the reference pool. Empty defaults to the release namespace. + namespace: "" + # Name the cloned pool takes in each org namespace. Empty defaults to name. + orgPoolName: "" + # warm.min override on the cloned pool. + warmMin: 0 # Usage metering (issues #164/#211/#602): the live per-org metering scraper # and the bearer-gated internal usage API the hosted console reads. OFF by # default: a self-host install that does not want metering renders none of it. diff --git a/docs/threat-model.md b/docs/threat-model.md index 4fe81f0a..5e30c877 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -1165,6 +1165,20 @@ deleted or drifted stack object enqueues the Org and is recreated; org deletion cascades through owner references (the cluster-scoped Org owns the cluster-scoped Namespace, a valid owner edge). +When `--org-pool-template-name` is set, the reconciler also clones a reference +SandboxPool's full spec (image, resources, placement, snapshots, network) into +each org namespace with `warm.min` overridden (default 0), named to match the +pool a create resolves. Cloning from one reference is what makes the per-org pool +schedule exactly where the reference does (a bare image would not carry the KVM +placement); `warm.min:0` keeps no idle warm husks, so N per-org pools cost nothing +at rest (the first fork per org cold-starts; the snapshot is content-addressed and +deduped per node). This does NOT move the isolation boundary: the pool and its +husks live inside the org's already-isolated namespace under the same default-deny +NetworkPolicy and ResourceQuota, so it grants no cross-tenant reach. It is the +enablement that lets the hosted deployment run per-org namespaces instead of one +shared namespace (which is what closes the shared-namespace referenced-object +class of issues in section 7b, the bare-name secret/workspace reference finding). + What isolates two orgs: **the separate namespace + the per-org default-deny NetworkPolicy + the per-org ResourceQuota ceiling + the microVM.** No single one of these is the whole boundary; the microVM remains the in-pod isolation diff --git a/internal/controller/org_builders_test.go b/internal/controller/org_builders_test.go index 5e9460ef..21562e8b 100644 --- a/internal/controller/org_builders_test.go +++ b/internal/controller/org_builders_test.go @@ -10,6 +10,53 @@ import ( "mitos.run/mitos/internal/tenant" ) +// TestBuildOrgPoolFromTemplate asserts the per-org pool clones the reference +// pool's full spec (image, resources, placement) and overrides ONLY warm.min to +// the configured floor, carries the org label and target name/namespace, so a +// per-org create in the org namespace has a schedulable pool to fork from (issue +// #288 multi-tenancy enablement). +func TestBuildOrgPoolFromTemplate(t *testing.T) { + tmpl := &v1.SandboxPool{ + Spec: v1.SandboxPoolSpec{ + Template: &v1.PoolTemplateSpec{ + Image: "ghcr.io/mitos-run/mitos-python:v1.13.0", + Resources: v1.SandboxResources{CPU: resource.MustParse("2"), Memory: resource.MustParse("512Mi")}, + }, + Warm: &v1.PoolWarm{Min: 8}, + Placement: &v1.PoolPlacement{ + NodeSelector: map[string]string{"mitos.run/kvm": "true"}, + }, + }, + } + org := &v1.Org{} + org.Name = "acme" + pool := buildOrgPoolFromTemplate(org, tenant.NamespaceForOrg("acme"), "python", tmpl, 0) + + if pool.Name != "python" { + t.Fatalf("name = %q, want python", pool.Name) + } + if pool.Namespace != "mitos-org-acme" { + t.Fatalf("namespace = %q, want mitos-org-acme", pool.Namespace) + } + if got := pool.Labels[tenant.OrgLabelKey]; got != "acme" { + t.Fatalf("org label = %q, want acme", got) + } + if pool.Spec.Template == nil || pool.Spec.Template.Image != "ghcr.io/mitos-run/mitos-python:v1.13.0" { + t.Fatalf("template image not cloned: %+v", pool.Spec.Template) + } + if pool.Spec.Placement == nil || pool.Spec.Placement.NodeSelector["mitos.run/kvm"] != "true" { + t.Fatalf("placement not cloned: %+v", pool.Spec.Placement) + } + if pool.Spec.Warm == nil || pool.Spec.Warm.Min != 0 { + t.Fatalf("warm.min = %+v, want a non-nil Warm with Min 0 (overridden)", pool.Spec.Warm) + } + // The clone must be independent: mutating it must not touch the template. + pool.Spec.Placement.NodeSelector["mitos.run/kvm"] = "mutated" + if tmpl.Spec.Placement.NodeSelector["mitos.run/kvm"] != "true" { + t.Fatalf("clone shares the template's placement map (not a deep copy)") + } +} + // TestBuildOrgDefaultDenyPolicy asserts the per-org NetworkPolicy is // default-deny in BOTH directions with a single DNS egress allow, applies to // every pod in the namespace, and carries the org label. diff --git a/internal/controller/org_controller.go b/internal/controller/org_controller.go index 64064f7a..f7ead423 100644 --- a/internal/controller/org_controller.go +++ b/internal/controller/org_controller.go @@ -65,6 +65,28 @@ type OrgReconciler struct { // DefaultMemory is the per-org aggregate memory limit ceiling applied when an // Org sets no Quota override. DefaultMemory resource.Quantity + + // PoolTemplateName is the name of a reference SandboxPool (in + // PoolTemplateNamespace) whose spec is cloned into every org namespace so a + // per-org create has a pool to fork from. Cloning the full spec (template, + // resources, placement, snapshots, network) from ONE source of truth is what + // makes the per-org pool schedulable on the same nodes as the reference pool, + // which a bare image cannot. Empty disables per-org pool provisioning (the + // self-host / bring-your-own-pool posture), so a single-tenant install is + // unaffected. + PoolTemplateName string + // PoolTemplateNamespace is where the reference pool lives. Empty defaults to + // the controller's own namespace (PoolSecretsNamespace, else "mitos"). + PoolTemplateNamespace string + // OrgPoolName is the name the cloned pool is created under in each org + // namespace. It MUST match the pool name a create resolves (the SDK passes it + // as the image/pool argument). Empty defaults to PoolTemplateName. + OrgPoolName string + // OrgPoolWarmMin overrides warm.min on the cloned per-org pool. Default 0: no + // dormant warm husks are held, so N per-org pools cost nothing at rest (the + // snapshot is content-addressed and deduped per node); the first fork per org + // cold-starts and the pool warms on demand. + OrgPoolWarmMin int32 } const ( @@ -128,9 +150,83 @@ func (r *OrgReconciler) ensureStack(ctx context.Context, org *v1.Org, ns string) if err := r.ensurePoolSecretsRoleBinding(ctx, org, ns); err != nil { return err } + if err := r.ensureOrgPool(ctx, org, ns); err != nil { + return err + } + return nil +} + +// ensureOrgPool clones the reference SandboxPool (PoolTemplateNamespace/ +// PoolTemplateName) into the org namespace when a template is configured. It is a +// no-op otherwise (the self-host / bring-your-own-pool posture), so a +// single-tenant install is unaffected. Cloning the full spec carries placement, +// resources, snapshots, and network from one source of truth, so the per-org +// pool schedules exactly where the reference pool does. The pool is created ONLY +// (never updated): once it exists the reconciler leaves its spec alone, so a +// warm.min bumped by the autoscaler or an operator persists. Owner-referenced to +// the Org for cascade deletion. +func (r *OrgReconciler) ensureOrgPool(ctx context.Context, org *v1.Org, ns string) error { + if r.PoolTemplateName == "" { + return nil + } + tns := r.PoolTemplateNamespace + if tns == "" { + tns = r.PoolSecretsNamespace + } + if tns == "" { + tns = defaultControllerNamespace + } + name := r.OrgPoolName + if name == "" { + name = r.PoolTemplateName + } + + // Create-once: if the per-org pool already exists, leave it alone. + existing := &v1.SandboxPool{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}} + if err := r.Get(ctx, client.ObjectKeyFromObject(existing), existing); err == nil { + return nil + } else if !apierrors.IsNotFound(err) { + return fmt.Errorf("get org pool %s/%s: %w", ns, name, err) + } + + var tmpl v1.SandboxPool + if err := r.Get(ctx, client.ObjectKey{Namespace: tns, Name: r.PoolTemplateName}, &tmpl); err != nil { + // A missing template is an operator misconfiguration, not a transient: fail + // the reconcile so it surfaces on the Org status with actionable context. + return fmt.Errorf("get org pool template %s/%s: %w", tns, r.PoolTemplateName, err) + } + + desired := buildOrgPoolFromTemplate(org, ns, name, &tmpl, r.OrgPoolWarmMin) + if err := r.setOwner(org, desired); err != nil { + return err + } + if err := r.Create(ctx, desired); err != nil && !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("create org pool %s/%s: %w", ns, name, err) + } return nil } +// buildOrgPoolFromTemplate builds a per-org SandboxPool by deep-copying the +// reference pool's spec and overriding only warm.min. Placement, resources, +// snapshots, template image, and network are carried verbatim so the per-org +// pool schedules exactly like the reference. warmMin defaults the pool to no +// idle warm husks (0) so N per-org pools cost nothing at rest. +func buildOrgPoolFromTemplate(org *v1.Org, ns, name string, tmpl *v1.SandboxPool, warmMin int32) *v1.SandboxPool { + spec := *tmpl.Spec.DeepCopy() + if spec.Warm == nil { + spec.Warm = &v1.PoolWarm{} + } + spec.Warm.Min = warmMin + return &v1.SandboxPool{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + Labels: tenant.OrgLabels(org.Name), + }, + Spec: spec, + } +} + // ensureNamespace creates or updates the org namespace with the org label and // the PSA privileged labels. // @@ -213,7 +309,7 @@ func buildOrgResourceQuota(org *v1.Org, ns string, defMaxSandboxes int32, defCPU } hard := corev1.ResourceList{ - corev1.ResourcePods: *resource.NewQuantity(int64(maxPods), resource.DecimalSI), + corev1.ResourcePods: *resource.NewQuantity(int64(maxPods), resource.DecimalSI), "count/sandboxes.mitos.run": *resource.NewQuantity(int64(maxSandboxes), resource.DecimalSI), corev1.ResourceLimitsCPU: cpu, corev1.ResourceLimitsMemory: mem, diff --git a/internal/controller/org_envtest_test.go b/internal/controller/org_envtest_test.go index 1e6f4a1b..b3e11c2a 100644 --- a/internal/controller/org_envtest_test.go +++ b/internal/controller/org_envtest_test.go @@ -8,6 +8,7 @@ import ( corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -157,6 +158,92 @@ func TestOrgReconcilerProvisionsIsolationStack(t *testing.T) { } } +// TestOrgReconcilerProvisionsPoolFromTemplate asserts that when a pool template +// is configured the reconciler clones it (full spec, warm.min overridden to 0) +// into the org namespace under OrgPoolName, owner-referenced to the Org, so a +// per-org create has a schedulable pool to fork from (issue #288). +func TestOrgReconcilerProvisionsPoolFromTemplate(t *testing.T) { + stamp := time.Now().UnixNano() + // Seed a reference pool in the controller namespace ("mitos"). Ensure the + // namespace exists first (envtest starts with none). + mitosNS := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "mitos"}} + if err := k8sClient.Create(ctx, mitosNS); err != nil && !apierrors.IsAlreadyExists(err) { + t.Fatalf("create mitos namespace: %v", err) + } + tmplName := fmt.Sprintf("python-%d", stamp) + tmpl := &v1.SandboxPool{ + ObjectMeta: metav1.ObjectMeta{Name: tmplName, Namespace: "mitos"}, + Spec: v1.SandboxPoolSpec{ + Template: &v1.PoolTemplateSpec{Image: "ghcr.io/mitos-run/mitos-python:v1.13.0"}, + Warm: &v1.PoolWarm{Min: 8}, + Placement: &v1.PoolPlacement{NodeSelector: map[string]string{"mitos.run/kvm": "true"}}, + }, + } + if err := k8sClient.Create(ctx, tmpl); err != nil { + t.Fatalf("create template pool: %v", err) + } + t.Cleanup(func() { _ = k8sClient.Delete(ctx, tmpl) }) + + orgID := fmt.Sprintf("pooled-%d", stamp) + org := &v1.Org{ObjectMeta: metav1.ObjectMeta{Name: orgID}} + if err := k8sClient.Create(ctx, org); err != nil { + t.Fatalf("create org: %v", err) + } + t.Cleanup(func() { _ = k8sClient.Delete(ctx, org) }) + + r := newOrgReconciler() + r.PoolTemplateName = tmplName + r.PoolTemplateNamespace = "mitos" + r.OrgPoolName = "python" + r.OrgPoolWarmMin = 0 + reconcileOrg(t, r, orgID) + + ns := tenant.NamespaceForOrg(orgID) + var pool v1.SandboxPool + if err := k8sClient.Get(ctx, types.NamespacedName{Name: "python", Namespace: ns}, &pool); err != nil { + t.Fatalf("get org pool: %v", err) + } + if pool.Spec.Template == nil || pool.Spec.Template.Image != "ghcr.io/mitos-run/mitos-python:v1.13.0" { + t.Errorf("pool template image = %+v, want cloned image", pool.Spec.Template) + } + if pool.Spec.Placement == nil || pool.Spec.Placement.NodeSelector["mitos.run/kvm"] != "true" { + t.Errorf("pool placement not cloned: %+v", pool.Spec.Placement) + } + if pool.Spec.Warm == nil || pool.Spec.Warm.Min != 0 { + t.Errorf("pool warm = %+v, want Min 0 (overridden)", pool.Spec.Warm) + } + if pool.Labels[tenant.OrgLabelKey] != orgID { + t.Errorf("pool org label = %q, want %q", pool.Labels[tenant.OrgLabelKey], orgID) + } + if !hasOrgOwner(pool.OwnerReferences, orgID) { + t.Errorf("pool missing Org owner reference: %+v", pool.OwnerReferences) + } +} + +// TestOrgReconcilerSkipsPoolWhenUnconfigured asserts that with no pool template +// (the self-host / bring-your-own-pool posture) NO pool is created, so a +// single-tenant install is unaffected. +func TestOrgReconcilerSkipsPoolWhenUnconfigured(t *testing.T) { + orgID := fmt.Sprintf("nopool-%d", time.Now().UnixNano()) + org := &v1.Org{ObjectMeta: metav1.ObjectMeta{Name: orgID}} + if err := k8sClient.Create(ctx, org); err != nil { + t.Fatalf("create org: %v", err) + } + t.Cleanup(func() { _ = k8sClient.Delete(ctx, org) }) + + r := newOrgReconciler() // PoolTemplateName empty + reconcileOrg(t, r, orgID) + + ns := tenant.NamespaceForOrg(orgID) + var list v1.SandboxPoolList + if err := k8sClient.List(ctx, &list, client.InNamespace(ns)); err != nil { + t.Fatalf("list pools: %v", err) + } + if len(list.Items) != 0 { + t.Fatalf("a pool was created despite no template: %+v", list.Items) + } +} + // TestOrgReconcilerTwoOrgsAreIsolated asserts two Orgs land in two DISTINCT // namespaces, each with its own full stack and ONLY its own org label. func TestOrgReconcilerTwoOrgsAreIsolated(t *testing.T) {