Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions crates/openshell-driver-kubernetes/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -38,11 +38,14 @@ The driver injects gateway callback configuration, sandbox identity, TLS client
material, and the supervisor SSH socket path into the workload. Driver-owned
values must override image-provided environment variables.

Sandbox pods run as `service_account_name` and keep
Sandbox pods run as `service_account_name` and default to
`automountServiceAccountToken: false`. The only Kubernetes token exposed to the
supervisor is an explicit, audience-bound projected token mounted at
supervisor by default is an explicit, audience-bound projected token mounted at
`/var/run/secrets/openshell/token` for the one-shot `IssueSandboxToken`
bootstrap exchange.
bootstrap exchange. Operators can opt into Kubernetes' default service account
token mount with `automount_service_account_token = true` when sandbox-local
tools need Kubernetes API access and the sandbox service account has explicit
least-privilege RBAC.

The gateway uses the supervisor relay for connect, exec, and file sync. Sandbox
pods do not need direct external ingress for SSH.
Expand Down
5 changes: 5 additions & 0 deletions crates/openshell-driver-kubernetes/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,10 @@ pub struct KubernetesComputeConfig {
/// Kubernetes `ServiceAccount` assigned to sandbox pods and accepted by
/// the gateway's `TokenReview` bootstrap authenticator.
pub service_account_name: String,
/// Whether sandbox pods should use Kubernetes' default ServiceAccount token
/// automount. Disabled by default so sandbox pods cannot access the
/// Kubernetes API unless the operator opts in and grants RBAC explicitly.
pub automount_service_account_token: bool,
pub default_image: String,
pub image_pull_policy: String,
/// Kubernetes `imagePullSecrets` names attached to sandbox pods.
Expand Down Expand Up @@ -226,6 +230,7 @@ impl Default for KubernetesComputeConfig {
Self {
namespace: DEFAULT_K8S_NAMESPACE.to_string(),
service_account_name: DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME.to_string(),
automount_service_account_token: false,
default_image: openshell_core::image::default_sandbox_image(),
// Default empty so the gateway omits `imagePullPolicy` from pod
// specs and Kubernetes applies its own default (Always for `latest`,
Expand Down
28 changes: 25 additions & 3 deletions crates/openshell-driver-kubernetes/src/driver.rs
Original file line number Diff line number Diff line change
Expand Up @@ -401,6 +401,7 @@ impl KubernetesComputeDriver {
supervisor_image_pull_policy: &self.config.supervisor_image_pull_policy,
supervisor_sideload_method: self.config.supervisor_sideload_method,
service_account_name: &self.config.service_account_name,
automount_service_account_token: self.config.automount_service_account_token,
sandbox_id: &sandbox.id,
sandbox_name: &sandbox.name,
grpc_endpoint: &self.config.grpc_endpoint,
Expand Down Expand Up @@ -1119,6 +1120,7 @@ struct SandboxPodParams<'a> {
supervisor_image_pull_policy: &'a str,
supervisor_sideload_method: SupervisorSideloadMethod,
service_account_name: &'a str,
automount_service_account_token: bool,
sandbox_id: &'a str,
sandbox_name: &'a str,
grpc_endpoint: &'a str,
Expand Down Expand Up @@ -1146,6 +1148,7 @@ impl Default for SandboxPodParams<'_> {
supervisor_image_pull_policy: "",
supervisor_sideload_method: SupervisorSideloadMethod::default(),
service_account_name: DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME,
automount_service_account_token: false,
sandbox_id: "",
sandbox_name: "",
grpc_endpoint: "",
Expand Down Expand Up @@ -1354,11 +1357,9 @@ fn sandbox_template_to_k8s(
);
}

// Disable service account token auto-mounting for security hardening.
// Sandbox pods should not have access to the Kubernetes API by default.
spec.insert(
"automountServiceAccountToken".to_string(),
serde_json::json!(false),
serde_json::json!(params.automount_service_account_token),
);

let mut container = serde_json::Map::new();
Expand Down Expand Up @@ -3105,6 +3106,27 @@ mod tests {
);
}

#[test]
fn sandbox_template_can_enable_service_account_token_automount() {
let params = SandboxPodParams {
automount_service_account_token: true,
..Default::default()
};
let pod_template = sandbox_template_to_k8s(
&SandboxTemplate::default(),
false,
&std::collections::HashMap::new(),
true,
&params,
);

assert_eq!(
pod_template["spec"]["automountServiceAccountToken"],
serde_json::json!(true),
"operators can opt sandboxes into Kubernetes API credentials"
);
}

#[test]
fn sandbox_template_omits_empty_image_pull_secrets() {
let pod_template = sandbox_template_to_k8s(
Expand Down
4 changes: 4 additions & 0 deletions crates/openshell-driver-kubernetes/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,9 @@ struct Args {
)]
sandbox_service_account: String,

#[arg(long, env = "OPENSHELL_K8S_AUTOMOUNT_SERVICE_ACCOUNT_TOKEN")]
automount_service_account_token: bool,

#[arg(long, env = "OPENSHELL_SANDBOX_IMAGE")]
sandbox_image: Option<String>,

Expand Down Expand Up @@ -109,6 +112,7 @@ async fn main() -> Result<()> {
let driver = KubernetesComputeDriver::new(KubernetesComputeConfig {
namespace: args.sandbox_namespace,
service_account_name: args.sandbox_service_account,
automount_service_account_token: args.automount_service_account_token,
default_image: args.sandbox_image.unwrap_or_default(),
image_pull_policy: args.sandbox_image_pull_policy.unwrap_or_default(),
image_pull_secrets: args.sandbox_image_pull_secrets,
Expand Down
1 change: 1 addition & 0 deletions deploy/helm/openshell/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,7 @@ add `ci/values-spire.yaml` to the OpenShell release values files.
| server.oidc.userRole | string | `""` | Role name for standard user access. |
| server.providerTokenGrants.spiffe.enabled | bool | `false` | Mount the SPIFFE Workload API socket into sandbox pods for dynamic provider token grants. |
| server.providerTokenGrants.spiffe.workloadApiSocketPath | string | `"/spiffe-workload-api/spire-agent.sock"` | Path to the SPIFFE Workload API socket mounted into sandbox pods. |
| server.sandboxAutomountServiceAccountToken | bool | `false` | Whether sandbox pods should use Kubernetes' default service account token automount. Keep false unless sandbox-local tools need Kubernetes API access and the sandbox service account has explicit least-privilege RBAC. |
| server.sandboxImage | string | `"ghcr.io/nvidia/openshell-community/sandboxes/base:latest"` | Default sandbox image used when requests do not specify one. |
| server.sandboxImagePullPolicy | string | `""` | Kubernetes imagePullPolicy for sandbox pods. Empty = Kubernetes default (Always for :latest, IfNotPresent otherwise). Set to "Always" for dev clusters so new images are picked up without manual eviction. |
| server.sandboxImagePullSecrets | list | `[]` | Image pull secrets attached to sandbox pods. Referenced Secrets must exist in the sandbox namespace. |
Expand Down
1 change: 1 addition & 0 deletions deploy/helm/openshell/templates/gateway-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,7 @@ data:
[openshell.drivers.kubernetes]
grpc_endpoint = {{ include "openshell.grpcEndpoint" . | quote }}
service_account_name = {{ include "openshell.sandboxServiceAccountName" . | quote }}
automount_service_account_token = {{ .Values.server.sandboxAutomountServiceAccountToken | default false }}
supervisor_sideload_method = {{ include "openshell.supervisorSideloadMethod" . | quote }}
sa_token_ttl_secs = {{ .Values.server.sandboxJwt.k8sSaTokenTtlSecs | default 3600 }}
{{- if .Values.server.providerTokenGrants.spiffe.enabled }}
Expand Down
16 changes: 16 additions & 0 deletions deploy/helm/openshell/tests/gateway_config_test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,22 @@ tests:
path: data["gateway.toml"]
pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?service_account_name\s*=\s*"openshell-sandbox"'

- it: disables sandbox service account token automount by default
template: templates/gateway-config.yaml
asserts:
- matchRegex:
path: data["gateway.toml"]
pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?automount_service_account_token\s*=\s*false'

- it: can enable sandbox service account token automount
template: templates/gateway-config.yaml
set:
server.sandboxAutomountServiceAccountToken: true
asserts:
- matchRegex:
path: data["gateway.toml"]
pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?automount_service_account_token\s*=\s*true'

- it: renders sandbox image pull secrets under [openshell.drivers.kubernetes]
template: templates/gateway-config.yaml
set:
Expand Down
4 changes: 4 additions & 0 deletions deploy/helm/openshell/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,10 @@ server:
# -- Image pull secrets attached to sandbox pods. Referenced Secrets must exist
# in the sandbox namespace.
sandboxImagePullSecrets: []
# -- Whether sandbox pods should use Kubernetes' default service account
# token automount. Keep false unless sandbox-local tools need Kubernetes API
# access and the sandbox service account has explicit least-privilege RBAC.
sandboxAutomountServiceAccountToken: false
# -- Default storage size for the workspace PVC in sandbox pods.
# Uses Kubernetes quantity syntax (e.g. "2Gi", "10Gi", "500Mi").
# Empty = built-in default (2Gi).
Expand Down
1 change: 1 addition & 0 deletions docs/kubernetes/setup.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,7 @@ The most commonly changed values are:
| `server.externalDbSecret` | Secret containing a PostgreSQL connection URI in the `uri` key. Use when the database is managed outside the chart. |
| `server.sandboxImage` | Default sandbox image used when a sandbox does not specify one. |
| `server.sandboxImagePullSecrets` | Image pull secrets attached to sandbox pods. Referenced Secrets must exist in the sandbox namespace. |
| `server.sandboxAutomountServiceAccountToken` | Opt sandbox pods into Kubernetes' default service account token mount. Keep false unless sandbox-local tools need Kubernetes API access and the sandbox service account has explicit least-privilege RBAC. |
| `server.grpcEndpoint` | Endpoint that sandbox supervisors use to call back to the gateway. Must be reachable from inside the cluster. |
| `server.appArmorProfile` | AppArmor profile requested for sandbox agent containers. Defaults to `Unconfined`. |
| `server.disableTls` | Run the gateway over plaintext HTTP. Use only behind a trusted transport. |
Expand Down
2 changes: 2 additions & 0 deletions docs/reference/gateway-config.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ default_image = "ghcr.io/nvidia/openshell/sandbox:latest"
supervisor_image = "ghcr.io/nvidia/openshell/supervisor:latest"
client_tls_secret_name = "openshell-client-tls"
service_account_name = "openshell-sandbox"
automount_service_account_token = false
host_gateway_ip = "10.0.0.1"
enable_user_namespaces = false
sa_token_ttl_secs = 3600
Expand Down Expand Up @@ -169,6 +170,7 @@ client_ca_path = "/etc/openshell-tls/client-ca/ca.crt"
[openshell.drivers.kubernetes]
namespace = "agents"
service_account_name = "openshell-sandbox"
automount_service_account_token = false
default_image = "ghcr.io/nvidia/openshell/sandbox:latest"
image_pull_policy = "IfNotPresent"
image_pull_secrets = ["regcred"]
Expand Down
1 change: 1 addition & 0 deletions docs/reference/sandbox-compute-drivers.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -256,6 +256,7 @@ For maintainer-level implementation details, refer to the [Kubernetes driver REA
| `compute_drivers = ["kubernetes"]` | Not applicable | Select the Kubernetes compute driver. |
| `[openshell.drivers.kubernetes].namespace` | `server.sandboxNamespace` | Set the namespace for sandbox resources. The Helm chart defaults to the release namespace when left empty. |
| `service_account_name` | `sandboxServiceAccount.name` | Set the Kubernetes service account assigned to sandbox pods and accepted by the gateway TokenReview bootstrap path. The Helm chart creates a dedicated sandbox service account by default. |
| `automount_service_account_token` | `server.sandboxAutomountServiceAccountToken` | Opt sandbox pods into Kubernetes' default service account token mount. Keep false unless sandbox-local tools need Kubernetes API access and the sandbox service account has explicit least-privilege RBAC. |
| `default_image` | `server.sandboxImage` | Set the default sandbox image. |
| `image_pull_policy` | `server.sandboxImagePullPolicy` | Set the Kubernetes image pull policy for sandbox pods. |
| `image_pull_secrets` | `server.sandboxImagePullSecrets` | Attach Kubernetes image pull secrets to sandbox pods. Referenced Secrets must exist in the sandbox namespace. |
Expand Down
Loading