diff --git a/CubeAPI/src/services/sandboxes.rs b/CubeAPI/src/services/sandboxes.rs index 27ac47029..3516faa75 100644 --- a/CubeAPI/src/services/sandboxes.rs +++ b/CubeAPI/src/services/sandboxes.rs @@ -18,8 +18,9 @@ use crate::{ }, error::{AppError, AppResult}, models::{ - EgressRule, LogLevel as ModelLogLevel, NewSandbox, Sandbox, SandboxDetail, SandboxLog, - SandboxLogEntry, SandboxLogs, SandboxLogsV2Response, SandboxNetworkConfig, SandboxState, + EgressRule, EnvVars, LogLevel as ModelLogLevel, NewSandbox, Sandbox, SandboxDetail, + SandboxLog, SandboxLogEntry, SandboxLogs, SandboxLogsV2Response, SandboxNetworkConfig, + SandboxState, }, }; @@ -28,6 +29,18 @@ const RET_CODE_HTTP_OK: i32 = 200; const RET_CODE_NOT_FOUND: i32 = 130404; const RET_CODE_CONFLICT: i32 = 130409; const HOSTDIR_MOUNT_KEY: &str = "host-mount"; +/// Annotation carrying create-time env_vars as a JSON object string {"K":"V"}. +/// The cube.master prefix ensures CubeMaster forwards it to Cubelet. +const CREATE_ENV_VARS_ANNOTATION: &str = "cube.master.sandbox.create_env_vars"; + +/// Encode create-time env_vars into the annotation value (a JSON object string). +/// Returns `None` when there is nothing to inject (absent or empty map), so the +/// annotation is only added when the caller actually passed variables. +fn encode_create_env_vars(env_vars: Option<&EnvVars>) -> Option { + let env_vars = env_vars.filter(|m| !m.is_empty())?; + // EnvVars is a String->String map, so serialization cannot fail. + serde_json::to_string(env_vars).ok() +} #[derive(Clone)] pub struct SandboxService { @@ -135,6 +148,15 @@ impl SandboxService { meta }); + // Carry create-time env_vars as sandbox runtime metadata for Cubelet: + // once the sandbox is ready Cubelet injects them through envd's native + // POST /init, so later commands.run can read them (same as E2B), without + // writing to rootfs/profile or the OCI container spec. The annotation uses + // the cube.master prefix so CubeMaster forwards it to Cubelet automatically. + if let Some(encoded) = encode_create_env_vars(body.env_vars.as_ref()) { + annotations.insert(CREATE_ENV_VARS_ANNOTATION.to_string(), encoded); + } + let cube_network_config = build_cube_network_config(body.allow_internet_access, body.network.as_ref())?; @@ -691,13 +713,42 @@ fn map_egress_rule(rule: &EgressRule) -> CubeEgressRule { mod tests { use std::collections::HashMap; - use super::{build_cube_network_config, filter_by_metadata, from_cubemaster_info}; + use super::{ + build_cube_network_config, encode_create_env_vars, filter_by_metadata, + from_cubemaster_info, CREATE_ENV_VARS_ANNOTATION, + }; use crate::cubemaster::{ListSandboxResponse, SandboxInfo}; use crate::models::{ - EgressRule, EgressRuleAction, EgressRuleInject, EgressRuleMatch, SandboxNetworkConfig, - SandboxState, + EgressRule, EgressRuleAction, EgressRuleInject, EgressRuleMatch, EnvVars, + SandboxNetworkConfig, SandboxState, }; + #[test] + fn create_env_vars_annotation_uses_cube_master_prefix() { + // CubeMaster only forwards annotations under the cube.master prefix to + // Cubelet, so this prefix is part of the wiring contract. + assert!(CREATE_ENV_VARS_ANNOTATION.starts_with("cube.master")); + } + + #[test] + fn encode_create_env_vars_skips_absent_and_empty() { + assert!(encode_create_env_vars(None).is_none()); + let empty: EnvVars = HashMap::new(); + assert!(encode_create_env_vars(Some(&empty)).is_none()); + } + + #[test] + fn encode_create_env_vars_serializes_to_json_object() { + let mut env = EnvVars::new(); + env.insert("FOO".to_string(), "bar".to_string()); + + let encoded = encode_create_env_vars(Some(&env)).expect("non-empty env should encode"); + let decoded: HashMap = + serde_json::from_str(&encoded).expect("annotation must be a JSON object"); + + assert_eq!(decoded.get("FOO"), Some(&"bar".to_string())); + } + #[test] fn metadata_filter_matches_all_pairs() { let metadata = HashMap::from([ diff --git a/CubeMaster/pkg/base/constants/constants.go b/CubeMaster/pkg/base/constants/constants.go index 4083dc13f..d801c4242 100644 --- a/CubeMaster/pkg/base/constants/constants.go +++ b/CubeMaster/pkg/base/constants/constants.go @@ -64,7 +64,13 @@ const ( CubeAnnotationsFallbackToSlowPath = "cube.master.fallback_to_slow_path" CubeAnnotationsSystemDiskSize = "cube.master.system_disk_size" - CubeAnnotationsAppSnapshotCreate = "cube.master.appsnapshot.create" + CubeAnnotationsAppSnapshotCreate = "cube.master.appsnapshot.create" + // CubeAnnotationSandboxCreateEnvVars carries create-time env_vars as a JSON + // object string. It is written by CubeAPI and consumed by Cubelet (which + // injects them via envd /init). As per-invocation runtime metadata it must be + // stripped on template commit so instance-level secrets are not persisted + // into template snapshots. + CubeAnnotationSandboxCreateEnvVars = "cube.master.sandbox.create_env_vars" CubeAnnotationAppSnapshotTemplateID = "cube.master.appsnapshot.template.id" CubeAnnotationAppSnapshotVersion = "cube.master.appsnapshot.version" CubeAnnotationAppSnapshotTemplateVersion = "cube.master.appsnapshot.template.version" diff --git a/CubeMaster/pkg/templatecenter/store.go b/CubeMaster/pkg/templatecenter/store.go index 7ab0b6aa6..d692cfe77 100644 --- a/CubeMaster/pkg/templatecenter/store.go +++ b/CubeMaster/pkg/templatecenter/store.go @@ -386,6 +386,9 @@ func normalizeStoredTemplateRequest(req *sandboxtypes.CreateCubeSandboxReq) (*sa // longer exist). delete(cloned.Annotations, constants.CubeAnnotationRuntimeSnapshotID) delete(cloned.Annotations, constants.CubeAnnotationRuntimeSnapshotAttachedAt) + // Create-time env_vars are per-instance runtime metadata (may contain + // secrets) and must not be persisted with the template snapshot. + delete(cloned.Annotations, constants.CubeAnnotationSandboxCreateEnvVars) cloned.Annotations[constants.CubeAnnotationAppSnapshotTemplateID] = templateID return cloned, nil } diff --git a/CubeMaster/pkg/templatecenter/store_test.go b/CubeMaster/pkg/templatecenter/store_test.go index 46dd79b8c..4d8643caa 100644 --- a/CubeMaster/pkg/templatecenter/store_test.go +++ b/CubeMaster/pkg/templatecenter/store_test.go @@ -60,6 +60,35 @@ func TestNormalizeStoredTemplateRequestStripsPhysicalAnnotations(t *testing.T) { assert.Empty(t, out.SnapshotDir) } +// TestNormalizeStoredTemplateRequestStripsCreateEnvVars verifies that the +// per-invocation create-time env_vars annotation (which may carry secrets) is +// scrubbed from the stored template request so it is never persisted into a +// template snapshot, while unrelated annotations are preserved. +func TestNormalizeStoredTemplateRequestStripsCreateEnvVars(t *testing.T) { + patches := gomonkey.NewPatches() + defer patches.Reset() + patches.ApplyFunc(NormalizeRequest, func(in *sandboxtypes.CreateCubeSandboxReq) (*sandboxtypes.CreateCubeSandboxReq, string, error) { + return in, "tpl-after-norm", nil + }) + + req := &sandboxtypes.CreateCubeSandboxReq{ + InstanceType: "cubebox", + Timeout: 1, + Annotations: map[string]string{ + constants.CubeAnnotationSandboxCreateEnvVars: `{"TOKEN":"secret"}`, + "unrelated": "keep-me", + }, + } + + out, err := normalizeStoredTemplateRequest(req) + require.NoError(t, err) + + _, present := out.Annotations[constants.CubeAnnotationSandboxCreateEnvVars] + assert.Falsef(t, present, "annotation %q must be stripped from stored template request", + constants.CubeAnnotationSandboxCreateEnvVars) + assert.Equal(t, "keep-me", out.Annotations["unrelated"]) +} + func TestResolveTemplateNodesFiltersRequestedHealthyNodes(t *testing.T) { patches := gomonkey.NewPatches() defer patches.Reset() diff --git a/Cubelet/services/cubebox/cube_container_create.go b/Cubelet/services/cubebox/cube_container_create.go index a4cbcd00c..e64fed9be 100644 --- a/Cubelet/services/cubebox/cube_container_create.go +++ b/Cubelet/services/cubebox/cube_container_create.go @@ -331,6 +331,12 @@ func (l *local) createContainers(ctx context.Context, flowOpts *workflow.CreateC if err := l.doProbe(param.ctxTmp, param.cntrReq, param.ci); err != nil { return err } + // Once the sandbox is ready, inject create-time env_vars into the guest + // envd (via its native /init) so later commands.run / run_code can read + // them. No-op when the annotation is absent. + if err := l.syncCreateEnvToEnvd(param.ctxTmp, sandBox, param.ci); err != nil { + return err + } err = l.cbriManager.PostCreateContainer(ctx, sandBox, param.ci) if err != nil { containerLog.Errorf("post create container failed, err: %v", err) diff --git a/Cubelet/services/cubebox/envd_init.go b/Cubelet/services/cubebox/envd_init.go new file mode 100644 index 000000000..b053ae169 --- /dev/null +++ b/Cubelet/services/cubebox/envd_init.go @@ -0,0 +1,155 @@ +// Copyright (c) 2024 Tencent Inc. +// SPDX-License-Identifier: Apache-2.0 +// + +package cubebox + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "strconv" + "time" + + "github.com/tencentcloud/CubeSandbox/Cubelet/pkg/log" + cubeboxstore "github.com/tencentcloud/CubeSandbox/Cubelet/pkg/store/cubebox" +) + +const ( + // createEnvVarsAnnotation is the contract with CubeAPI: it carries the + // create-time env_vars as a JSON object string {"K":"V"}. CubeMaster forwards + // it here automatically via the cube.master prefix passthrough. + createEnvVarsAnnotation = "cube.master.sandbox.create_env_vars" + + // Retry budget for /init: envd may not be listening yet right after the + // sandbox becomes ready, so we short-poll until it is up or we time out. + envdInitMaxWait = 15 * time.Second + envdInitInterval = 200 * time.Millisecond + envdInitReqTimeout = 3 * time.Second +) + +// envdServerPort is the HTTP port envd listens on inside the guest (E2B envd is +// fixed at 49983). It is a var so tests can point /init at a local stub server. +var envdServerPort = 49983 + +// envdInitBody is the envd POST /init request body (e2b-dev/infra). We only use +// envVars and timestamp here: +// - envVars: stored into envd's global defaults.EnvVars and merged into child +// process environments on Process/Start; +// - timestamp: used by envd to accept only newer requests, and to correct the +// guest clock as a side effect. +type envdInitBody struct { + EnvVars map[string]string `json:"envVars,omitempty"` + Timestamp string `json:"timestamp,omitempty"` +} + +// syncCreateEnvToEnvd injects the create-time env_vars (carried via annotation) +// into the guest envd. +// +// It mirrors the E2B orchestrator initEnvd flow: the host side connects directly +// to the guest's ip:49983 and calls the native POST /init. envd stores these +// variables into its global defaults.EnvVars, so processes it later starts +// (commands.run / run_code) inherit them (precedence: global < per-command, where +// per-command can override). envd merges additively and never clears existing +// entries, so any env already present in the runtime is preserved. +// +// Nothing is written to rootfs/profile and nothing goes into the OCI container +// spec, so neither the image/template nor the template snapshot is polluted with +// instance-level secrets. Sandboxes without create env are a no-op. +func (l *local) syncCreateEnvToEnvd(ctx context.Context, sandBox *cubeboxstore.CubeBox, + ci *cubeboxstore.Container) error { + raw, ok := sandBox.Annotations[createEnvVarsAnnotation] + if !ok || raw == "" { + return nil + } + + envVars := map[string]string{} + if err := json.Unmarshal([]byte(raw), &envVars); err != nil { + return fmt.Errorf("parse create env_vars annotation failed: %w", err) + } + if len(envVars) == 0 { + return nil + } + + ip := ci.IP + if ip == "" || ip == "" { + ip = sandBox.IP + } + if ip == "" || ip == "" { + return fmt.Errorf("sync create env to envd: empty sandbox IP") + } + + addr := fmt.Sprintf("http://%s/init", net.JoinHostPort(ip, strconv.Itoa(envdServerPort))) + body, err := json.Marshal(envdInitBody{ + EnvVars: envVars, + Timestamp: time.Now().UTC().Format(time.RFC3339Nano), + }) + if err != nil { + return fmt.Errorf("marshal envd init body failed: %w", err) + } + + start := time.Now() + deadline := start.Add(envdInitMaxWait) + client := &http.Client{Timeout: envdInitReqTimeout} + + var ( + lastErr error + attempt int + ) + for { + attempt++ + lastErr = postEnvdInit(ctx, client, addr, body) + if lastErr == nil { + log.G(ctx).Infof("envd /init ok: sandbox=%s ip=%s vars=%d attempts=%d cost=%v", + sandBox.SandboxID, ip, len(envVars), attempt, time.Since(start)) + return nil + } + + if ctx.Err() != nil { + lastErr = ctx.Err() + break + } + if time.Now().After(deadline) { + break + } + select { + case <-ctx.Done(): + lastErr = ctx.Err() + case <-time.After(envdInitInterval): + } + if ctx.Err() != nil { + break + } + } + + return fmt.Errorf("envd /init failed: sandbox=%s ip=%s attempts=%d cost=%v: %w", + sandBox.SandboxID, ip, attempt, time.Since(start), lastErr) +} + +// postEnvdInit sends one /init request; only 204/200 is treated as success. +func postEnvdInit(ctx context.Context, client *http.Client, addr string, body []byte) error { + reqCtx, cancel := context.WithTimeout(ctx, envdInitReqTimeout) + defer cancel() + + req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, addr, bytes.NewReader(body)) + if err != nil { + return fmt.Errorf("build envd init request failed: %w", err) + } + req.Header.Set("Content-Type", "application/json") + + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + io.Copy(io.Discard, resp.Body) + + if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK { + return fmt.Errorf("envd /init unexpected status %d", resp.StatusCode) + } + return nil +} diff --git a/Cubelet/services/cubebox/envd_init_test.go b/Cubelet/services/cubebox/envd_init_test.go new file mode 100644 index 000000000..2ee61be67 --- /dev/null +++ b/Cubelet/services/cubebox/envd_init_test.go @@ -0,0 +1,138 @@ +// Copyright (c) 2024 Tencent Inc. +// SPDX-License-Identifier: Apache-2.0 +// + +package cubebox + +import ( + "context" + "encoding/json" + "net" + "net/http" + "net/http/httptest" + "net/url" + "strconv" + "sync/atomic" + "testing" + + cubeboxstore "github.com/tencentcloud/CubeSandbox/Cubelet/pkg/store/cubebox" +) + +// startEnvdStub starts an httptest server acting as the guest envd /init +// endpoint, repoints envdServerPort at it, and returns the host the sandbox IP +// should be set to. The original port is restored when the test ends. +func startEnvdStub(t *testing.T, handler http.HandlerFunc) (server *httptest.Server, host string) { + t.Helper() + server = httptest.NewServer(handler) + t.Cleanup(server.Close) + + u, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("parse stub url: %v", err) + } + h, portStr, err := net.SplitHostPort(u.Host) + if err != nil { + t.Fatalf("split stub host: %v", err) + } + port, err := strconv.Atoi(portStr) + if err != nil { + t.Fatalf("parse stub port: %v", err) + } + + orig := envdServerPort + envdServerPort = port + t.Cleanup(func() { envdServerPort = orig }) + + return server, h +} + +func newEnvSandbox(annotationValue, ip string) (*cubeboxstore.CubeBox, *cubeboxstore.Container) { + annotations := map[string]string{} + if annotationValue != "" { + annotations[createEnvVarsAnnotation] = annotationValue + } + sb := &cubeboxstore.CubeBox{ + Metadata: cubeboxstore.Metadata{ + SandboxID: "sb-test", + Annotations: annotations, + }, + IP: ip, + } + ci := &cubeboxstore.Container{IP: ip} + return sb, ci +} + +func TestSyncCreateEnvToEnvdNoAnnotationIsNoop(t *testing.T) { + var called int32 + _, host := startEnvdStub(t, func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&called, 1) + w.WriteHeader(http.StatusNoContent) + }) + + sb, ci := newEnvSandbox("", host) + l := &local{} + if err := l.syncCreateEnvToEnvd(context.Background(), sb, ci); err != nil { + t.Fatalf("expected no-op nil, got %v", err) + } + if atomic.LoadInt32(&called) != 0 { + t.Fatalf("envd /init must not be called without the annotation") + } +} + +func TestSyncCreateEnvToEnvdInvalidJSON(t *testing.T) { + sb, ci := newEnvSandbox("not-json", "127.0.0.1") + l := &local{} + if err := l.syncCreateEnvToEnvd(context.Background(), sb, ci); err == nil { + t.Fatal("expected error for invalid env_vars annotation JSON") + } +} + +func TestSyncCreateEnvToEnvdEmptyIP(t *testing.T) { + sb, ci := newEnvSandbox(`{"FOO":"bar"}`, "") + l := &local{} + if err := l.syncCreateEnvToEnvd(context.Background(), sb, ci); err == nil { + t.Fatal("expected error when sandbox IP is empty") + } +} + +func TestSyncCreateEnvToEnvdSuccessSendsEnvVars(t *testing.T) { + var got envdInitBody + _, host := startEnvdStub(t, func(w http.ResponseWriter, r *http.Request) { + _ = json.NewDecoder(r.Body).Decode(&got) + w.WriteHeader(http.StatusNoContent) + }) + + sb, ci := newEnvSandbox(`{"FOO":"bar","TOKEN":"x"}`, host) + l := &local{} + if err := l.syncCreateEnvToEnvd(context.Background(), sb, ci); err != nil { + t.Fatalf("expected success, got %v", err) + } + + if got.EnvVars["FOO"] != "bar" || got.EnvVars["TOKEN"] != "x" { + t.Fatalf("envd /init received unexpected envVars: %+v", got.EnvVars) + } + if got.Timestamp == "" { + t.Fatal("envd /init request must carry a timestamp") + } +} + +func TestSyncCreateEnvToEnvdRetriesThenSucceeds(t *testing.T) { + var attempts int32 + _, host := startEnvdStub(t, func(w http.ResponseWriter, r *http.Request) { + // Fail the first attempt to exercise the retry loop, then succeed. + if atomic.AddInt32(&attempts, 1) == 1 { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + w.WriteHeader(http.StatusNoContent) + }) + + sb, ci := newEnvSandbox(`{"FOO":"bar"}`, host) + l := &local{} + if err := l.syncCreateEnvToEnvd(context.Background(), sb, ci); err != nil { + t.Fatalf("expected eventual success, got %v", err) + } + if atomic.LoadInt32(&attempts) < 2 { + t.Fatalf("expected at least 2 attempts, got %d", attempts) + } +}