diff --git a/CubeAPI/src/services/sandboxes.rs b/CubeAPI/src/services/sandboxes.rs index 27ac4702..689fbed7 100644 --- a/CubeAPI/src/services/sandboxes.rs +++ b/CubeAPI/src/services/sandboxes.rs @@ -28,6 +28,9 @@ const RET_CODE_HTTP_OK: i32 = 200; const RET_CODE_NOT_FOUND: i32 = 130404; const RET_CODE_CONFLICT: i32 = 130409; const HOSTDIR_MOUNT_KEY: &str = "host-mount"; +/// Annotation carrying create-time env_vars as a JSON object string {"K":"V"}. +/// The cube.master prefix ensures CubeMaster forwards it to Cubelet. +const CREATE_ENV_VARS_ANNOTATION: &str = "cube.master.sandbox.create_env_vars"; #[derive(Clone)] pub struct SandboxService { @@ -135,6 +138,16 @@ impl SandboxService { meta }); + // Carry create-time env_vars as sandbox runtime metadata for Cubelet: + // once the sandbox is ready Cubelet injects them through envd's native + // POST /init, so later commands.run can read them (same as E2B), without + // writing to rootfs/profile or the OCI container spec. The annotation uses + // the cube.master prefix so CubeMaster forwards it to Cubelet automatically. + if let Some(env_vars) = body.env_vars.as_ref().filter(|m| !m.is_empty()) { + let encoded = serde_json::to_string(env_vars).map_err(internal_error)?; + annotations.insert(CREATE_ENV_VARS_ANNOTATION.to_string(), encoded); + } + let cube_network_config = build_cube_network_config(body.allow_internet_access, body.network.as_ref())?; diff --git a/CubeMaster/pkg/base/constants/constants.go b/CubeMaster/pkg/base/constants/constants.go index 4083dc13..d801c424 100644 --- a/CubeMaster/pkg/base/constants/constants.go +++ b/CubeMaster/pkg/base/constants/constants.go @@ -64,7 +64,13 @@ const ( CubeAnnotationsFallbackToSlowPath = "cube.master.fallback_to_slow_path" CubeAnnotationsSystemDiskSize = "cube.master.system_disk_size" - CubeAnnotationsAppSnapshotCreate = "cube.master.appsnapshot.create" + CubeAnnotationsAppSnapshotCreate = "cube.master.appsnapshot.create" + // CubeAnnotationSandboxCreateEnvVars carries create-time env_vars as a JSON + // object string. It is written by CubeAPI and consumed by Cubelet (which + // injects them via envd /init). As per-invocation runtime metadata it must be + // stripped on template commit so instance-level secrets are not persisted + // into template snapshots. + CubeAnnotationSandboxCreateEnvVars = "cube.master.sandbox.create_env_vars" CubeAnnotationAppSnapshotTemplateID = "cube.master.appsnapshot.template.id" CubeAnnotationAppSnapshotVersion = "cube.master.appsnapshot.version" CubeAnnotationAppSnapshotTemplateVersion = "cube.master.appsnapshot.template.version" diff --git a/CubeMaster/pkg/templatecenter/store.go b/CubeMaster/pkg/templatecenter/store.go index 7ab0b6aa..d692cfe7 100644 --- a/CubeMaster/pkg/templatecenter/store.go +++ b/CubeMaster/pkg/templatecenter/store.go @@ -386,6 +386,9 @@ func normalizeStoredTemplateRequest(req *sandboxtypes.CreateCubeSandboxReq) (*sa // longer exist). delete(cloned.Annotations, constants.CubeAnnotationRuntimeSnapshotID) delete(cloned.Annotations, constants.CubeAnnotationRuntimeSnapshotAttachedAt) + // Create-time env_vars are per-instance runtime metadata (may contain + // secrets) and must not be persisted with the template snapshot. + delete(cloned.Annotations, constants.CubeAnnotationSandboxCreateEnvVars) cloned.Annotations[constants.CubeAnnotationAppSnapshotTemplateID] = templateID return cloned, nil } diff --git a/Cubelet/services/cubebox/cube_container_create.go b/Cubelet/services/cubebox/cube_container_create.go index a4cbcd00..e64fed9b 100644 --- a/Cubelet/services/cubebox/cube_container_create.go +++ b/Cubelet/services/cubebox/cube_container_create.go @@ -331,6 +331,12 @@ func (l *local) createContainers(ctx context.Context, flowOpts *workflow.CreateC if err := l.doProbe(param.ctxTmp, param.cntrReq, param.ci); err != nil { return err } + // Once the sandbox is ready, inject create-time env_vars into the guest + // envd (via its native /init) so later commands.run / run_code can read + // them. No-op when the annotation is absent. + if err := l.syncCreateEnvToEnvd(param.ctxTmp, sandBox, param.ci); err != nil { + return err + } err = l.cbriManager.PostCreateContainer(ctx, sandBox, param.ci) if err != nil { containerLog.Errorf("post create container failed, err: %v", err) diff --git a/Cubelet/services/cubebox/envd_init.go b/Cubelet/services/cubebox/envd_init.go new file mode 100644 index 00000000..c670342c --- /dev/null +++ b/Cubelet/services/cubebox/envd_init.go @@ -0,0 +1,154 @@ +// Copyright (c) 2024 Tencent Inc. +// SPDX-License-Identifier: Apache-2.0 +// + +package cubebox + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "strconv" + "time" + + "github.com/tencentcloud/CubeSandbox/Cubelet/pkg/log" + cubeboxstore "github.com/tencentcloud/CubeSandbox/Cubelet/pkg/store/cubebox" +) + +const ( + // createEnvVarsAnnotation is the contract with CubeAPI: it carries the + // create-time env_vars as a JSON object string {"K":"V"}. CubeMaster forwards + // it here automatically via the cube.master prefix passthrough. + createEnvVarsAnnotation = "cube.master.sandbox.create_env_vars" + // envdServerPort is the HTTP port envd listens on inside the guest + // (E2B envd is fixed at 49983). + envdServerPort = 49983 + + // Retry budget for /init: envd may not be listening yet right after the + // sandbox becomes ready, so we short-poll until it is up or we time out. + envdInitMaxWait = 15 * time.Second + envdInitInterval = 200 * time.Millisecond + envdInitReqTimeout = 3 * time.Second +) + +// envdInitBody is the envd POST /init request body (e2b-dev/infra). We only use +// envVars and timestamp here: +// - envVars: stored into envd's global defaults.EnvVars and merged into child +// process environments on Process/Start; +// - timestamp: used by envd to accept only newer requests, and to correct the +// guest clock as a side effect. +type envdInitBody struct { + EnvVars map[string]string `json:"envVars,omitempty"` + Timestamp string `json:"timestamp,omitempty"` +} + +// syncCreateEnvToEnvd injects the create-time env_vars (carried via annotation) +// into the guest envd. +// +// It mirrors the E2B orchestrator initEnvd flow: the host side connects directly +// to the guest's ip:49983 and calls the native POST /init. envd stores these +// variables into its global defaults.EnvVars, so processes it later starts +// (commands.run / run_code) inherit them (precedence: global < per-command, where +// per-command can override). envd merges additively and never clears existing +// entries, so any env already present in the runtime is preserved. +// +// Nothing is written to rootfs/profile and nothing goes into the OCI container +// spec, so neither the image/template nor the template snapshot is polluted with +// instance-level secrets. Sandboxes without create env are a no-op. +func (l *local) syncCreateEnvToEnvd(ctx context.Context, sandBox *cubeboxstore.CubeBox, + ci *cubeboxstore.Container) error { + raw, ok := sandBox.Annotations[createEnvVarsAnnotation] + if !ok || raw == "" { + return nil + } + + envVars := map[string]string{} + if err := json.Unmarshal([]byte(raw), &envVars); err != nil { + return fmt.Errorf("parse create env_vars annotation failed: %w", err) + } + if len(envVars) == 0 { + return nil + } + + ip := ci.IP + if ip == "" || ip == "" { + ip = sandBox.IP + } + if ip == "" || ip == "" { + return fmt.Errorf("sync create env to envd: empty sandbox IP") + } + + addr := fmt.Sprintf("http://%s/init", net.JoinHostPort(ip, strconv.Itoa(envdServerPort))) + body, err := json.Marshal(envdInitBody{ + EnvVars: envVars, + Timestamp: time.Now().UTC().Format(time.RFC3339Nano), + }) + if err != nil { + return fmt.Errorf("marshal envd init body failed: %w", err) + } + + start := time.Now() + deadline := start.Add(envdInitMaxWait) + client := &http.Client{Timeout: envdInitReqTimeout} + + var ( + lastErr error + attempt int + ) + for { + attempt++ + lastErr = postEnvdInit(ctx, client, addr, body) + if lastErr == nil { + log.G(ctx).Infof("envd /init ok: sandbox=%s ip=%s vars=%d attempts=%d cost=%v", + sandBox.SandboxID, ip, len(envVars), attempt, time.Since(start)) + return nil + } + + if ctx.Err() != nil { + lastErr = ctx.Err() + break + } + if time.Now().After(deadline) { + break + } + select { + case <-ctx.Done(): + lastErr = ctx.Err() + case <-time.After(envdInitInterval): + } + if ctx.Err() != nil { + break + } + } + + return fmt.Errorf("envd /init failed: sandbox=%s ip=%s attempts=%d cost=%v: %w", + sandBox.SandboxID, ip, attempt, time.Since(start), lastErr) +} + +// postEnvdInit sends one /init request; only 204/200 is treated as success. +func postEnvdInit(ctx context.Context, client *http.Client, addr string, body []byte) error { + reqCtx, cancel := context.WithTimeout(ctx, envdInitReqTimeout) + defer cancel() + + req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, addr, bytes.NewReader(body)) + if err != nil { + return fmt.Errorf("build envd init request failed: %w", err) + } + req.Header.Set("Content-Type", "application/json") + + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + io.Copy(io.Discard, resp.Body) + + if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK { + return fmt.Errorf("envd /init unexpected status %d", resp.StatusCode) + } + return nil +}