From ef433078f72ee67e2bc18985012c3e9622f08309 Mon Sep 17 00:00:00 2001 From: xiaojunxiang Date: Sun, 7 Jun 2026 01:03:48 +0800 Subject: [PATCH] fix: make Sandbox.create(env_vars) visible to commands.run Inject create-time env vars via per-container restore bind mounts and POSIX single-quoted profile exports in Cubelet. Extend CubeAPI sandbox create mapping, add Cubelet netfile/cubebox helpers, and cover SDK plus Cubelet behavior with tests. Signed-off-by: xiaojunxiang --- CubeAPI/src/services/sandboxes.rs | 70 ++++++++- .../service/httpservice/cube/cubeboxutil.go | 11 +- .../httpservice/cube/cubeboxutil_test.go | 60 ++++++++ Cubelet/pkg/container/netfile/netfile.go | 140 ++++++++++++++++- Cubelet/pkg/container/netfile/netfile_test.go | 71 ++++++++- Cubelet/services/cubebox/annotation.go | 2 +- .../services/cubebox/cube_container_create.go | 145 ++++++++++++++++++ Cubelet/storage/hostdir.go | 10 +- sdk/python/tests/test_env_injection_e2e.py | 56 +++++++ 9 files changed, 547 insertions(+), 18 deletions(-) create mode 100644 CubeMaster/pkg/service/httpservice/cube/cubeboxutil_test.go create mode 100644 sdk/python/tests/test_env_injection_e2e.py diff --git a/CubeAPI/src/services/sandboxes.rs b/CubeAPI/src/services/sandboxes.rs index 00419caa..bc7c477a 100644 --- a/CubeAPI/src/services/sandboxes.rs +++ b/CubeAPI/src/services/sandboxes.rs @@ -9,10 +9,10 @@ use uuid::Uuid; use crate::{ constants::ENVD_VERSION, cubemaster::{ - datetime_from_unix_nanos, extract_template_id, CreateSandboxRequest, CubeMasterClient, - CubeMasterError, CubeVSContext, DeleteSandboxRequest, ListSandboxRequest, SandboxInfo, - SandboxLogsRequest, SandboxRefreshRequest, SandboxStatus, SandboxTimeoutRequest, - SandboxUpdateRequest, + datetime_from_unix_nanos, extract_template_id, ContainerSpec, CreateSandboxRequest, + CubeMasterClient, CubeMasterError, CubeVSContext, DeleteSandboxRequest, EnvVar, ImageSpec, + ListSandboxRequest, SandboxInfo, SandboxLogsRequest, SandboxRefreshRequest, SandboxStatus, + SandboxTimeoutRequest, SandboxUpdateRequest, }, error::{AppError, AppResult}, models::{ @@ -140,7 +140,7 @@ impl SandboxService { annotations, labels, volumes: None, - containers: vec![], + containers: build_containers_from_env_vars(body.env_vars.as_ref()), exposed_ports: vec![], network_type: Some("tap".to_string()), cubevs_context: build_cubevs_context(body.allow_internet_access, body.network.as_ref()), @@ -608,6 +608,41 @@ fn new_request_id() -> String { Uuid::new_v4().to_string() } +/// Map SDK `envVars` into a placeholder container so CubeMaster merges them +/// with the template container via `applyTemplateToContainer`. +fn build_containers_from_env_vars(env_vars: Option<&HashMap>) -> Vec { + let Some(env_vars) = env_vars.filter(|vars| !vars.is_empty()) else { + return Vec::new(); + }; + + let envs = env_vars + .iter() + .map(|(key, value)| EnvVar { + key: key.clone(), + value: value.clone(), + }) + .collect(); + + vec![ContainerSpec { + name: None, + image: ImageSpec { + image: String::new(), + storage_media: None, + }, + command: None, + args: None, + working_dir: None, + resources: None, + envs: Some(envs), + volume_mounts: None, + dns_config: None, + r_limit: None, + security_context: None, + probe: None, + annotations: None, + }] +} + pub(crate) fn build_cubevs_context( allow_internet_access: Option, network: Option<&SandboxNetworkConfig>, @@ -635,7 +670,7 @@ pub(crate) fn build_cubevs_context( mod tests { use std::collections::HashMap; - use super::{build_cubevs_context, filter_by_metadata, from_cubemaster_info}; + use super::{build_containers_from_env_vars, build_cubevs_context, filter_by_metadata, from_cubemaster_info}; use crate::cubemaster::{ListSandboxResponse, SandboxInfo}; use crate::models::{SandboxNetworkConfig, SandboxState}; @@ -680,6 +715,7 @@ mod tests { status: "running".to_string(), started_at: None, end_at: None, + create_at: 0, cpu_count: 2, memory_mb: 2048, template_id: "tpl-1".to_string(), @@ -692,6 +728,28 @@ mod tests { assert_eq!(listed.template_id, "tpl-1"); } + #[test] + fn build_containers_from_env_vars_maps_sdk_payload() { + let mut env_vars = HashMap::from([ + ("CUBE_CREATE_ENV_TEST".to_string(), "injected-at-create".to_string()), + ("MY_APP_TOKEN".to_string(), "token-abc-123".to_string()), + ]); + + let containers = build_containers_from_env_vars(Some(&env_vars)); + assert_eq!(containers.len(), 1); + let envs = containers[0] + .envs + .as_ref() + .expect("create env vars should produce container envs"); + assert_eq!(envs.len(), 2); + assert!(envs.iter().any(|e| e.key == "CUBE_CREATE_ENV_TEST" && e.value == "injected-at-create")); + assert!(envs.iter().any(|e| e.key == "MY_APP_TOKEN" && e.value == "token-abc-123")); + + env_vars.clear(); + assert!(build_containers_from_env_vars(Some(&env_vars)).is_empty()); + assert!(build_containers_from_env_vars(None).is_empty()); + } + #[test] fn listed_sandbox_maps_paused_container_state_from_cubemaster_list() { let payload = serde_json::json!({ diff --git a/CubeMaster/pkg/service/httpservice/cube/cubeboxutil.go b/CubeMaster/pkg/service/httpservice/cube/cubeboxutil.go index 9786d67c..d2e59204 100644 --- a/CubeMaster/pkg/service/httpservice/cube/cubeboxutil.go +++ b/CubeMaster/pkg/service/httpservice/cube/cubeboxutil.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "maps" + "slices" "strconv" "strings" "time" @@ -123,9 +124,17 @@ func applyTemplateToContainer(ctr *types.Container, templateCtr *types.Container ctr.Sysctls = templateCtr.Sysctls ctr.SecurityContext = templateCtr.SecurityContext - ctr.Envs = append(ctr.Envs, templateCtr.Envs...) + ctr.Envs = append(slices.Clone(templateCtr.Envs), ctr.Envs...) applyTemplateVolumeMounts(templateCtr, ctr) + // Create-time overrides may send env-only placeholder containers from CubeAPI. + if len(ctr.Command) == 0 { + ctr.Command = templateCtr.Command + } + if len(ctr.Args) == 0 { + ctr.Args = templateCtr.Args + } + if !isContainerReqWhiteTag("WorkingDir") { ctr.WorkingDir = templateCtr.WorkingDir } diff --git a/CubeMaster/pkg/service/httpservice/cube/cubeboxutil_test.go b/CubeMaster/pkg/service/httpservice/cube/cubeboxutil_test.go new file mode 100644 index 00000000..835963c9 --- /dev/null +++ b/CubeMaster/pkg/service/httpservice/cube/cubeboxutil_test.go @@ -0,0 +1,60 @@ +// Copyright (c) 2024 Tencent Inc. +// SPDX-License-Identifier: Apache-2.0 +// + +package cube + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/tencentcloud/CubeSandbox/CubeMaster/pkg/service/sandbox/types" +) + +func TestApplyTemplateToContainerCreateTimeEnvOverridesTemplateDefaults(t *testing.T) { + template := &types.Container{ + Name: "main", + Command: []string{"/bin/sh", "-c", "sleep infinity"}, + Args: []string{"ignored"}, + Envs: []*types.KeyValue{ + {Key: "BASE_ENV", Value: "from-template"}, + {Key: "OVERRIDE_ME", Value: "template-value"}, + }, + Image: &types.ImageSpec{Image: "tpl-image"}, + } + + request := &types.Container{ + Envs: []*types.KeyValue{ + {Key: "OVERRIDE_ME", Value: "create-value"}, + {Key: "CREATE_ONLY", Value: "create-only"}, + }, + } + + require.NoError(t, applyTemplateToContainer(request, template, 0)) + + assert.Equal(t, []string{"/bin/sh", "-c", "sleep infinity"}, request.Command) + assert.Equal(t, []string{"ignored"}, request.Args) + require.Len(t, request.Envs, 4) + assert.Equal(t, &types.KeyValue{Key: "BASE_ENV", Value: "from-template"}, request.Envs[0]) + assert.Equal(t, &types.KeyValue{Key: "OVERRIDE_ME", Value: "template-value"}, request.Envs[1]) + assert.Equal(t, &types.KeyValue{Key: "OVERRIDE_ME", Value: "create-value"}, request.Envs[2]) + assert.Equal(t, &types.KeyValue{Key: "CREATE_ONLY", Value: "create-only"}, request.Envs[3]) +} + +func TestApplyTemplateToContainerPreservesExplicitCommandAndArgs(t *testing.T) { + template := &types.Container{ + Command: []string{"/entrypoint.sh"}, + Args: []string{"template-arg"}, + Image: &types.ImageSpec{Image: "tpl-image"}, + } + request := &types.Container{ + Command: []string{"/custom-entrypoint"}, + Args: []string{"request-arg"}, + } + + require.NoError(t, applyTemplateToContainer(request, template, 0)) + + assert.Equal(t, []string{"/custom-entrypoint"}, request.Command) + assert.Equal(t, []string{"request-arg"}, request.Args) +} diff --git a/Cubelet/pkg/container/netfile/netfile.go b/Cubelet/pkg/container/netfile/netfile.go index d5df93b8..fbf13a45 100644 --- a/Cubelet/pkg/container/netfile/netfile.go +++ b/Cubelet/pkg/container/netfile/netfile.go @@ -14,6 +14,8 @@ import ( "path/filepath" "sort" "strings" + "sync" + "time" "github.com/containerd/containerd/v2/pkg/oci" jsoniter "github.com/json-iterator/go" @@ -31,10 +33,14 @@ var ( ) const ( - netfilePathResolv = "/etc/resolv.conf" - netfilePathHosts = "/etc/hosts" - hostnameFilePath = "/etc/hostname" - defaultDNSIP = "119.29.29.29" + netfilePathResolv = "/etc/resolv.conf" + netfilePathHosts = "/etc/hosts" + hostnameFilePath = "/etc/hostname" + createEnvProfilePath = "/etc/profile.d/99-cube-create-env.sh" + CreateEnvProfilePath = createEnvProfilePath + CreateEnvInternalVolumeName = "cube-internal-create-env" + CreateEnvHostBase = "/data/cubelet/create-env" + defaultDNSIP = "119.29.29.29" ) func Init(oldDir string) error { @@ -57,6 +63,7 @@ type CubeboxNetfile struct { Hostname string ContainerNetfiles map[string]ContainerNetfile + mu sync.Mutex } func (c *CubeboxNetfile) WriteToHost() error { @@ -110,6 +117,12 @@ func (cn *CubeboxNetfile) CreateNetfiles(req *cubebox.RunCubeSandboxRequest) err }, }, } + if envProfile := genCreateEnvProfile(c.GetEnvs()); len(envProfile) > 0 { + netfiles[c.Name].Files[createEnvProfilePath] = FileContent{ + Path: createEnvProfilePath, + Content: envProfile, + } + } } cn.ContainerNetfiles = netfiles return nil @@ -159,7 +172,43 @@ func (cn *CubeboxNetfile) ContainerVirtiofsMounts(containerName string) []virtio return mounts } -func (cn *CubeboxNetfile) OciContainerNetfileSpec(ctx context.Context, containerName string) oci.SpecOpts { +// CreateEnvVolumeName returns the hostdir volume name for per-container create env injection. +func CreateEnvVolumeName(containerKey string) string { + if containerKey == "" { + return CreateEnvInternalVolumeName + } + return CreateEnvInternalVolumeName + "-" + containerKey +} + +// EnsureCreateEnvProfile refreshes create-time env injection file from the +// latest container envs. Netfile Create runs before cubebox merges envs, so +// profile.d must be rebuilt when OCI annotations are generated. +func (cn *CubeboxNetfile) EnsureCreateEnvProfile(containerName string, envs []*cubebox.KeyValue) { + if cn == nil || cn.ContainerNetfiles == nil { + return + } + cn.mu.Lock() + defer cn.mu.Unlock() + cf, ok := cn.ContainerNetfiles[containerName] + if !ok { + return + } + envProfile := genCreateEnvProfile(envs) + if len(envProfile) == 0 { + delete(cf.Files, createEnvProfilePath) + } else { + if cf.Files == nil { + cf.Files = make(map[string]FileContent) + } + cf.Files[createEnvProfilePath] = FileContent{ + Path: createEnvProfilePath, + Content: envProfile, + } + } + cn.ContainerNetfiles[containerName] = cf +} + +func (cn *CubeboxNetfile) OciContainerNetfileSpec(ctx context.Context, containerName string, envs []*cubebox.KeyValue) oci.SpecOpts { if cn.RootPath != "" { return nil } @@ -172,6 +221,12 @@ func (cn *CubeboxNetfile) OciContainerNetfileSpec(ctx context.Context, container for _, f := range cf.Files { files = append(files, f) } + if envProfile := genCreateEnvProfile(envs); len(envProfile) > 0 { + files = append(files, FileContent{ + Path: createEnvProfilePath, + Content: envProfile, + }) + } d, err := jsoniter.MarshalToString(files) if err != nil { @@ -290,6 +345,81 @@ func defaultDNSServers() []string { return append([]string(nil), cfg.Common.DefaultDNSServers...) } +// GenCreateEnvProfile builds shell exports for create-time env injection. +func GenCreateEnvProfile(envs []*cubebox.KeyValue) []byte { + return genCreateEnvProfile(envs) +} + +// shellSingleQuoted wraps s in POSIX single quotes so profile.d sourcing does not expand $, `, etc. +func shellSingleQuoted(s string) string { + return "'" + strings.ReplaceAll(s, "'", "'\\''") + "'" +} + +func genCreateEnvProfile(envs []*cubebox.KeyValue) []byte { + if len(envs) == 0 { + return nil + } + var b strings.Builder + b.WriteString("# Generated by Cubelet for create-time env injection\n") + wrote := false + for _, kv := range envs { + if kv == nil || kv.Key == "" { + continue + } + if strings.HasPrefix(kv.Key, "CUBE_CONTAINER_") || kv.Key == "__BRIEF_SUMMARY__" { + continue + } + b.WriteString("export ") + b.WriteString(kv.Key) + b.WriteString("=") + b.WriteString(shellSingleQuoted(kv.Value)) + b.WriteString("\n") + wrote = true + } + if !wrote { + return nil + } + return []byte(b.String()) +} + +const rootfsEnvProfileMountTimeout = 30 * time.Second + +// WriteCreateEnvProfileToBlockDevice writes create-time env exports into the +// sandbox writable rootfs ext4 device. AppSnapshot memory restore skips shim +// custom-file netfiles, so profile.d must be seeded on the rootfs directly. +func WriteCreateEnvProfileToBlockDevice(ctx context.Context, devicePath string, envs []*cubebox.KeyValue) error { + content := genCreateEnvProfile(envs) + if len(content) == 0 || strings.TrimSpace(devicePath) == "" { + return nil + } + mountDir, err := os.MkdirTemp("", "cube-create-env-mount-*") + if err != nil { + return fmt.Errorf("create mount dir for create env profile: %w", err) + } + defer os.RemoveAll(mountDir) + + profilePath := filepath.Join(mountDir, "etc", "profile.d", filepath.Base(createEnvProfilePath)) + cmds := [][]string{ + {"mount", devicePath, mountDir}, + {"mkdir", "-p", filepath.Dir(profilePath)}, + } + for _, cmd := range cmds { + if _, stderr, execErr := utils.ExecV(cmd, rootfsEnvProfileMountTimeout); execErr != nil { + _, _, _ = utils.ExecV([]string{"umount", mountDir}, rootfsEnvProfileMountTimeout) + return fmt.Errorf("prepare rootfs for create env profile (%v): %s", cmd, stderr) + } + } + if err := os.WriteFile(profilePath, content, 0644); err != nil { + _, _, _ = utils.ExecV([]string{"umount", mountDir}, rootfsEnvProfileMountTimeout) + return fmt.Errorf("write create env profile to rootfs: %w", err) + } + if _, stderr, err := utils.ExecV([]string{"umount", mountDir}, rootfsEnvProfileMountTimeout); err != nil { + return fmt.Errorf("umount rootfs after create env profile: %s", stderr) + } + log.G(ctx).Infof("write create env profile to rootfs device %s", devicePath) + return nil +} + func Clean(ctx context.Context, containerID string) error { // 1.2.1 路径穿越防护：校验 containerID 不含路径穿越字符 dir, err := utils.SafeJoinPath(oldNetfilePath, containerID) diff --git a/Cubelet/pkg/container/netfile/netfile_test.go b/Cubelet/pkg/container/netfile/netfile_test.go index eb8bdfe4..b65aebb4 100644 --- a/Cubelet/pkg/container/netfile/netfile_test.go +++ b/Cubelet/pkg/container/netfile/netfile_test.go @@ -257,16 +257,16 @@ func TestCubeboxNetfile_OciContainerNetfileSpec(t *testing.T) { }, } - specOpts := cn.OciContainerNetfileSpec(ctx, "container1") + specOpts := cn.OciContainerNetfileSpec(ctx, "container1", nil) assert.NotNil(t, specOpts) - specOpts = cn.OciContainerNetfileSpec(ctx, "nonexistent") + specOpts = cn.OciContainerNetfileSpec(ctx, "nonexistent", nil) assert.Nil(t, specOpts) cn.ContainerNetfiles["empty-container"] = ContainerNetfile{ Files: map[string]FileContent{}, } - specOpts = cn.OciContainerNetfileSpec(ctx, "empty-container") + specOpts = cn.OciContainerNetfileSpec(ctx, "empty-container", nil) assert.Nil(t, specOpts) } @@ -377,6 +377,69 @@ func TestResolveEffectiveDNSServers(t *testing.T) { } } +func TestGenCreateEnvProfileExportsCreateTimeVars(t *testing.T) { + content := string(GenCreateEnvProfile([]*cubebox.KeyValue{ + {Key: "MY_APP_TOKEN", Value: "token-abc-123"}, + {Key: "CUBE_CONTAINER_MAIN", Value: "ignored-internal"}, + {Key: "__BRIEF_SUMMARY__", Value: "ignored-summary"}, + {Key: "EMPTY_OK", Value: ""}, + {Key: "SHELL_LITERAL", Value: "hello $USER `cmd` 'quote'"}, + })) + assert.Contains(t, content, "export MY_APP_TOKEN='token-abc-123'") + assert.Contains(t, content, "export EMPTY_OK=''") + assert.Contains(t, content, "export SHELL_LITERAL='hello $USER `cmd` '\\''quote'\\'''") + assert.NotContains(t, content, "CUBE_CONTAINER_MAIN") + assert.NotContains(t, content, "__BRIEF_SUMMARY__") +} + +func TestCreateEnvVolumeName(t *testing.T) { + assert.Equal(t, CreateEnvInternalVolumeName, CreateEnvVolumeName("")) + assert.Equal(t, CreateEnvInternalVolumeName+"-sidecar", CreateEnvVolumeName("sidecar")) +} + +func TestCreateNetfilesAddsCreateEnvProfile(t *testing.T) { + cn := &CubeboxNetfile{Hostname: "test-hostname"} + req := &cubebox.RunCubeSandboxRequest{ + Containers: []*cubebox.ContainerConfig{ + { + Name: "main", + Envs: []*cubebox.KeyValue{ + {Key: "CUBE_CREATE_ENV_TEST", Value: "injected-at-create"}, + }, + }, + }, + } + + require.NoError(t, cn.CreateNetfiles(req)) + + files := cn.ContainerNetfiles["main"].Files + require.Contains(t, files, CreateEnvProfilePath) + assert.Contains(t, string(files[CreateEnvProfilePath].Content), "export CUBE_CREATE_ENV_TEST='injected-at-create'") +} + +func TestEnsureCreateEnvProfileRefreshesFromLatestEnvs(t *testing.T) { + cn := &CubeboxNetfile{ + ContainerNetfiles: map[string]ContainerNetfile{ + "main": { + Files: map[string]FileContent{ + CreateEnvProfilePath: { + Path: CreateEnvProfilePath, + Content: []byte("stale"), + }, + }, + }, + }, + } + + cn.EnsureCreateEnvProfile("main", []*cubebox.KeyValue{ + {Key: "REFRESHED", Value: "new-value"}, + }) + + content := string(cn.ContainerNetfiles["main"].Files[CreateEnvProfilePath].Content) + assert.Contains(t, content, "export REFRESHED='new-value'") + assert.NotContains(t, content, "stale") +} + func TestFileContent_EmptyContent(t *testing.T) { cn := &CubeboxNetfile{ ContainerNetfiles: map[string]ContainerNetfile{ @@ -392,6 +455,6 @@ func TestFileContent_EmptyContent(t *testing.T) { } ctx := context.Background() - specOpts := cn.OciContainerNetfileSpec(ctx, "container1") + specOpts := cn.OciContainerNetfileSpec(ctx, "container1", nil) assert.NotNil(t, specOpts) } diff --git a/Cubelet/services/cubebox/annotation.go b/Cubelet/services/cubebox/annotation.go index 6ca7a8fc..4fe57d5c 100644 --- a/Cubelet/services/cubebox/annotation.go +++ b/Cubelet/services/cubebox/annotation.go @@ -174,7 +174,7 @@ func (l *local) genNetworkAnnotationOpt(ctx context.Context, specOpts = append(specOpts, opts.NetworkInfo.OCISpecOpts()) } if opts.NetFile != nil { - if spec := opts.NetFile.OciContainerNetfileSpec(ctx, containerReq.Name); spec != nil { + if spec := opts.NetFile.OciContainerNetfileSpec(ctx, containerReq.Name, containerReq.GetEnvs()); spec != nil { specOpts = append(specOpts, spec) } diff --git a/Cubelet/services/cubebox/cube_container_create.go b/Cubelet/services/cubebox/cube_container_create.go index 229b680d..cd03a150 100644 --- a/Cubelet/services/cubebox/cube_container_create.go +++ b/Cubelet/services/cubebox/cube_container_create.go @@ -31,6 +31,7 @@ import ( "github.com/opencontainers/runtime-spec/specs-go" "k8s.io/apimachinery/pkg/api/resource" runtime "k8s.io/cri-api/pkg/apis/runtime/v1" + "golang.org/x/sys/unix" "github.com/tencentcloud/CubeSandbox/Cubelet/api/services/cubebox/v1" "github.com/tencentcloud/CubeSandbox/Cubelet/api/services/errorcode/v1" @@ -459,6 +460,11 @@ func (l *local) createCubeboxContainer(ctx context.Context, flowOpts *workflow.C } } + if err := injectCreateEnvProfileToRootfs(ctx, flowOpts, realReq); err != nil { + return fmt.Errorf("inject create env profile to rootfs failed: %w", err) + } + syncSavedContainerConfigs(realReq, sandBox) + if err := l.cubeboxManger.Save(ctx, sandBox, cubes.WithNoEvent); err != nil { log.G(ctx).Warnf("saveSandBoxInfo failed.%s", err.Error()) return ret.Err(errorcode.ErrorCode_UpdateLocalMetaDataFailed, err.Error()) @@ -466,6 +472,145 @@ func (l *local) createCubeboxContainer(ctx context.Context, flowOpts *workflow.C return nil } +func injectCreateEnvProfileToRootfs(ctx context.Context, flowOpts *workflow.CreateContext, realReq *cubebox.RunCubeSandboxRequest) error { + if flowOpts == nil || realReq == nil { + return nil + } + if flowOpts.IsRetoreSnapshot() { + return injectCreateEnvProfileForRestore(ctx, flowOpts, realReq) + } + if flowOpts.StorageInfo == nil { + return nil + } + storageInfo, ok := flowOpts.StorageInfo.(*storage.StorageInfo) + if !ok || len(storageInfo.Volumes) == 0 { + return nil + } + for _, containerReq := range realReq.GetContainers() { + var blkPath string + for _, vm := range containerReq.GetVolumeMounts() { + if vm.GetContainerPath() != "/" { + continue + } + if vol, ok := storageInfo.Volumes[vm.GetName()]; ok && vol.FilePath != "" { + blkPath = vol.FilePath + break + } + } + if blkPath == "" { + continue + } + return localnetfile.WriteCreateEnvProfileToBlockDevice(ctx, blkPath, containerReq.GetEnvs()) + } + return nil +} + +func syncSavedContainerConfigs(realReq *cubebox.RunCubeSandboxRequest, sandBox *cubeboxstore.CubeBox) { + for _, cntrReq := range realReq.GetContainers() { + ci, err := sandBox.Get(cntrReq.Id) + if err != nil { + continue + } + ci.Config = makeContainerConfigToSave(cntrReq) + if sandBox.FirstContainerName == ci.ID { + sandBox.Config = ci.Config + } + } +} + +func injectCreateEnvProfileForRestore(ctx context.Context, flowOpts *workflow.CreateContext, realReq *cubebox.RunCubeSandboxRequest) error { + if flowOpts.StorageInfo == nil { + return nil + } + storageInfo, ok := flowOpts.StorageInfo.(*storage.StorageInfo) + if !ok { + return nil + } + sandboxID := flowOpts.SandboxID + if sandboxID == "" { + return nil + } + + shareDir := filepath.Join(storage.HostDirBasePath, sandboxID, "ro") + sharePrepared := false + if storageInfo.HostDirBackendInfos == nil { + storageInfo.HostDirBackendInfos = make(map[string]*storage.HostDirBackendInfo) + } + + for _, containerReq := range realReq.GetContainers() { + content := localnetfile.GenCreateEnvProfile(containerReq.GetEnvs()) + if len(content) == 0 { + continue + } + + containerKey := containerReq.GetName() + if containerKey == "" { + containerKey = containerReq.GetId() + } + if containerKey == "" { + continue + } + volumeName := localnetfile.CreateEnvVolumeName(containerKey) + + hostDir := filepath.Join(localnetfile.CreateEnvHostBase, sandboxID, containerKey) + if err := os.MkdirAll(hostDir, 0o755); err != nil { + return fmt.Errorf("mkdir create env host dir %s: %w", hostDir, err) + } + hostFile := filepath.Join(hostDir, filepath.Base(localnetfile.CreateEnvProfilePath)) + if err := os.WriteFile(hostFile, content, 0o644); err != nil { + return fmt.Errorf("write create env profile %s: %w", hostFile, err) + } + + if !sharePrepared { + if err := os.MkdirAll(shareDir, 0o755); err != nil { + return fmt.Errorf("mkdir create env share dir %s: %w", shareDir, err) + } + sharePrepared = true + } + + bindDest := filepath.Join(shareDir, volumeName) + if err := os.WriteFile(bindDest, content, 0o644); err != nil { + return fmt.Errorf("seed create env bind dest %s: %w", bindDest, err) + } + if err := unix.Mount(hostFile, bindDest, "", unix.MS_BIND, ""); err != nil { + return fmt.Errorf("bind mount create env profile %s -> %s: %w", hostFile, bindDest, err) + } + roFlags := uintptr(unix.MS_BIND | unix.MS_REMOUNT | unix.MS_RDONLY) + if err := unix.Mount("", bindDest, "", roFlags, ""); err != nil { + return fmt.Errorf("remount create env profile ro %s: %w", bindDest, err) + } + + backendKey := volumeName + "/profile" + storageInfo.HostDirBackendInfos[backendKey] = &storage.HostDirBackendInfo{ + VolumeName: volumeName, + ShareDir: shareDir, + BindPath: bindDest, + ReadOnly: true, + } + log.G(ctx).Infof("prepared create env profile exec mount: container=%s host=%s share=%s dest=%s", + containerKey, hostFile, shareDir, localnetfile.CreateEnvProfilePath) + + appendCreateEnvVolumeMount(containerReq, volumeName) + } + return nil +} + +func appendCreateEnvVolumeMount(containerReq *cubebox.ContainerConfig, volumeName string) { + if volumeName == "" { + volumeName = localnetfile.CreateEnvInternalVolumeName + } + for _, vm := range containerReq.GetVolumeMounts() { + if vm.GetName() == volumeName { + return + } + } + containerReq.VolumeMounts = append(containerReq.VolumeMounts, &cubebox.VolumeMounts{ + Name: volumeName, + ContainerPath: localnetfile.CreateEnvProfilePath, + Readonly: true, + }) +} + func (l *local) generateContainerID(ctx context.Context, flowOpts *workflow.CreateContext, index int) (context.Context, string) { var cid string ctxTmp := context.WithValue(ctx, constants.KCubeIndexContext, strconv.Itoa(index)) diff --git a/Cubelet/storage/hostdir.go b/Cubelet/storage/hostdir.go index 3d1409a3..17e36cc9 100644 --- a/Cubelet/storage/hostdir.go +++ b/Cubelet/storage/hostdir.go @@ -18,7 +18,11 @@ import ( "golang.org/x/sys/unix" ) -var hostDirBasePath = "/data/cubelet/hostdir" +var ( + hostDirBasePath = "/data/cubelet/hostdir" + HostDirBasePath = hostDirBasePath + createEnvHostBase = "/data/cubelet/create-env" +) type HostDirBackendInfo struct { VolumeName string `json:"volume_name"` @@ -137,5 +141,9 @@ func (l *local) cleanupHostDirVolumes(ctx context.Context, info *StorageInfo) er log.G(ctx).Warnf("cleanupHostDirVolumes: removeAll %s: %v", sandboxDir, err) return err } + createEnvDir := filepath.Join(createEnvHostBase, info.SandboxID) + if err := os.RemoveAll(createEnvDir); err != nil { + log.G(ctx).Warnf("cleanupHostDirVolumes: removeAll %s: %v", createEnvDir, err) + } return nil } diff --git a/sdk/python/tests/test_env_injection_e2e.py b/sdk/python/tests/test_env_injection_e2e.py new file mode 100644 index 00000000..e6680ef6 --- /dev/null +++ b/sdk/python/tests/test_env_injection_e2e.py @@ -0,0 +1,56 @@ +# Copyright (c) 2026 Tencent Inc. +# SPDX-License-Identifier: Apache-2.0 +"""E2E: Sandbox.create(env_vars=...) 注入的变量可被 commands.run 使用.""" + +from __future__ import annotations + +import os +import uuid + +import pytest + +from cubesandbox import Config, Sandbox + + +pytestmark = pytest.mark.e2e + + +def _option(pytestconfig: pytest.Config, option: str, env: str, default: str | None = None) -> str | None: + return pytestconfig.getoption(option) or os.environ.get(env) or default + + +def _require_e2e(pytestconfig: pytest.Config) -> None: + if not pytestconfig.getoption("--run-e2e") and os.environ.get("CUBE_E2E") != "1": + pytest.skip("use --run-e2e or set CUBE_E2E=1 to run live CubeAPI e2e tests") + + +def _config(pytestconfig: pytest.Config) -> Config: + return Config( + api_url=_option(pytestconfig, "--cube-api-url", "CUBE_API_URL", "http://127.0.0.1:3000"), + template_id=_option(pytestconfig, "--cube-template-id", "CUBE_TEMPLATE_ID"), + proxy_node_ip=os.environ.get("CUBE_PROXY_NODE_IP", "127.0.0.1"), + timeout=600, + ) + + +def test_create_env_vars_available_in_commands_run(pytestconfig: pytest.Config) -> None: + _require_e2e(pytestconfig) + config = _config(pytestconfig) + if not config.template_id: + pytest.skip("set CUBE_TEMPLATE_ID or pass --cube-template-id") + + env_key = f"CUBE_PY_CREATE_ENV_{uuid.uuid4().hex[:8].upper()}" + env_value = f"injected-{uuid.uuid4().hex[:8]}" + sb = Sandbox.create(env_vars={env_key: env_value, "MY_APP_TOKEN": "token-abc-123"}, config=config) + + try: + # commands.run 内部使用 bash -l -c, 与 SDK 语义一致 + got = sb.commands.run(f"printenv {env_key}") + assert got.exit_code == 0, got.stderr + assert env_value in got.stdout + + token = sb.commands.run("printenv MY_APP_TOKEN") + assert token.exit_code == 0, token.stderr + assert "token-abc-123" in token.stdout + finally: + sb.kill()