Nothing, Deprecated #548
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,154 @@ | ||
| // Copyright (c) 2024 Tencent Inc. | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
| // | ||
|
|
||
| package cubebox | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "context" | ||
| "encoding/json" | ||
| "fmt" | ||
| "io" | ||
| "net" | ||
| "net/http" | ||
| "strconv" | ||
| "time" | ||
|
|
||
| "github.com/tencentcloud/CubeSandbox/Cubelet/pkg/log" | ||
| cubeboxstore "github.com/tencentcloud/CubeSandbox/Cubelet/pkg/store/cubebox" | ||
| ) | ||
|
|
||
| const ( | ||
| // createEnvVarsAnnotation is the contract with CubeAPI: it carries the | ||
| // create-time env_vars as a JSON object string {"K":"V"}. CubeMaster forwards | ||
| // it here automatically via the cube.master prefix passthrough. | ||
| createEnvVarsAnnotation = "cube.master.sandbox.create_env_vars" | ||
| // envdServerPort is the HTTP port envd listens on inside the guest | ||
| // (E2B envd is fixed at 49983). | ||
| envdServerPort = 49983 | ||
|
|
||
| // Retry budget for /init: envd may not be listening yet right after the | ||
| // sandbox becomes ready, so we short-poll until it is up or we time out. | ||
| envdInitMaxWait = 15 * time.Second | ||
| envdInitInterval = 200 * time.Millisecond | ||
| envdInitReqTimeout = 3 * time.Second | ||
| ) | ||
|
|
||
| // envdInitBody is the envd POST /init request body (e2b-dev/infra). We only use | ||
| // envVars and timestamp here: | ||
| // - envVars: stored into envd's global defaults.EnvVars and merged into child | ||
| // process environments on Process/Start; | ||
| // - timestamp: used by envd to accept only newer requests, and to correct the | ||
| // guest clock as a side effect. | ||
| type envdInitBody struct { | ||
| EnvVars map[string]string `json:"envVars,omitempty"` | ||
| Timestamp string `json:"timestamp,omitempty"` | ||
| } | ||
|
|
||
| // syncCreateEnvToEnvd injects the create-time env_vars (carried via annotation) | ||
| // into the guest envd. | ||
| // | ||
| // It mirrors the E2B orchestrator initEnvd flow: the host side connects directly | ||
| // to the guest's ip:49983 and calls the native POST /init. envd stores these | ||
| // variables into its global defaults.EnvVars, so processes it later starts | ||
| // (commands.run / run_code) inherit them (precedence: global < per-command, where | ||
| // per-command can override). envd merges additively and never clears existing | ||
| // entries, so any env already present in the runtime is preserved. | ||
| // | ||
| // Nothing is written to rootfs/profile and nothing goes into the OCI container | ||
| // spec, so neither the image/template nor the template snapshot is polluted with | ||
| // instance-level secrets. Sandboxes without create env are a no-op. | ||
| func (l *local) syncCreateEnvToEnvd(ctx context.Context, sandBox *cubeboxstore.CubeBox, | ||
| ci *cubeboxstore.Container) error { | ||
| raw, ok := sandBox.Annotations[createEnvVarsAnnotation] | ||
| if !ok || raw == "" { | ||
| return nil | ||
| } | ||
|
|
||
| envVars := map[string]string{} | ||
| if err := json.Unmarshal([]byte(raw), &envVars); err != nil { | ||
| return fmt.Errorf("parse create env_vars annotation failed: %w", err) | ||
| } | ||
| if len(envVars) == 0 { | ||
| return nil | ||
| } | ||
|
|
||
| ip := ci.IP | ||
| if ip == "" || ip == "<nil>" { | ||
| ip = sandBox.IP | ||
| } | ||
| if ip == "" || ip == "<nil>" { | ||
| return fmt.Errorf("sync create env to envd: empty sandbox IP") | ||
| } | ||
|
|
||
| addr := fmt.Sprintf("http://%s/init", net.JoinHostPort(ip, strconv.Itoa(envdServerPort))) | ||
| body, err := json.Marshal(envdInitBody{ | ||
| EnvVars: envVars, | ||
| Timestamp: time.Now().UTC().Format(time.RFC3339Nano), | ||
| }) | ||
| if err != nil { | ||
| return fmt.Errorf("marshal envd init body failed: %w", err) | ||
| } | ||
|
|
||
| start := time.Now() | ||
| deadline := start.Add(envdInitMaxWait) | ||
| client := &http.Client{Timeout: envdInitReqTimeout} | ||
|
There was a problem hiding this comment. Code quality: HTTP client created per invocation, no connection reuse A new Suggestion: Lift var envdHTTPClient = &http.Client{
Timeout: envdInitReqTimeout,
Transport: &http.Transport{
MaxIdleConnsPerHost: 4,
IdleConnTimeout: 30 * time.Second,
DialContext: (&net.Dialer{
Timeout: 2 * time.Second,
}).DialContext,
},
} |
||
|
|
||
| var ( | ||
| lastErr error | ||
| attempt int | ||
| ) | ||
| for { | ||
| attempt++ | ||
| lastErr = postEnvdInit(ctx, client, addr, body) | ||
| if lastErr == nil { | ||
| log.G(ctx).Infof("envd /init ok: sandbox=%s ip=%s vars=%d attempts=%d cost=%v", | ||
| sandBox.SandboxID, ip, len(envVars), attempt, time.Since(start)) | ||
| return nil | ||
| } | ||
|
|
||
| if ctx.Err() != nil { | ||
| lastErr = ctx.Err() | ||
| break | ||
| } | ||
| if time.Now().After(deadline) { | ||
| break | ||
| } | ||
| select { | ||
| case <-ctx.Done(): | ||
| lastErr = ctx.Err() | ||
| case <-time.After(envdInitInterval): | ||
| } | ||
| if ctx.Err() != nil { | ||
| break | ||
| } | ||
| } | ||
|
Comment on lines
+111
to
+126
There was a problem hiding this comment. Code quality: Redundant context-cancellation checks in retry loop The retry loop checks Suggestion: Simplify to a for time.Since(start) < envdInitMaxWait {
attempt++
if err := postEnvdInit(ctx, client, addr, body); err == nil {
// log success...
return nil
}
lastErr = err
select {
case <-ctx.Done():
lastErr = ctx.Err()
case <-time.After(envdInitInterval):
}
}
return fmt.Errorf("envd /init failed: ...")This eliminates the three-way context check and makes the control flow self-documenting. |
||
|
|
||
| return fmt.Errorf("envd /init failed: sandbox=%s ip=%s attempts=%d cost=%v: %w", | ||
| sandBox.SandboxID, ip, attempt, time.Since(start), lastErr) | ||
| } | ||
|
|
||
| // postEnvdInit sends one /init request; only 204/200 is treated as success. | ||
| func postEnvdInit(ctx context.Context, client *http.Client, addr string, body []byte) error { | ||
| reqCtx, cancel := context.WithTimeout(ctx, envdInitReqTimeout) | ||
| defer cancel() | ||
|
|
||
| req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, addr, bytes.NewReader(body)) | ||
| if err != nil { | ||
| return fmt.Errorf("build envd init request failed: %w", err) | ||
| } | ||
| req.Header.Set("Content-Type", "application/json") | ||
|
|
||
| resp, err := client.Do(req) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer resp.Body.Close() | ||
| io.Copy(io.Discard, resp.Body) | ||
|
There was a problem hiding this comment. Code quality:
Also, |
||
|
|
||
| if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK { | ||
| return fmt.Errorf("envd /init unexpected status %d", resp.StatusCode) | ||
| } | ||
| return nil | ||
| } | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Performance:
syncCreateEnvToEnvdis called inside the per-container loop, should be per-sandboxsyncCreateEnvToEnvdcallsPOST /initon the envd instance inside the guest — and envd is a single instance per sandbox, not per container. Calling it N times for an N-container sandbox is redundant and multiplies latency: with the 15-second retry budget (envdInitMaxWait), a 3-container sandbox could stall for up to 45s in the worst case.This should be moved outside the
for _, param := range paramsloop, called once after all containers have been created and probed. SincesyncCreateEnvToEnvdusesci.IPas the primary IP source (falling back tosandBox.IP), you'd need to resolve the IP outside the loop — but the first/probe container's IP should be sufficient for the envd connection.