diff --git a/conf/app.conf b/conf/app.conf index d146190..d3f1a27 100644 --- a/conf/app.conf +++ b/conf/app.conf @@ -22,6 +22,8 @@ socks5Proxy = 127.0.0.1:10808 ; -- Helm images (optional) ------------------------------------------------- ; Set true only when implicit/latest chart images must use IfNotPresent. helmImplicitLatestPullPolicy = false +; Maximum time Helm waits for an application and its jobs to become ready. +helmInstallTimeout = 20m ; -- Control plane ---------------------------------------------------------- apiserverPort = 6443 diff --git a/object/helm_operation.go b/object/helm_operation.go index 4959b9a..3d7221f 100644 --- a/object/helm_operation.go +++ b/object/helm_operation.go @@ -191,6 +191,14 @@ func StartHelmOperationTaskContext(ctx context.Context, id int64, phase string) return nil } +func TouchHelmOperationTaskContext(ctx context.Context, id int64) error { + _, err := ormer.Engine.Context(ctx).ID(id). + Where("status IN (?, ?)", HelmOperationStatusPending, HelmOperationStatusRunning). + Cols("updated_at"). + Update(&HelmOperationTask{UpdatedAt: time.Now().UTC()}) + return err +} + func UpdateHelmOperationTaskPhase(id int64, phase string) error { return UpdateHelmOperationTaskPhaseContext(context.Background(), id, phase) } diff --git a/object/helm_operation_recorder.go b/object/helm_operation_recorder.go index 7b7d97d..55c50ea 100644 --- a/object/helm_operation_recorder.go +++ b/object/helm_operation_recorder.go @@ -22,6 +22,7 @@ const ( helmOperationFinishTimeout = HelmOperationPersistenceTimeout helmOperationFinishAttempts = 3 helmOperationFinishRetryDelay = 100 * time.Millisecond + helmOperationHeartbeatInterval = time.Minute ) type HelmOperationRecorder struct { @@ -36,32 +37,36 @@ type HelmOperationRecorder struct { finish sync.Once finishErr error - persistLogs func(context.Context, int64, []*HelmOperationLog) error - finishTask func(context.Context, int64, bool, string) error - terminalMatches func(context.Context, int64, bool, string) (bool, error) - persistTimeout time.Duration - shutdownTimeout time.Duration - finishTimeout time.Duration - finishRetryDelay time.Duration - persistCtx context.Context - cancelPersist context.CancelFunc + persistLogs func(context.Context, int64, []*HelmOperationLog) error + finishTask func(context.Context, int64, bool, string) error + terminalMatches func(context.Context, int64, bool, string) (bool, error) + touchTask func(context.Context, int64) error + persistTimeout time.Duration + heartbeatInterval time.Duration + shutdownTimeout time.Duration + finishTimeout time.Duration + finishRetryDelay time.Duration + persistCtx context.Context + cancelPersist context.CancelFunc } func NewHelmOperationRecorder(taskID int64) *HelmOperationRecorder { persistCtx, cancelPersist := context.WithCancel(context.Background()) recorder := &HelmOperationRecorder{ - taskID: taskID, - queue: make(chan *HelmOperationLog, helmOperationLogBatchSize*2), - done: make(chan struct{}), - persistLogs: addHelmOperationLogsContext, - finishTask: FinishHelmOperationTaskContext, - terminalMatches: HelmOperationTaskHasTerminalOutcomeContext, - persistTimeout: helmOperationLogPersistTimeout, - shutdownTimeout: helmOperationRecorderShutdown, - finishTimeout: helmOperationFinishTimeout, - finishRetryDelay: helmOperationFinishRetryDelay, - persistCtx: persistCtx, - cancelPersist: cancelPersist, + taskID: taskID, + queue: make(chan *HelmOperationLog, helmOperationLogBatchSize*2), + done: make(chan struct{}), + persistLogs: addHelmOperationLogsContext, + finishTask: FinishHelmOperationTaskContext, + terminalMatches: HelmOperationTaskHasTerminalOutcomeContext, + touchTask: TouchHelmOperationTaskContext, + persistTimeout: helmOperationLogPersistTimeout, + heartbeatInterval: helmOperationHeartbeatInterval, + shutdownTimeout: helmOperationRecorderShutdown, + finishTimeout: helmOperationFinishTimeout, + finishRetryDelay: helmOperationFinishRetryDelay, + persistCtx: persistCtx, + cancelPersist: cancelPersist, } go recorder.run() return recorder @@ -219,6 +224,8 @@ func (r *HelmOperationRecorder) run() { }() ticker := time.NewTicker(helmOperationLogFlushInterval) defer ticker.Stop() + heartbeatTicker := time.NewTicker(r.heartbeatInterval) + defer heartbeatTicker.Stop() batch := make([]*HelmOperationLog, 0, helmOperationLogBatchSize) flush := func() { if len(batch) == 0 { @@ -247,6 +254,13 @@ func (r *HelmOperationRecorder) run() { } case <-ticker.C: flush() + case <-heartbeatTicker.C: + ctx, cancel := context.WithTimeout(r.persistCtx, r.persistTimeout) + err := r.touchTask(ctx, r.taskID) + cancel() + if err != nil && !errors.Is(err, context.Canceled) { + logs.Warning("refresh Helm operation task %d heartbeat: %v", r.taskID, err) + } } } } diff --git a/store/helm.go b/store/helm.go index d66b4f2..33f4449 100644 --- a/store/helm.go +++ b/store/helm.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "fmt" - "io" "net/http" "net/url" "sort" @@ -44,15 +43,13 @@ import ( ) const ( - helmOperationTimeout = 5 * time.Minute - helmInstallTimeout = 10 * time.Minute - helmCompatibilityTimeout = 2 * time.Minute - helmChartLoadTimeout = 2 * time.Minute - helmDiagnosticsTimeout = 15 * time.Second - helmInstallOperationTimeout = helmChartLoadTimeout + helmCompatibilityTimeout + helmInstallTimeout + helmDiagnosticsTimeout - helmDiagnosticsMaxEvents = 20 - helmDiagnosticsMessageLen = 240 - helmDiagnosticsEventLen = 360 + helmOperationTimeout = 5 * time.Minute + helmCompatibilityTimeout = 2 * time.Minute + helmChartLoadTimeout = 30 * time.Minute + helmDiagnosticsTimeout = 15 * time.Second + helmDiagnosticsMaxEvents = 20 + helmDiagnosticsMessageLen = 240 + helmDiagnosticsEventLen = 360 ) // ---------- Types ---------- @@ -283,20 +280,11 @@ func httpClientWithContext(ctx context.Context, base *http.Client) *http.Client return &client } -func helmGet(ctx context.Context, url string) (*http.Response, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - return nil, err - } - req.Header.Set("User-Agent", "Helm/3.21.2") - return proxypkg.ProxyHttpClient.Do(req) -} - // ---------- Repo index ---------- func fetchIndexFile(ctx context.Context, repoURL string) (*repo.IndexFile, error) { indexURL := strings.TrimRight(repoURL, "/") + "/index.yaml" - resp, err := helmGet(ctx, indexURL) + data, err := downloadHelmArtifact(ctx, indexURL) if err != nil { return nil, fmt.Errorf( "fetch index %q: %w", @@ -304,14 +292,6 @@ func fetchIndexFile(ctx context.Context, repoURL string) (*repo.IndexFile, error sanitizeErrorMessage(err, indexURL, repoURL), ) } - defer resp.Body.Close() - if resp.StatusCode != 200 { - return nil, fmt.Errorf("index returned HTTP %d", resp.StatusCode) - } - data, err := io.ReadAll(resp.Body) - if err != nil { - return nil, err - } // Use sigs.k8s.io/yaml (YAML→JSON→struct) so that embedded pointer fields // like *chart.Metadata inside ChartVersion are properly allocated. Plain // gopkg.in/yaml.v3 leaves those pointers nil, causing panics in SortEntries. @@ -426,7 +406,11 @@ func isOCIChartTag(tag string) bool { } func newOCIRegistryClient(ctx context.Context) (*registry.Client, error) { - return registry.NewClient(registry.ClientOptHTTPClient(httpClientWithContext(ctx, proxypkg.ProxyHttpClient))) + client := httpClientWithContext(ctx, proxypkg.ProxyHttpClient) + if client.Timeout <= 0 { + client.Timeout = helmDownloadAttemptTimeout + } + return registry.NewClient(registry.ClientOptHTTPClient(client)) } // pullOCIChart pulls the chart hosted at repoURL, resolving to the newest published @@ -456,7 +440,12 @@ func pullOCIChartFromRegistry(ctx context.Context, repoURL, version string) (*re if resolvedVersion == "" { if !strings.Contains(ref, "@") { - tags, err := rc.Tags(ref) + var tags []string + err = retryOCIRegistryOperation(ctx, func() error { + var tagsErr error + tags, tagsErr = rc.Tags(ref) + return tagsErr + }) if err != nil { return nil, fmt.Errorf( "list oci tags for %q: %w", @@ -481,8 +470,18 @@ func pullOCIChartFromRegistry(ctx context.Context, repoURL, version string) (*re } pullRef = fmt.Sprintf("%s:%s", ref, resolvedVersion) } + if cached, ok, err := cachedOCIChartPull(pullRef); err != nil { + return nil, err + } else if ok { + return cached, nil + } - pull, err := rc.Pull(pullRef, registry.PullOptWithChart(true)) + var pull *registry.PullResult + err = retryOCIRegistryOperation(ctx, func() error { + var pullErr error + pull, pullErr = rc.Pull(pullRef, registry.PullOptWithChart(true)) + return pullErr + }) if err != nil { return nil, fmt.Errorf( "pull oci chart %q: %w", @@ -490,9 +489,35 @@ func pullOCIChartFromRegistry(ctx context.Context, repoURL, version string) (*re sanitizeErrorMessage(err, repoURL, ref, pullRef), ) } + cacheOCIChartPull(pullRef, pull) return pull, nil } +func cachedOCIChartPull(pullRef string) (*registry.PullResult, bool, error) { + data, ok := defaultHelmArtifactCache.get("oci-chart:" + pullRef) + if !ok { + return nil, false, nil + } + loaded, err := loader.LoadArchive(bytes.NewReader(data)) + if err != nil { + return nil, true, fmt.Errorf("load cached oci chart %q: %w", redactURLForError(pullRef), err) + } + return ®istry.PullResult{ + Chart: ®istry.DescriptorPullSummaryWithMeta{ + DescriptorPullSummary: registry.DescriptorPullSummary{Data: data, Size: int64(len(data))}, + Meta: loaded.Metadata, + }, + Ref: pullRef, + }, true, nil +} + +func cacheOCIChartPull(pullRef string, pull *registry.PullResult) { + if pull == nil || pull.Chart == nil || len(pull.Chart.Data) == 0 { + return + } + defaultHelmArtifactCache.put("oci-chart:"+pullRef, pull.Chart.Data) +} + func latestOCISemverTag(tags []string) string { type versionedTag struct { tag string @@ -743,7 +768,6 @@ func loadChartWithContext(parent context.Context, chartName, repoURL, version st } ctx, cancel := context.WithTimeout(parent, helmChartLoadTimeout) defer cancel() - if isOCIRepo(repoURL) { ch, err := loadOCIChart(ctx, repoURL, version) if err != nil { @@ -805,7 +829,7 @@ func loadChartWithContext(parent context.Context, chartName, repoURL, version st chartURL = strings.TrimRight(repoURL, "/") + "/" + strings.TrimLeft(chartURL, "/") } - resp, err := helmGet(ctx, chartURL) + data, err := downloadHelmArtifact(ctx, chartURL) if err != nil { return nil, fmt.Errorf( "load chart %q from repo %q version %q: download chart archive %q failed: %w", @@ -816,29 +840,6 @@ func loadChartWithContext(parent context.Context, chartName, repoURL, version st sanitizeErrorMessage(err, chartURL, repoURL), ) } - defer resp.Body.Close() - if resp.StatusCode != 200 { - return nil, fmt.Errorf( - "load chart %q from repo %q version %q: chart archive %q returned HTTP %d", - chartName, - redactURLForError(repoURL), - entry.Version, - redactURLForError(chartURL), - resp.StatusCode, - ) - } - - data, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf( - "load chart %q from repo %q version %q: read chart archive %q failed: %w", - chartName, - redactURLForError(repoURL), - entry.Version, - redactURLForError(chartURL), - err, - ) - } ch, err := loader.LoadArchive(bytes.NewReader(data)) if err != nil { return nil, fmt.Errorf( @@ -1267,6 +1268,10 @@ func GetHelmReleases(cfg *rest.Config, namespace string) ([]HelmReleaseSummary, } func InstallHelmChart(cfg *rest.Config, releaseName, namespace, chartName, repoURL, version, valuesYAML string) error { + installTimeout, err := configuredHelmInstallTimeout() + if err != nil { + return err + } actionConfig, err := newHelmConfig(cfg, namespace) if err != nil { return err @@ -1295,13 +1300,7 @@ func InstallHelmChart(cfg *rest.Config, releaseName, namespace, chartName, repoU return err } install := action.NewInstall(actionConfig) - install.ReleaseName = releaseName - install.Namespace = namespace - install.CreateNamespace = true - install.Wait = true - install.WaitForJobs = true - install.Timeout = helmInstallTimeout - install.PostRenderer = configuredLocalImagePullPolicyPostRenderer() + configureHelmInstall(install, releaseName, namespace, installTimeout) _, err = install.Run(ch, vals) if err != nil { @@ -1336,8 +1335,6 @@ func InstallHelmChartStream(ctx context.Context, lifecycle HelmInstallLifecycle, if streamCtx == nil { streamCtx = context.Background() } - installCtx, cancelInstall := context.WithTimeout(context.WithoutCancel(streamCtx), helmInstallOperationTimeout) - defer cancelInstall() send := func(line string) bool { if err := lifecycle.RecordLog(line); err != nil { logrus.Warnf("failed to persist Helm operation log: %v", err) @@ -1367,6 +1364,19 @@ func InstallHelmChartStream(ctx context.Context, lifecycle HelmInstallLifecycle, } return } + installTimeout, err := configuredHelmInstallTimeout() + if err != nil { + send("ERROR: " + err.Error()) + if finishErr := lifecycle.Finish(err); finishErr != nil { + logrus.Errorf("failed to finish Helm operation after configuration error: %v", finishErr) + } + return + } + installCtx, cancelInstall := context.WithTimeout( + context.WithoutCancel(streamCtx), + helmChartLoadTimeout+helmCompatibilityTimeout+installTimeout+helmDiagnosticsTimeout, + ) + defer cancelInstall() helmChart, err := loadChartWithContext(installCtx, chartName, repoURL, version) if err != nil { send("ERROR: " + err.Error()) @@ -1411,13 +1421,7 @@ func InstallHelmChartStream(ctx context.Context, lifecycle HelmInstallLifecycle, return } install := action.NewInstall(actionConfig) - install.ReleaseName = releaseName - install.Namespace = namespace - install.CreateNamespace = true - install.Wait = true - install.WaitForJobs = true - install.Timeout = helmInstallTimeout - install.PostRenderer = configuredLocalImagePullPolicyPostRenderer() + configureHelmInstall(install, releaseName, namespace, installTimeout) if _, err = install.RunWithContext(installCtx, helmChart, vals); err != nil { for _, line := range helmReleaseDiagnostics(installCtx, cfg, releaseName, namespace) { send(line) diff --git a/store/helm_download.go b/store/helm_download.go new file mode 100644 index 0000000..1c49f73 --- /dev/null +++ b/store/helm_download.go @@ -0,0 +1,269 @@ +package store + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "net/http" + "sync" + "time" + + proxypkg "github.com/casosorg/casos/proxy" + "oras.land/oras-go/v2/registry/remote/errcode" +) + +const ( + helmDownloadAttemptTimeout = 2 * time.Minute + helmDownloadCacheTTL = 15 * time.Minute + helmDownloadCacheEntries = 32 + helmDownloadCacheBytes = 128 << 20 +) + +var defaultHelmArtifactCache = newHelmArtifactCache(helmDownloadCacheTTL, helmDownloadCacheEntries, helmDownloadCacheBytes) + +func downloadHelmArtifact(ctx context.Context, rawURL string) ([]byte, error) { + downloader := newHelmArtifactDownloader( + proxypkg.ProxyHttpClient, + helmDownloadAttemptTimeout, + []time.Duration{time.Second, 3 * time.Second}, + defaultHelmArtifactCache, + ) + return downloader.download(ctx, rawURL) +} + +type helmHTTPStatusError struct { + statusCode int +} + +func (e *helmHTTPStatusError) Error() string { + return fmt.Sprintf("HTTP %d", e.statusCode) +} + +type helmArtifactDownloader struct { + client *http.Client + attemptTimeout time.Duration + retryDelays []time.Duration + cache *helmArtifactCache +} + +func newHelmArtifactDownloader(client *http.Client, attemptTimeout time.Duration, retryDelays []time.Duration, cache *helmArtifactCache) *helmArtifactDownloader { + if client == nil { + client = http.DefaultClient + } + return &helmArtifactDownloader{ + client: client, + attemptTimeout: attemptTimeout, + retryDelays: append([]time.Duration(nil), retryDelays...), + cache: cache, + } +} + +func (d *helmArtifactDownloader) download(ctx context.Context, rawURL string) ([]byte, error) { + if ctx == nil { + ctx = context.Background() + } + if d.cache != nil { + if data, ok := d.cache.get(rawURL); ok { + return data, nil + } + } + + for attempt := 0; ; attempt++ { + data, err := d.downloadOnce(ctx, rawURL) + if err == nil { + if d.cache != nil { + d.cache.put(rawURL, data) + } + return data, nil + } + if ctx.Err() != nil || attempt >= len(d.retryDelays) || !isRetryableHelmDownloadError(err) { + return nil, err + } + if err := waitForHelmDownloadRetry(ctx, d.retryDelays[attempt]); err != nil { + return nil, err + } + } +} + +func (d *helmArtifactDownloader) downloadOnce(ctx context.Context, rawURL string) ([]byte, error) { + attemptCtx := ctx + cancel := func() {} + if d.attemptTimeout > 0 { + attemptCtx, cancel = context.WithTimeout(ctx, d.attemptTimeout) + } + defer cancel() + + req, err := http.NewRequestWithContext(attemptCtx, http.MethodGet, rawURL, nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", "Helm/3.21.2") + resp, err := d.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, &helmHTTPStatusError{statusCode: resp.StatusCode} + } + return io.ReadAll(resp.Body) +} + +func isRetryableHelmDownloadError(err error) bool { + if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { + return true + } + var networkError net.Error + if errors.As(err, &networkError) && (networkError.Timeout() || networkError.Temporary()) { + return true + } + var statusError *helmHTTPStatusError + if errors.As(err, &statusError) { + return statusError.statusCode == http.StatusRequestTimeout || + statusError.statusCode == http.StatusTooManyRequests || + (statusError.statusCode >= http.StatusInternalServerError && statusError.statusCode < 600) + } + return false +} + +func waitForHelmDownloadRetry(ctx context.Context, delay time.Duration) error { + if delay <= 0 { + return nil + } + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +func retryOCIRegistryOperation(ctx context.Context, operation func() error) error { + return retryOCIRegistryOperationWithDelays(ctx, operation, []time.Duration{ + 500 * time.Millisecond, + 1500 * time.Millisecond, + }) +} + +func retryOCIRegistryOperationWithDelays(ctx context.Context, operation func() error, delays []time.Duration) error { + if ctx == nil { + ctx = context.Background() + } + for attempt := 0; ; attempt++ { + err := operation() + if err == nil { + return nil + } + if ctx.Err() != nil || attempt >= len(delays) || !isRetryableOCIRegistryError(err) { + return err + } + if err := waitForHelmDownloadRetry(ctx, delays[attempt]); err != nil { + return err + } + } +} + +func isRetryableOCIRegistryError(err error) bool { + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { + return true + } + var networkError net.Error + if errors.As(err, &networkError) && (networkError.Timeout() || networkError.Temporary()) { + return true + } + var operationError *net.OpError + if errors.As(err, &operationError) { + return true + } + var responseError *errcode.ErrorResponse + if errors.As(err, &responseError) { + return responseError.StatusCode == http.StatusRequestTimeout || + responseError.StatusCode == http.StatusTooManyRequests || + (responseError.StatusCode >= http.StatusInternalServerError && responseError.StatusCode < 600) + } + return false +} + +type helmArtifactCacheEntry struct { + data []byte + storedAt time.Time + expiresAt time.Time +} + +type helmArtifactCache struct { + mu sync.Mutex + entries map[string]helmArtifactCacheEntry + ttl time.Duration + maxEntries int + maxBytes int + usedBytes int +} + +func newHelmArtifactCache(ttl time.Duration, maxEntries, maxBytes int) *helmArtifactCache { + return &helmArtifactCache{ + entries: make(map[string]helmArtifactCacheEntry), + ttl: ttl, + maxEntries: maxEntries, + maxBytes: maxBytes, + } +} + +func (c *helmArtifactCache) get(key string) ([]byte, bool) { + if c == nil || c.ttl <= 0 || c.maxEntries <= 0 || c.maxBytes <= 0 { + return nil, false + } + c.mu.Lock() + defer c.mu.Unlock() + entry, ok := c.entries[key] + if !ok { + return nil, false + } + if time.Now().After(entry.expiresAt) { + c.deleteLocked(key, entry) + return nil, false + } + return append([]byte(nil), entry.data...), true +} + +func (c *helmArtifactCache) put(key string, data []byte) { + if c == nil || c.ttl <= 0 || c.maxEntries <= 0 || c.maxBytes <= 0 || len(data) > c.maxBytes { + return + } + now := time.Now() + c.mu.Lock() + defer c.mu.Unlock() + for existingKey, entry := range c.entries { + if now.After(entry.expiresAt) { + c.deleteLocked(existingKey, entry) + } + } + if entry, ok := c.entries[key]; ok { + c.deleteLocked(key, entry) + } + for len(c.entries) >= c.maxEntries || c.usedBytes+len(data) > c.maxBytes { + oldestKey := "" + var oldestEntry helmArtifactCacheEntry + for existingKey, entry := range c.entries { + if oldestKey == "" || entry.storedAt.Before(oldestEntry.storedAt) { + oldestKey = existingKey + oldestEntry = entry + } + } + if oldestKey == "" { + break + } + c.deleteLocked(oldestKey, oldestEntry) + } + cloned := append([]byte(nil), data...) + c.entries[key] = helmArtifactCacheEntry{data: cloned, storedAt: now, expiresAt: now.Add(c.ttl)} + c.usedBytes += len(cloned) +} + +func (c *helmArtifactCache) deleteLocked(key string, entry helmArtifactCacheEntry) { + delete(c.entries, key) + c.usedBytes -= len(entry.data) +} diff --git a/store/helm_install.go b/store/helm_install.go new file mode 100644 index 0000000..99b9c02 --- /dev/null +++ b/store/helm_install.go @@ -0,0 +1,43 @@ +package store + +import ( + "fmt" + "strings" + "time" + + "helm.sh/helm/v3/pkg/action" + + "github.com/casosorg/casos/conf" +) + +const defaultHelmInstallTimeout = 20 * time.Minute + +func configuredHelmInstallTimeout() (time.Duration, error) { + return parseHelmInstallTimeout(conf.GetConfigString("helmInstallTimeout")) +} + +func parseHelmInstallTimeout(raw string) (time.Duration, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return defaultHelmInstallTimeout, nil + } + timeout, err := time.ParseDuration(raw) + if err != nil { + return 0, fmt.Errorf("invalid helmInstallTimeout %q: %w", raw, err) + } + if timeout <= 0 { + return 0, fmt.Errorf("helmInstallTimeout must be greater than zero") + } + return timeout, nil +} + +func configureHelmInstall(install *action.Install, releaseName, namespace string, timeout time.Duration) { + install.ReleaseName = releaseName + install.Namespace = namespace + install.CreateNamespace = true + install.Wait = true + install.WaitForJobs = true + install.Atomic = true + install.Timeout = timeout + install.PostRenderer = configuredLocalImagePullPolicyPostRenderer() +}