diff --git a/store/helm.go b/store/helm.go index 88c5d1d..38d8552 100644 --- a/store/helm.go +++ b/store/helm.go @@ -46,6 +46,7 @@ import ( const ( helmOperationTimeout = 5 * time.Minute helmInstallTimeout = 10 * time.Minute + helmCompatibilityTimeout = 2 * time.Minute helmDiagnosticsTimeout = 15 * time.Second helmDiagnosticsMaxEvents = 20 helmDiagnosticsMessageLen = 240 @@ -317,6 +318,9 @@ func FetchRepoIndex(repoURL string) ([]HelmChartSummary, error) { continue } v := versions[0] + if !isInstallableHelmChartMetadata(v.Metadata) { + continue + } charts = append(charts, HelmChartSummary{ Name: name, Version: v.Version, @@ -478,6 +482,9 @@ func fetchOCIChartSummary(repoURL string) ([]HelmChartSummary, error) { if meta == nil { return nil, fmt.Errorf("chart metadata not found for %s", redactURLForError(repoURL)) } + if !isInstallableHelmChartMetadata(meta) { + return []HelmChartSummary{}, nil + } return []HelmChartSummary{ { Name: meta.Name, @@ -490,6 +497,10 @@ func fetchOCIChartSummary(repoURL string) ([]HelmChartSummary, error) { }, nil } +func isInstallableHelmChartMetadata(metadata *chart.Metadata) bool { + return metadata != nil && !strings.EqualFold(strings.TrimSpace(metadata.Type), "library") +} + // ---------- Chart loader ---------- func redactURLForError(raw string) string { @@ -1208,17 +1219,24 @@ func InstallHelmChart(cfg *rest.Config, releaseName, namespace, chartName, repoU } attachHelmCapabilities(actionConfig, cfg, namespace, helmWarningLog) + if err := validateHelmChartCompatibility(context.Background(), cfg, actionConfig, releaseName, namespace, ch, vals); err != nil { + return err + } install := action.NewInstall(actionConfig) install.ReleaseName = releaseName install.Namespace = namespace install.CreateNamespace = true install.Wait = true + install.WaitForJobs = true install.Timeout = helmInstallTimeout _, err = install.Run(ch, vals) if err != nil { return withHelmReleaseDiagnostics(context.Background(), cfg, releaseName, namespace, err) } + reportHelmReadiness(inspectHelmReleaseResources(context.Background(), cfg, releaseName, namespace), func(message string) { + logrus.Warn(message) + }) return nil } @@ -1292,6 +1310,11 @@ func InstallHelmChartStream(ctx context.Context, lifecycle HelmInstallLifecycle, return } attachHelmCapabilities(actionConfig, cfg, namespace, logFn) + if err := validateHelmChartCompatibility(installCtx, cfg, actionConfig, releaseName, namespace, helmChart, vals); err != nil { + send("ERROR: " + err.Error()) + _ = lifecycle.Finish(err) + return + } if err := lifecycle.MarkInstalling(); err != nil { send("ERROR: " + err.Error()) if finishErr := lifecycle.Finish(err); finishErr != nil { @@ -1304,6 +1327,7 @@ func InstallHelmChartStream(ctx context.Context, lifecycle HelmInstallLifecycle, install.Namespace = namespace install.CreateNamespace = true install.Wait = true + install.WaitForJobs = true install.Timeout = helmInstallTimeout if _, err = install.RunWithContext(installCtx, helmChart, vals); err != nil { for _, line := range helmReleaseDiagnostics(installCtx, cfg, releaseName, namespace) { @@ -1315,6 +1339,9 @@ func InstallHelmChartStream(ctx context.Context, lifecycle HelmInstallLifecycle, } return } + reportHelmReadiness(inspectHelmReleaseResources(installCtx, cfg, releaseName, namespace), func(message string) { + send("WARNING: " + message) + }) if err := lifecycle.Finish(nil); err != nil { logrus.Warnf("failed to finish Helm operation: %v", err) select { @@ -1346,13 +1373,23 @@ func UpgradeHelmRelease(cfg *rest.Config, releaseName, namespace, chartName, rep } attachHelmCapabilities(actionConfig, cfg, namespace, helmWarningLog) + if err := validateHelmReleaseCompatibility(context.Background(), actionConfig, releaseName, namespace, ch, vals); err != nil { + return err + } upgrade := action.NewUpgrade(actionConfig) upgrade.Namespace = namespace upgrade.Wait = true + upgrade.WaitForJobs = true upgrade.Timeout = helmOperationTimeout _, err = upgrade.Run(releaseName, ch, vals) - return err + if err != nil { + return withHelmReleaseDiagnostics(context.Background(), cfg, releaseName, namespace, err) + } + reportHelmReadiness(inspectHelmReleaseResources(context.Background(), cfg, releaseName, namespace), func(message string) { + logrus.Warn(message) + }) + return nil } func RollbackHelmRelease(cfg *rest.Config, releaseName, namespace string, revision int) error { @@ -1360,11 +1397,21 @@ func RollbackHelmRelease(cfg *rest.Config, releaseName, namespace string, revisi if err != nil { return err } + if err := validateHelmRollbackCompatibility(context.Background(), actionConfig, releaseName, namespace, revision); err != nil { + return err + } rollback := action.NewRollback(actionConfig) rollback.Version = revision rollback.Wait = true + rollback.WaitForJobs = true rollback.Timeout = helmOperationTimeout - return rollback.Run(releaseName) + if err := rollback.Run(releaseName); err != nil { + return withHelmReleaseDiagnostics(context.Background(), cfg, releaseName, namespace, err) + } + reportHelmReadiness(inspectHelmReleaseResources(context.Background(), cfg, releaseName, namespace), func(message string) { + logrus.Warn(message) + }) + return nil } func UninstallHelmRelease(cfg *rest.Config, releaseName, namespace string) error { diff --git a/store/helm_compatibility.go b/store/helm_compatibility.go new file mode 100644 index 0000000..80f42fc --- /dev/null +++ b/store/helm_compatibility.go @@ -0,0 +1,118 @@ +package store + +import ( + "context" + "fmt" + + "helm.sh/helm/v3/pkg/action" + "helm.sh/helm/v3/pkg/chart" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" +) + +func validateHelmChartCompatibility(ctx context.Context, cfg *rest.Config, actionConfig *action.Configuration, releaseName, namespace string, chartToInstall *chart.Chart, values map[string]interface{}) error { + if chartToInstall == nil || chartToInstall.Metadata == nil { + return fmt.Errorf("chart metadata is missing") + } + if !isInstallableHelmChartMetadata(chartToInstall.Metadata) { + return fmt.Errorf("chart %s is a library chart and cannot be installed as an application", chartToInstall.Name()) + } + if ctx == nil { + ctx = context.Background() + } + ctx, cancel := context.WithTimeout(ctx, helmCompatibilityTimeout) + defer cancel() + namespaceExists, err := helmNamespaceExists(ctx, cfg, namespace) + if err != nil { + return err + } + // Helm's dry-run validates rendering and resource structure against the + // target cluster. Remote template lookups are only safe after the release + // namespace exists; the real install remains the source of truth. + dryRun := newHelmCompatibilityDryRun(actionConfig, releaseName, namespace, namespaceExists) + _, err = dryRun.RunWithContext(ctx, chartToInstall, values) + if err != nil { + return fmt.Errorf("render chart %s for compatibility check: %w", chartToInstall.Name(), err) + } + return nil +} + +func validateHelmReleaseCompatibility(ctx context.Context, actionConfig *action.Configuration, releaseName, namespace string, chartToInstall *chart.Chart, values map[string]interface{}) error { + if chartToInstall == nil || chartToInstall.Metadata == nil { + return fmt.Errorf("chart metadata is missing") + } + if !isInstallableHelmChartMetadata(chartToInstall.Metadata) { + return fmt.Errorf("chart %s is a library chart and cannot be installed as an application", chartToInstall.Name()) + } + if ctx == nil { + ctx = context.Background() + } + ctx, cancel := context.WithTimeout(ctx, helmCompatibilityTimeout) + defer cancel() + + dryRun := action.NewUpgrade(actionConfig) + dryRun.Namespace = namespace + dryRun.DryRun = true + dryRun.DryRunOption = "server" + dryRun.Wait = true + dryRun.WaitForJobs = true + dryRun.Timeout = helmCompatibilityTimeout + if _, err := dryRun.RunWithContext(ctx, releaseName, chartToInstall, values); err != nil { + return fmt.Errorf("render Helm release %s for compatibility check: %w", releaseName, err) + } + return nil +} + +func validateHelmRollbackCompatibility(ctx context.Context, actionConfig *action.Configuration, releaseName, namespace string, revision int) error { + currentRelease, err := actionConfig.Releases.Last(releaseName) + if err != nil { + return fmt.Errorf("read current Helm release %s: %w", releaseName, err) + } + targetRevision := revision + if targetRevision == 0 { + targetRevision = currentRelease.Version - 1 + } + if targetRevision <= 0 { + return fmt.Errorf("release %s has no prior revision to roll back to", releaseName) + } + target, err := actionConfig.Releases.Get(releaseName, targetRevision) + if err != nil { + return fmt.Errorf("read Helm release %s revision %d: %w", releaseName, targetRevision, err) + } + if err := validateHelmReleaseCompatibility(ctx, actionConfig, releaseName, namespace, target.Chart, target.Config); err != nil { + return err + } + return nil +} + +func newHelmCompatibilityDryRun(actionConfig *action.Configuration, releaseName, namespace string, remoteLookup bool) *action.Install { + dryRun := action.NewInstall(actionConfig) + dryRun.ReleaseName = releaseName + dryRun.Namespace = namespace + dryRun.CreateNamespace = true + dryRun.DryRun = true + dryRun.DryRunOption = "client" + if remoteLookup { + dryRun.DryRunOption = "server" + } + dryRun.Timeout = helmCompatibilityTimeout + return dryRun +} + +func helmNamespaceExists(ctx context.Context, cfg *rest.Config, namespace string) (bool, error) { + if cfg == nil { + return false, fmt.Errorf("Helm compatibility REST config is nil") + } + client, err := kubernetes.NewForConfig(cfg) + if err != nil { + return false, fmt.Errorf("create Helm compatibility client: %w", err) + } + if _, err := client.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{}); apierrors.IsNotFound(err) { + return false, nil + } else if err != nil { + return false, fmt.Errorf("check Helm release namespace %s: %w", namespace, err) + } + return true, nil +} diff --git a/store/helm_readiness.go b/store/helm_readiness.go new file mode 100644 index 0000000..8cc5078 --- /dev/null +++ b/store/helm_readiness.go @@ -0,0 +1,283 @@ +package store + +import ( + "context" + "errors" + "fmt" + "io" + "sort" + "strings" + + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" + utilyaml "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" +) + +const helmManifestDecoderBuffer = 4096 + +// inspectHelmReleaseResources reports access-path health after Helm has already +// decided that an operation succeeded. Its result must never change Helm state. +func inspectHelmReleaseResources(parent context.Context, cfg *rest.Config, releaseName, namespace string) error { + if cfg == nil { + return errors.New("Helm readiness REST config is nil") + } + if parent == nil { + parent = context.Background() + } + client, err := kubernetes.NewForConfig(cfg) + if err != nil { + return fmt.Errorf("create Helm readiness client: %w", err) + } + refs, err := helmReleaseResourceRefs(cfg, releaseName, namespace) + if err != nil { + return err + } + if len(refs) == 0 { + return nil + } + + ctx, cancel := context.WithTimeout(parent, helmDiagnosticsTimeout) + defer cancel() + return validateHelmReleaseResourcesWithRefs(ctx, client, releaseName, namespace, refs) +} + +func reportHelmReadiness(readinessErr error, warn func(string)) { + if readinessErr == nil || warn == nil { + return + } + warn("Helm operation succeeded, but some access-path resources are not ready yet: " + readinessErr.Error()) +} + +type helmResourceRef struct { + kind string + name string + namespace string +} + +var helmReadinessResourceKinds = map[string]struct{}{ + "Ingress": {}, + "Service": {}, +} + +func helmReleaseResourceRefs(cfg *rest.Config, releaseName, namespace string) ([]helmResourceRef, error) { + actionConfig, err := newHelmConfig(cfg, namespace) + if err != nil { + return nil, fmt.Errorf("create Helm release store: %w", err) + } + release, err := actionConfig.Releases.Last(releaseName) + if err != nil { + return nil, fmt.Errorf("read Helm release manifest: %w", err) + } + decoder := utilyaml.NewYAMLOrJSONDecoder(strings.NewReader(release.Manifest), helmManifestDecoderBuffer) + refs := make([]helmResourceRef, 0) + seen := make(map[string]struct{}) + for { + var raw map[string]interface{} + if err := decoder.Decode(&raw); err != nil { + if errors.Is(err, io.EOF) { + break + } + return nil, fmt.Errorf("decode Helm release manifest: %w", err) + } + if len(raw) == 0 { + continue + } + object := unstructured.Unstructured{Object: raw} + if _, ok := helmReadinessResourceKinds[object.GetKind()]; !ok || object.GetName() == "" { + continue + } + if object.GetAnnotations()["helm.sh/hook"] != "" { + continue + } + refNamespace := object.GetNamespace() + if refNamespace == "" { + refNamespace = namespace + } + ref := helmResourceRef{kind: object.GetKind(), name: object.GetName(), namespace: refNamespace} + key := strings.Join([]string{ref.kind, ref.namespace, ref.name}, "/") + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + refs = append(refs, ref) + } + return refs, nil +} + +func validateHelmReleaseResourcesWithRefs(ctx context.Context, client kubernetes.Interface, releaseName, namespace string, refs []helmResourceRef) error { + if client == nil { + return errors.New("Helm readiness Kubernetes client is nil") + } + problems := make([]string, 0) + appendProblem := func(problem string) { + problems = append(problems, problem) + } + + for _, ref := range refsForKind(refs, "Service") { + service, err := client.CoreV1().Services(ref.namespace).Get(ctx, ref.name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + appendProblem(fmt.Sprintf("Service %s/%s is missing", ref.namespace, ref.name)) + continue + } + if err != nil { + return fmt.Errorf("get Helm release Service %s/%s: %w", ref.namespace, ref.name, err) + } + if err := checkServiceReadiness(ctx, client, service, appendProblem); err != nil { + return err + } + } + + for _, ref := range refsForKind(refs, "Ingress") { + ingress, err := client.NetworkingV1().Ingresses(ref.namespace).Get(ctx, ref.name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + appendProblem(fmt.Sprintf("Ingress %s/%s is missing", ref.namespace, ref.name)) + continue + } + if err != nil { + return fmt.Errorf("get Helm release Ingress %s/%s: %w", ref.namespace, ref.name, err) + } + if err := checkIngressReadiness(ctx, client, ingress, appendProblem); err != nil { + return err + } + } + + if len(problems) == 0 { + return nil + } + sort.Strings(problems) + return fmt.Errorf("Helm release %s/%s access path is not ready yet: %s", namespace, releaseName, strings.Join(problems, "; ")) +} + +func serviceHasReadyEndpointSlice(ctx context.Context, client kubernetes.Interface, service corev1.Service) (bool, error) { + slices, err := client.DiscoveryV1().EndpointSlices(service.Namespace).List(ctx, metav1.ListOptions{ + LabelSelector: labels.Set{"kubernetes.io/service-name": service.Name}.AsSelector().String(), + }) + if apierrors.IsNotFound(err) { + return serviceHasReadyEndpoints(ctx, client, service) + } + if err != nil { + return false, err + } + for _, endpointSlice := range slices.Items { + for _, endpoint := range endpointSlice.Endpoints { + // Kubernetes defines nil Ready as true for backward compatibility. + ready := endpoint.Conditions.Ready == nil || *endpoint.Conditions.Ready + terminating := endpoint.Conditions.Terminating != nil && *endpoint.Conditions.Terminating + if ready && !terminating { + return true, nil + } + } + } + return false, nil +} + +func serviceHasReadyEndpoints(ctx context.Context, client kubernetes.Interface, service corev1.Service) (bool, error) { + endpoints, err := client.CoreV1().Endpoints(service.Namespace).Get(ctx, service.Name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return false, nil + } + if err != nil { + return false, err + } + for _, subset := range endpoints.Subsets { + if len(subset.Addresses) > 0 { + return true, nil + } + } + return false, nil +} + +func checkServiceReadiness(ctx context.Context, client kubernetes.Interface, service *corev1.Service, appendProblem func(string)) error { + if !serviceRequiresReadyEndpoint(*service) { + return nil + } + ready, err := serviceHasReadyEndpointSlice(ctx, client, *service) + if err != nil { + return fmt.Errorf("check Service %s/%s endpoints: %w", service.Namespace, service.Name, err) + } + if !ready { + appendProblem(fmt.Sprintf("Service %s/%s has no ready EndpointSlice", service.Namespace, service.Name)) + } + return nil +} + +func serviceRequiresReadyEndpoint(service corev1.Service) bool { + return service.Spec.Type != corev1.ServiceTypeExternalName +} + +func checkIngressReadiness(ctx context.Context, client kubernetes.Interface, ingress *networkingv1.Ingress, appendProblem func(string)) error { + if ingress.Spec.IngressClassName != nil && strings.TrimSpace(*ingress.Spec.IngressClassName) != "" { + if _, err := client.NetworkingV1().IngressClasses().Get(ctx, *ingress.Spec.IngressClassName, metav1.GetOptions{}); apierrors.IsNotFound(err) { + appendProblem(fmt.Sprintf("Ingress %s/%s references missing IngressClass %s", ingress.Namespace, ingress.Name, *ingress.Spec.IngressClassName)) + } else if err != nil { + return fmt.Errorf("check IngressClass for %s/%s: %w", ingress.Namespace, ingress.Name, err) + } + } + + services := make(map[string]struct{}) + checkBackend := func(serviceName string) error { + if serviceName == "" { + return nil + } + if _, ok := services[serviceName]; ok { + return nil + } + services[serviceName] = struct{}{} + service, err := client.CoreV1().Services(ingress.Namespace).Get(ctx, serviceName, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + appendProblem(fmt.Sprintf("Ingress %s/%s backend Service %s/%s is missing", ingress.Namespace, ingress.Name, ingress.Namespace, serviceName)) + return nil + } + if err != nil { + return fmt.Errorf("get Ingress backend Service %s/%s: %w", ingress.Namespace, serviceName, err) + } + if !serviceRequiresReadyEndpoint(*service) { + return nil + } + ready, err := serviceHasReadyEndpointSlice(ctx, client, *service) + if err != nil { + return fmt.Errorf("check Ingress backend Service %s/%s endpoints: %w", ingress.Namespace, serviceName, err) + } + if !ready { + appendProblem(fmt.Sprintf("Ingress %s/%s backend Service %s/%s has no ready EndpointSlice", ingress.Namespace, ingress.Name, ingress.Namespace, serviceName)) + } + return nil + } + if ingress.Spec.DefaultBackend != nil && ingress.Spec.DefaultBackend.Service != nil { + if err := checkBackend(ingress.Spec.DefaultBackend.Service.Name); err != nil { + return err + } + } + for _, rule := range ingress.Spec.Rules { + if rule.HTTP == nil { + continue + } + for _, path := range rule.HTTP.Paths { + if path.Backend.Service == nil { + // Resource backends are controller-specific and have no generic + // Kubernetes readiness signal for CasOS to inspect. + continue + } + if err := checkBackend(path.Backend.Service.Name); err != nil { + return err + } + } + } + return nil +} + +func refsForKind(refs []helmResourceRef, kind string) []helmResourceRef { + result := make([]helmResourceRef, 0) + for _, ref := range refs { + if ref.kind == kind { + result = append(result, ref) + } + } + return result +}