diff --git a/pkg/asset/cluster/tfvars/tfvars.go b/pkg/asset/cluster/tfvars/tfvars.go index 12e16a4a92a..2d8bc87cb12 100644 --- a/pkg/asset/cluster/tfvars/tfvars.go +++ b/pkg/asset/cluster/tfvars/tfvars.go @@ -27,7 +27,6 @@ import ( gcpbootstrap "github.com/openshift/installer/pkg/asset/ignition/bootstrap/gcp" "github.com/openshift/installer/pkg/asset/ignition/machine" "github.com/openshift/installer/pkg/asset/installconfig" - awsconfig "github.com/openshift/installer/pkg/asset/installconfig/aws" aztypes "github.com/openshift/installer/pkg/asset/installconfig/azure" gcpconfig "github.com/openshift/installer/pkg/asset/installconfig/gcp" ibmcloudconfig "github.com/openshift/installer/pkg/asset/installconfig/ibmcloud" @@ -38,7 +37,6 @@ import ( "github.com/openshift/installer/pkg/asset/releaseimage" "github.com/openshift/installer/pkg/asset/rhcos" "github.com/openshift/installer/pkg/tfvars" - awstfvars "github.com/openshift/installer/pkg/tfvars/aws" azuretfvars "github.com/openshift/installer/pkg/tfvars/azure" baremetaltfvars "github.com/openshift/installer/pkg/tfvars/baremetal" gcptfvars "github.com/openshift/installer/pkg/tfvars/gcp" @@ -222,135 +220,7 @@ func (t *TerraformVariables) Generate(ctx context.Context, parents asset.Parents switch platform { case aws.Name: - var vpc string - var privateSubnets []string - var publicSubnets []string - - if len(installConfig.Config.Platform.AWS.VPC.Subnets) > 0 { - subnets, err := installConfig.AWS.PrivateSubnets(ctx) - if err != nil { - return err - } - - for id := range subnets { - privateSubnets = append(privateSubnets, id) - } - - subnets, err = installConfig.AWS.PublicSubnets(ctx) - if err != nil { - return err - } - - for id := range subnets { - publicSubnets = append(publicSubnets, id) - } - - vpc, err = installConfig.AWS.VPCID(ctx) - if err != nil { - return err - } - } - - object := "bootstrap.ign" - bucket := fmt.Sprintf("%s-bootstrap", clusterID.InfraID) - - platformAWS := installConfig.Config.Platform.AWS - client, err := awsconfig.NewS3Client(ctx, awsconfig.EndpointOptions{ - Region: platformAWS.Region, - Endpoints: platformAWS.ServiceEndpoints, - }) - if err != nil { - return fmt.Errorf("failed to create s3 client: %w", err) - } - - url, err := awsconfig.PresignedS3URL(ctx, client, bucket, object) - if err != nil { - return err - } - masters, err := mastersAsset.Machines() - if err != nil { - return err - } - masterConfigs := make([]*machinev1beta1.AWSMachineProviderConfig, len(masters)) - for i, m := range masters { - masterConfigs[i] = m.Spec.ProviderSpec.Value.Object.(*machinev1beta1.AWSMachineProviderConfig) //nolint:errcheck // legacy, pre-linter - } - workers, err := workersAsset.MachineSets() - if err != nil { - return err - } - - workerConfigs := make([]*machinev1beta1.AWSMachineProviderConfig, len(workers)) - for i, m := range workers { - workerConfigs[i] = m.Spec.Template.Spec.ProviderSpec.Value.Object.(*machinev1beta1.AWSMachineProviderConfig) //nolint:errcheck // legacy, pre-linter - } - osImage := strings.SplitN(rhcosImage.ControlPlane, ",", 2) - osImageID := osImage[0] - osImageRegion := installConfig.Config.AWS.Region - if len(osImage) == 2 { - osImageRegion = osImage[1] - } - - workerIAMRoleName := "" - if mp := installConfig.Config.WorkerMachinePool(); mp != nil { - awsMP := &aws.MachinePool{} - awsMP.Set(installConfig.Config.AWS.DefaultMachinePlatform) - awsMP.Set(mp.Platform.AWS) - workerIAMRoleName = awsMP.IAMRole - } - - var securityGroups []string - if mp := installConfig.Config.AWS.DefaultMachinePlatform; mp != nil { - securityGroups = mp.AdditionalSecurityGroupIDs - } - masterIAMRoleName := "" - if mp := installConfig.Config.ControlPlane; mp != nil { - awsMP := &aws.MachinePool{} - awsMP.Set(installConfig.Config.AWS.DefaultMachinePlatform) - awsMP.Set(mp.Platform.AWS) - masterIAMRoleName = awsMP.IAMRole - if len(awsMP.AdditionalSecurityGroupIDs) > 0 { - securityGroups = awsMP.AdditionalSecurityGroupIDs - } - } - - // AWS Zones is used to determine which route table the edge zone will be associated. - allZones, err := installConfig.AWS.AllZones(ctx) - if err != nil { - return err - } - - data, err := awstfvars.TFVars(awstfvars.TFVarsSources{ - VPC: vpc, - PrivateSubnets: privateSubnets, - PublicSubnets: publicSubnets, - AvailabilityZones: allZones, - InternalZone: installConfig.Config.AWS.HostedZone, - InternalZoneRole: installConfig.Config.AWS.HostedZoneRole, - Services: installConfig.Config.AWS.ServiceEndpoints, - Publish: installConfig.Config.Publish, - MasterConfigs: masterConfigs, - WorkerConfigs: workerConfigs, - AMIID: osImageID, - AMIRegion: osImageRegion, - IgnitionBucket: bucket, - IgnitionPresignedURL: url, - AdditionalTrustBundle: installConfig.Config.AdditionalTrustBundle, - MasterIAMRoleName: masterIAMRoleName, - WorkerIAMRoleName: workerIAMRoleName, - Architecture: installConfig.Config.ControlPlane.Architecture, - Proxy: installConfig.Config.Proxy, - PreserveBootstrapIgnition: installConfig.Config.AWS.BestEffortDeleteIgnition, - MasterSecurityGroups: securityGroups, - PublicIpv4Pool: installConfig.Config.AWS.PublicIpv4Pool, - }) - if err != nil { - return errors.Wrapf(err, "failed to get %s Terraform variables", platform) - } - t.FileList = append(t.FileList, &asset.File{ - Filename: TfPlatformVarsFileName, - Data: data, - }) + // AWS uses CAPI for infrastructure provisioning; no Terraform variables needed. case azure.Name: session, err := installConfig.Azure.Session() if err != nil { diff --git a/pkg/asset/installconfig/installconfig.go b/pkg/asset/installconfig/installconfig.go index 4adc60179b1..1f1bdee56e8 100644 --- a/pkg/asset/installconfig/installconfig.go +++ b/pkg/asset/installconfig/installconfig.go @@ -206,7 +206,7 @@ func (a *InstallConfig) finishAWS() error { if totalEdgeSubnets == 0 { return nil } - if edgePool := defaults.CreateEdgeMachinePoolDefaults(a.Config.Compute, &a.Config.Platform, totalEdgeSubnets); edgePool != nil { + if edgePool := defaults.CreateEdgeMachinePoolDefaults(a.Config.Compute, &a.Config.Platform, totalEdgeSubnets, a.Config.EnabledFeatureGates()); edgePool != nil { a.Config.Compute = append(a.Config.Compute, *edgePool) } } diff --git a/pkg/asset/machines/aws/awsmachines.go b/pkg/asset/machines/aws/awsmachines.go index 673a935a828..bb0086b415d 100644 --- a/pkg/asset/machines/aws/awsmachines.go +++ b/pkg/asset/machines/aws/awsmachines.go @@ -39,6 +39,97 @@ type MachineInput struct { Config *types.InstallConfig } +// CAPIMachineSpecInput defines inputs for building an AWSMachineSpec. +type CAPIMachineSpecInput struct { + InstanceType string + AMI string + IAMInstanceProfile string + Subnet *capa.AWSResourceReference + PublicIP bool + Tags capa.Tags + EC2RootVolume awstypes.EC2RootVolume + KMSKeyARN string + IMDS capa.HTTPTokensState + SecurityGroups []capa.AWSResourceReference + AdditionalSecurityGroupIDs []string + CPUOptions *awstypes.CPUOptions + Ignition *capa.Ignition + DedicatedHostID string + IPFamily network.IPFamily +} + +// GenerateCAPIMachineSpec constructs a capa.AWSMachineSpec from the provided inputs. +func GenerateCAPIMachineSpec(in *CAPIMachineSpecInput) capa.AWSMachineSpec { + spec := capa.AWSMachineSpec{ + Ignition: in.Ignition, + InstanceType: in.InstanceType, + AMI: capa.AMIReference{ID: ptr.To(in.AMI)}, + SSHKeyName: ptr.To(""), + IAMInstanceProfile: in.IAMInstanceProfile, + Subnet: in.Subnet, + PublicIP: ptr.To(in.PublicIP), + AdditionalTags: in.Tags, + RootVolume: &capa.Volume{ + Size: int64(in.EC2RootVolume.Size), + Type: capa.VolumeType(in.EC2RootVolume.Type), + IOPS: int64(in.EC2RootVolume.IOPS), + Encrypted: ptr.To(true), + EncryptionKey: in.KMSKeyARN, + }, + InstanceMetadataOptions: &capa.InstanceMetadataOptions{ + HTTPTokens: in.IMDS, + HTTPEndpoint: capa.InstanceMetadataEndpointStateEnabled, + }, + } + + if throughput := in.EC2RootVolume.Throughput; throughput != nil { + spec.RootVolume.Throughput = ptr.To(int64(*throughput)) + } + + spec.AdditionalSecurityGroups = append(spec.AdditionalSecurityGroups, in.SecurityGroups...) + for _, sg := range in.AdditionalSecurityGroupIDs { + spec.AdditionalSecurityGroups = append( + spec.AdditionalSecurityGroups, + capa.AWSResourceReference{ID: ptr.To(sg)}, + ) + } + + if in.CPUOptions != nil { + cpuOptions := capa.CPUOptions{} + if in.CPUOptions.ConfidentialCompute != nil { + cpuOptions.ConfidentialCompute = capa.AWSConfidentialComputePolicy(*in.CPUOptions.ConfidentialCompute) + } + spec.CPUOptions = cpuOptions + } + + if in.DedicatedHostID != "" { + spec.Tenancy = "host" + spec.HostAffinity = ptr.To("host") + spec.HostID = ptr.To(in.DedicatedHostID) + } + + if in.IPFamily.DualStackEnabled() { + // Only resource-name supports A and AAAA records for private host names + // See: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/hostname-types.html#ec2-instance-private-hostnames + spec.PrivateDNSName = &capa.PrivateDNSName{ + EnableResourceNameDNSAAAARecord: ptr.To(true), + EnableResourceNameDNSARecord: ptr.To(true), + HostnameType: ptr.To("resource-name"), + } + spec.InstanceMetadataOptions.HTTPProtocolIPv6 = capa.InstanceMetadataEndpointStateEnabled + + // AssignPrimaryIPv6 is required for IPv6 primary to register instances to IPv6 target groups + switch in.IPFamily { + case network.DualStackIPv6Primary: + spec.AssignPrimaryIPv6 = ptr.To(capa.PrimaryIPv6AssignmentStateEnabled) + case network.DualStackIPv4Primary: + spec.AssignPrimaryIPv6 = ptr.To(capa.PrimaryIPv6AssignmentStateDisabled) + } + } + + return spec +} + // GenerateMachines returns manifests and runtime objects to provision the control plane (including bootstrap, if applicable) nodes using CAPI. func GenerateMachines(clusterID string, in *MachineInput) ([]*asset.RuntimeFile, error) { if poolPlatform := in.Pool.Platform.Name(); poolPlatform != awstypes.Name { @@ -97,55 +188,25 @@ func GenerateMachines(clusterID string, in *MachineInput) ([]*asset.RuntimeFile, "cluster.x-k8s.io/control-plane": "", }, }, - Spec: capa.AWSMachineSpec{ - Ignition: in.Ignition, - UncompressedUserData: ptr.To(true), - InstanceType: mpool.InstanceType, - AMI: capa.AMIReference{ID: ptr.To(mpool.AMIID)}, - SSHKeyName: ptr.To(""), - IAMInstanceProfile: instanceProfile, - Subnet: subnet, - PublicIP: ptr.To(in.PublicIP), - AdditionalTags: in.Tags, - RootVolume: &capa.Volume{ - Size: int64(mpool.EC2RootVolume.Size), - Type: capa.VolumeType(mpool.EC2RootVolume.Type), - IOPS: int64(mpool.EC2RootVolume.IOPS), - Encrypted: ptr.To(true), - EncryptionKey: mpool.KMSKeyARN, - }, - InstanceMetadataOptions: &capa.InstanceMetadataOptions{ - HTTPTokens: imds, - HTTPEndpoint: capa.InstanceMetadataEndpointStateEnabled, - }, - }, + Spec: GenerateCAPIMachineSpec(&CAPIMachineSpecInput{ + InstanceType: mpool.InstanceType, + AMI: mpool.AMIID, + IAMInstanceProfile: instanceProfile, + Subnet: subnet, + PublicIP: in.PublicIP, + Tags: in.Tags, + EC2RootVolume: mpool.EC2RootVolume, + KMSKeyARN: mpool.KMSKeyARN, + IMDS: imds, + AdditionalSecurityGroupIDs: mpool.AdditionalSecurityGroupIDs, + CPUOptions: mpool.CPUOptions, + Ignition: in.Ignition, + IPFamily: in.IPFamily, + }), } awsMachine.SetGroupVersionKind(capa.GroupVersion.WithKind("AWSMachine")) utils.SetMachineOSStreamLabels(awsMachine, in.Config) - if throughput := mpool.EC2RootVolume.Throughput; throughput != nil { - awsMachine.Spec.RootVolume.Throughput = ptr.To(int64(*throughput)) - } - - if in.IPFamily.DualStackEnabled() { - awsMachine.Spec.PrivateDNSName = &capa.PrivateDNSName{ - EnableResourceNameDNSAAAARecord: ptr.To(true), - EnableResourceNameDNSARecord: ptr.To(true), - // Only resource-name supports A and AAAA records for private host names - // See: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/hostname-types.html#ec2-instance-private-hostnames - HostnameType: ptr.To("resource-name"), - } - awsMachine.Spec.InstanceMetadataOptions.HTTPProtocolIPv6 = capa.InstanceMetadataEndpointStateEnabled - - // AssignPrimaryIPv6 is required for IPv6 primary to register instances to IPv6 target groups - switch in.IPFamily { - case network.DualStackIPv6Primary: - awsMachine.Spec.AssignPrimaryIPv6 = ptr.To(capa.PrimaryIPv6AssignmentStateEnabled) - case network.DualStackIPv4Primary: - awsMachine.Spec.AssignPrimaryIPv6 = ptr.To(capa.PrimaryIPv6AssignmentStateDisabled) - } - } - if in.Role == "bootstrap" { awsMachine.Name = capiutils.GenerateBoostrapMachineName(clusterID) awsMachine.Labels["install.openshift.io/bootstrap"] = "" @@ -159,24 +220,6 @@ func GenerateMachines(clusterID string, in *MachineInput) ([]*asset.RuntimeFile, } } - // Handle additional security groups. - for _, sg := range mpool.AdditionalSecurityGroupIDs { - awsMachine.Spec.AdditionalSecurityGroups = append( - awsMachine.Spec.AdditionalSecurityGroups, - capa.AWSResourceReference{ID: ptr.To(sg)}, - ) - } - - if mpool.CPUOptions != nil { - cpuOptions := capa.CPUOptions{} - - if mpool.CPUOptions.ConfidentialCompute != nil { - cpuOptions.ConfidentialCompute = capa.AWSConfidentialComputePolicy(*mpool.CPUOptions.ConfidentialCompute) - } - - awsMachine.Spec.CPUOptions = cpuOptions - } - result = append(result, &asset.RuntimeFile{ File: asset.File{Filename: fmt.Sprintf("10_inframachine_%s.yaml", awsMachine.Name)}, Object: awsMachine, diff --git a/pkg/asset/machines/aws/awsmachines_test.go b/pkg/asset/machines/aws/awsmachines_test.go index 79b562e7209..824d53cfe18 100644 --- a/pkg/asset/machines/aws/awsmachines_test.go +++ b/pkg/asset/machines/aws/awsmachines_test.go @@ -117,7 +117,6 @@ func TestGenerateMachines(t *testing.T) { RootVolume: &capa.Volume{ Encrypted: ptr.To(true), }, - UncompressedUserData: ptr.To(true), Ignition: &capa.Ignition{ StorageType: capa.IgnitionStorageTypeOptionUnencryptedUserData, }, @@ -173,7 +172,6 @@ func TestGenerateMachines(t *testing.T) { RootVolume: &capa.Volume{ Encrypted: ptr.To(true), }, - UncompressedUserData: ptr.To(true), Ignition: &capa.Ignition{ StorageType: capa.IgnitionStorageTypeOptionUnencryptedUserData, }, @@ -231,7 +229,6 @@ func TestGenerateMachines(t *testing.T) { RootVolume: &capa.Volume{ Encrypted: ptr.To(true), }, - UncompressedUserData: ptr.To(true), Ignition: &capa.Ignition{ StorageType: capa.IgnitionStorageTypeOptionUnencryptedUserData, }, diff --git a/pkg/asset/machines/aws/clusterapi_machinesets.go b/pkg/asset/machines/aws/clusterapi_machinesets.go new file mode 100644 index 00000000000..657c6448f40 --- /dev/null +++ b/pkg/asset/machines/aws/clusterapi_machinesets.go @@ -0,0 +1,199 @@ +// Package aws generates Machine objects for aws. +package aws + +import ( + "fmt" + "maps" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + capa "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" + capi "sigs.k8s.io/cluster-api/api/core/v1beta2" + + "github.com/openshift/installer/pkg/types/aws" + "github.com/openshift/installer/pkg/utils" +) + +// ClusterAPIMachineSets returns CAPI MachineSet and AWSMachineTemplate resources. +// This mirrors the MAPI MachineSets() function but produces CAPI-native types. +func ClusterAPIMachineSets(in *MachineSetInput) ([]capa.AWSMachineTemplate, []capi.MachineSet, error) { + if poolPlatform := in.Pool.Platform.Name(); poolPlatform != aws.Name { + return nil, nil, fmt.Errorf("non-AWS machine-pool: %q", poolPlatform) + } + mpool := in.Pool.Platform.AWS + azs := mpool.Zones + + total := int64(0) + if in.Pool.Replicas != nil { + total = *in.Pool.Replicas + } + numOfAZs := int64(len(azs)) + + var templates []capa.AWSMachineTemplate + var machineSets []capi.MachineSet + + imds := capa.HTTPTokensStateOptional + if mpool.EC2Metadata.Authentication == "Required" { + imds = capa.HTTPTokensStateRequired + } + + instanceProfile := mpool.IAMProfile + if len(instanceProfile) == 0 { + instanceProfile = fmt.Sprintf("%s-worker-profile", in.ClusterID) + } + + tags, err := CapaTagsFromUserTags(in.ClusterID, in.InstallConfigPlatformAWS.UserTags) + if err != nil { + return nil, nil, fmt.Errorf("failed to create CAPA tags from user tags: %w", err) + } + + for idx, az := range mpool.Zones { + replicas := int32(total / numOfAZs) + if int64(idx) < total%numOfAZs { + replicas++ + } + + nodeLabels := map[string]string{ + "node-role.kubernetes.io/worker": "", + } + instanceType := mpool.InstanceType + publicSubnet := in.PublicSubnet + subnetRef := &capa.AWSResourceReference{} + + if len(in.Subnets) > 0 { + subnet, ok := in.Subnets[az] + if !ok { + return nil, nil, fmt.Errorf("no subnet for zone %s", az) + } + publicSubnet = subnet.Public + subnetRef.ID = ptr.To(subnet.ID) + } else { + subnetInternetScope := "private" + if publicSubnet { + subnetInternetScope = "public" + } + subnetRef.Filters = []capa.Filter{ + { + Name: "tag:Name", + Values: []string{fmt.Sprintf("%s-subnet-%s-%s", in.ClusterID, subnetInternetScope, az)}, + }, + } + } + + // TODO: edge pools do not share same instance type and regular cluster workloads. + // The instance type is selected based in the offerings for the location. + // The labels and taints are set to prevent regular workloads. + // https://github.com/openshift/enhancements/blob/master/enhancements/installer/aws-custom-edge-machineset-local-zones.md + // FIXME: node taints on Machine/MachineSet is only supported in CAPI v1.12+ with feature gate MachineTaintPropagation. + // Until we bump the CAPI version, edge machines can only be provisioned via MAPI. + + dedicatedHost := DedicatedHost(in.Hosts, mpool.HostPlacement, az) + + name := fmt.Sprintf("%s-%s-%s", in.ClusterID, in.Pool.Name, az) + + // Build AWSMachineTemplate for this zone + template := capa.AWSMachineTemplate{ + TypeMeta: metav1.TypeMeta{ + APIVersion: capa.GroupVersion.String(), + Kind: "AWSMachineTemplate", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "openshift-cluster-api", + Labels: map[string]string{ + "cluster.x-k8s.io/cluster-name": in.ClusterID, + }, + }, + Spec: capa.AWSMachineTemplateSpec{ + Template: capa.AWSMachineTemplateResource{ + Spec: GenerateCAPIMachineSpec(&CAPIMachineSpecInput{ + InstanceType: instanceType, + AMI: mpool.AMIID, + IAMInstanceProfile: instanceProfile, + Subnet: subnetRef, + PublicIP: publicSubnet, + Tags: tags, + EC2RootVolume: mpool.EC2RootVolume, + KMSKeyARN: mpool.KMSKeyARN, + IMDS: imds, + SecurityGroups: []capa.AWSResourceReference{ + { + Filters: []capa.Filter{{ + Name: "tag:Name", + Values: []string{fmt.Sprintf("%s-node", in.ClusterID)}, + }}, + }, + { + Filters: []capa.Filter{{ + Name: "tag:Name", + Values: []string{fmt.Sprintf("%s-lb", in.ClusterID)}, + }}, + }, + }, + AdditionalSecurityGroupIDs: mpool.AdditionalSecurityGroupIDs, + CPUOptions: mpool.CPUOptions, + Ignition: &capa.Ignition{ + Version: "3.2", + // Worker machines should get ignition from the MCS on the control plane nodes + StorageType: capa.IgnitionStorageTypeOptionUnencryptedUserData, + }, + DedicatedHostID: dedicatedHost, + IPFamily: in.InstallConfigPlatformAWS.IPFamily, + }), + }, + }, + } + templates = append(templates, template) + + // Build CAPI MachineSet referencing the template + machineSet := capi.MachineSet{ + TypeMeta: metav1.TypeMeta{ + APIVersion: capi.GroupVersion.String(), + Kind: "MachineSet", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "openshift-cluster-api", + Labels: map[string]string{ + "cluster.x-k8s.io/cluster-name": in.ClusterID, + }, + }, + Spec: capi.MachineSetSpec{ + ClusterName: in.ClusterID, + Replicas: &replicas, + Selector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + "cluster.x-k8s.io/cluster-name": in.ClusterID, + "cluster.x-k8s.io/set-name": name, + }, + }, + Template: capi.MachineTemplateSpec{ + ObjectMeta: capi.ObjectMeta{ + Labels: map[string]string{ + "cluster.x-k8s.io/cluster-name": in.ClusterID, + "cluster.x-k8s.io/set-name": name, + }, + }, + Spec: capi.MachineSpec{ + ClusterName: in.ClusterID, + Bootstrap: capi.Bootstrap{ + DataSecretName: ptr.To(in.UserDataSecret), + }, + InfrastructureRef: capi.ContractVersionedObjectReference{ + APIGroup: capa.GroupVersion.Group, + Kind: "AWSMachineTemplate", + Name: name, + }, + FailureDomain: az, + }, + }, + }, + } + // Machine labels will be synced from the Machine to the corresponding Node + maps.Copy(machineSet.Spec.Template.ObjectMeta.Labels, nodeLabels) + utils.SetCAPIMachineSetOSStreamLabels(&machineSet, in.Config) + + machineSets = append(machineSets, machineSet) + } + return templates, machineSets, nil +} diff --git a/pkg/asset/machines/userdata.go b/pkg/asset/machines/userdata.go index 8f08ea47204..da44e8ce95a 100644 --- a/pkg/asset/machines/userdata.go +++ b/pkg/asset/machines/userdata.go @@ -19,8 +19,8 @@ data: userData: {{.content}} `)) -// UserDataSecret generates the user data secret that contains the -// master or worker pointer ignition. +// UserDataSecret generates the user data secret in openshift-machine-api. +// For CAPI, the cluster-capi-operator syncs it to openshift-cluster-api. func UserDataSecret(name string, content []byte) ([]byte, error) { encodedData := map[string]string{ "name": name, diff --git a/pkg/asset/machines/worker.go b/pkg/asset/machines/worker.go index 1ce12f915c2..57b2432f39a 100644 --- a/pkg/asset/machines/worker.go +++ b/pkg/asset/machines/worker.go @@ -14,7 +14,9 @@ import ( "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/pointer" - capz "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" //nolint:staticcheck //CORS-3563 + capa "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" + capz "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" + capi "sigs.k8s.io/cluster-api/api/core/v1beta2" "sigs.k8s.io/yaml" configv1 "github.com/openshift/api/config/v1" @@ -73,6 +75,12 @@ const ( // workerMachineSetFileName is the format string for constructing the worker MachineSet filenames. workerMachineSetFileName = "99_openshift-cluster-api_worker-machineset-%s.yaml" + // workerCAPIMachineSetFileName is the format string for constructing the CAPI worker MachineSet filenames. + workerCAPIMachineSetFileName = "99_openshift-cluster-api_worker-capi-machineset-%s.yaml" + + // workerMachineTemplateFileName is the format string for constructing the worker MachineTemplate filenames. + workerMachineTemplateFileName = "99_openshift-cluster-api_worker-machinetemplate-%s.yaml" + // workerMachineFileName is the format string for constructing the worker Machine filenames. workerMachineFileName = "99_openshift-cluster-api_worker-machines-%s.yaml" @@ -91,10 +99,12 @@ const ( ) var ( - workerMachineSetFileNamePattern = fmt.Sprintf(workerMachineSetFileName, "*") - workerMachineFileNamePattern = fmt.Sprintf(workerMachineFileName, "*") - workerIPClaimFileNamePattern = fmt.Sprintf(ipClaimFileName, "*worker*") - workerIPAddressFileNamePattern = fmt.Sprintf(ipAddressFileName, "*worker*") + workerMachineSetFileNamePattern = fmt.Sprintf(workerMachineSetFileName, "*") + workerCAPIMachineSetFileNamePattern = fmt.Sprintf(workerCAPIMachineSetFileName, "*") + workerMachineTemplateFileNamePattern = fmt.Sprintf(workerMachineTemplateFileName, "*") + workerMachineFileNamePattern = fmt.Sprintf(workerMachineFileName, "*") + workerIPClaimFileNamePattern = fmt.Sprintf(ipClaimFileName, "*worker*") + workerIPAddressFileNamePattern = fmt.Sprintf(ipAddressFileName, "*worker*") _ asset.WritableAsset = (*Worker)(nil) ) @@ -285,12 +295,14 @@ func awsSetPreferredInstanceByEdgeZone(ctx context.Context, defaultTypes []strin // Worker generates the machinesets for `worker` machine pool. type Worker struct { - UserDataFile *asset.File - MachineConfigFiles []*asset.File - MachineSetFiles []*asset.File - MachineFiles []*asset.File - IPClaimFiles []*asset.File - IPAddrFiles []*asset.File + UserDataFile *asset.File + MachineConfigFiles []*asset.File + MachineSetFiles []*asset.File + MachineTemplateFiles []*asset.File + CAPIMachineSetFiles []*asset.File + MachineFiles []*asset.File + IPClaimFiles []*asset.File + IPAddrFiles []*asset.File } // Name returns a human friendly name for the Worker Asset. @@ -327,7 +339,13 @@ func (w *Worker) Generate(ctx context.Context, dependencies asset.Parents) error workerUserDataSecretName := "worker-user-data" machineConfigs := []*mcfgv1.MachineConfig{} - var ipClaims, ipAddrs, machines, machineSets []runtime.Object + + var ipClaims, ipAddrs, machines []runtime.Object + // MAPI machineset manifests + var machineSets []runtime.Object + // CAPI machineset and machine template manifests + var machineTemplates, capiMachineSets []runtime.Object + var err error ic := installConfig.Config @@ -541,7 +559,8 @@ func (w *Worker) Generate(ctx context.Context, dependencies asset.Parents) error } pool.Platform.AWS = &mpool - sets, err := aws.MachineSets(&aws.MachineSetInput{ + + input := &aws.MachineSetInput{ ClusterID: clusterID.InfraID, InstallConfigPlatformAWS: installConfig.Config.Platform.AWS, Subnets: subnets, @@ -552,12 +571,27 @@ func (w *Worker) Generate(ctx context.Context, dependencies asset.Parents) error UserDataSecret: workerUserDataSecretName, Hosts: dHosts, Config: installConfig.Config, - }) - if err != nil { - return errors.Wrap(err, "failed to create worker machine objects") } - for _, set := range sets { - machineSets = append(machineSets, set) + + if pool.Management == types.ClusterAPI { + templates, sets, err := aws.ClusterAPIMachineSets(input) + if err != nil { + return fmt.Errorf("failed to create CAPI worker machineset objects: %w", err) + } + for _, template := range templates { + machineTemplates = append(machineTemplates, &template) + } + for _, set := range sets { + capiMachineSets = append(capiMachineSets, &set) + } + } else { + sets, err := aws.MachineSets(input) + if err != nil { + return fmt.Errorf("failed to create worker machine objects: %w", err) + } + for _, set := range sets { + machineSets = append(machineSets, set) + } } case azuretypes.Name: mpool := defaultAzureMachinePoolPlatform(installConfig.Config.Platform.Azure.CloudName) @@ -818,6 +852,12 @@ func (w *Worker) Generate(ctx context.Context, dependencies asset.Parents) error if w.MachineSetFiles, err = serialize(machineSets, workerMachineSetFileName, false); err != nil { return fmt.Errorf("failed to serialize worker machine sets: %w", err) } + if w.MachineTemplateFiles, err = serialize(machineTemplates, workerMachineTemplateFileName, false); err != nil { + return fmt.Errorf("failed to serialize worker machine templates: %w", err) + } + if w.CAPIMachineSetFiles, err = serialize(capiMachineSets, workerCAPIMachineSetFileName, false); err != nil { + return fmt.Errorf("failed to serialize worker CAPI machine sets: %w", err) + } if w.IPClaimFiles, err = serialize(ipClaims, ipClaimFileName, true); err != nil { return fmt.Errorf("failed to serialize worker ip claims: %w", err) } @@ -838,6 +878,8 @@ func (w *Worker) Files() []*asset.File { } files = append(files, w.MachineConfigFiles...) files = append(files, w.MachineSetFiles...) + files = append(files, w.MachineTemplateFiles...) + files = append(files, w.CAPIMachineSetFiles...) files = append(files, w.MachineFiles...) files = append(files, w.IPClaimFiles...) files = append(files, w.IPAddrFiles...) @@ -867,6 +909,18 @@ func (w *Worker) Load(f asset.FileFetcher) (found bool, err error) { w.MachineSetFiles = fileList + fileList, err = f.FetchByPattern(filepath.Join(directory, workerMachineTemplateFileNamePattern)) + if err != nil { + return true, err + } + w.MachineTemplateFiles = fileList + + fileList, err = f.FetchByPattern(filepath.Join(directory, workerCAPIMachineSetFileNamePattern)) + if err != nil { + return true, err + } + w.CAPIMachineSetFiles = fileList + fileList, err = f.FetchByPattern(filepath.Join(directory, workerMachineFileNamePattern)) if err != nil { return true, err @@ -940,6 +994,32 @@ func (w *Worker) MachineSets() ([]machinev1beta1.MachineSet, error) { return machineSets, nil } +// CAPIMachineSets returns deserialized CAPI MachineSet manifest structures. +func (w *Worker) CAPIMachineSets() ([]capi.MachineSet, error) { + machineSets := make([]capi.MachineSet, 0, len(w.CAPIMachineSetFiles)) + for i, file := range w.CAPIMachineSetFiles { + machineSet := &capi.MachineSet{} + if err := yaml.Unmarshal(file.Data, machineSet); err != nil { + return nil, errors.Wrapf(err, "unmarshal CAPI worker machineset %d", i) + } + machineSets = append(machineSets, *machineSet) + } + return machineSets, nil +} + +// CAPIMachineTemplates returns deserialized CAPI AWSMachineTemplate manifest structures. +func (w *Worker) CAPIMachineTemplates() ([]capa.AWSMachineTemplate, error) { + templates := make([]capa.AWSMachineTemplate, 0, len(w.MachineTemplateFiles)) + for i, file := range w.MachineTemplateFiles { + template := &capa.AWSMachineTemplate{} + if err := yaml.Unmarshal(file.Data, template); err != nil { + return nil, errors.Wrapf(err, "unmarshal CAPI worker machine template %d", i) + } + templates = append(templates, *template) + } + return templates, nil +} + // serialize marshals a list of runtime.Object manifests into asset files. // When useObjectName is true, the object's metadata name is used in the filename, // e.g. "99_openshift-machine-api_claim-cluster-worker-0-claim-0-0.yaml". diff --git a/pkg/asset/quota/aws/aws.go b/pkg/asset/quota/aws/aws.go index 015b7b03dcd..9a8d6bc4798 100644 --- a/pkg/asset/quota/aws/aws.go +++ b/pkg/asset/quota/aws/aws.go @@ -8,32 +8,24 @@ import ( "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/util/sets" + capa "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" + capi "sigs.k8s.io/cluster-api/api/core/v1beta2" - machineapi "github.com/openshift/api/machine/v1beta1" + machinev1beta1 "github.com/openshift/api/machine/v1beta1" + quotatypes "github.com/openshift/installer/pkg/asset/quota/types" "github.com/openshift/installer/pkg/quota" "github.com/openshift/installer/pkg/types" ) // Constraints returns a list of quota constraints based on the InstallConfig. // These constraints can be used to check if there is enough quota for creating a cluster -// for the isntall config. -func Constraints(config *types.InstallConfig, controlPlanes []machineapi.Machine, computes []machineapi.MachineSet, instanceTypes map[string]InstanceTypeInfo) []quota.Constraint { - ctrplConfigs := make([]*machineapi.AWSMachineProviderConfig, len(controlPlanes)) - for i, m := range controlPlanes { - ctrplConfigs[i] = m.Spec.ProviderSpec.Value.Object.(*machineapi.AWSMachineProviderConfig) - } - computeReplicas := make([]int64, len(computes)) - computeConfigs := make([]*machineapi.AWSMachineProviderConfig, len(computes)) - for i, w := range computes { - computeReplicas[i] = int64(*w.Spec.Replicas) - computeConfigs[i] = w.Spec.Template.Spec.ProviderSpec.Value.Object.(*machineapi.AWSMachineProviderConfig) - } - +// for the install config. +func Constraints(config *types.InstallConfig, controlPlanes []quotatypes.MachineInfo, computes []quotatypes.MachineInfo, instanceTypes map[string]InstanceTypeInfo) []quota.Constraint { var ret []quota.Constraint for _, gen := range []constraintGenerator{ - network(config, append(ctrplConfigs, computeConfigs...)), - controlPlane(config, ctrplConfigs, instanceTypes), - compute(config, computeReplicas, computeConfigs, instanceTypes), + network(config, append(controlPlanes, computes...)), + controlPlane(config, controlPlanes, instanceTypes), + compute(config, computes, instanceTypes), others, } { ret = append(ret, gen()...) @@ -63,11 +55,11 @@ func aggregate(quotas []quota.Constraint) []quota.Constraint { // constraintGenerator generates a list of constraints. type constraintGenerator func() []quota.Constraint -func network(config *types.InstallConfig, machines []*machineapi.AWSMachineProviderConfig) func() []quota.Constraint { +func network(config *types.InstallConfig, machines []quotatypes.MachineInfo) func() []quota.Constraint { return func() []quota.Constraint { zones := sets.NewString() for _, m := range machines { - zones.Insert(m.Placement.AvailabilityZone) + zones.Insert(m.AvailabilityZone) } var ret []quota.Constraint @@ -102,7 +94,7 @@ func network(config *types.InstallConfig, machines []*machineapi.AWSMachineProvi } } -func controlPlane(config *types.InstallConfig, machines []*machineapi.AWSMachineProviderConfig, instanceTypes map[string]InstanceTypeInfo) func() []quota.Constraint { +func controlPlane(config *types.InstallConfig, machines []quotatypes.MachineInfo, instanceTypes map[string]InstanceTypeInfo) func() []quota.Constraint { return func() []quota.Constraint { var ret []quota.Constraint for _, m := range machines { @@ -114,12 +106,12 @@ func controlPlane(config *types.InstallConfig, machines []*machineapi.AWSMachine } } -func compute(config *types.InstallConfig, replicas []int64, machines []*machineapi.AWSMachineProviderConfig, instanceTypes map[string]InstanceTypeInfo) func() []quota.Constraint { +func compute(config *types.InstallConfig, machines []quotatypes.MachineInfo, instanceTypes map[string]InstanceTypeInfo) func() []quota.Constraint { return func() []quota.Constraint { var ret []quota.Constraint - for idx, m := range machines { + for _, m := range machines { q := machineTypeToQuota(m.InstanceType, instanceTypes) - q.Count = q.Count * replicas[idx] + q.Count *= m.Replicas q.Region = config.Platform.AWS.Region ret = append(ret, q) } @@ -156,3 +148,48 @@ func machineTypeToQuota(t string, instanceTypes map[string]InstanceTypeInfo) quo return quota.Constraint{Name: "ec2/L-7295265B", Count: 0} } } + +// MachineInfoFromMAPIMachines converts MAPI Machine objects to MachineInfo. +func MachineInfoFromMAPIMachines(mapiMachines []machinev1beta1.Machine) []quotatypes.MachineInfo { + infos := make([]quotatypes.MachineInfo, 0, len(mapiMachines)) + for _, m := range mapiMachines { + providerConfig := m.Spec.ProviderSpec.Value.Object.(*machinev1beta1.AWSMachineProviderConfig) + infos = append(infos, quotatypes.MachineInfo{ + InstanceType: providerConfig.InstanceType, + AvailabilityZone: providerConfig.Placement.AvailabilityZone, + Replicas: 1, + }) + } + return infos +} + +// MachineInfoFromMAPIMachineSets converts MAPI MachineSet objects to MachineInfo. +func MachineInfoFromMAPIMachineSets(mapiMachineSets []machinev1beta1.MachineSet) []quotatypes.MachineInfo { + infos := make([]quotatypes.MachineInfo, 0, len(mapiMachineSets)) + for _, ms := range mapiMachineSets { + providerConfig := ms.Spec.Template.Spec.ProviderSpec.Value.Object.(*machinev1beta1.AWSMachineProviderConfig) + infos = append(infos, quotatypes.MachineInfo{ + InstanceType: providerConfig.InstanceType, + AvailabilityZone: providerConfig.Placement.AvailabilityZone, + Replicas: int64(*ms.Spec.Replicas), + }) + } + return infos +} + +// MachineInfoFromCAPIMachineSets converts CAPI MachineSet and AWSMachineTemplate objects to MachineInfo. +func MachineInfoFromCAPIMachineSets(capiMachineSets []capi.MachineSet, capiTemplates []capa.AWSMachineTemplate) []quotatypes.MachineInfo { + templateInstanceTypes := make(map[string]string, len(capiTemplates)) + for _, t := range capiTemplates { + templateInstanceTypes[t.Name] = t.Spec.Template.Spec.InstanceType + } + infos := make([]quotatypes.MachineInfo, 0, len(capiMachineSets)) + for _, ms := range capiMachineSets { + infos = append(infos, quotatypes.MachineInfo{ + InstanceType: templateInstanceTypes[ms.Spec.Template.Spec.InfrastructureRef.Name], + AvailabilityZone: ms.Spec.Template.Spec.FailureDomain, + Replicas: int64(*ms.Spec.Replicas), + }) + } + return infos +} diff --git a/pkg/asset/quota/quota.go b/pkg/asset/quota/quota.go index f6d220df906..d215b102c36 100644 --- a/pkg/asset/quota/quota.go +++ b/pkg/asset/quota/quota.go @@ -70,6 +70,15 @@ func (a *PlatformQuotaCheck) Generate(ctx context.Context, dependencies asset.Pa return err } + capiWorkers, err := workersAsset.CAPIMachineSets() + if err != nil { + return err + } + capiTemplates, err := workersAsset.CAPIMachineTemplates() + if err != nil { + return err + } + platform := ic.Config.Platform.Name() switch platform { case typesaws.Name: @@ -95,7 +104,14 @@ func (a *PlatformQuotaCheck) Generate(ctx context.Context, dependencies asset.Pa if err != nil { return errors.Wrapf(err, "failed to load instance types for %s", ic.AWS.Region) } - reports, err := quota.Check(q, aws.Constraints(ic.Config, masters, workers, instanceTypes)) + + masterInfos := aws.MachineInfoFromMAPIMachines(masters) + // Edge pools always use MAPI at the moment; worker pools may use CAPI. The split is + // handled at generation time (MachineSetFiles vs CAPIMachineSetFiles), + // so we simply collect from both sources — empty slices are no-ops. + workerInfos := append(aws.MachineInfoFromMAPIMachineSets(workers), aws.MachineInfoFromCAPIMachineSets(capiWorkers, capiTemplates)...) + + reports, err := quota.Check(q, aws.Constraints(ic.Config, masterInfos, workerInfos, instanceTypes)) if err != nil { return summarizeFailingReport(reports) } diff --git a/pkg/asset/quota/types/types.go b/pkg/asset/quota/types/types.go new file mode 100644 index 00000000000..8435a519229 --- /dev/null +++ b/pkg/asset/quota/types/types.go @@ -0,0 +1,9 @@ +package types + +// MachineInfo holds the quota-relevant fields for a machine, +// abstracting over both MAPI and CAPI representations. +type MachineInfo struct { + InstanceType string + AvailabilityZone string + Replicas int64 +} diff --git a/pkg/tfvars/aws/OWNERS b/pkg/tfvars/aws/OWNERS deleted file mode 100644 index 6e59d685aa6..00000000000 --- a/pkg/tfvars/aws/OWNERS +++ /dev/null @@ -1,7 +0,0 @@ -# See the OWNERS docs: https://git.k8s.io/community/contributors/guide/owners.md -# This file just uses aliases defined in OWNERS_ALIASES. - -approvers: - - aws-approvers -reviewers: - - aws-reviewers diff --git a/pkg/tfvars/aws/aws.go b/pkg/tfvars/aws/aws.go deleted file mode 100644 index 1d9ba7ad87d..00000000000 --- a/pkg/tfvars/aws/aws.go +++ /dev/null @@ -1,272 +0,0 @@ -// Package aws contains AWS-specific Terraform-variable logic. -package aws - -import ( - "encoding/json" - "fmt" - "sort" - "strings" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - - machinev1beta1 "github.com/openshift/api/machine/v1beta1" - "github.com/openshift/installer/pkg/asset/ignition/bootstrap" - icaws "github.com/openshift/installer/pkg/asset/installconfig/aws" - "github.com/openshift/installer/pkg/types" - typesaws "github.com/openshift/installer/pkg/types/aws" -) - -// Config contains the AWS platform data for terraform. -type Config struct { - AMI string `json:"aws_ami"` - AMIRegion string `json:"aws_ami_region"` - CustomEndpoints map[string]string `json:"custom_endpoints,omitempty"` - ExtraTags map[string]string `json:"aws_extra_tags,omitempty"` - BootstrapInstanceType string `json:"aws_bootstrap_instance_type,omitempty"` - MasterInstanceType string `json:"aws_master_instance_type,omitempty"` - MasterAvailabilityZones []string `json:"aws_master_availability_zones"` - WorkerAvailabilityZones []string `json:"aws_worker_availability_zones"` - EdgeLocalZones []string `json:"aws_edge_local_zones,omitempty"` - EdgeZonesGatewayIndex map[string]int `json:"aws_edge_parent_zones_index,omitempty"` - EdgeZonesType map[string]string `json:"aws_edge_zones_type,omitempty"` - IOPS int64 `json:"aws_master_root_volume_iops"` - Size int64 `json:"aws_master_root_volume_size,omitempty"` - Type string `json:"aws_master_root_volume_type,omitempty"` - Encrypted bool `json:"aws_master_root_volume_encrypted"` - KMSKeyID string `json:"aws_master_root_volume_kms_key_id,omitempty"` - Region string `json:"aws_region,omitempty"` - VPC string `json:"aws_vpc,omitempty"` - PrivateSubnets []string `json:"aws_private_subnets,omitempty"` - PublicSubnets *[]string `json:"aws_public_subnets,omitempty"` - InternalZone string `json:"aws_internal_zone,omitempty"` - InternalZoneRole string `json:"aws_internal_zone_role,omitempty"` - PublishStrategy string `json:"aws_publish_strategy,omitempty"` - IgnitionBucket string `json:"aws_ignition_bucket"` - BootstrapIgnitionStub string `json:"aws_bootstrap_stub_ignition"` - MasterIAMRoleName string `json:"aws_master_iam_role_name,omitempty"` - WorkerIAMRoleName string `json:"aws_worker_iam_role_name,omitempty"` - MasterMetadataAuthentication string `json:"aws_master_instance_metadata_authentication,omitempty"` - BootstrapMetadataAuthentication string `json:"aws_bootstrap_instance_metadata_authentication,omitempty"` - PreserveBootstrapIgnition bool `json:"aws_preserve_bootstrap_ignition"` - MasterSecurityGroups []string `json:"aws_master_security_groups,omitempty"` - PublicIpv4Pool string `json:"aws_public_ipv4_pool"` - MasterUseSpotInstance bool `json:"aws_master_use_spot_instance,omitempty"` -} - -// TFVarsSources contains the parameters to be converted into Terraform variables -type TFVarsSources struct { - VPC string - PrivateSubnets, PublicSubnets []string - InternalZone, InternalZoneRole string - Services []typesaws.ServiceEndpoint - AvailabilityZones icaws.Zones - - Publish types.PublishingStrategy - - AMIID, AMIRegion string - - MasterConfigs, WorkerConfigs []*machinev1beta1.AWSMachineProviderConfig - - IgnitionBucket, IgnitionPresignedURL string - - AdditionalTrustBundle string - - MasterIAMRoleName, WorkerIAMRoleName string - - MasterMetadataAuthentication string - - Architecture types.Architecture - - Proxy *types.Proxy - - PreserveBootstrapIgnition bool - - MasterSecurityGroups []string - - PublicIpv4Pool string -} - -// TFVars generates AWS-specific Terraform variables launching the cluster. -func TFVars(sources TFVarsSources) ([]byte, error) { - masterConfig := sources.MasterConfigs[0] - - endpoints := make(map[string]string) - for _, service := range sources.Services { - service := service - endpoints[service.Name] = service.URL - } - - tags := make(map[string]string, len(masterConfig.Tags)) - for _, tag := range masterConfig.Tags { - tags[tag.Name] = tag.Value - } - - exists := struct{}{} - allAvailabilityZonesMap := map[string]struct{}{} - masterAvailabilityZones := make([]string, len(sources.MasterConfigs)) - for i, c := range sources.MasterConfigs { - masterAvailabilityZones[i] = c.Placement.AvailabilityZone - allAvailabilityZonesMap[c.Placement.AvailabilityZone] = exists - } - - availabilityZoneMap := map[string]struct{}{} - edgeLocalZoneMap := map[string]struct{}{} - for _, c := range sources.WorkerConfigs { - zoneName := c.Placement.AvailabilityZone - if _, ok := sources.AvailabilityZones[zoneName]; !ok { - return nil, errors.New(fmt.Sprintf("unable to find the zone when generating terraform vars: %s", zoneName)) - } - if sources.AvailabilityZones[zoneName].Type == typesaws.LocalZoneType || - sources.AvailabilityZones[zoneName].Type == typesaws.WavelengthZoneType { - edgeLocalZoneMap[zoneName] = exists - continue - } - availabilityZoneMap[zoneName] = exists - allAvailabilityZonesMap[zoneName] = exists - } - - workerAvailabilityZones := make([]string, 0, len(availabilityZoneMap)) - for zone := range availabilityZoneMap { - workerAvailabilityZones = append(workerAvailabilityZones, zone) - } - - allAvailabilityZones := make([]string, 0, len(allAvailabilityZonesMap)) - for zone := range allAvailabilityZonesMap { - allAvailabilityZones = append(allAvailabilityZones, zone) - } - - // Create map for edge zone and parent's zone index. - // AWS Local Zones does not support private Nat Gateways, to egress internet - // traffic from the zone, so the parent's zone route table will be - // used to associate private subnets created in the edge zones. - // The allAvailabilityZones holds all Availability Zone type (in the Region) - // for the cluster, where the terraform creates network resources - // (NAT Gateway). The index of that list will be used to determine the - // parent's zone route table ID, when exists, otherwise the default - // private route table will be used. - // TODO(when Local Zone supports Nat Gateway): create private route table - // by Local Zone location. - sort.Strings(allAvailabilityZones) - edgeLocalZones := make([]string, 0, len(edgeLocalZoneMap)) - edgeZonesGatewayIndexMap := make(map[string]int, len(edgeLocalZoneMap)) - edgeZonesType := make(map[string]string, len(edgeLocalZoneMap)) - // new VPC - if len(sources.PrivateSubnets) == 0 { - for zone := range edgeLocalZoneMap { - parent := sources.AvailabilityZones[zone].ParentZoneName - gwIndex := 0 - for idx, az := range allAvailabilityZones { - if az == parent { - gwIndex = idx - break - } - } - edgeLocalZones = append(edgeLocalZones, zone) - edgeZonesGatewayIndexMap[zone] = gwIndex - edgeZonesType[zone] = sources.AvailabilityZones[zone].Type - } - } - - if len(masterConfig.BlockDevices) == 0 { - return nil, errors.New("block device slice cannot be empty") - } - - rootVolume := masterConfig.BlockDevices[0] - if rootVolume.EBS == nil { - return nil, errors.New("EBS information must be configured for the root volume") - } - - if rootVolume.EBS.VolumeType == nil { - return nil, errors.New("EBS volume type must be configured for the root volume") - } - - if rootVolume.EBS.VolumeSize == nil { - return nil, errors.New("EBS volume size must be configured for the root volume") - } - - if *rootVolume.EBS.VolumeType == "io1" && rootVolume.EBS.Iops == nil { - return nil, errors.New("EBS IOPS must be configured for the io1 root volume") - } - - useSpotInstances := masterConfig.SpotMarketOptions != nil - if useSpotInstances { - logrus.Warn("Found Spot instance configuration. Please be warned, this is not advised.") - } - - cfg := &Config{ - CustomEndpoints: endpoints, - Region: masterConfig.Placement.Region, - ExtraTags: tags, - MasterAvailabilityZones: masterAvailabilityZones, - WorkerAvailabilityZones: workerAvailabilityZones, - EdgeLocalZones: edgeLocalZones, - EdgeZonesGatewayIndex: edgeZonesGatewayIndexMap, - EdgeZonesType: edgeZonesType, - BootstrapInstanceType: masterConfig.InstanceType, - MasterInstanceType: masterConfig.InstanceType, - Size: *rootVolume.EBS.VolumeSize, - Type: *rootVolume.EBS.VolumeType, - VPC: sources.VPC, - PrivateSubnets: sources.PrivateSubnets, - InternalZone: sources.InternalZone, - InternalZoneRole: sources.InternalZoneRole, - PublishStrategy: string(sources.Publish), - IgnitionBucket: sources.IgnitionBucket, - MasterIAMRoleName: sources.MasterIAMRoleName, - WorkerIAMRoleName: sources.WorkerIAMRoleName, - PreserveBootstrapIgnition: sources.PreserveBootstrapIgnition, - MasterSecurityGroups: sources.MasterSecurityGroups, - PublicIpv4Pool: sources.PublicIpv4Pool, - MasterUseSpotInstance: useSpotInstances, - } - - stubIgn, err := bootstrap.GenerateIgnitionShimWithCertBundleAndProxy(sources.IgnitionPresignedURL, sources.AdditionalTrustBundle, sources.Proxy) - if err != nil { - return nil, errors.Wrap(err, "failed to create stub Ignition config for bootstrap") - } - - // Check the size of the raw ignition stub is less than 16KB for aws user-data - // see https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-add-user-data.html - if len(stubIgn) > 16000 { - return nil, fmt.Errorf("rendered bootstrap ignition shim exceeds the 16KB limit for AWS user data -- try reducing the size of your CA cert bundle") - } - cfg.BootstrapIgnitionStub = string(stubIgn) - - if len(sources.PublicSubnets) == 0 { - if cfg.VPC != "" { - cfg.PublicSubnets = &[]string{} - } - } else { - cfg.PublicSubnets = &sources.PublicSubnets - } - - if rootVolume.EBS.Iops != nil { - cfg.IOPS = *rootVolume.EBS.Iops - } - - cfg.Encrypted = true - if rootVolume.EBS.Encrypted != nil { - cfg.Encrypted = *rootVolume.EBS.Encrypted - } - if rootVolume.EBS.KMSKey.ID != nil && *rootVolume.EBS.KMSKey.ID != "" { - cfg.KMSKeyID = *rootVolume.EBS.KMSKey.ID - } else if rootVolume.EBS.KMSKey.ARN != nil && *rootVolume.EBS.KMSKey.ARN != "" { - cfg.KMSKeyID = *rootVolume.EBS.KMSKey.ARN - } - - if masterConfig.AMI.ID != nil && *masterConfig.AMI.ID != "" { - cfg.AMI = *masterConfig.AMI.ID - cfg.AMIRegion = masterConfig.Placement.Region - } else { - cfg.AMI = sources.AMIID - cfg.AMIRegion = sources.AMIRegion - } - - if masterConfig.MetadataServiceOptions.Authentication != "" { - cfg.MasterMetadataAuthentication = strings.ToLower(string(masterConfig.MetadataServiceOptions.Authentication)) - cfg.BootstrapMetadataAuthentication = cfg.MasterMetadataAuthentication - } - - return json.MarshalIndent(cfg, "", " ") -} diff --git a/pkg/types/defaults/installconfig.go b/pkg/types/defaults/installconfig.go index 6e88ba5aba7..eaf3e15a71c 100644 --- a/pkg/types/defaults/installconfig.go +++ b/pkg/types/defaults/installconfig.go @@ -68,11 +68,11 @@ func SetInstallConfigDefaults(c *types.InstallConfig) { c.ControlPlane = &types.MachinePool{} } c.ControlPlane.Name = "master" - SetMachinePoolDefaults(c.ControlPlane, &c.Platform) + SetMachinePoolDefaults(c.ControlPlane, &c.Platform, c.EnabledFeatureGates()) if c.Arbiter != nil { c.Arbiter.Name = "arbiter" - SetMachinePoolDefaults(c.Arbiter, &c.Platform) + SetMachinePoolDefaults(c.Arbiter, &c.Platform, c.EnabledFeatureGates()) } defaultComputePoolUndefined := true @@ -86,7 +86,7 @@ func SetInstallConfigDefaults(c *types.InstallConfig) { c.Compute = append(c.Compute, types.MachinePool{Name: types.MachinePoolComputeRoleName}) } for i := range c.Compute { - SetMachinePoolDefaults(&c.Compute[i], &c.Platform) + SetMachinePoolDefaults(&c.Compute[i], &c.Platform, c.EnabledFeatureGates()) } if c.CredentialsMode == "" { diff --git a/pkg/types/defaults/machinepools.go b/pkg/types/defaults/machinepools.go index cf778e1e670..74eedc57a64 100644 --- a/pkg/types/defaults/machinepools.go +++ b/pkg/types/defaults/machinepools.go @@ -3,18 +3,20 @@ package defaults import ( "net" + "github.com/openshift/api/features" "github.com/openshift/installer/pkg/types" "github.com/openshift/installer/pkg/types/aws" awsdefaults "github.com/openshift/installer/pkg/types/aws/defaults" "github.com/openshift/installer/pkg/types/azure" azuredefaults "github.com/openshift/installer/pkg/types/azure/defaults" + "github.com/openshift/installer/pkg/types/featuregates" "github.com/openshift/installer/pkg/types/gcp" gcpdefaults "github.com/openshift/installer/pkg/types/gcp/defaults" "github.com/openshift/installer/pkg/version" ) // SetMachinePoolDefaults sets the defaults for the machine pool. -func SetMachinePoolDefaults(p *types.MachinePool, platform *types.Platform) { +func SetMachinePoolDefaults(p *types.MachinePool, platform *types.Platform, fgates featuregates.FeatureGate) { defaultReplicaCount := int64(3) if p.Name == types.MachinePoolEdgeRoleName || p.Name == types.MachinePoolArbiterRoleName { defaultReplicaCount = 0 @@ -39,6 +41,16 @@ func SetMachinePoolDefaults(p *types.MachinePool, platform *types.Platform) { } } + // Set management to ClusterAPI if the appropriate feature gate is enabled and management is unspecified + if p.Management == "" { + if p.Name == types.MachinePoolControlPlaneRoleName && fgates.Enabled(features.FeatureGateClusterAPIControlPlaneInstall) { + p.Management = types.ClusterAPI + } + if p.Name == types.MachinePoolComputeRoleName && fgates.Enabled(features.FeatureGateClusterAPIComputeInstall) { + p.Management = types.ClusterAPI + } + } + switch platform.Name() { case aws.Name: if p.Platform.AWS == nil && platform.AWS.DefaultMachinePlatform != nil { @@ -69,7 +81,7 @@ func hasEdgePoolConfig(pools []types.MachinePool) bool { } // CreateEdgeMachinePoolDefaults create the edge compute pool when it is not already defined. -func CreateEdgeMachinePoolDefaults(pools []types.MachinePool, platform *types.Platform, replicas int64) *types.MachinePool { +func CreateEdgeMachinePoolDefaults(pools []types.MachinePool, platform *types.Platform, replicas int64, fgates featuregates.FeatureGate) *types.MachinePool { if hasEdgePoolConfig(pools) { return nil } @@ -77,6 +89,6 @@ func CreateEdgeMachinePoolDefaults(pools []types.MachinePool, platform *types.Pl Name: types.MachinePoolEdgeRoleName, Replicas: &replicas, } - SetMachinePoolDefaults(pool, platform) + SetMachinePoolDefaults(pool, platform, fgates) return pool } diff --git a/pkg/types/defaults/machinepools_test.go b/pkg/types/defaults/machinepools_test.go index 0a256446470..7a2db9073a0 100644 --- a/pkg/types/defaults/machinepools_test.go +++ b/pkg/types/defaults/machinepools_test.go @@ -5,6 +5,7 @@ import ( "github.com/stretchr/testify/assert" + configv1 "github.com/openshift/api/config/v1" "github.com/openshift/installer/pkg/types" ) @@ -133,12 +134,77 @@ func TestSetMahcinePoolDefaults(t *testing.T) { } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - SetMachinePoolDefaults(tc.pool, tc.platform) + // Use default feature set (no special features enabled) + config := &types.InstallConfig{} + SetMachinePoolDefaults(tc.pool, tc.platform, config.EnabledFeatureGates()) assert.Equal(t, tc.expected, tc.pool, "unexpected machine pool") }) } } +func TestSetMachinePoolDefaultsWithFeatureGates(t *testing.T) { + cases := []struct { + name string + pool *types.MachinePool + platform *types.Platform + featureSet configv1.FeatureSet + expectedManagement types.MachineManagementAPI + }{ + { + name: "control plane with DevPreviewNoUpgrade feature set", + pool: &types.MachinePool{Name: types.MachinePoolControlPlaneRoleName}, + platform: &types.Platform{}, + featureSet: configv1.DevPreviewNoUpgrade, + expectedManagement: types.ClusterAPI, + }, + { + name: "control plane with default feature set", + pool: &types.MachinePool{Name: types.MachinePoolControlPlaneRoleName}, + platform: &types.Platform{}, + featureSet: configv1.Default, + expectedManagement: "", + }, + { + name: "compute with DevPreviewNoUpgrade feature set", + pool: &types.MachinePool{Name: types.MachinePoolComputeRoleName}, + platform: &types.Platform{}, + featureSet: configv1.DevPreviewNoUpgrade, + expectedManagement: types.ClusterAPI, + }, + { + name: "compute with default feature set", + pool: &types.MachinePool{Name: types.MachinePoolComputeRoleName}, + platform: &types.Platform{}, + featureSet: configv1.Default, + expectedManagement: "", + }, + { + name: "control plane with management already set", + pool: &types.MachinePool{Name: types.MachinePoolControlPlaneRoleName, Management: types.MachineAPI}, + platform: &types.Platform{}, + featureSet: configv1.DevPreviewNoUpgrade, + expectedManagement: types.MachineAPI, + }, + { + name: "compute with management already set", + pool: &types.MachinePool{Name: types.MachinePoolComputeRoleName, Management: types.MachineAPI}, + platform: &types.Platform{}, + featureSet: configv1.DevPreviewNoUpgrade, + expectedManagement: types.MachineAPI, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + config := &types.InstallConfig{ + FeatureSet: tc.featureSet, + } + SetMachinePoolDefaults(tc.pool, tc.platform, config.EnabledFeatureGates()) + assert.Equal(t, tc.expectedManagement, tc.pool.Management, "unexpected management API") + }) + } +} + func TestHasEdgePoolConfig(t *testing.T) { cases := []struct { name string diff --git a/pkg/types/validation/installconfig.go b/pkg/types/validation/installconfig.go index d6c088ee596..b91ca9bd33e 100644 --- a/pkg/types/validation/installconfig.go +++ b/pkg/types/validation/installconfig.go @@ -900,6 +900,9 @@ func validateCompute(platform *types.Platform, control *types.MachinePool, pools case types.MachinePoolComputeRoleName: case types.MachinePoolEdgeRoleName: allErrs = append(allErrs, validateComputeEdge(platform, p.Name, poolFldPath, poolFldPath)...) + if p.Management == types.ClusterAPI { + allErrs = append(allErrs, field.Invalid(poolFldPath.Child("management"), p.Management, "edge compute pools cannot be managed by Cluster API")) + } default: allErrs = append(allErrs, field.NotSupported(poolFldPath.Child("name"), p.Name, []string{types.MachinePoolComputeRoleName, types.MachinePoolEdgeRoleName})) } diff --git a/pkg/types/validation/installconfig_test.go b/pkg/types/validation/installconfig_test.go index ef7e2372c13..4e87872a41d 100644 --- a/pkg/types/validation/installconfig_test.go +++ b/pkg/types/validation/installconfig_test.go @@ -856,6 +856,19 @@ func TestValidateInstallConfig(t *testing.T) { }(), expectedError: `^compute\[1\]\.name: Duplicate value: "worker"$`, }, + { + name: "edge compute with cluster api", + installConfig: func() *types.InstallConfig { + c := validInstallConfig() + c.Compute = append(c.Compute, func() types.MachinePool { + p := *validMachinePool("edge") + p.Management = types.ClusterAPI + return p + }()) + return c + }(), + expectedError: `^compute\[1\]\.management: Invalid value: "ClusterAPI": edge compute pools cannot be managed by Cluster API$`, + }, { name: "no compute replicas", installConfig: func() *types.InstallConfig { diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 86af3fe5812..e414fef2f1f 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -2,6 +2,7 @@ package utils import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + capi "sigs.k8s.io/cluster-api/api/core/v1beta2" "github.com/openshift/api/features" machinev1 "github.com/openshift/api/machine/v1" @@ -43,6 +44,24 @@ func SetMachineSetOSStreamLabels(machineSet *machineapi.MachineSet, ic *types.In machineSet.Spec.Template.Labels[types.OSStreamLabelKey] = string(ic.OSImageStream) } +// SetCAPIMachineSetOSStreamLabels adds the OS image stream label to a CAPI MachineSet's +// metadata and Spec.Template if the OSStreams feature gate is enabled. +func SetCAPIMachineSetOSStreamLabels(machineSet *capi.MachineSet, ic *types.InstallConfig) { + if ic == nil || !ic.Enabled(features.FeatureGateOSStreams) { + return + } + labels := machineSet.GetLabels() + if labels == nil { + labels = make(map[string]string) + } + labels[types.OSStreamLabelKey] = string(ic.OSImageStream) + machineSet.SetLabels(labels) + if machineSet.Spec.Template.ObjectMeta.Labels == nil { + machineSet.Spec.Template.ObjectMeta.Labels = make(map[string]string) + } + machineSet.Spec.Template.ObjectMeta.Labels[types.OSStreamLabelKey] = string(ic.OSImageStream) +} + // SetCPMSOSStreamLabels adds the OS image stream label to a ControlPlaneMachineSet's // metadata and Spec.Template if the OSStreams feature gate is enabled. func SetCPMSOSStreamLabels(cpms *machinev1.ControlPlaneMachineSet, ic *types.InstallConfig) {