diff --git a/workflows/cloudscale/decommission-steps.yml b/workflows/cloudscale/decommission-steps.yml index 450414a..079eac8 100644 --- a/workflows/cloudscale/decommission-steps.yml +++ b/workflows/cloudscale/decommission-steps.yml @@ -79,6 +79,11 @@ spells: HIERADATA_REPO_TOKEN=${INPUT_gitlab_api_token} EOF + if [[ ! -d catalog ]] + then + commodore catalog compile "$INPUT_commodore_cluster_id" + fi + tf_image=$(\ yq eval ".parameters.openshift4_terraform.images.terraform.image" \ dependencies/openshift4-terraform/class/defaults.yml) @@ -195,7 +200,10 @@ spells: "-backend-config=unlock_method=DELETE" \ "-backend-config=retry_wait_min=5" - terraform state rm "module.cluster.module.lb.module.hiera[0].gitfile_checkout.appuio_hieradata" + if terraform state list | grep "gitfile_checkout.appuio_hieradata" > /dev/null + then + terraform state rm "module.cluster.module.lb.module.hiera[0].gitfile_checkout.appuio_hieradata" + fi # Suppress errors on the first run; it is expected to fail terraform destroy --auto-approve || true diff --git a/workflows/cloudscale/image-steps.yml b/workflows/cloudscale/image-steps.yml index f217753..f076eee 100644 --- a/workflows/cloudscale/image-steps.yml +++ b/workflows/cloudscale/image-steps.yml @@ -115,7 +115,7 @@ spells: "$(echo "$INPUT_bucket_user" | jq -r '.keys[0].access_key')" \ "$(echo "$INPUT_bucket_user" | jq -r '.keys[0].secret_key')" - echo "Uploading Red Hat CoreOS image '$INPUT_image_path' to S3 bucket '${INPUT_commodore_cluster_id}-image-registry'..." + echo "Uploading Red Hat CoreOS image '$INPUT_image_path' to S3 bucket '${INPUT_commodore_cluster_id}-bootstrap-ignition'..." mc cp "rhcos-${INPUT_image_major}.${INPUT_image_minor}.qcow2" "${INPUT_commodore_cluster_id}/${INPUT_commodore_cluster_id}-bootstrap-ignition/" echo "Upload completed." diff --git a/workflows/cloudscale/init-steps.yml b/workflows/cloudscale/init-steps.yml index 183a496..8827ab6 100644 --- a/workflows/cloudscale/init-steps.yml +++ b/workflows/cloudscale/init-steps.yml @@ -61,10 +61,12 @@ spells: if [[ $( curl -sH "Authorization: Bearer ${INPUT_cloudscale_token}" https://api.cloudscale.ch/v1/flavors -o /dev/null -w"%{http_code}" ) != 200 ]] then echo "Cloudscale token not valid!" + exit 1 fi if [[ $( curl -sH "Authorization: Bearer ${INPUT_cloudscale_token_floaty}" https://api.cloudscale.ch/v1/flavors -o /dev/null -w"%{http_code}" ) != 200 ]] then echo "Cloudscale Floaty token not valid!" + exit 1 fi - match: Then I set secrets in Vault description: |- diff --git a/workflows/exoscale-decommission.workflow b/workflows/exoscale-decommission.workflow index c7228ae..0868c23 100644 --- a/workflows/exoscale-decommission.workflow +++ b/workflows/exoscale-decommission.workflow @@ -8,6 +8,7 @@ Then I confirm cluster deletion Then I disable the OpsGenie heartbeat And I disable Project Syn And I delete all Load Balancer services +And I delete all persistent volumes Then I save the loadbalancer metadata And I downtime the loadbalancers in icinga And I decommission Terraform resources diff --git a/workflows/exoscale/decommission-steps.yml b/workflows/exoscale/decommission-steps.yml index d89b8ba..e7f4653 100644 --- a/workflows/exoscale/decommission-steps.yml +++ b/workflows/exoscale/decommission-steps.yml @@ -42,6 +42,11 @@ spells: HIERADATA_REPO_TOKEN=${INPUT_gitlab_api_token} EOF + if [[ ! -d catalog ]] + then + commodore catalog compile "$INPUT_commodore_cluster_id" + fi + tf_image=$(\ yq eval ".parameters.openshift4_terraform.images.terraform.image" \ dependencies/openshift4-terraform/class/defaults.yml) @@ -165,7 +170,10 @@ spells: "-backend-config=unlock_method=DELETE" \ "-backend-config=retry_wait_min=5" - terraform state rm "module.cluster.module.lb.module.hiera[0].gitfile_checkout.appuio_hieradata" + if terraform state list | grep "gitfile_checkout.appuio_hieradata" > /dev/null + then + terraform state rm "module.cluster.module.lb.module.hiera[0].gitfile_checkout.appuio_hieradata" + fi # Suppress errors on the first run; it is expected to fail terraform destroy --auto-approve || true diff --git a/workflows/shared/decommission-steps.yml b/workflows/shared/decommission-steps.yml index cb798bf..423b9a8 100644 --- a/workflows/shared/decommission-steps.yml +++ b/workflows/shared/decommission-steps.yml @@ -9,6 +9,7 @@ spells: Usually of the form `.` - name: passbolt_passphrase + type: local-sensitive description: |- Your password for Passbolt. @@ -90,7 +91,7 @@ spells: kubectl delete machineautoscaler -A --all - match: And I delete all persistent volumes description: |- - This step deletes all persistent volumes on the cluster, so that the corresponding Cloudscale resources can be decommissioned by the controller. + This step deletes all persistent volumes on the cluster, so that the corresponding CSP resources can be decommissioned by the controller. By cordoning all non-master nodes and deleting all their pods (except the csi driver pods) we ensure that no new PVs are created, while the existing ones can be cleaned up. Deleting all pods has the additional benefit that we don’t have to deal with PDBs when deleting the machinesets in the next step. inputs: @@ -154,20 +155,13 @@ spells: inputs: - name: lb_fqdn_1 - name: lb_fqdn_2 + - name: control_vshn_api_token run: | set -euo pipefail - echo '###################################################################################' - echo '# #' - echo "# Please manually delete the cluster's LoadBalancer servers before proceeding. #" - echo '# #' - echo '###################################################################################' - echo - echo You can go to: - echo https://control.vshn.net/servers/definitions/appuio/${INPUT_lb_fqdn_1}/delete - echo https://control.vshn.net/servers/definitions/appuio/${INPUT_lb_fqdn_2}/delete - sleep 2 - # NOTE(aa): This step is currently annoying to automate, but once ticket PORTAL-253 is resolved, - # it should be easy. + echo "Deleting server definitions ..." + curl -XDELETE -H"$AUTH" https://control.vshn.net/api/servers/1/appuio/${INPUT_lb_fqdn_1} + curl -XDELETE -H"$AUTH" https://control.vshn.net/api/servers/1/appuio/${INPUT_lb_fqdn_2} + echo "Server definitions deleted." - match: And I decommission the LoadBalancers description: |- diff --git a/workflows/shared/image-steps.yml b/workflows/shared/image-steps.yml index bac7283..36ba0e1 100644 --- a/workflows/shared/image-steps.yml +++ b/workflows/shared/image-steps.yml @@ -36,5 +36,47 @@ spells: echo Downloading OpenShift image "$MATCH_image_name" to "$image_path" + echo curl -L "https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/${MAJOR}.${MINOR}/${MATCH_image_name}/rhcos-${MATCH_image_name}-x86_64-openstack.x86_64.qcow2.gz" curl -L "https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/${MAJOR}.${MINOR}/${MATCH_image_name}/rhcos-${MATCH_image_name}-x86_64-openstack.x86_64.qcow2.gz" | gzip -d > "$image_path" env -i "image_path=$image_path" >> "$OUTPUT" + +spells: +- match: Then I download the OpenShift OVA image for version "(?P[^"]+)" + description: |- + This step downloads the OpenShift OVA image for the version specified by in the step. + + If the image already exists locally, it skips the download. + outputs: + - name: image_path + - name: image_major + - name: image_minor + - name: image_patch + run: | + set -euo pipefail + + . "$GANDALF_SPELLBOOK_DIR"/scripts/semver.sh + + MAJOR=0 + MINOR=0 + PATCH=0 + SPECIAL="" + semverParseInto "$MATCH_image_name" MAJOR MINOR PATCH SPECIAL + + image_path="rhcos-$MAJOR.$MINOR.ova" + + env -i "image_major=$MAJOR" >> "$OUTPUT" + env -i "image_minor=$MINOR" >> "$OUTPUT" + env -i "image_patch=$PATCH" >> "$OUTPUT" + + echo "Image is $image_path" + + if [ -f "$image_path" ]; then + echo "Image $image_path already exists, skipping download." + env -i "image_path=$image_path" >> "$OUTPUT" + exit 0 + fi + + echo Downloading OpenShift image "$MATCH_image_name" to "$image_path" + + curl -L "https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/${MAJOR}.${MINOR}/${MATCH_image_name}/rhcos-${MATCH_image_name}-x86_64-vmware.x86_64.ova" > "$image_path" + env -i "image_path=$image_path" >> "$OUTPUT" diff --git a/workflows/xelon-decommission.workflow b/workflows/xelon-decommission.workflow new file mode 100644 index 0000000..9cf2680 --- /dev/null +++ b/workflows/xelon-decommission.workflow @@ -0,0 +1,4 @@ +Given I have all prerequisites installed +Then I destroy Terraform resources +And I delete all S3 buckets +And I delete the cluster's API tokens diff --git a/workflows/xelon.workflow b/workflows/xelon.workflow new file mode 100644 index 0000000..e9f15c1 --- /dev/null +++ b/workflows/xelon.workflow @@ -0,0 +1,13 @@ +Given I have all prerequisites installed +And I download the `openshift-install` binary for version "4.21" +And Cloudscale API tokens +And Xelon API tokens +And a personal VSHN GitLab access token +Then I download the OpenShift OVA image for version "4.21.0" +And I import the image into Xelon +And I set up required S3 buckets +Then I configure the OpenShift installer +Then I prepare for terraform +Then I create the bootstrap node +Then I fix the load balancer +Then I create the control plane nodes diff --git a/workflows/xelon/cilium/apiextensions.k8s.io_v1_customresourcedefinition_ciliumconfigs.cilium.io.yaml b/workflows/xelon/cilium/apiextensions.k8s.io_v1_customresourcedefinition_ciliumconfigs.cilium.io.yaml new file mode 100644 index 0000000..7c41384 --- /dev/null +++ b/workflows/xelon/cilium/apiextensions.k8s.io_v1_customresourcedefinition_ciliumconfigs.cilium.io.yaml @@ -0,0 +1,116 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.5 + name: ciliumconfigs.cilium.io +spec: + group: cilium.io + names: + categories: + - all + - cilium + kind: CiliumConfig + listKind: CiliumConfigList + plural: ciliumconfigs + shortNames: + - cconf + - cconfs + singular: ciliumconfig + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: CiliumConfig defines the configuration of Isovalent Networking + for Kubernetes and all its components + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CiliumConfigSpec defines the desired state of CiliumConfig + type: object + x-kubernetes-preserve-unknown-fields: true + status: + description: CiliumConfigStatus defines the observed state of CiliumConfig + properties: + conditions: + description: Conditions provides details on the state of the component + items: + description: Condition contains details for one aspect of the + current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, + Unknown. + enum: + - 'True' + - 'False' + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/workflows/xelon/cilium/ciliumconfig.yaml b/workflows/xelon/cilium/ciliumconfig.yaml new file mode 100644 index 0000000..5b9879b --- /dev/null +++ b/workflows/xelon/cilium/ciliumconfig.yaml @@ -0,0 +1,82 @@ +apiVersion: cilium.io/v1alpha1 +kind: CiliumConfig +metadata: + labels: + app.kubernetes.io/name: clife + name: ciliumconfig +spec: + bgpControlPlane: + enabled: false + secretsNamespace: + name: cilium + bpf: + masquerade: true + clustermesh: + apiserver: + metrics: + serviceMonitor: + enabled: true + cni: + binPath: /var/lib/cni/bin + confPath: /var/run/multus/cni/net.d + egressGateway: + enabled: true + endpointRoutes: + enabled: true + enterprise: + egressGatewayHA: + enabled: true + featureGate: + approved: + - BGPControlPlane + - EgressGatewayHA + - EgressGatewayIPv4 + - EncryptionWireguard + - EnterpriseBGPControlPlane + - L2Announcement + - LocalRedirectPolicy + envoy: + enabled: true + hubble: + metrics: + enabled: + - dns:sourceContext=workload|namespace|reserved-identity;destinationContext=workload|namespace|reserved-identity + - drop:sourceContext=workload|namespace|reserved-identity;destinationContext=workload|namespace|reserved-identity + - httpV2:sourceContext=workload|namespace|reserved-identity;destinationContext=workload|namespace|reserved-identity + serviceMonitor: + enabled: true + relay: + enabled: true + tls: + enabled: true + ipam: + mode: cluster-pool + operator: + clusterPoolIPv4MaskSize: 24 + clusterPoolIPv4PodCIDRList: + - 10.0.0.0/8 + k8sClientRateLimit: + burst: 30 + qps: 15 + k8sServiceHost: ${APIINT} + k8sServicePort: '6443' + kubeProxyReplacement: 'true' + l2announcements: + enabled: false + l7Proxy: true + localRedirectPolicy: true + operator: + prometheus: + enabled: false + serviceMonitor: + enabled: true + resources: + limits: + memory: 500Mi + requests: + cpu: 100m + memory: 300Mi + prometheus: + enabled: true + serviceMonitor: + enabled: true diff --git a/workflows/xelon/cilium/operatorgroup.yaml b/workflows/xelon/cilium/operatorgroup.yaml new file mode 100644 index 0000000..4ce3fdf --- /dev/null +++ b/workflows/xelon/cilium/operatorgroup.yaml @@ -0,0 +1,6 @@ +apiVersion: operators.coreos.com/v1alpha2 +kind: OperatorGroup +metadata: + name: clife + namespace: cilium +spec: null diff --git a/workflows/xelon/cilium/rbac.authorization.k8s.io_v1_clusterrole_clife-ciliumconfig-admin-role.yaml b/workflows/xelon/cilium/rbac.authorization.k8s.io_v1_clusterrole_clife-ciliumconfig-admin-role.yaml new file mode 100644 index 0000000..48852a2 --- /dev/null +++ b/workflows/xelon/cilium/rbac.authorization.k8s.io_v1_clusterrole_clife-ciliumconfig-admin-role.yaml @@ -0,0 +1,20 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: clife + name: clife-ciliumconfig-admin-role +rules: + - apiGroups: + - cilium.io + resources: + - ciliumconfigs + verbs: + - '*' + - apiGroups: + - cilium.io + resources: + - ciliumconfigs/status + verbs: + - get diff --git a/workflows/xelon/cilium/rbac.authorization.k8s.io_v1_clusterrole_clife-ciliumconfig-editor-role.yaml b/workflows/xelon/cilium/rbac.authorization.k8s.io_v1_clusterrole_clife-ciliumconfig-editor-role.yaml new file mode 100644 index 0000000..8819b65 --- /dev/null +++ b/workflows/xelon/cilium/rbac.authorization.k8s.io_v1_clusterrole_clife-ciliumconfig-editor-role.yaml @@ -0,0 +1,26 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: clife + name: clife-ciliumconfig-editor-role +rules: + - apiGroups: + - cilium.io + resources: + - ciliumconfigs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - cilium.io + resources: + - ciliumconfigs/status + verbs: + - get diff --git a/workflows/xelon/cilium/rbac.authorization.k8s.io_v1_clusterrole_clife-ciliumconfig-viewer-role.yaml b/workflows/xelon/cilium/rbac.authorization.k8s.io_v1_clusterrole_clife-ciliumconfig-viewer-role.yaml new file mode 100644 index 0000000..55421f4 --- /dev/null +++ b/workflows/xelon/cilium/rbac.authorization.k8s.io_v1_clusterrole_clife-ciliumconfig-viewer-role.yaml @@ -0,0 +1,22 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: clife + name: clife-ciliumconfig-viewer-role +rules: + - apiGroups: + - cilium.io + resources: + - ciliumconfigs + verbs: + - get + - list + - watch + - apiGroups: + - cilium.io + resources: + - ciliumconfigs/status + verbs: + - get diff --git a/workflows/xelon/cilium/rbac.authorization.k8s.io_v1_clusterrole_clife-manager-role.yaml b/workflows/xelon/cilium/rbac.authorization.k8s.io_v1_clusterrole_clife-manager-role.yaml new file mode 100644 index 0000000..37ecf26 --- /dev/null +++ b/workflows/xelon/cilium/rbac.authorization.k8s.io_v1_clusterrole_clife-manager-role.yaml @@ -0,0 +1,175 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: clife-manager-role +rules: + - apiGroups: + - '' + resources: + - configmaps + - endpoints + - namespaces + - persistentvolumeclaims + - resourcequotas + - secrets + - serviceaccounts + - services + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - apps + resources: + - daemonsets + - deployments + - statefulsets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - batch + resources: + - cronjobs + - jobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - cert-manager.io + resources: + - certificates + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - cilium.io + resources: + - ciliumconfigs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - cilium.io + resources: + - ciliumconfigs/finalizers + verbs: + - update + - apiGroups: + - cilium.io + resources: + - ciliumconfigs/status + verbs: + - get + - patch + - update + - apiGroups: + - gateway.networking.k8s.io + resources: + - gatewayclasses + - httproutes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - monitoring.coreos.com + resources: + - servicemonitors + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - networking.k8s.io + resources: + - ingressclasses + - ingresses + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - policy + resources: + - poddisruptionbudgets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - rbac.authorization.k8s.io + resources: + - clusterrolebindings + - rolebindings + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - rbac.authorization.k8s.io + resources: + - clusterroles + - roles + verbs: + - bind + - create + - delete + - escalate + - get + - list + - patch + - update + - watch + - apiGroups: + - security.openshift.io + resourceNames: + - hostnetwork-v2 + resources: + - securitycontextconstraints + verbs: + - use diff --git a/workflows/xelon/cilium/rbac.authorization.k8s.io_v1_clusterrole_clife-metrics-auth-role.yaml b/workflows/xelon/cilium/rbac.authorization.k8s.io_v1_clusterrole_clife-metrics-auth-role.yaml new file mode 100644 index 0000000..2601e57 --- /dev/null +++ b/workflows/xelon/cilium/rbac.authorization.k8s.io_v1_clusterrole_clife-metrics-auth-role.yaml @@ -0,0 +1,20 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: clife + name: clife-metrics-auth-role +rules: + - apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create + - apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create diff --git a/workflows/xelon/cilium/rbac.authorization.k8s.io_v1_clusterrole_clife-metrics-reader.yaml b/workflows/xelon/cilium/rbac.authorization.k8s.io_v1_clusterrole_clife-metrics-reader.yaml new file mode 100644 index 0000000..82827d2 --- /dev/null +++ b/workflows/xelon/cilium/rbac.authorization.k8s.io_v1_clusterrole_clife-metrics-reader.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: clife + name: clife-metrics-reader +rules: + - nonResourceURLs: + - /metrics + verbs: + - get diff --git a/workflows/xelon/cilium/rbac.authorization.k8s.io_v1_clusterrolebinding_clife-manager-rolebinding.yaml b/workflows/xelon/cilium/rbac.authorization.k8s.io_v1_clusterrolebinding_clife-manager-rolebinding.yaml new file mode 100644 index 0000000..2b16599 --- /dev/null +++ b/workflows/xelon/cilium/rbac.authorization.k8s.io_v1_clusterrolebinding_clife-manager-rolebinding.yaml @@ -0,0 +1,15 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: clife + name: clife-manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: clife-manager-role +subjects: + - kind: ServiceAccount + name: clife-controller-manager + namespace: cilium diff --git a/workflows/xelon/cilium/rbac.authorization.k8s.io_v1_clusterrolebinding_clife-metrics-auth-rolebinding.yaml b/workflows/xelon/cilium/rbac.authorization.k8s.io_v1_clusterrolebinding_clife-metrics-auth-rolebinding.yaml new file mode 100644 index 0000000..f30f62b --- /dev/null +++ b/workflows/xelon/cilium/rbac.authorization.k8s.io_v1_clusterrolebinding_clife-metrics-auth-rolebinding.yaml @@ -0,0 +1,15 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: clife + name: clife-metrics-auth-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: clife-metrics-auth-role +subjects: + - kind: ServiceAccount + name: clife-controller-manager + namespace: cilium diff --git a/workflows/xelon/cilium/rbac.authorization.k8s.io_v1_role_clife-leader-election-role.yaml b/workflows/xelon/cilium/rbac.authorization.k8s.io_v1_role_clife-leader-election-role.yaml new file mode 100644 index 0000000..8d78b4f --- /dev/null +++ b/workflows/xelon/cilium/rbac.authorization.k8s.io_v1_role_clife-leader-election-role.yaml @@ -0,0 +1,40 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: clife + name: clife-leader-election-role + namespace: cilium +rules: + - apiGroups: + - '' + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete + - apiGroups: + - '' + resources: + - events + verbs: + - create + - patch diff --git a/workflows/xelon/cilium/rbac.authorization.k8s.io_v1_rolebinding_clife-leader-election-rolebinding.yaml b/workflows/xelon/cilium/rbac.authorization.k8s.io_v1_rolebinding_clife-leader-election-rolebinding.yaml new file mode 100644 index 0000000..1652ade --- /dev/null +++ b/workflows/xelon/cilium/rbac.authorization.k8s.io_v1_rolebinding_clife-leader-election-rolebinding.yaml @@ -0,0 +1,16 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: clife + name: clife-leader-election-rolebinding + namespace: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: clife-leader-election-role +subjects: + - kind: ServiceAccount + name: clife-controller-manager + namespace: cilium diff --git a/workflows/xelon/cilium/subscription.yaml b/workflows/xelon/cilium/subscription.yaml new file mode 100644 index 0000000..40b6103 --- /dev/null +++ b/workflows/xelon/cilium/subscription.yaml @@ -0,0 +1,24 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: clife + namespace: cilium +spec: + channel: '1.18' + config: + env: + - name: KUBERNETES_SERVICE_HOST + value: ${API_INT} + - name: KUBERNETES_SERVICE_PORT + value: '6443' + resources: + limits: + cpu: 100m + memory: 500Mi + requests: + cpu: 100m + memory: 250Mi + installPlanApproval: Manual + name: clife + source: certified-operators + sourceNamespace: openshift-marketplace diff --git a/workflows/xelon/cilium/v1_namespace_cilium.yaml b/workflows/xelon/cilium/v1_namespace_cilium.yaml new file mode 100644 index 0000000..f10a1d9 --- /dev/null +++ b/workflows/xelon/cilium/v1_namespace_cilium.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Namespace +metadata: + annotations: + openshift.io/node-selector: '' + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: clife + control-plane: controller-manager + openshift.io/cluster-logging: 'true' + openshift.io/cluster-monitoring: 'true' + openshift.io/run-level: '0' + pod-security.kubernetes.io/audit: privileged + name: cilium diff --git a/workflows/xelon/cilium/v1_service_clife-metrics.yaml b/workflows/xelon/cilium/v1_service_clife-metrics.yaml new file mode 100644 index 0000000..9105d45 --- /dev/null +++ b/workflows/xelon/cilium/v1_service_clife-metrics.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: clife + control-plane: controller-manager + name: clife-metrics + namespace: cilium +spec: + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: 18443 + selector: + control-plane: controller-manager diff --git a/workflows/xelon/cilium/v1_serviceaccount_clife-controller-manager.yaml b/workflows/xelon/cilium/v1_serviceaccount_clife-controller-manager.yaml new file mode 100644 index 0000000..a168feb --- /dev/null +++ b/workflows/xelon/cilium/v1_serviceaccount_clife-controller-manager.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: clife + name: clife-controller-manager + namespace: cilium diff --git a/workflows/xelon/decommission-steps.yml b/workflows/xelon/decommission-steps.yml new file mode 100644 index 0000000..19f5ba9 --- /dev/null +++ b/workflows/xelon/decommission-steps.yml @@ -0,0 +1,83 @@ +spells: +- match: Then I destroy Terraform resources + description: This step runs terraform destroy. + inputs: + - name: xelon_token + - name: xelon_client_id + - name: ignition_bootstrap + run: | + set -euo pipefail + + installer_dir="$(pwd)/target" + + cat < terraform.env + TF_VAR_xelon_token=${INPUT_xelon_token} + TF_VAR_xelon_client_id=${INPUT_xelon_client_id} + TF_VAR_ignition_bootstrap=${INPUT_ignition_bootstrap} + EOF + tf_image="registry.gitlab.com/gitlab-org/terraform-images/releases/terraform" + tf_tag='1.5.7' + + echo "Using Terraform image: ${tf_image}:${tf_tag}" + + base_dir=$(pwd) + terraform() { + touch .terraformrc + docker run --rm -e REAL_UID="$(id -u)" -e TF_CLI_CONFIG_FILE=/tf/.terraformrc --env-file "${base_dir}/terraform.env" -w /tf -v "$(pwd):/tf" --ulimit memlock=-1 "${tf_image}:${tf_tag}" /tf/terraform.sh "${@}" + } + + + pushd terraform + + terraform init + + terraform destroy -auto-approve || true + + # NOTE(aa): Destruction tends to fail the first time due to race condition between deleting the network and deleting IPs from the network. + terraform destroy + +- match: And I delete all S3 buckets + description: |- + This step deletes the cluster's associated S3 buckets from Cloudscale. + inputs: + - name: cloudscale_token + type: sensitive + - name: clsc_region + - name: commodore_cluster_id + run: | + set -euo pipefail + # Use already exiting bucket user + response=$(curl -sH "Authorization: Bearer $INPUT_cloudscale_token" \ + https://api.cloudscale.ch/v1/objects-users | \ + jq -e ".[] | select(.display_name == \"${INPUT_commodore_cluster_id}\")") + + # configure minio client to use the bucket + mc alias set \ + "${INPUT_commodore_cluster_id}" "https://objects.${INPUT_clsc_region}.cloudscale.ch" \ + "$(echo "$response" | jq -r '.keys[0].access_key')" \ + "$(echo "$response" | jq -r '.keys[0].secret_key')" + + # delete bootstrap-ignition bucket (should already be deleted after setup) + mc rb "${INPUT_commodore_cluster_id}/${INPUT_commodore_cluster_id}-bootstrap-ignition" --force || true + +- match: And I delete the cluster's API tokens + description: |- + This step deletes the cluster's associated Cloudscale API tokens from Cloudscale. + inputs: + - name: cloudscale_token + run: | + set -euo pipefail + echo '##############################################################' + echo '# #' + echo '# Please delete the Cloudscale token #' + echo '# associated with this cluster via the Cloudscale console. #' + echo '# #' + echo '##############################################################' + + echo -n "Checking for deletion of main Cloudscale token ..." + while [[ $(curl -sH "Authorization: Bearer ${INPUT_cloudscale_token}" https://api.cloudscale.ch/v1/flavors -o /dev/null -w"%{http_code}") != 401 ]] + do + echo -n . + sleep 3 + done + echo " Success." diff --git a/workflows/xelon/image-steps.yml b/workflows/xelon/image-steps.yml new file mode 100644 index 0000000..ee8614b --- /dev/null +++ b/workflows/xelon/image-steps.yml @@ -0,0 +1,80 @@ +spells: +- match: And I set up required S3 buckets + description: |- + This step sets up the required S3 buckets for the OpenShift cluster installation. + + It uses the MinIO Client (`mc`) to create the necessary buckets if they do not already exist. + inputs: + - name: cloudscale_token + - name: commodore_cluster_id + - name: clsc_region + outputs: + - name: bucket_user + type: sensitive + run: | + set -euo pipefail + + response=$(curl -sH "Authorization: Bearer ${INPUT_cloudscale_token}" \ + https://api.cloudscale.ch/v1/objects-users | \ + jq -e ".[] | select(.display_name == \"${INPUT_commodore_cluster_id}\")" ||:) + if [ -z "$response" ]; then + echo "Creating Cloudscale S3 user for cluster ID '${INPUT_commodore_cluster_id}'..." + response=$(curl -sH "Authorization: Bearer ${INPUT_cloudscale_token}" \ + -F display_name=${INPUT_commodore_cluster_id} \ + https://api.cloudscale.ch/v1/objects-users) + echo "Created user with id $(echo "$response" | jq -r .id)" + else + echo "Cloudscale S3 user for cluster ID '${INPUT_commodore_cluster_id}' already exists. id: $(echo "$response" | jq -r .id)" + fi + + echo -n "Waiting for S3 credentials to become available ..." + until mc alias set \ + "${INPUT_commodore_cluster_id}" "https://objects.${INPUT_clsc_region}.cloudscale.ch" \ + "$(echo "$response" | jq -r '.keys[0].access_key')" \ + "$(echo "$response" | jq -r '.keys[0].secret_key')" + do + echo -n . + sleep 5 + done + echo "OK" + + mc mb --ignore-existing \ + "${INPUT_commodore_cluster_id}/${INPUT_commodore_cluster_id}-bootstrap-ignition" + + keyid=$(mc alias list ${INPUT_commodore_cluster_id} -json | jq -r .accessKey) + export AWS_ACCESS_KEY_ID="${keyid}" + secretkey=$(mc alias list ${INPUT_commodore_cluster_id} -json | jq -r .secretKey) + export AWS_SECRET_ACCESS_KEY="${secretkey}" + + echo "S3 buckets are set up." + + env -i "bucket_user=$(echo "$response" | jq -c .)" >> "$OUTPUT" + +- match: And I import the image into Xelon + description: |- + This step uploads the Red Hat CoreOS image to Xelon and imports it as a template. + inputs: + - name: image_path + - name: commodore_cluster_id + - name: image_major + - name: image_minor + outputs: + - name: rhcos_template_name + run: | + set -euo pipefail + + rm -rf extracted_image + mkdir extracted_image + tar -C extracted_image -xf "${INPUT_image_path}" + + echo '##############################################################' + echo '# #' + echo '# Please import the image into Xelon VDC using the Web UI. #' + echo '# #' + echo '##############################################################' + echo + echo 'You can go to https://vdc.xelon.ch/import-template' + echo 'You will find all the files to import in '"$( pwd )/extracted_image" + echo + echo 'When you are done, record the template name in the outputs.' + sleep 2 diff --git a/workflows/xelon/init-steps.yml b/workflows/xelon/init-steps.yml new file mode 100644 index 0000000..f4f4f6f --- /dev/null +++ b/workflows/xelon/init-steps.yml @@ -0,0 +1,529 @@ +spells: +- match: Given I have all prerequisites installed + description: |- + This step checks if all necessary prerequisites are installed on your system, + including 'yq' (version 4 or higher, by Mike Farah) and 'oc' (OpenShift CLI). + run: | + set -euo pipefail + echo "TODO: requirements are copied from cloudscale, please update!" + echo + echo "Checking prerequisites..." + + if which yq >/dev/null 2>&1 ; then { echo "✅ yq is installed."; } ; else { echo "❌ yq is not installed. Please install yq to proceed."; exit 1; } ; fi + if yq --version | grep -E 'version v[4-9]\.' | grep 'mikefarah' >/dev/null 2>&1 ; then { echo "✅ yq by mikefarah version 4 or higher is installed."; } ; else { echo "❌ yq version 4 or higher is required. Please upgrade yq to proceed."; exit 1; } ; fi + + if which jq >/dev/null 2>&1 ; then { echo "✅ jq is installed."; } ; else { echo "❌ jq is not installed. Please install jq to proceed."; exit 1; } ; fi + + if which oc >/dev/null 2>&1 ; then { echo "✅ oc (OpenShift CLI) is installed."; } ; else { echo "❌ oc (OpenShift CLI) is not installed. Please install oc to proceed."; exit 1; } ; fi + + if which vault >/dev/null 2>&1 ; then { echo "✅ vault (HashiCorp Vault) is installed."; } ; else { echo "❌ vault (HashiCorp Vault) is not installed. Please install vault to proceed."; exit 1; } ; fi + + if which curl >/dev/null 2>&1 ; then { echo "✅ curl is installed."; } ; else { echo "❌ curl is not installed. Please install curl to proceed."; exit 1; } ; fi + + if which docker >/dev/null 2>&1 ; then { echo "✅ docker is installed."; } ; else { echo "❌ docker is not installed. Please install docker to proceed."; exit 1; } ; fi + + if which glab >/dev/null 2>&1 ; then { echo "✅ glab (GitLab CLI) is installed."; } ; else { echo "❌ glab (GitLab CLI) is not installed. Please install glab to proceed."; exit 1; } ; fi + + if which host >/dev/null 2>&1 ; then { echo "✅ host (DNS lookup utility) is installed."; } ; else { echo "❌ host (DNS lookup utility) is not installed. Please install host to proceed."; exit 1; } ; fi + + if which mc >/dev/null 2>&1 ; then { echo "✅ mc (MinIO Client) is installed."; } ; else { echo "❌ mc (MinIO Client) is not installed. Please install mc >= RELEASE.2024-01-18T07-03-39Z to proceed."; exit 1; } ; fi + mc_version=$(mc --version | grep -Eo 'RELEASE[^ ]+') + if echo "$mc_version" | grep -E 'RELEASE\.202[4-9]-' >/dev/null 2>&1 ; then { echo "✅ mc version ${mc_version} is sufficient."; } ; else { echo "❌ mc version ${mc_version} is insufficient. Please upgrade mc to >= RELEASE.2024-01-18T07-03-39Z to proceed."; exit 1; } ; fi + + if which aws >/dev/null 2>&1 ; then { echo "✅ aws (AWS CLI) is installed."; } ; else { echo "❌ aws (AWS CLI) is not installed. Please install aws to proceed. Our recommended installer is uv: 'uv tool install awscli'"; exit 1; } ; fi + + if which restic >/dev/null 2>&1 ; then { echo "✅ restic (Backup CLI) is installed."; } ; else { echo "❌ restic (Backup CLI) is not installed. Please install restic to proceed."; exit 1; } ; fi + + if which emergency-credentials-receive >/dev/null 2>&1 ; then { echo "✅ emergency-credentials-receive (Cluster emergency access helper) is installed."; } ; else { echo "❌ emergency-credentials-receive is not installed. Please install it from https://github.com/vshn/emergency-credentials-receive ."; exit 1; } ; fi + + if which commodore >/dev/null 2>&1 ; then { echo "✅ commodore (Project Syn) is installed."; } ; else { echo "❌ commodore (Project Syn) is not installed. Please install it with 'uv tool install syn-commodore && commodore tool install --missing' ."; exit 1; } ; fi + + echo "TODO: requirements are copied from cloudscale, please update!" + echo "✅ All prerequisites are met." +- match: And Cloudscale API tokens + inputs: + - name: cloudscale_token + type: sensitive + description: |- + Cloudscale API token with read+write permissions. + + Used for the ignition bucket + EVENTUALLY we'll want to move to using Xelon's S3 for this + description: |- + Create 1 new cloudscale API tokens with read+write permissions and name them + {{ .commodore_cluster_id }} and {{ .commodore_cluster_id }}_floaty in a random test cloudscale project + + This step currently does not validate whether the tokens have write permission. + run: | + set -euo pipefail + if [[ $( curl -sH "Authorization: Bearer ${INPUT_cloudscale_token}" https://api.cloudscale.ch/v1/flavors -o /dev/null -w"%{http_code}" ) != 200 ]] + then + echo "Cloudscale token not valid!" + exit 1 + fi +- match: And Xelon API tokens + inputs: + - name: xelon_token + type: sensitive + description: |- + Xelon API token with read+write permissions. + + Go to https://vdc.xelon.ch/user-account and click "Add Service" + - name: xelon_client_id + type: sensitive + description: |- + Xelon API client ID + + You receive this when you create the service token. + description: |- + Create 1 new Xelon Service Token for your account. + + This step currently does not validate whether the tokens have write permission. + run: | + set -euo pipefail + if [[ $( curl -sH "Authorization: Bearer ${INPUT_xelon_token}" -H"X-User-Id: $INPUT_xelon_client_id" https://hq.xelon.ch/api/service/tenants -o /dev/null -w"%{http_code}" ) != 200 ]] + then + echo "Xelon token not valid!" + exit 1 + fi + + +- match: Then I configure the OpenShift installer + description: |- + This step configures the OpenShift installer for the Xelon cluster by generating + the necessary installation files using Commodore. + inputs: + - name: commodore_cluster_id + - name: commodore_tenant_id + - name: base_domain + - name: cluster_domain + - name: vault_address + - name: vault_login_method + - name: redhat_pull_secret + - name: clsc_region + - name: bucket_user + - name: cloudscale_token + - name: openshift_install_bin + outputs: + - name: ignition_bootstrap + - name: ssh_public_key_path + run: | + set -euo pipefail + + openshift-install() { + "${INPUT_openshift_install_bin}" "${@}" + } + + export VAULT_ADDR="${INPUT_vault_address}" + vault login -method="${INPUT_vault_login_method}" + + ssh_private_key="$(pwd)/ssh_${INPUT_commodore_cluster_id}" + ssh_public_key="${ssh_private_key}.pub" + + env -i "ssh_public_key_path=$ssh_public_key" >> "$OUTPUT" + + if vault kv get -format=json clusters/kv/${INPUT_commodore_tenant_id}/${INPUT_commodore_cluster_id}/xelon/ssh >/dev/null 2>&1; then + echo "SSH keypair for cluster ${INPUT_commodore_cluster_id} already exists in Vault, skipping generation." + + vault kv get -format=json clusters/kv/${INPUT_commodore_tenant_id}/${INPUT_commodore_cluster_id}/xelon/ssh | \ + jq -r '.data.data.private_key|@base64d' > "${ssh_private_key}" + + chmod 600 "${ssh_private_key}" + ssh-keygen -f "${ssh_private_key}" -y > "${ssh_public_key}" + + else + echo "Generating new SSH keypair for cluster ${INPUT_commodore_cluster_id}." + + ssh-keygen -C "vault@${INPUT_commodore_cluster_id}" -t ed25519 -f "$ssh_private_key" -N '' + + base64_no_wrap='base64' + if [[ "$OSTYPE" == "linux"* ]]; then + base64_no_wrap='base64 --wrap 0' + fi + + # vault kv put clusters/kv/${INPUT_commodore_tenant_id}/${INPUT_commodore_cluster_id}/xelon/ssh \ + # private_key="$(cat "$ssh_private_key" | eval "$base64_no_wrap")" + fi + + echo Adding SSH private key to ssh-agent... + echo You might need to start the ssh-agent first using: eval "\$(ssh-agent)" + echo ssh-add "$ssh_private_key" + ssh-add "$ssh_private_key" + + installer_dir="$(pwd)/target" + rm -rf "${installer_dir}" + mkdir -p "${installer_dir}" + + cat > "${installer_dir}/install-config.yaml" < "${installer_dir}/manifests/cilium_$(basename "$f")" + done + + + echo "CILIUMCONFIG:" + cat "${installer_dir}"/manifests/cilium_ciliumconfig* + echo + echo "SUBSCRIPTION:" + cat "${installer_dir}"/manifests/cilium_subscription* + echo + + # shellcheck disable=2016 + # We don't want the shell to execute network.operator.openshift.io as a + # command, so we need single quotes here. + echo 'Generating initial `network.operator.openshift.io` resource...' + yq '{ + "apiVersion": "operator.openshift.io/v1", + "kind": "Network", + "metadata": { + "name": "cluster" + }, + "spec": { + "deployKubeProxy": false, + "clusterNetwork": .spec.clusterNetwork, + "externalIP": { + "policy": {} + }, + "networkType": "Cilium", + "serviceNetwork": .spec.serviceNetwork + }}' "${installer_dir}/manifests/cluster-network-02-config.yml" \ + > "${installer_dir}/manifests/cilium_cluster-network-operator.yaml" + + gen_cluster_domain=$(yq e '.spec.baseDomain' \ + "${installer_dir}/manifests/cluster-dns-02-config.yml") + if [ "$gen_cluster_domain" != "$INPUT_cluster_domain" ]; then + echo -e "\033[0;31mGenerated cluster domain doesn't match expected cluster domain: Got '$gen_cluster_domain', want '$INPUT_cluster_domain'\033[0;0m" + sleep 1 + exit 1 + else + echo -e "\033[0;32mGenerated cluster domain matches expected cluster domain.\033[0;0m" + sleep 1 + fi + + echo Running OpenShift installer to create ignition configs... + openshift-install --dir "${installer_dir}" \ + create ignition-configs + + mc alias set \ + "${INPUT_commodore_cluster_id}" "https://objects.${INPUT_clsc_region}.cloudscale.ch" \ + "$(echo "$INPUT_bucket_user" | jq -r '.keys[0].access_key')" \ + "$(echo "$INPUT_bucket_user" | jq -r '.keys[0].secret_key')" + + mc cp "${installer_dir}/bootstrap.ign" "${INPUT_commodore_cluster_id}/${INPUT_commodore_cluster_id}-bootstrap-ignition/" + + ignition_bootstrap=$(mc share download \ + --json --expire=24h \ + "${INPUT_commodore_cluster_id}/${INPUT_commodore_cluster_id}-bootstrap-ignition/bootstrap.ign" | jq -r '.share') + + env -i "ignition_bootstrap=$ignition_bootstrap" >> "$OUTPUT" + + echo "✅ OpenShift installer configured successfully." + +- match: Then I prepare for terraform + description: |- + This step creates the terraform environment + inputs: + - name: commodore_cluster_id + - name: ignition_bootstrap + - name: xelon_token + - name: xelon_client_id + - name: csp_region + - name: base_domain + - name: rhcos_template_name + - name: terraform_git_commit + description: | + Git ref for the terraform repo - if you don't know what to use, use master + See https://github.com/appuio/terraform-openshift4-xelon + - name: ssh_public_key_path + run: | + set -euo pipefail + + mkdir -p terraform + installer_dir="$(pwd)/target" + + sshsig="$( cat "${INPUT_ssh_public_key_path}" )" + jq -r '.ignition.security.tls.certificateAuthorities[0].source' "${installer_dir}/master.ign" | awk -F ',' '{ print $2 }' | base64 --decode > ignition.ca + ignitionca="$( awk '{printf "%s\\n", $0}' ignition.ca )" + cat < terraform/main.tf.json + { + "module": { + "cluster": { + "base_domain": "${INPUT_base_domain}", + "cluster_id": "${INPUT_commodore_cluster_id}", + "xelon_cloud": "${INPUT_csp_region}", + "xelon_token": "\${var.xelon_token}", + "xelon_client_id": "\${var.xelon_client_id}", + "ignition_bootstrap": "\${var.ignition_bootstrap}", + "ignition_ca": "${ignitionca}", + "rhcos_template_name": "${INPUT_rhcos_template_name}", + "source": "git::https://github.com/appuio/terraform-openshift4-xelon.git//?ref=${INPUT_terraform_git_commit}", + "ssh_key": "${sshsig}" + } + } + } + EOF + + cat <<"EOF" > terraform/variables.tf.json + { + "variable": { + "ignition_bootstrap": { + "default": "" + }, + "xelon_token": { + "default": "" + }, + "xelon_client_id": { + "default": "" + } + } + } + EOF + + cat <<"EOF" > terraform/outputs.tf.json + { + "output": { + "api_ip": { + "value": "${module.cluster.api_ip}" + }, + "ingress_ip": { + "value": "${module.cluster.ingress_ip}" + }, + "cluster_dns": { + "value": "${module.cluster.dns_entries}" + }, + "subnet_uuid": { + "value": "${module.cluster.network_id}" + } + } + } + EOF + + cat <<"EOF" > terraform/terraform.sh + #!/bin/sh + # Intended to run as root, this script ensures that the non-privileged user + # that we want to run as in the end can use git-https and curl. + # TODO: build custom Terraform CI image based on GitLab's image. + adduser -D -s /bin/sh -u "${REAL_UID}" -h /tf terraform + apk add --no-cache curl >/dev/null 2>&1 + export GIT_ASKPASS=/tf/git-askpass.sh + # Note: busybox `su` can't directly execute a binary, so we use the secondary + # script which only does `exec terraform`. + su -p terraform /tf/tf.sh -- "$@" + EOF + chmod +x terraform/terraform.sh + + cat <<"EOF" > terraform/tf.sh + #!/bin/sh + exec terraform "$@" + EOF + chmod +x terraform/tf.sh + +- match: Then I create the bootstrap node + description: |- + This step provisions the bootstrap node for the Xelon OpenShift cluster using Terraform. + inputs: + - name: xelon_token + - name: xelon_client_id + - name: ignition_bootstrap + run: | + set -euo pipefail + + installer_dir="$(pwd)/target" + + cat < terraform.env + TF_VAR_xelon_token=${INPUT_xelon_token} + TF_VAR_xelon_client_id=${INPUT_xelon_client_id} + TF_VAR_ignition_bootstrap=${INPUT_ignition_bootstrap} + EOF + tf_image="registry.gitlab.com/gitlab-org/terraform-images/releases/terraform" + tf_tag='1.5.7' + + echo "Using Terraform image: ${tf_image}:${tf_tag}" + + base_dir=$(pwd) + terraform() { + touch .terraformrc + docker run --rm -e REAL_UID="$(id -u)" -e TF_CLI_CONFIG_FILE=/tf/.terraformrc --env-file "${base_dir}/terraform.env" -w /tf -v "$(pwd):/tf" --ulimit memlock=-1 "${tf_image}:${tf_tag}" /tf/terraform.sh "${@}" + } + + + pushd terraform + + terraform init + + cat > override.tf </dev/null; do + echo -n "." + sleep 5 + done && echo "✅ API is up" + env -i "kubeconfig_path=${installer_dir}/auth/kubeconfig" >> "$OUTPUT" + +- match: Then I create the control plane nodes + description: |- + This step provisions the control plane nodes for the Xelon OpenShift cluster using Terraform. + inputs: + - name: xelon_token + - name: xelon_client_id + - name: ignition_bootstrap + - name: cluster_domain + - name: kubeconfig_path + run: | + set -euo pipefail + + installer_dir="$(pwd)/target" + + cat < terraform.env + TF_VAR_xelon_token=${INPUT_xelon_token} + TF_VAR_xelon_client_id=${INPUT_xelon_client_id} + TF_VAR_ignition_bootstrap=${INPUT_ignition_bootstrap} + EOF + tf_image="registry.gitlab.com/gitlab-org/terraform-images/releases/terraform" + tf_tag='1.5.7' + + echo "Using Terraform image: ${tf_image}:${tf_tag}" + + base_dir=$(pwd) + terraform() { + touch .terraformrc + docker run --rm -e REAL_UID="$(id -u)" -e TF_CLI_CONFIG_FILE=/tf/.terraformrc --env-file "${base_dir}/terraform.env" -w /tf -v "$(pwd):/tf" --ulimit memlock=-1 "${tf_image}:${tf_tag}" /tf/terraform.sh "${@}" + } + + + pushd terraform + + terraform init + + cat > override.tf <