Skip to content

Commit 5352198

Browse files
Pearl1594nvazquez
Pearl1594
authored and
nvazquez
committed
CKS: Externalize control and worker node setup wait time and installation attempts
1 parent 67c75e1 commit 5352198

7 files changed

Lines changed: 145 additions & 7 deletions

File tree

‎plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2288,7 +2288,11 @@ public ConfigKey<?>[] getConfigKeys() {
22882288
KubernetesClusterUpgradeRetries,
22892289
KubernetesClusterAddNodeTimeout,
22902290
KubernetesClusterExperimentalFeaturesEnabled,
2291-
KubernetesMaxClusterSize
2291+
KubernetesMaxClusterSize,
2292+
KubernetesControlNodeInstallAttemptWait,
2293+
KubernetesControlNodeInstallReattempts,
2294+
KubernetesWorkerNodeInstallAttemptWait,
2295+
KubernetesWorkerNodeInstallReattempts
22922296
};
22932297
}
22942298
}

‎plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,30 @@ public interface KubernetesClusterService extends PluggableService, Configurable
105105
true,
106106
ConfigKey.Scope.Account,
107107
KubernetesServiceEnabled.key());
108+
static final ConfigKey<Long> KubernetesControlNodeInstallAttemptWait = new ConfigKey<Long>("Advanced", Long.class,
109+
"cloud.kubernetes.control.node.install.attempt.wait.duration",
110+
"15",
111+
"Time in seconds for the installation process to wait before it re-attempts",
112+
true,
113+
KubernetesServiceEnabled.key());
114+
static final ConfigKey<Long> KubernetesControlNodeInstallReattempts = new ConfigKey<Long>("Advanced", Long.class,
115+
"cloud.kubernetes.control.node.install.reattempt.count",
116+
"100",
117+
"Number of times the offline installation of K8S will be re-attempted",
118+
true,
119+
KubernetesServiceEnabled.key());
120+
final ConfigKey<Long> KubernetesWorkerNodeInstallAttemptWait = new ConfigKey<Long>("Advanced", Long.class,
121+
"cloud.kubernetes.worker.node.install.attempt.wait.duration",
122+
"30",
123+
"Time in seconds for the installation process to wait before it re-attempts",
124+
true,
125+
KubernetesServiceEnabled.key());
126+
static final ConfigKey<Long> KubernetesWorkerNodeInstallReattempts = new ConfigKey<Long>("Advanced", Long.class,
127+
"cloud.kubernetes.worker.node.install.reattempt.count",
128+
"40",
129+
"Number of times the offline installation of K8S will be re-attempted",
130+
true,
131+
KubernetesServiceEnabled.key());
108132

109133
KubernetesCluster findById(final Long id);
110134

‎plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,14 +36,19 @@
3636

3737
import javax.inject.Inject;
3838

39+
import com.cloud.kubernetes.cluster.KubernetesClusterDetailsVO;
3940
import com.cloud.kubernetes.cluster.KubernetesClusterHelper.KubernetesClusterNodeType;
41+
import com.cloud.kubernetes.cluster.KubernetesClusterService;
42+
import com.cloud.kubernetes.cluster.utils.KubernetesClusterUtil;
4043
import com.cloud.network.rules.FirewallManager;
4144
import com.cloud.network.rules.RulesService;
4245
import com.cloud.network.rules.dao.PortForwardingRulesDao;
4346
import com.cloud.offering.NetworkOffering;
4447
import com.cloud.offerings.dao.NetworkOfferingDao;
48+
import com.cloud.user.SSHKeyPairVO;
4549
import com.cloud.utils.db.TransactionCallbackWithException;
4650
import com.cloud.utils.net.Ip;
51+
import org.apache.cloudstack.api.ApiConstants;
4752
import org.apache.cloudstack.api.BaseCmd;
4853
import org.apache.cloudstack.api.command.user.firewall.CreateFirewallRuleCmd;
4954
import org.apache.cloudstack.api.command.user.network.CreateNetworkACLCmd;
@@ -166,6 +171,79 @@ protected void init() {
166171
kubernetesClusterNodeNamePrefix = getKubernetesClusterNodeNamePrefix();
167172
}
168173

174+
private String getKubernetesNodeConfig(final String joinIp, final boolean ejectIso) throws IOException {
175+
String k8sNodeConfig = readResourceFile("/conf/k8s-node.yml");
176+
final String sshPubKey = "{{ k8s.ssh.pub.key }}";
177+
final String joinIpKey = "{{ k8s_control_node.join_ip }}";
178+
final String clusterTokenKey = "{{ k8s_control_node.cluster.token }}";
179+
final String ejectIsoKey = "{{ k8s.eject.iso }}";
180+
final String installWaitTime = "{{ k8s.install.wait.time }}";
181+
final String installReattemptsCount = "{{ k8s.install.reattempts.count }}";
182+
183+
final Long waitTime = KubernetesClusterService.KubernetesWorkerNodeInstallAttemptWait.value();
184+
final Long reattempts = KubernetesClusterService.KubernetesWorkerNodeInstallReattempts.value();
185+
String pubKey = "- \"" + configurationDao.getValue("ssh.publickey") + "\"";
186+
String sshKeyPair = kubernetesCluster.getKeyPair();
187+
if (StringUtils.isNotEmpty(sshKeyPair)) {
188+
SSHKeyPairVO sshkp = sshKeyPairDao.findByName(owner.getAccountId(), owner.getDomainId(), sshKeyPair);
189+
if (sshkp != null) {
190+
pubKey += "\n - \"" + sshkp.getPublicKey() + "\"";
191+
}
192+
}
193+
k8sNodeConfig = k8sNodeConfig.replace(sshPubKey, pubKey);
194+
k8sNodeConfig = k8sNodeConfig.replace(joinIpKey, joinIp);
195+
k8sNodeConfig = k8sNodeConfig.replace(clusterTokenKey, KubernetesClusterUtil.generateClusterToken(kubernetesCluster));
196+
k8sNodeConfig = k8sNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso));
197+
k8sNodeConfig = k8sNodeConfig.replace(installWaitTime, String.valueOf(waitTime));
198+
k8sNodeConfig = k8sNodeConfig.replace(installReattemptsCount, String.valueOf(reattempts));
199+
k8sNodeConfig = updateKubeConfigWithRegistryDetails(k8sNodeConfig);
200+
201+
return k8sNodeConfig;
202+
}
203+
204+
protected String updateKubeConfigWithRegistryDetails(String k8sConfig) {
205+
/* genarate /etc/containerd/config.toml file on the nodes only if Kubernetes cluster is created to
206+
* use docker private registry */
207+
String registryUsername = null;
208+
String registryPassword = null;
209+
String registryUrl = null;
210+
211+
List<KubernetesClusterDetailsVO> details = kubernetesClusterDetailsDao.listDetails(kubernetesCluster.getId());
212+
for (KubernetesClusterDetailsVO detail : details) {
213+
if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_USER_NAME)) {
214+
registryUsername = detail.getValue();
215+
}
216+
if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_PASSWORD)) {
217+
registryPassword = detail.getValue();
218+
}
219+
if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_URL)) {
220+
registryUrl = detail.getValue();
221+
}
222+
}
223+
224+
if (StringUtils.isNoneEmpty(registryUsername, registryPassword, registryUrl)) {
225+
// Update runcmd in the cloud-init configuration to run a script that updates the containerd config with provided registry details
226+
String runCmd = "- bash -x /opt/bin/setup-containerd";
227+
228+
String registryEp = registryUrl.split("://")[1];
229+
k8sConfig = k8sConfig.replace("- containerd config default > /etc/containerd/config.toml", runCmd);
230+
final String registryUrlKey = "{{registry.url}}";
231+
final String registryUrlEpKey = "{{registry.url.endpoint}}";
232+
final String registryAuthKey = "{{registry.token}}";
233+
final String registryUname = "{{registry.username}}";
234+
final String registryPsswd = "{{registry.password}}";
235+
236+
final String usernamePasswordKey = registryUsername + ":" + registryPassword;
237+
String base64Auth = Base64.encodeBase64String(usernamePasswordKey.getBytes(com.cloud.utils.StringUtils.getPreferredCharset()));
238+
k8sConfig = k8sConfig.replace(registryUrlKey, registryUrl);
239+
k8sConfig = k8sConfig.replace(registryUrlEpKey, registryEp);
240+
k8sConfig = k8sConfig.replace(registryUname, registryUsername);
241+
k8sConfig = k8sConfig.replace(registryPsswd, registryPassword);
242+
k8sConfig = k8sConfig.replace(registryAuthKey, base64Auth);
243+
}
244+
return k8sConfig;
245+
}
246+
169247
protected DeployDestination plan(final long nodesCount, final DataCenter zone, final ServiceOffering offering) throws InsufficientServerCapacityException {
170248
final int cpu_requested = offering.getCpu() * offering.getSpeed();
171249
final long ram_requested = offering.getRamSize() * 1024L * 1024L;

‎plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,9 @@ private String getKubernetesControlNodeConfig(final String controlNodeIp, final
141141
final String clusterToken = "{{ k8s_control_node.cluster.token }}";
142142
final String clusterInitArgsKey = "{{ k8s_control_node.cluster.initargs }}";
143143
final String ejectIsoKey = "{{ k8s.eject.iso }}";
144+
final String installWaitTime = "{{ k8s.install.wait.time }}";
145+
final String installReattemptsCount = "{{ k8s.install.reattempts.count }}";
146+
144147
final List<String> addresses = new ArrayList<>();
145148
addresses.add(controlNodeIp);
146149
if (!serverIp.equals(controlNodeIp)) {
@@ -152,6 +155,8 @@ private String getKubernetesControlNodeConfig(final String controlNodeIp, final
152155
final String tlsClientCert = CertUtils.x509CertificateToPem(certificate.getClientCertificate());
153156
final String tlsPrivateKey = CertUtils.privateKeyToPem(certificate.getPrivateKey());
154157
final String tlsCaCert = CertUtils.x509CertificatesToPem(certificate.getCaCertificates());
158+
final Long waitTime = KubernetesClusterService.KubernetesControlNodeInstallAttemptWait.value();
159+
final Long reattempts = KubernetesClusterService.KubernetesControlNodeInstallReattempts.value();
155160
k8sControlNodeConfig = k8sControlNodeConfig.replace(apiServerCert, tlsClientCert.replace("\n", "\n "));
156161
k8sControlNodeConfig = k8sControlNodeConfig.replace(apiServerKey, tlsPrivateKey.replace("\n", "\n "));
157162
k8sControlNodeConfig = k8sControlNodeConfig.replace(caCert, tlsCaCert.replace("\n", "\n "));
@@ -163,6 +168,8 @@ private String getKubernetesControlNodeConfig(final String controlNodeIp, final
163168
pubKey += "\n - \"" + sshkp.getPublicKey() + "\"";
164169
}
165170
}
171+
k8sControlNodeConfig = k8sControlNodeConfig.replace(installWaitTime, String.valueOf(waitTime));
172+
k8sControlNodeConfig = k8sControlNodeConfig.replace(installReattemptsCount, String.valueOf(reattempts));
166173
k8sControlNodeConfig = k8sControlNodeConfig.replace(sshPubKey, pubKey);
167174
k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterToken, KubernetesClusterUtil.generateClusterToken(kubernetesCluster));
168175
String initArgs = "";
@@ -244,6 +251,11 @@ private String getKubernetesAdditionalControlNodeConfig(final String joinIp, fin
244251
final String sshPubKey = "{{ k8s.ssh.pub.key }}";
245252
final String clusterHACertificateKey = "{{ k8s_control_node.cluster.ha.certificate.key }}";
246253
final String ejectIsoKey = "{{ k8s.eject.iso }}";
254+
final String installWaitTime = "{{ k8s.install.wait.time }}";
255+
final String installReattemptsCount = "{{ k8s.install.reattempts.count }}";
256+
257+
final Long waitTime = KubernetesClusterService.KubernetesControlNodeInstallAttemptWait.value();
258+
final Long reattempts = KubernetesClusterService.KubernetesControlNodeInstallReattempts.value();
247259
String pubKey = "- \"" + configurationDao.getValue("ssh.publickey") + "\"";
248260
String sshKeyPair = kubernetesCluster.getKeyPair();
249261
if (StringUtils.isNotEmpty(sshKeyPair)) {
@@ -252,6 +264,8 @@ private String getKubernetesAdditionalControlNodeConfig(final String joinIp, fin
252264
pubKey += "\n - \"" + sshkp.getPublicKey() + "\"";
253265
}
254266
}
267+
k8sControlNodeConfig = k8sControlNodeConfig.replace(installWaitTime, String.valueOf(waitTime));
268+
k8sControlNodeConfig = k8sControlNodeConfig.replace(installReattemptsCount, String.valueOf(reattempts));
255269
k8sControlNodeConfig = k8sControlNodeConfig.replace(sshPubKey, pubKey);
256270
k8sControlNodeConfig = k8sControlNodeConfig.replace(joinIpKey, joinIp);
257271
k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterTokenKey, KubernetesClusterUtil.generateClusterToken(kubernetesCluster));

‎plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,14 @@ write_files:
4242
ATTEMPT_ONLINE_INSTALL=false
4343
setup_complete=false
4444
45-
OFFLINE_INSTALL_ATTEMPT_SLEEP=15
46-
MAX_OFFLINE_INSTALL_ATTEMPTS=100
45+
OFFLINE_INSTALL_ATTEMPT_SLEEP={{ k8s.install.wait.time }}
46+
MAX_OFFLINE_INSTALL_ATTEMPTS={{ k8s.install.reattempts.count }}
47+
if [[ -z $OFFLINE_INSTALL_ATTEMPT_SLEEP || $OFFLINE_INSTALL_ATTEMPT_SLEEP -eq 0 ]]; then
48+
OFFLINE_INSTALL_ATTEMPT_SLEEP=15
49+
fi
50+
if [[ -z $MAX_OFFLINE_INSTALL_ATTEMPTS || $MAX_OFFLINE_INSTALL_ATTEMPTS -eq 0 ]]; then
51+
MAX_OFFLINE_INSTALL_ATTEMPTS=100
52+
fi
4753
offline_attempts=1
4854
MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3
4955
EJECT_ISO_FROM_OS={{ k8s.eject.iso }}

‎plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,8 +62,14 @@ write_files:
6262
ATTEMPT_ONLINE_INSTALL=false
6363
setup_complete=false
6464
65-
OFFLINE_INSTALL_ATTEMPT_SLEEP=15
66-
MAX_OFFLINE_INSTALL_ATTEMPTS=100
65+
OFFLINE_INSTALL_ATTEMPT_SLEEP={{ k8s.install.wait.time }}
66+
MAX_OFFLINE_INSTALL_ATTEMPTS={{ k8s.install.reattempts.count }}
67+
if [[ -z $OFFLINE_INSTALL_ATTEMPT_SLEEP || $OFFLINE_INSTALL_ATTEMPT_SLEEP -eq 0 ]]; then
68+
OFFLINE_INSTALL_ATTEMPT_SLEEP=15
69+
fi
70+
if [[ -z $MAX_OFFLINE_INSTALL_ATTEMPTS || $MAX_OFFLINE_INSTALL_ATTEMPTS -eq 0 ]]; then
71+
MAX_OFFLINE_INSTALL_ATTEMPTS=100
72+
fi
6773
offline_attempts=1
6874
MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3
6975
EJECT_ISO_FROM_OS={{ k8s.eject.iso }}

‎plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,14 @@ write_files:
4242
ATTEMPT_ONLINE_INSTALL=false
4343
setup_complete=false
4444
45-
OFFLINE_INSTALL_ATTEMPT_SLEEP=30
46-
MAX_OFFLINE_INSTALL_ATTEMPTS=40
45+
OFFLINE_INSTALL_ATTEMPT_SLEEP={{ k8s.install.wait.time }}
46+
MAX_OFFLINE_INSTALL_ATTEMPTS={{ k8s.install.reattempts.count }}
47+
if [[ -z $OFFLINE_INSTALL_ATTEMPT_SLEEP || $OFFLINE_INSTALL_ATTEMPT_SLEEP -eq 0 ]]; then
48+
OFFLINE_INSTALL_ATTEMPT_SLEEP=30
49+
fi
50+
if [[ -z $MAX_OFFLINE_INSTALL_ATTEMPTS || $MAX_OFFLINE_INSTALL_ATTEMPTS -eq 0 ]]; then
51+
MAX_OFFLINE_INSTALL_ATTEMPTS=40
52+
fi
4753
offline_attempts=1
4854
MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3
4955
EJECT_ISO_FROM_OS={{ k8s.eject.iso }}

0 commit comments

Comments
 (0)